From d8ab0e721366e8fdb85ee257a036d431bac2ada5 Mon Sep 17 00:00:00 2001 From: Stas Panasiuk Date: Tue, 28 May 2024 08:56:43 -0400 Subject: [PATCH 01/15] Added editorconfig --- .github/workflows/build_and_test.yml | 2 +- flink-cyber/.editorconfig | 1482 ++++++++++++++++++++++++++ flink-cyber/pom.xml | 28 +- 3 files changed, 1509 insertions(+), 3 deletions(-) create mode 100644 flink-cyber/.editorconfig diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c2dd13f8..9b042f4e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -28,7 +28,7 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Build and Test with Maven - run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B package --file flink-cyber/pom.xml + run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B verify --file flink-cyber/pom.xml # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive - name: Update dependency graph diff --git a/flink-cyber/.editorconfig b/flink-cyber/.editorconfig new file mode 100644 index 00000000..33f83bed --- /dev/null +++ b/flink-cyber/.editorconfig @@ -0,0 +1,1482 @@ +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = false +max_line_length = 120 +tab_width = 4 +ij_continuation_indent_size = 8 +ij_formatter_off_tag = @formatter:off +ij_formatter_on_tag = @formatter:on +ij_formatter_tags_enabled = true +ij_smart_tabs = false +# ij_visual_guides = +ij_wrap_on_typing = false + +[*.css] +ij_css_align_closing_brace_with_properties = false +ij_css_blank_lines_around_nested_selector = 1 +ij_css_blank_lines_between_blocks = 1 +ij_css_block_comment_add_space = false +ij_css_brace_placement = end_of_line +ij_css_enforce_quotes_on_format = false +ij_css_hex_color_long_format = false +ij_css_hex_color_lower_case = false +ij_css_hex_color_short_format = false +ij_css_hex_color_upper_case = false +ij_css_keep_blank_lines_in_code = 2 +ij_css_keep_indents_on_empty_lines = false +ij_css_keep_single_line_blocks = false +ij_css_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow +ij_css_space_after_colon = true +ij_css_space_before_opening_brace = true +ij_css_use_double_quotes = true +ij_css_value_alignment = do_not_align + +[*.java] +ij_java_align_consecutive_assignments = false +ij_java_align_consecutive_variable_declarations = false +ij_java_align_group_field_declarations = false +ij_java_align_multiline_annotation_parameters = false +ij_java_align_multiline_array_initializer_expression = false +ij_java_align_multiline_assignment = false +ij_java_align_multiline_binary_operation = true +ij_java_align_multiline_chained_methods = false +ij_java_align_multiline_deconstruction_list_components = true +ij_java_align_multiline_extends_list = false +ij_java_align_multiline_for = true +ij_java_align_multiline_method_parentheses = false +ij_java_align_multiline_parameters = true +ij_java_align_multiline_parameters_in_calls = false +ij_java_align_multiline_parenthesized_expression = false +ij_java_align_multiline_records = true +ij_java_align_multiline_resources = true +ij_java_align_multiline_ternary_operation = false +ij_java_align_multiline_text_blocks = false +ij_java_align_multiline_throws_list = false +ij_java_align_subsequent_simple_methods = false +ij_java_align_throws_keyword = false +ij_java_align_types_in_multi_catch = true +ij_java_annotation_parameter_wrap = off +ij_java_array_initializer_new_line_after_left_brace = false +ij_java_array_initializer_right_brace_on_new_line = false +ij_java_array_initializer_wrap = off +ij_java_assert_statement_colon_on_next_line = false +ij_java_assert_statement_wrap = off +ij_java_assignment_wrap = off +ij_java_binary_operation_sign_on_next_line = false +ij_java_binary_operation_wrap = off +ij_java_blank_lines_after_anonymous_class_header = 0 +ij_java_blank_lines_after_class_header = 0 +ij_java_blank_lines_after_imports = 1 +ij_java_blank_lines_after_package = 1 +ij_java_blank_lines_around_class = 1 +ij_java_blank_lines_around_field = 0 +ij_java_blank_lines_around_field_in_interface = 0 +ij_java_blank_lines_around_initializer = 1 +ij_java_blank_lines_around_method = 1 +ij_java_blank_lines_around_method_in_interface = 1 +ij_java_blank_lines_before_class_end = 0 +ij_java_blank_lines_before_imports = 1 +ij_java_blank_lines_before_method_body = 0 +ij_java_blank_lines_before_package = 0 +ij_java_block_brace_style = end_of_line +ij_java_block_comment_add_space = false +ij_java_block_comment_at_first_column = true +# ij_java_builder_methods = +ij_java_call_parameters_new_line_after_left_paren = false +ij_java_call_parameters_right_paren_on_new_line = false +ij_java_call_parameters_wrap = off +ij_java_case_statement_on_separate_line = true +ij_java_catch_on_new_line = false +ij_java_class_annotation_wrap = split_into_lines +ij_java_class_brace_style = end_of_line +ij_java_class_count_to_use_import_on_demand = 10 +ij_java_class_names_in_javadoc = 1 +ij_java_deconstruction_list_wrap = normal +ij_java_do_not_indent_top_level_class_members = false +ij_java_do_not_wrap_after_single_annotation = false +ij_java_do_not_wrap_after_single_annotation_in_parameter = false +ij_java_do_while_brace_force = never +ij_java_doc_add_blank_line_after_description = true +ij_java_doc_add_blank_line_after_param_comments = false +ij_java_doc_add_blank_line_after_return = false +ij_java_doc_add_p_tag_on_empty_lines = true +ij_java_doc_align_exception_comments = true +ij_java_doc_align_param_comments = true +ij_java_doc_do_not_wrap_if_one_line = false +ij_java_doc_enable_formatting = true +ij_java_doc_enable_leading_asterisks = true +ij_java_doc_indent_on_continuation = false +ij_java_doc_keep_empty_lines = true +ij_java_doc_keep_empty_parameter_tag = true +ij_java_doc_keep_empty_return_tag = true +ij_java_doc_keep_empty_throws_tag = true +ij_java_doc_keep_invalid_tags = true +ij_java_doc_param_description_on_new_line = false +ij_java_doc_preserve_line_breaks = false +ij_java_doc_use_throws_not_exception_tag = true +ij_java_else_on_new_line = false +# ij_java_entity_dd_prefix = +ij_java_entity_dd_suffix = EJB +# ij_java_entity_eb_prefix = +ij_java_entity_eb_suffix = Bean +# ij_java_entity_hi_prefix = +ij_java_entity_hi_suffix = Home +ij_java_entity_lhi_prefix = Local +ij_java_entity_lhi_suffix = Home +ij_java_entity_li_prefix = Local +# ij_java_entity_li_suffix = +ij_java_entity_pk_class = java.lang.String +# ij_java_entity_ri_prefix = +# ij_java_entity_ri_suffix = +# ij_java_entity_vo_prefix = +ij_java_entity_vo_suffix = VO +ij_java_enum_constants_wrap = off +ij_java_enum_field_annotation_wrap = off +ij_java_extends_keyword_wrap = off +ij_java_extends_list_wrap = off +ij_java_field_annotation_wrap = split_into_lines +# ij_java_field_name_prefix = +# ij_java_field_name_suffix = +# ij_java_filter_class_prefix = +# ij_java_filter_class_suffix = +# ij_java_filter_dd_prefix = +# ij_java_filter_dd_suffix = +ij_java_finally_on_new_line = false +ij_java_for_brace_force = never +ij_java_for_statement_new_line_after_left_paren = false +ij_java_for_statement_right_paren_on_new_line = false +ij_java_for_statement_wrap = off +ij_java_generate_final_locals = false +ij_java_generate_final_parameters = false +ij_java_if_brace_force = never +ij_java_imports_layout = *,|,javax.**,java.**,|,$* +ij_java_indent_case_from_switch = true +ij_java_insert_inner_class_imports = false +ij_java_insert_override_annotation = true +ij_java_keep_blank_lines_before_right_brace = 2 +ij_java_keep_blank_lines_between_package_declaration_and_header = 2 +ij_java_keep_blank_lines_in_code = 2 +ij_java_keep_blank_lines_in_declarations = 2 +ij_java_keep_builder_methods_indents = false +ij_java_keep_control_statement_in_one_line = true +ij_java_keep_first_column_comment = true +ij_java_keep_indents_on_empty_lines = false +ij_java_keep_line_breaks = true +ij_java_keep_multiple_expressions_in_one_line = false +ij_java_keep_simple_blocks_in_one_line = false +ij_java_keep_simple_classes_in_one_line = false +ij_java_keep_simple_lambdas_in_one_line = false +ij_java_keep_simple_methods_in_one_line = false +ij_java_label_indent_absolute = false +ij_java_label_indent_size = 0 +ij_java_lambda_brace_style = end_of_line +ij_java_layout_static_imports_separately = true +ij_java_line_comment_add_space = false +ij_java_line_comment_add_space_on_reformat = false +ij_java_line_comment_at_first_column = true +# ij_java_listener_class_prefix = +# ij_java_listener_class_suffix = +# ij_java_local_variable_name_prefix = +# ij_java_local_variable_name_suffix = +# ij_java_message_dd_prefix = +ij_java_message_dd_suffix = EJB +# ij_java_message_eb_prefix = +ij_java_message_eb_suffix = Bean +ij_java_method_annotation_wrap = split_into_lines +ij_java_method_brace_style = end_of_line +ij_java_method_call_chain_wrap = off +ij_java_method_parameters_new_line_after_left_paren = false +ij_java_method_parameters_right_paren_on_new_line = false +ij_java_method_parameters_wrap = off +ij_java_modifier_list_wrap = false +ij_java_multi_catch_types_wrap = normal +ij_java_names_count_to_use_import_on_demand = 5 +ij_java_new_line_after_lparen_in_annotation = false +ij_java_new_line_after_lparen_in_deconstruction_pattern = true +ij_java_new_line_after_lparen_in_record_header = false +ij_java_new_line_when_body_is_presented = false +ij_java_packages_to_use_import_on_demand = java.awt.*,javax.swing.* +ij_java_parameter_annotation_wrap = off +# ij_java_parameter_name_prefix = +# ij_java_parameter_name_suffix = +ij_java_parentheses_expression_new_line_after_left_paren = false +ij_java_parentheses_expression_right_paren_on_new_line = false +ij_java_place_assignment_sign_on_next_line = false +ij_java_prefer_longer_names = true +ij_java_prefer_parameters_wrap = false +ij_java_record_components_wrap = normal +# ij_java_repeat_annotations = +ij_java_repeat_synchronized = true +ij_java_replace_instanceof_and_cast = false +ij_java_replace_null_check = true +ij_java_replace_sum_lambda_with_method_ref = true +ij_java_resource_list_new_line_after_left_paren = false +ij_java_resource_list_right_paren_on_new_line = false +ij_java_resource_list_wrap = off +ij_java_rparen_on_new_line_in_annotation = false +ij_java_rparen_on_new_line_in_deconstruction_pattern = true +ij_java_rparen_on_new_line_in_record_header = false +# ij_java_servlet_class_prefix = +# ij_java_servlet_class_suffix = +# ij_java_servlet_dd_prefix = +# ij_java_servlet_dd_suffix = +# ij_java_session_dd_prefix = +ij_java_session_dd_suffix = EJB +# ij_java_session_eb_prefix = +ij_java_session_eb_suffix = Bean +# ij_java_session_hi_prefix = +ij_java_session_hi_suffix = Home +ij_java_session_lhi_prefix = Local +ij_java_session_lhi_suffix = Home +ij_java_session_li_prefix = Local +# ij_java_session_li_suffix = +# ij_java_session_ri_prefix = +# ij_java_session_ri_suffix = +# ij_java_session_si_prefix = +ij_java_session_si_suffix = Service +ij_java_space_after_closing_angle_bracket_in_type_argument = false +ij_java_space_after_colon = true +ij_java_space_after_comma = true +ij_java_space_after_comma_in_type_arguments = true +ij_java_space_after_for_semicolon = true +ij_java_space_after_quest = true +ij_java_space_after_type_cast = true +ij_java_space_before_annotation_array_initializer_left_brace = false +ij_java_space_before_annotation_parameter_list = false +ij_java_space_before_array_initializer_left_brace = false +ij_java_space_before_catch_keyword = true +ij_java_space_before_catch_left_brace = true +ij_java_space_before_catch_parentheses = true +ij_java_space_before_class_left_brace = true +ij_java_space_before_colon = true +ij_java_space_before_colon_in_foreach = true +ij_java_space_before_comma = false +ij_java_space_before_deconstruction_list = false +ij_java_space_before_do_left_brace = true +ij_java_space_before_else_keyword = true +ij_java_space_before_else_left_brace = true +ij_java_space_before_finally_keyword = true +ij_java_space_before_finally_left_brace = true +ij_java_space_before_for_left_brace = true +ij_java_space_before_for_parentheses = true +ij_java_space_before_for_semicolon = false +ij_java_space_before_if_left_brace = true +ij_java_space_before_if_parentheses = true +ij_java_space_before_method_call_parentheses = false +ij_java_space_before_method_left_brace = true +ij_java_space_before_method_parentheses = false +ij_java_space_before_opening_angle_bracket_in_type_parameter = false +ij_java_space_before_quest = true +ij_java_space_before_switch_left_brace = true +ij_java_space_before_switch_parentheses = true +ij_java_space_before_synchronized_left_brace = true +ij_java_space_before_synchronized_parentheses = true +ij_java_space_before_try_left_brace = true +ij_java_space_before_try_parentheses = true +ij_java_space_before_type_parameter_list = false +ij_java_space_before_while_keyword = true +ij_java_space_before_while_left_brace = true +ij_java_space_before_while_parentheses = true +ij_java_space_inside_one_line_enum_braces = false +ij_java_space_within_empty_array_initializer_braces = false +ij_java_space_within_empty_method_call_parentheses = false +ij_java_space_within_empty_method_parentheses = false +ij_java_spaces_around_additive_operators = true +ij_java_spaces_around_annotation_eq = true +ij_java_spaces_around_assignment_operators = true +ij_java_spaces_around_bitwise_operators = true +ij_java_spaces_around_equality_operators = true +ij_java_spaces_around_lambda_arrow = true +ij_java_spaces_around_logical_operators = true +ij_java_spaces_around_method_ref_dbl_colon = false +ij_java_spaces_around_multiplicative_operators = true +ij_java_spaces_around_relational_operators = true +ij_java_spaces_around_shift_operators = true +ij_java_spaces_around_type_bounds_in_type_parameters = true +ij_java_spaces_around_unary_operator = false +ij_java_spaces_within_angle_brackets = false +ij_java_spaces_within_annotation_parentheses = false +ij_java_spaces_within_array_initializer_braces = false +ij_java_spaces_within_braces = false +ij_java_spaces_within_brackets = false +ij_java_spaces_within_cast_parentheses = false +ij_java_spaces_within_catch_parentheses = false +ij_java_spaces_within_deconstruction_list = false +ij_java_spaces_within_for_parentheses = false +ij_java_spaces_within_if_parentheses = false +ij_java_spaces_within_method_call_parentheses = false +ij_java_spaces_within_method_parentheses = false +ij_java_spaces_within_parentheses = false +ij_java_spaces_within_record_header = false +ij_java_spaces_within_switch_parentheses = false +ij_java_spaces_within_synchronized_parentheses = false +ij_java_spaces_within_try_parentheses = false +ij_java_spaces_within_while_parentheses = false +ij_java_special_else_if_treatment = true +# ij_java_static_field_name_prefix = +# ij_java_static_field_name_suffix = +# ij_java_subclass_name_prefix = +ij_java_subclass_name_suffix = Impl +ij_java_switch_expressions_wrap = normal +ij_java_ternary_operation_signs_on_next_line = false +ij_java_ternary_operation_wrap = off +# ij_java_test_name_prefix = +ij_java_test_name_suffix = Test +ij_java_throws_keyword_wrap = off +ij_java_throws_list_wrap = off +ij_java_use_external_annotations = false +ij_java_use_fq_class_names = false +ij_java_use_relative_indents = false +ij_java_use_single_class_imports = true +ij_java_variable_annotation_wrap = off +ij_java_visibility = public +ij_java_while_brace_force = never +ij_java_while_on_new_line = false +ij_java_wrap_comments = false +ij_java_wrap_first_method_in_call_chain = false +ij_java_wrap_long_lines = false +ij_java_wrap_semicolon_after_call_chain = false + +[*.less] +indent_size = 2 +ij_less_align_closing_brace_with_properties = false +ij_less_blank_lines_around_nested_selector = 1 +ij_less_blank_lines_between_blocks = 1 +ij_less_block_comment_add_space = false +ij_less_brace_placement = 0 +ij_less_enforce_quotes_on_format = false +ij_less_hex_color_long_format = false +ij_less_hex_color_lower_case = false +ij_less_hex_color_short_format = false +ij_less_hex_color_upper_case = false +ij_less_keep_blank_lines_in_code = 2 +ij_less_keep_indents_on_empty_lines = false +ij_less_keep_single_line_blocks = false +ij_less_line_comment_add_space = false +ij_less_line_comment_at_first_column = false +ij_less_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow +ij_less_space_after_colon = true +ij_less_space_before_opening_brace = true +ij_less_use_double_quotes = true +ij_less_value_alignment = 0 + +[*.proto] +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 4 +ij_protobuf_keep_blank_lines_in_code = 2 +ij_protobuf_keep_indents_on_empty_lines = false +ij_protobuf_keep_line_breaks = true +ij_protobuf_space_after_comma = true +ij_protobuf_space_before_comma = false +ij_protobuf_spaces_around_assignment_operators = true +ij_protobuf_spaces_within_braces = false +ij_protobuf_spaces_within_brackets = false + +[*.sass] +indent_size = 2 +ij_sass_align_closing_brace_with_properties = false +ij_sass_blank_lines_around_nested_selector = 1 +ij_sass_blank_lines_between_blocks = 1 +ij_sass_brace_placement = 0 +ij_sass_enforce_quotes_on_format = false +ij_sass_hex_color_long_format = false +ij_sass_hex_color_lower_case = false +ij_sass_hex_color_short_format = false +ij_sass_hex_color_upper_case = false +ij_sass_keep_blank_lines_in_code = 2 +ij_sass_keep_indents_on_empty_lines = false +ij_sass_keep_single_line_blocks = false +ij_sass_line_comment_add_space = false +ij_sass_line_comment_at_first_column = false +ij_sass_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow +ij_sass_space_after_colon = true +ij_sass_space_before_opening_brace = true +ij_sass_use_double_quotes = true +ij_sass_value_alignment = 0 + +[*.scala] +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 2 +ij_scala_align_composite_pattern = true +ij_scala_align_extends_with = 0 +ij_scala_align_group_field_declarations = false +ij_scala_align_if_else = false +ij_scala_align_in_columns_case_branch = false +ij_scala_align_multiline_binary_operation = false +ij_scala_align_multiline_chained_methods = false +ij_scala_align_multiline_for = true +ij_scala_align_multiline_parameters = true +ij_scala_align_multiline_parameters_in_calls = false +ij_scala_align_multiline_parenthesized_expression = false +ij_scala_align_parameter_types_in_multiline_declarations = 0 +ij_scala_align_tuple_elements = false +ij_scala_alternate_continuation_indent_for_params = 4 +ij_scala_binary_operation_wrap = off +ij_scala_blank_lines_after_anonymous_class_header = 0 +ij_scala_blank_lines_after_class_header = 0 +ij_scala_blank_lines_after_imports = 1 +ij_scala_blank_lines_after_package = 1 +ij_scala_blank_lines_around_class = 1 +ij_scala_blank_lines_around_class_in_inner_scopes = 0 +ij_scala_blank_lines_around_field = 0 +ij_scala_blank_lines_around_field_in_inner_scopes = 0 +ij_scala_blank_lines_around_field_in_interface = 0 +ij_scala_blank_lines_around_method = 1 +ij_scala_blank_lines_around_method_in_inner_scopes = 1 +ij_scala_blank_lines_around_method_in_interface = 1 +ij_scala_blank_lines_before_class_end = 0 +ij_scala_blank_lines_before_imports = 1 +ij_scala_blank_lines_before_method_body = 0 +ij_scala_blank_lines_before_package = 0 +ij_scala_block_brace_style = end_of_line +ij_scala_block_comment_add_space = false +ij_scala_block_comment_at_first_column = true +ij_scala_call_parameters_new_line_after_lparen = 0 +ij_scala_call_parameters_right_paren_on_new_line = false +ij_scala_call_parameters_wrap = off +ij_scala_case_clause_brace_force = never +ij_scala_catch_on_new_line = false +ij_scala_class_annotation_wrap = split_into_lines +ij_scala_class_brace_style = end_of_line +ij_scala_closure_brace_force = never +ij_scala_do_not_align_block_expr_params = true +ij_scala_do_not_indent_case_clause_body = false +ij_scala_do_not_indent_tuples_close_brace = true +ij_scala_do_while_brace_force = never +ij_scala_else_on_new_line = false +ij_scala_enable_scaladoc_formatting = true +ij_scala_enforce_functional_syntax_for_unit = true +ij_scala_extends_keyword_wrap = off +ij_scala_extends_list_wrap = off +ij_scala_field_annotation_wrap = split_into_lines +ij_scala_finally_brace_force = never +ij_scala_finally_on_new_line = false +ij_scala_for_brace_force = never +ij_scala_for_statement_wrap = off +ij_scala_formatter = 0 +ij_scala_if_brace_force = never +# ij_scala_implicit_value_class_prefix = +ij_scala_implicit_value_class_suffix = Ops +ij_scala_indent_braced_function_args = true +ij_scala_indent_case_from_switch = true +ij_scala_indent_fewer_braces_in_method_call_chains = false +ij_scala_indent_first_parameter = true +ij_scala_indent_first_parameter_clause = false +ij_scala_indent_type_arguments = true +ij_scala_indent_type_parameters = true +ij_scala_indent_yield_after_one_line_enumerators = true +ij_scala_keep_blank_lines_before_right_brace = 2 +ij_scala_keep_blank_lines_in_code = 2 +ij_scala_keep_blank_lines_in_declarations = 2 +ij_scala_keep_comments_on_same_line = true +ij_scala_keep_first_column_comment = false +ij_scala_keep_indents_on_empty_lines = false +ij_scala_keep_line_breaks = true +ij_scala_keep_one_line_lambdas_in_arg_list = false +ij_scala_keep_simple_blocks_in_one_line = false +ij_scala_keep_simple_methods_in_one_line = false +ij_scala_keep_xml_formatting = false +ij_scala_line_comment_add_space = false +ij_scala_line_comment_at_first_column = true +ij_scala_method_annotation_wrap = split_into_lines +ij_scala_method_brace_force = never +ij_scala_method_brace_style = end_of_line +ij_scala_method_call_chain_wrap = off +ij_scala_method_parameters_new_line_after_left_paren = false +ij_scala_method_parameters_right_paren_on_new_line = false +ij_scala_method_parameters_wrap = off +ij_scala_modifier_list_wrap = false +ij_scala_multiline_string_align_dangling_closing_quotes = false +ij_scala_multiline_string_closing_quotes_on_new_line = false +ij_scala_multiline_string_insert_margin_on_enter = true +ij_scala_multiline_string_margin_char = | +ij_scala_multiline_string_margin_indent = 2 +ij_scala_multiline_string_opening_quotes_on_new_line = true +ij_scala_multiline_string_process_margin_on_copy_paste = true +ij_scala_new_line_after_case_clause_arrow_when_multiline_body = false +ij_scala_newline_after_annotations = false +ij_scala_not_continuation_indent_for_params = false +ij_scala_parameter_annotation_wrap = off +ij_scala_parentheses_expression_new_line_after_left_paren = false +ij_scala_parentheses_expression_right_paren_on_new_line = false +ij_scala_place_closure_parameters_on_new_line = false +ij_scala_place_self_type_on_new_line = true +ij_scala_prefer_parameters_wrap = false +ij_scala_preserve_space_after_method_declaration_name = false +ij_scala_reformat_on_compile = false +ij_scala_replace_case_arrow_with_unicode_char = false +ij_scala_replace_for_generator_arrow_with_unicode_char = false +ij_scala_replace_lambda_with_greek_letter = false +ij_scala_replace_map_arrow_with_unicode_char = false +# ij_scala_scalafmt_config_path = +ij_scala_scalafmt_fallback_to_default_settings = false +ij_scala_scalafmt_reformat_on_files_save = false +ij_scala_scalafmt_show_invalid_code_warnings = true +ij_scala_scalafmt_use_intellij_formatter_for_range_format = true +ij_scala_sd_align_exception_comments = true +ij_scala_sd_align_list_item_content = true +ij_scala_sd_align_other_tags_comments = true +ij_scala_sd_align_parameters_comments = true +ij_scala_sd_align_return_comments = true +ij_scala_sd_blank_line_after_parameters_comments = false +ij_scala_sd_blank_line_after_return_comments = false +ij_scala_sd_blank_line_before_parameters = false +ij_scala_sd_blank_line_before_tags = true +ij_scala_sd_blank_line_between_parameters = false +ij_scala_sd_keep_blank_lines_between_tags = false +ij_scala_sd_preserve_spaces_in_tags = false +ij_scala_space_after_comma = true +ij_scala_space_after_for_semicolon = true +ij_scala_space_after_modifiers_constructor = false +ij_scala_space_after_type_colon = true +ij_scala_space_before_brace_method_call = true +ij_scala_space_before_class_left_brace = true +ij_scala_space_before_for_parentheses = true +ij_scala_space_before_if_parentheses = true +ij_scala_space_before_infix_like_method_parentheses = false +ij_scala_space_before_infix_method_call_parentheses = false +ij_scala_space_before_infix_operator_like_method_call_parentheses = true +ij_scala_space_before_method_call_parentheses = false +ij_scala_space_before_method_left_brace = true +ij_scala_space_before_method_parentheses = false +ij_scala_space_before_type_colon = false +ij_scala_space_before_type_parameter_in_def_list = false +ij_scala_space_before_type_parameter_leading_context_bound_colon = false +ij_scala_space_before_type_parameter_leading_context_bound_colon_hk = true +ij_scala_space_before_type_parameter_list = false +ij_scala_space_before_type_parameter_rest_context_bound_colons = true +ij_scala_space_before_while_parentheses = true +ij_scala_space_inside_closure_braces = true +ij_scala_space_inside_self_type_braces = true +ij_scala_space_within_empty_method_call_parentheses = false +ij_scala_spaces_around_at_in_patterns = false +ij_scala_spaces_in_imports = false +ij_scala_spaces_in_one_line_blocks = false +ij_scala_spaces_within_brackets = false +ij_scala_spaces_within_for_parentheses = false +ij_scala_spaces_within_if_parentheses = false +ij_scala_spaces_within_method_call_parentheses = false +ij_scala_spaces_within_method_parentheses = false +ij_scala_spaces_within_parentheses = false +ij_scala_spaces_within_while_parentheses = false +ij_scala_special_else_if_treatment = true +ij_scala_trailing_comma_arg_list_enabled = true +ij_scala_trailing_comma_import_selector_enabled = false +ij_scala_trailing_comma_mode = trailing_comma_keep +ij_scala_trailing_comma_params_enabled = true +ij_scala_trailing_comma_pattern_arg_list_enabled = false +ij_scala_trailing_comma_tuple_enabled = false +ij_scala_trailing_comma_tuple_type_enabled = false +ij_scala_trailing_comma_type_params_enabled = false +ij_scala_try_brace_force = never +ij_scala_type_annotation_exclude_constant = true +ij_scala_type_annotation_exclude_in_dialect_sources = true +ij_scala_type_annotation_exclude_in_test_sources = false +ij_scala_type_annotation_exclude_member_of_anonymous_class = false +ij_scala_type_annotation_exclude_member_of_private_class = false +ij_scala_type_annotation_exclude_when_type_is_stable = true +ij_scala_type_annotation_function_parameter = false +ij_scala_type_annotation_implicit_modifier = true +ij_scala_type_annotation_local_definition = false +ij_scala_type_annotation_private_member = false +ij_scala_type_annotation_protected_member = true +ij_scala_type_annotation_public_member = true +ij_scala_type_annotation_structural_type = true +ij_scala_type_annotation_underscore_parameter = false +ij_scala_type_annotation_unit_type = true +ij_scala_use_alternate_continuation_indent_for_params = false +ij_scala_use_scala3_indentation_based_syntax = true +ij_scala_use_scaladoc2_formatting = false +ij_scala_variable_annotation_wrap = off +ij_scala_while_brace_force = never +ij_scala_while_on_new_line = false +ij_scala_wrap_before_with_keyword = false +ij_scala_wrap_first_method_in_call_chain = false +ij_scala_wrap_long_lines = false + +[*.scss] +indent_size = 2 +ij_scss_align_closing_brace_with_properties = false +ij_scss_blank_lines_around_nested_selector = 1 +ij_scss_blank_lines_between_blocks = 1 +ij_scss_block_comment_add_space = false +ij_scss_brace_placement = 0 +ij_scss_enforce_quotes_on_format = false +ij_scss_hex_color_long_format = false +ij_scss_hex_color_lower_case = false +ij_scss_hex_color_short_format = false +ij_scss_hex_color_upper_case = false +ij_scss_keep_blank_lines_in_code = 2 +ij_scss_keep_indents_on_empty_lines = false +ij_scss_keep_single_line_blocks = false +ij_scss_line_comment_add_space = false +ij_scss_line_comment_at_first_column = false +ij_scss_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow +ij_scss_space_after_colon = true +ij_scss_space_before_opening_brace = true +ij_scss_use_double_quotes = true +ij_scss_value_alignment = 0 + +[*.vue] +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 4 +ij_vue_indent_children_of_top_level = template +ij_vue_interpolation_new_line_after_start_delimiter = true +ij_vue_interpolation_new_line_before_end_delimiter = true +ij_vue_interpolation_wrap = off +ij_vue_keep_indents_on_empty_lines = false +ij_vue_spaces_within_interpolation_expressions = true + +[.editorconfig] +ij_editorconfig_align_group_field_declarations = false +ij_editorconfig_space_after_colon = false +ij_editorconfig_space_after_comma = true +ij_editorconfig_space_before_colon = false +ij_editorconfig_space_before_comma = false +ij_editorconfig_spaces_around_assignment_operators = true + +[{*.ant,*.fxml,*.jhm,*.jnlp,*.jrxml,*.pom,*.rng,*.tld,*.wadl,*.wsdl,*.xml,*.xsd,*.xsl,*.xslt,*.xul}] +ij_xml_align_attributes = true +ij_xml_align_text = false +ij_xml_attribute_wrap = normal +ij_xml_block_comment_add_space = false +ij_xml_block_comment_at_first_column = true +ij_xml_keep_blank_lines = 2 +ij_xml_keep_indents_on_empty_lines = false +ij_xml_keep_line_breaks = true +ij_xml_keep_line_breaks_in_text = true +ij_xml_keep_whitespaces = false +ij_xml_keep_whitespaces_around_cdata = preserve +ij_xml_keep_whitespaces_inside_cdata = false +ij_xml_line_comment_at_first_column = true +ij_xml_space_after_tag_name = false +ij_xml_space_around_equals_in_attribute = false +ij_xml_space_inside_empty_tag = false +ij_xml_text_wrap = normal + +[{*.ats,*.cts,*.mts,*.ts}] +ij_continuation_indent_size = 4 +ij_typescript_align_imports = false +ij_typescript_align_multiline_array_initializer_expression = false +ij_typescript_align_multiline_binary_operation = false +ij_typescript_align_multiline_chained_methods = false +ij_typescript_align_multiline_extends_list = false +ij_typescript_align_multiline_for = true +ij_typescript_align_multiline_parameters = true +ij_typescript_align_multiline_parameters_in_calls = false +ij_typescript_align_multiline_ternary_operation = false +ij_typescript_align_object_properties = 0 +ij_typescript_align_union_types = false +ij_typescript_align_var_statements = 0 +ij_typescript_array_initializer_new_line_after_left_brace = false +ij_typescript_array_initializer_right_brace_on_new_line = false +ij_typescript_array_initializer_wrap = off +ij_typescript_assignment_wrap = off +ij_typescript_binary_operation_sign_on_next_line = false +ij_typescript_binary_operation_wrap = off +ij_typescript_blacklist_imports = rxjs/Rx,node_modules/**,**/node_modules/**,@angular/material,@angular/material/typings/** +ij_typescript_blank_lines_after_imports = 1 +ij_typescript_blank_lines_around_class = 1 +ij_typescript_blank_lines_around_field = 0 +ij_typescript_blank_lines_around_field_in_interface = 0 +ij_typescript_blank_lines_around_function = 1 +ij_typescript_blank_lines_around_method = 1 +ij_typescript_blank_lines_around_method_in_interface = 1 +ij_typescript_block_brace_style = end_of_line +ij_typescript_block_comment_add_space = false +ij_typescript_block_comment_at_first_column = true +ij_typescript_call_parameters_new_line_after_left_paren = false +ij_typescript_call_parameters_right_paren_on_new_line = false +ij_typescript_call_parameters_wrap = off +ij_typescript_catch_on_new_line = false +ij_typescript_chained_call_dot_on_new_line = true +ij_typescript_class_brace_style = end_of_line +ij_typescript_comma_on_new_line = false +ij_typescript_do_while_brace_force = never +ij_typescript_else_on_new_line = false +ij_typescript_enforce_trailing_comma = keep +ij_typescript_enum_constants_wrap = on_every_item +ij_typescript_extends_keyword_wrap = off +ij_typescript_extends_list_wrap = off +ij_typescript_field_prefix = _ +ij_typescript_file_name_style = relaxed +ij_typescript_finally_on_new_line = false +ij_typescript_for_brace_force = never +ij_typescript_for_statement_new_line_after_left_paren = false +ij_typescript_for_statement_right_paren_on_new_line = false +ij_typescript_for_statement_wrap = off +ij_typescript_force_quote_style = false +ij_typescript_force_semicolon_style = false +ij_typescript_function_expression_brace_style = end_of_line +ij_typescript_if_brace_force = never +ij_typescript_import_merge_members = global +ij_typescript_import_prefer_absolute_path = global +ij_typescript_import_sort_members = true +ij_typescript_import_sort_module_name = false +ij_typescript_import_use_node_resolution = true +ij_typescript_imports_wrap = on_every_item +ij_typescript_indent_case_from_switch = true +ij_typescript_indent_chained_calls = true +ij_typescript_indent_package_children = 0 +ij_typescript_jsdoc_include_types = false +ij_typescript_jsx_attribute_value = braces +ij_typescript_keep_blank_lines_in_code = 2 +ij_typescript_keep_first_column_comment = true +ij_typescript_keep_indents_on_empty_lines = false +ij_typescript_keep_line_breaks = true +ij_typescript_keep_simple_blocks_in_one_line = false +ij_typescript_keep_simple_methods_in_one_line = false +ij_typescript_line_comment_add_space = true +ij_typescript_line_comment_at_first_column = false +ij_typescript_method_brace_style = end_of_line +ij_typescript_method_call_chain_wrap = off +ij_typescript_method_parameters_new_line_after_left_paren = false +ij_typescript_method_parameters_right_paren_on_new_line = false +ij_typescript_method_parameters_wrap = off +ij_typescript_object_literal_wrap = on_every_item +ij_typescript_object_types_wrap = on_every_item +ij_typescript_parentheses_expression_new_line_after_left_paren = false +ij_typescript_parentheses_expression_right_paren_on_new_line = false +ij_typescript_place_assignment_sign_on_next_line = false +ij_typescript_prefer_as_type_cast = false +ij_typescript_prefer_explicit_types_function_expression_returns = false +ij_typescript_prefer_explicit_types_function_returns = false +ij_typescript_prefer_explicit_types_vars_fields = false +ij_typescript_prefer_parameters_wrap = false +# ij_typescript_property_prefix = +ij_typescript_reformat_c_style_comments = false +ij_typescript_space_after_colon = true +ij_typescript_space_after_comma = true +ij_typescript_space_after_dots_in_rest_parameter = false +ij_typescript_space_after_generator_mult = true +ij_typescript_space_after_property_colon = true +ij_typescript_space_after_quest = true +ij_typescript_space_after_type_colon = true +ij_typescript_space_after_unary_not = false +ij_typescript_space_before_async_arrow_lparen = true +ij_typescript_space_before_catch_keyword = true +ij_typescript_space_before_catch_left_brace = true +ij_typescript_space_before_catch_parentheses = true +ij_typescript_space_before_class_lbrace = true +ij_typescript_space_before_class_left_brace = true +ij_typescript_space_before_colon = true +ij_typescript_space_before_comma = false +ij_typescript_space_before_do_left_brace = true +ij_typescript_space_before_else_keyword = true +ij_typescript_space_before_else_left_brace = true +ij_typescript_space_before_finally_keyword = true +ij_typescript_space_before_finally_left_brace = true +ij_typescript_space_before_for_left_brace = true +ij_typescript_space_before_for_parentheses = true +ij_typescript_space_before_for_semicolon = false +ij_typescript_space_before_function_left_parenth = true +ij_typescript_space_before_generator_mult = false +ij_typescript_space_before_if_left_brace = true +ij_typescript_space_before_if_parentheses = true +ij_typescript_space_before_method_call_parentheses = false +ij_typescript_space_before_method_left_brace = true +ij_typescript_space_before_method_parentheses = false +ij_typescript_space_before_property_colon = false +ij_typescript_space_before_quest = true +ij_typescript_space_before_switch_left_brace = true +ij_typescript_space_before_switch_parentheses = true +ij_typescript_space_before_try_left_brace = true +ij_typescript_space_before_type_colon = false +ij_typescript_space_before_unary_not = false +ij_typescript_space_before_while_keyword = true +ij_typescript_space_before_while_left_brace = true +ij_typescript_space_before_while_parentheses = true +ij_typescript_spaces_around_additive_operators = true +ij_typescript_spaces_around_arrow_function_operator = true +ij_typescript_spaces_around_assignment_operators = true +ij_typescript_spaces_around_bitwise_operators = true +ij_typescript_spaces_around_equality_operators = true +ij_typescript_spaces_around_logical_operators = true +ij_typescript_spaces_around_multiplicative_operators = true +ij_typescript_spaces_around_relational_operators = true +ij_typescript_spaces_around_shift_operators = true +ij_typescript_spaces_around_unary_operator = false +ij_typescript_spaces_within_array_initializer_brackets = false +ij_typescript_spaces_within_brackets = false +ij_typescript_spaces_within_catch_parentheses = false +ij_typescript_spaces_within_for_parentheses = false +ij_typescript_spaces_within_if_parentheses = false +ij_typescript_spaces_within_imports = false +ij_typescript_spaces_within_interpolation_expressions = false +ij_typescript_spaces_within_method_call_parentheses = false +ij_typescript_spaces_within_method_parentheses = false +ij_typescript_spaces_within_object_literal_braces = false +ij_typescript_spaces_within_object_type_braces = true +ij_typescript_spaces_within_parentheses = false +ij_typescript_spaces_within_switch_parentheses = false +ij_typescript_spaces_within_type_assertion = false +ij_typescript_spaces_within_union_types = true +ij_typescript_spaces_within_while_parentheses = false +ij_typescript_special_else_if_treatment = true +ij_typescript_ternary_operation_signs_on_next_line = false +ij_typescript_ternary_operation_wrap = off +ij_typescript_union_types_wrap = on_every_item +ij_typescript_use_chained_calls_group_indents = false +ij_typescript_use_double_quotes = true +ij_typescript_use_explicit_js_extension = auto +ij_typescript_use_import_type = auto +ij_typescript_use_path_mapping = always +ij_typescript_use_public_modifier = false +ij_typescript_use_semicolon_after_statement = true +ij_typescript_var_declaration_wrap = normal +ij_typescript_while_brace_force = never +ij_typescript_while_on_new_line = false +ij_typescript_wrap_comments = false + +[{*.bash,*.sh,*.zsh}] +indent_size = 2 +tab_width = 2 +ij_shell_binary_ops_start_line = false +ij_shell_keep_column_alignment_padding = false +ij_shell_minify_program = false +ij_shell_redirect_followed_by_space = false +ij_shell_switch_cases_indented = false +ij_shell_use_unix_line_separator = true + +[{*.cjs,*.js}] +ij_continuation_indent_size = 4 +ij_javascript_align_imports = false +ij_javascript_align_multiline_array_initializer_expression = false +ij_javascript_align_multiline_binary_operation = false +ij_javascript_align_multiline_chained_methods = false +ij_javascript_align_multiline_extends_list = false +ij_javascript_align_multiline_for = true +ij_javascript_align_multiline_parameters = true +ij_javascript_align_multiline_parameters_in_calls = false +ij_javascript_align_multiline_ternary_operation = false +ij_javascript_align_object_properties = 0 +ij_javascript_align_union_types = false +ij_javascript_align_var_statements = 0 +ij_javascript_array_initializer_new_line_after_left_brace = false +ij_javascript_array_initializer_right_brace_on_new_line = false +ij_javascript_array_initializer_wrap = off +ij_javascript_assignment_wrap = off +ij_javascript_binary_operation_sign_on_next_line = false +ij_javascript_binary_operation_wrap = off +ij_javascript_blacklist_imports = rxjs/Rx,node_modules/**,**/node_modules/**,@angular/material,@angular/material/typings/** +ij_javascript_blank_lines_after_imports = 1 +ij_javascript_blank_lines_around_class = 1 +ij_javascript_blank_lines_around_field = 0 +ij_javascript_blank_lines_around_function = 1 +ij_javascript_blank_lines_around_method = 1 +ij_javascript_block_brace_style = end_of_line +ij_javascript_block_comment_add_space = false +ij_javascript_block_comment_at_first_column = true +ij_javascript_call_parameters_new_line_after_left_paren = false +ij_javascript_call_parameters_right_paren_on_new_line = false +ij_javascript_call_parameters_wrap = off +ij_javascript_catch_on_new_line = false +ij_javascript_chained_call_dot_on_new_line = true +ij_javascript_class_brace_style = end_of_line +ij_javascript_comma_on_new_line = false +ij_javascript_do_while_brace_force = never +ij_javascript_else_on_new_line = false +ij_javascript_enforce_trailing_comma = keep +ij_javascript_extends_keyword_wrap = off +ij_javascript_extends_list_wrap = off +ij_javascript_field_prefix = _ +ij_javascript_file_name_style = relaxed +ij_javascript_finally_on_new_line = false +ij_javascript_for_brace_force = never +ij_javascript_for_statement_new_line_after_left_paren = false +ij_javascript_for_statement_right_paren_on_new_line = false +ij_javascript_for_statement_wrap = off +ij_javascript_force_quote_style = false +ij_javascript_force_semicolon_style = false +ij_javascript_function_expression_brace_style = end_of_line +ij_javascript_if_brace_force = never +ij_javascript_import_merge_members = global +ij_javascript_import_prefer_absolute_path = global +ij_javascript_import_sort_members = true +ij_javascript_import_sort_module_name = false +ij_javascript_import_use_node_resolution = true +ij_javascript_imports_wrap = on_every_item +ij_javascript_indent_case_from_switch = true +ij_javascript_indent_chained_calls = true +ij_javascript_indent_package_children = 0 +ij_javascript_jsx_attribute_value = braces +ij_javascript_keep_blank_lines_in_code = 2 +ij_javascript_keep_first_column_comment = true +ij_javascript_keep_indents_on_empty_lines = false +ij_javascript_keep_line_breaks = true +ij_javascript_keep_simple_blocks_in_one_line = false +ij_javascript_keep_simple_methods_in_one_line = false +ij_javascript_line_comment_add_space = true +ij_javascript_line_comment_at_first_column = false +ij_javascript_method_brace_style = end_of_line +ij_javascript_method_call_chain_wrap = off +ij_javascript_method_parameters_new_line_after_left_paren = false +ij_javascript_method_parameters_right_paren_on_new_line = false +ij_javascript_method_parameters_wrap = off +ij_javascript_object_literal_wrap = on_every_item +ij_javascript_object_types_wrap = on_every_item +ij_javascript_parentheses_expression_new_line_after_left_paren = false +ij_javascript_parentheses_expression_right_paren_on_new_line = false +ij_javascript_place_assignment_sign_on_next_line = false +ij_javascript_prefer_as_type_cast = false +ij_javascript_prefer_explicit_types_function_expression_returns = false +ij_javascript_prefer_explicit_types_function_returns = false +ij_javascript_prefer_explicit_types_vars_fields = false +ij_javascript_prefer_parameters_wrap = false +# ij_javascript_property_prefix = +ij_javascript_reformat_c_style_comments = false +ij_javascript_space_after_colon = true +ij_javascript_space_after_comma = true +ij_javascript_space_after_dots_in_rest_parameter = false +ij_javascript_space_after_generator_mult = true +ij_javascript_space_after_property_colon = true +ij_javascript_space_after_quest = true +ij_javascript_space_after_type_colon = true +ij_javascript_space_after_unary_not = false +ij_javascript_space_before_async_arrow_lparen = true +ij_javascript_space_before_catch_keyword = true +ij_javascript_space_before_catch_left_brace = true +ij_javascript_space_before_catch_parentheses = true +ij_javascript_space_before_class_lbrace = true +ij_javascript_space_before_class_left_brace = true +ij_javascript_space_before_colon = true +ij_javascript_space_before_comma = false +ij_javascript_space_before_do_left_brace = true +ij_javascript_space_before_else_keyword = true +ij_javascript_space_before_else_left_brace = true +ij_javascript_space_before_finally_keyword = true +ij_javascript_space_before_finally_left_brace = true +ij_javascript_space_before_for_left_brace = true +ij_javascript_space_before_for_parentheses = true +ij_javascript_space_before_for_semicolon = false +ij_javascript_space_before_function_left_parenth = true +ij_javascript_space_before_generator_mult = false +ij_javascript_space_before_if_left_brace = true +ij_javascript_space_before_if_parentheses = true +ij_javascript_space_before_method_call_parentheses = false +ij_javascript_space_before_method_left_brace = true +ij_javascript_space_before_method_parentheses = false +ij_javascript_space_before_property_colon = false +ij_javascript_space_before_quest = true +ij_javascript_space_before_switch_left_brace = true +ij_javascript_space_before_switch_parentheses = true +ij_javascript_space_before_try_left_brace = true +ij_javascript_space_before_type_colon = false +ij_javascript_space_before_unary_not = false +ij_javascript_space_before_while_keyword = true +ij_javascript_space_before_while_left_brace = true +ij_javascript_space_before_while_parentheses = true +ij_javascript_spaces_around_additive_operators = true +ij_javascript_spaces_around_arrow_function_operator = true +ij_javascript_spaces_around_assignment_operators = true +ij_javascript_spaces_around_bitwise_operators = true +ij_javascript_spaces_around_equality_operators = true +ij_javascript_spaces_around_logical_operators = true +ij_javascript_spaces_around_multiplicative_operators = true +ij_javascript_spaces_around_relational_operators = true +ij_javascript_spaces_around_shift_operators = true +ij_javascript_spaces_around_unary_operator = false +ij_javascript_spaces_within_array_initializer_brackets = false +ij_javascript_spaces_within_brackets = false +ij_javascript_spaces_within_catch_parentheses = false +ij_javascript_spaces_within_for_parentheses = false +ij_javascript_spaces_within_if_parentheses = false +ij_javascript_spaces_within_imports = false +ij_javascript_spaces_within_interpolation_expressions = false +ij_javascript_spaces_within_method_call_parentheses = false +ij_javascript_spaces_within_method_parentheses = false +ij_javascript_spaces_within_object_literal_braces = false +ij_javascript_spaces_within_object_type_braces = true +ij_javascript_spaces_within_parentheses = false +ij_javascript_spaces_within_switch_parentheses = false +ij_javascript_spaces_within_type_assertion = false +ij_javascript_spaces_within_union_types = true +ij_javascript_spaces_within_while_parentheses = false +ij_javascript_special_else_if_treatment = true +ij_javascript_ternary_operation_signs_on_next_line = false +ij_javascript_ternary_operation_wrap = off +ij_javascript_union_types_wrap = on_every_item +ij_javascript_use_chained_calls_group_indents = false +ij_javascript_use_double_quotes = true +ij_javascript_use_explicit_js_extension = auto +ij_javascript_use_import_type = auto +ij_javascript_use_path_mapping = always +ij_javascript_use_public_modifier = false +ij_javascript_use_semicolon_after_statement = true +ij_javascript_var_declaration_wrap = normal +ij_javascript_while_brace_force = never +ij_javascript_while_on_new_line = false +ij_javascript_wrap_comments = false + +[{*.ft,*.vm,*.vsl}] +ij_vtl_keep_indents_on_empty_lines = false + +[{*.gant,*.groovy,*.gy}] +ij_groovy_align_group_field_declarations = false +ij_groovy_align_multiline_array_initializer_expression = false +ij_groovy_align_multiline_assignment = false +ij_groovy_align_multiline_binary_operation = false +ij_groovy_align_multiline_chained_methods = false +ij_groovy_align_multiline_extends_list = false +ij_groovy_align_multiline_for = true +ij_groovy_align_multiline_list_or_map = true +ij_groovy_align_multiline_method_parentheses = false +ij_groovy_align_multiline_parameters = true +ij_groovy_align_multiline_parameters_in_calls = false +ij_groovy_align_multiline_resources = true +ij_groovy_align_multiline_ternary_operation = false +ij_groovy_align_multiline_throws_list = false +ij_groovy_align_named_args_in_map = true +ij_groovy_align_throws_keyword = false +ij_groovy_array_initializer_new_line_after_left_brace = false +ij_groovy_array_initializer_right_brace_on_new_line = false +ij_groovy_array_initializer_wrap = off +ij_groovy_assert_statement_wrap = off +ij_groovy_assignment_wrap = off +ij_groovy_binary_operation_wrap = off +ij_groovy_blank_lines_after_class_header = 0 +ij_groovy_blank_lines_after_imports = 1 +ij_groovy_blank_lines_after_package = 1 +ij_groovy_blank_lines_around_class = 1 +ij_groovy_blank_lines_around_field = 0 +ij_groovy_blank_lines_around_field_in_interface = 0 +ij_groovy_blank_lines_around_method = 1 +ij_groovy_blank_lines_around_method_in_interface = 1 +ij_groovy_blank_lines_before_imports = 1 +ij_groovy_blank_lines_before_method_body = 0 +ij_groovy_blank_lines_before_package = 0 +ij_groovy_block_brace_style = end_of_line +ij_groovy_block_comment_add_space = false +ij_groovy_block_comment_at_first_column = true +ij_groovy_call_parameters_new_line_after_left_paren = false +ij_groovy_call_parameters_right_paren_on_new_line = false +ij_groovy_call_parameters_wrap = off +ij_groovy_catch_on_new_line = false +ij_groovy_class_annotation_wrap = split_into_lines +ij_groovy_class_brace_style = end_of_line +ij_groovy_class_count_to_use_import_on_demand = 5 +ij_groovy_do_while_brace_force = never +ij_groovy_else_on_new_line = false +ij_groovy_enable_groovydoc_formatting = true +ij_groovy_enum_constants_wrap = off +ij_groovy_extends_keyword_wrap = off +ij_groovy_extends_list_wrap = off +ij_groovy_field_annotation_wrap = split_into_lines +ij_groovy_finally_on_new_line = false +ij_groovy_for_brace_force = never +ij_groovy_for_statement_new_line_after_left_paren = false +ij_groovy_for_statement_right_paren_on_new_line = false +ij_groovy_for_statement_wrap = off +ij_groovy_ginq_general_clause_wrap_policy = 2 +ij_groovy_ginq_having_wrap_policy = 1 +ij_groovy_ginq_indent_having_clause = true +ij_groovy_ginq_indent_on_clause = true +ij_groovy_ginq_on_wrap_policy = 1 +ij_groovy_ginq_space_after_keyword = true +ij_groovy_if_brace_force = never +ij_groovy_import_annotation_wrap = 2 +ij_groovy_imports_layout = *,|,javax.**,java.**,|,$* +ij_groovy_indent_case_from_switch = true +ij_groovy_indent_label_blocks = true +ij_groovy_insert_inner_class_imports = false +ij_groovy_keep_blank_lines_before_right_brace = 2 +ij_groovy_keep_blank_lines_in_code = 2 +ij_groovy_keep_blank_lines_in_declarations = 2 +ij_groovy_keep_control_statement_in_one_line = true +ij_groovy_keep_first_column_comment = true +ij_groovy_keep_indents_on_empty_lines = false +ij_groovy_keep_line_breaks = true +ij_groovy_keep_multiple_expressions_in_one_line = false +ij_groovy_keep_simple_blocks_in_one_line = false +ij_groovy_keep_simple_classes_in_one_line = true +ij_groovy_keep_simple_lambdas_in_one_line = true +ij_groovy_keep_simple_methods_in_one_line = true +ij_groovy_label_indent_absolute = false +ij_groovy_label_indent_size = 0 +ij_groovy_lambda_brace_style = end_of_line +ij_groovy_layout_static_imports_separately = true +ij_groovy_line_comment_add_space = false +ij_groovy_line_comment_add_space_on_reformat = false +ij_groovy_line_comment_at_first_column = true +ij_groovy_method_annotation_wrap = split_into_lines +ij_groovy_method_brace_style = end_of_line +ij_groovy_method_call_chain_wrap = off +ij_groovy_method_parameters_new_line_after_left_paren = false +ij_groovy_method_parameters_right_paren_on_new_line = false +ij_groovy_method_parameters_wrap = off +ij_groovy_modifier_list_wrap = false +ij_groovy_names_count_to_use_import_on_demand = 3 +ij_groovy_packages_to_use_import_on_demand = java.awt.*,javax.swing.* +ij_groovy_parameter_annotation_wrap = off +ij_groovy_parentheses_expression_new_line_after_left_paren = false +ij_groovy_parentheses_expression_right_paren_on_new_line = false +ij_groovy_prefer_parameters_wrap = false +ij_groovy_resource_list_new_line_after_left_paren = false +ij_groovy_resource_list_right_paren_on_new_line = false +ij_groovy_resource_list_wrap = off +ij_groovy_space_after_assert_separator = true +ij_groovy_space_after_colon = true +ij_groovy_space_after_comma = true +ij_groovy_space_after_comma_in_type_arguments = true +ij_groovy_space_after_for_semicolon = true +ij_groovy_space_after_quest = true +ij_groovy_space_after_type_cast = true +ij_groovy_space_before_annotation_parameter_list = false +ij_groovy_space_before_array_initializer_left_brace = false +ij_groovy_space_before_assert_separator = false +ij_groovy_space_before_catch_keyword = true +ij_groovy_space_before_catch_left_brace = true +ij_groovy_space_before_catch_parentheses = true +ij_groovy_space_before_class_left_brace = true +ij_groovy_space_before_closure_left_brace = true +ij_groovy_space_before_colon = true +ij_groovy_space_before_comma = false +ij_groovy_space_before_do_left_brace = true +ij_groovy_space_before_else_keyword = true +ij_groovy_space_before_else_left_brace = true +ij_groovy_space_before_finally_keyword = true +ij_groovy_space_before_finally_left_brace = true +ij_groovy_space_before_for_left_brace = true +ij_groovy_space_before_for_parentheses = true +ij_groovy_space_before_for_semicolon = false +ij_groovy_space_before_if_left_brace = true +ij_groovy_space_before_if_parentheses = true +ij_groovy_space_before_method_call_parentheses = false +ij_groovy_space_before_method_left_brace = true +ij_groovy_space_before_method_parentheses = false +ij_groovy_space_before_quest = true +ij_groovy_space_before_record_parentheses = false +ij_groovy_space_before_switch_left_brace = true +ij_groovy_space_before_switch_parentheses = true +ij_groovy_space_before_synchronized_left_brace = true +ij_groovy_space_before_synchronized_parentheses = true +ij_groovy_space_before_try_left_brace = true +ij_groovy_space_before_try_parentheses = true +ij_groovy_space_before_while_keyword = true +ij_groovy_space_before_while_left_brace = true +ij_groovy_space_before_while_parentheses = true +ij_groovy_space_in_named_argument = true +ij_groovy_space_in_named_argument_before_colon = false +ij_groovy_space_within_empty_array_initializer_braces = false +ij_groovy_space_within_empty_method_call_parentheses = false +ij_groovy_spaces_around_additive_operators = true +ij_groovy_spaces_around_assignment_operators = true +ij_groovy_spaces_around_bitwise_operators = true +ij_groovy_spaces_around_equality_operators = true +ij_groovy_spaces_around_lambda_arrow = true +ij_groovy_spaces_around_logical_operators = true +ij_groovy_spaces_around_multiplicative_operators = true +ij_groovy_spaces_around_regex_operators = true +ij_groovy_spaces_around_relational_operators = true +ij_groovy_spaces_around_shift_operators = true +ij_groovy_spaces_within_annotation_parentheses = false +ij_groovy_spaces_within_array_initializer_braces = false +ij_groovy_spaces_within_braces = true +ij_groovy_spaces_within_brackets = false +ij_groovy_spaces_within_cast_parentheses = false +ij_groovy_spaces_within_catch_parentheses = false +ij_groovy_spaces_within_for_parentheses = false +ij_groovy_spaces_within_gstring_injection_braces = false +ij_groovy_spaces_within_if_parentheses = false +ij_groovy_spaces_within_list_or_map = false +ij_groovy_spaces_within_method_call_parentheses = false +ij_groovy_spaces_within_method_parentheses = false +ij_groovy_spaces_within_parentheses = false +ij_groovy_spaces_within_switch_parentheses = false +ij_groovy_spaces_within_synchronized_parentheses = false +ij_groovy_spaces_within_try_parentheses = false +ij_groovy_spaces_within_tuple_expression = false +ij_groovy_spaces_within_while_parentheses = false +ij_groovy_special_else_if_treatment = true +ij_groovy_ternary_operation_wrap = off +ij_groovy_throws_keyword_wrap = off +ij_groovy_throws_list_wrap = off +ij_groovy_use_flying_geese_braces = false +ij_groovy_use_fq_class_names = false +ij_groovy_use_fq_class_names_in_javadoc = true +ij_groovy_use_relative_indents = false +ij_groovy_use_single_class_imports = true +ij_groovy_variable_annotation_wrap = off +ij_groovy_while_brace_force = never +ij_groovy_while_on_new_line = false +ij_groovy_wrap_chain_calls_after_dot = false +ij_groovy_wrap_long_lines = false + +[{*.har,*.jsb2,*.jsb3,*.json,*.jsonc,*.postman_collection,*.postman_collection.json,*.postman_environment,*.postman_environment.json,.babelrc,.eslintrc,.prettierrc,.stylelintrc,.ws-context,bowerrc,jest.config}] +indent_size = 2 +ij_json_array_wrapping = split_into_lines +ij_json_keep_blank_lines_in_code = 0 +ij_json_keep_indents_on_empty_lines = false +ij_json_keep_line_breaks = true +ij_json_keep_trailing_comma = false +ij_json_object_wrapping = split_into_lines +ij_json_property_alignment = do_not_align +ij_json_space_after_colon = true +ij_json_space_after_comma = true +ij_json_space_before_colon = false +ij_json_space_before_comma = false +ij_json_spaces_within_braces = false +ij_json_spaces_within_brackets = false +ij_json_wrap_long_lines = false + +[{*.htm,*.html,*.sht,*.shtm,*.shtml}] +ij_html_add_new_line_before_tags = body,div,p,form,h1,h2,h3 +ij_html_align_attributes = true +ij_html_align_text = false +ij_html_attribute_wrap = normal +ij_html_block_comment_add_space = false +ij_html_block_comment_at_first_column = true +ij_html_do_not_align_children_of_min_lines = 0 +ij_html_do_not_break_if_inline_tags = title,h1,h2,h3,h4,h5,h6,p +ij_html_do_not_indent_children_of_tags = html,body,thead,tbody,tfoot +ij_html_enforce_quotes = false +ij_html_inline_tags = a,abbr,acronym,b,basefont,bdo,big,br,cite,cite,code,dfn,em,font,i,img,input,kbd,label,q,s,samp,select,small,span,strike,strong,sub,sup,textarea,tt,u,var +ij_html_keep_blank_lines = 2 +ij_html_keep_indents_on_empty_lines = false +ij_html_keep_line_breaks = true +ij_html_keep_line_breaks_in_text = true +ij_html_keep_whitespaces = false +ij_html_keep_whitespaces_inside = span,pre,textarea +ij_html_line_comment_at_first_column = true +ij_html_new_line_after_last_attribute = never +ij_html_new_line_before_first_attribute = never +ij_html_quote_style = double +ij_html_remove_new_line_before_tags = br +ij_html_space_after_tag_name = false +ij_html_space_around_equality_in_attribute = false +ij_html_space_inside_empty_tag = false +ij_html_text_wrap = normal + +[{*.http,*.rest}] +indent_size = 0 +ij_continuation_indent_size = 4 +ij_http-request_call_parameters_wrap = normal +ij_http-request_method_parameters_wrap = split_into_lines +ij_http-request_space_before_comma = true +ij_http-request_spaces_around_assignment_operators = true + +[{*.jsf,*.jsp,*.jspf,*.tag,*.tagf,*.xjsp}] +ij_jsp_jsp_prefer_comma_separated_import_list = false +ij_jsp_keep_indents_on_empty_lines = false + +[{*.jspx,*.tagx}] +ij_jspx_keep_indents_on_empty_lines = false + +[{*.kt,*.kts}] +ij_kotlin_align_in_columns_case_branch = false +ij_kotlin_align_multiline_binary_operation = false +ij_kotlin_align_multiline_extends_list = false +ij_kotlin_align_multiline_method_parentheses = false +ij_kotlin_align_multiline_parameters = true +ij_kotlin_align_multiline_parameters_in_calls = false +ij_kotlin_allow_trailing_comma = false +ij_kotlin_allow_trailing_comma_on_call_site = false +ij_kotlin_assignment_wrap = normal +ij_kotlin_blank_lines_after_class_header = 0 +ij_kotlin_blank_lines_around_block_when_branches = 0 +ij_kotlin_blank_lines_before_declaration_with_comment_or_annotation_on_separate_line = 1 +ij_kotlin_block_comment_add_space = false +ij_kotlin_block_comment_at_first_column = true +ij_kotlin_call_parameters_new_line_after_left_paren = true +ij_kotlin_call_parameters_right_paren_on_new_line = true +ij_kotlin_call_parameters_wrap = on_every_item +ij_kotlin_catch_on_new_line = false +ij_kotlin_class_annotation_wrap = split_into_lines +ij_kotlin_continuation_indent_for_chained_calls = false +ij_kotlin_continuation_indent_for_expression_bodies = false +ij_kotlin_continuation_indent_in_argument_lists = false +ij_kotlin_continuation_indent_in_elvis = false +ij_kotlin_continuation_indent_in_if_conditions = false +ij_kotlin_continuation_indent_in_parameter_lists = false +ij_kotlin_continuation_indent_in_supertype_lists = false +ij_kotlin_else_on_new_line = false +ij_kotlin_enum_constants_wrap = off +ij_kotlin_extends_list_wrap = normal +ij_kotlin_field_annotation_wrap = split_into_lines +ij_kotlin_finally_on_new_line = false +ij_kotlin_if_rparen_on_new_line = true +ij_kotlin_import_nested_classes = false +ij_kotlin_imports_layout = *,java.**,javax.**,kotlin.**,^ +ij_kotlin_insert_whitespaces_in_simple_one_line_method = true +ij_kotlin_keep_blank_lines_before_right_brace = 2 +ij_kotlin_keep_blank_lines_in_code = 2 +ij_kotlin_keep_blank_lines_in_declarations = 2 +ij_kotlin_keep_first_column_comment = true +ij_kotlin_keep_indents_on_empty_lines = false +ij_kotlin_keep_line_breaks = true +ij_kotlin_lbrace_on_next_line = false +ij_kotlin_line_break_after_multiline_when_entry = true +ij_kotlin_line_comment_add_space = false +ij_kotlin_line_comment_add_space_on_reformat = false +ij_kotlin_line_comment_at_first_column = true +ij_kotlin_method_annotation_wrap = split_into_lines +ij_kotlin_method_call_chain_wrap = normal +ij_kotlin_method_parameters_new_line_after_left_paren = true +ij_kotlin_method_parameters_right_paren_on_new_line = true +ij_kotlin_method_parameters_wrap = on_every_item +ij_kotlin_name_count_to_use_star_import = 5 +ij_kotlin_name_count_to_use_star_import_for_members = 3 +ij_kotlin_packages_to_use_import_on_demand = java.util.*,kotlinx.android.synthetic.**,io.ktor.** +ij_kotlin_parameter_annotation_wrap = off +ij_kotlin_space_after_comma = true +ij_kotlin_space_after_extend_colon = true +ij_kotlin_space_after_type_colon = true +ij_kotlin_space_before_catch_parentheses = true +ij_kotlin_space_before_comma = false +ij_kotlin_space_before_extend_colon = true +ij_kotlin_space_before_for_parentheses = true +ij_kotlin_space_before_if_parentheses = true +ij_kotlin_space_before_lambda_arrow = true +ij_kotlin_space_before_type_colon = false +ij_kotlin_space_before_when_parentheses = true +ij_kotlin_space_before_while_parentheses = true +ij_kotlin_spaces_around_additive_operators = true +ij_kotlin_spaces_around_assignment_operators = true +ij_kotlin_spaces_around_equality_operators = true +ij_kotlin_spaces_around_function_type_arrow = true +ij_kotlin_spaces_around_logical_operators = true +ij_kotlin_spaces_around_multiplicative_operators = true +ij_kotlin_spaces_around_range = false +ij_kotlin_spaces_around_relational_operators = true +ij_kotlin_spaces_around_unary_operator = false +ij_kotlin_spaces_around_when_arrow = true +ij_kotlin_variable_annotation_wrap = off +ij_kotlin_while_on_new_line = false +ij_kotlin_wrap_elvis_expressions = 1 +ij_kotlin_wrap_expression_body_functions = 1 +ij_kotlin_wrap_first_method_in_call_chain = false + +[{*.markdown,*.md}] +ij_markdown_force_one_space_after_blockquote_symbol = true +ij_markdown_force_one_space_after_header_symbol = true +ij_markdown_force_one_space_after_list_bullet = true +ij_markdown_force_one_space_between_words = true +ij_markdown_format_tables = true +ij_markdown_insert_quote_arrows_on_wrap = true +ij_markdown_keep_indents_on_empty_lines = false +ij_markdown_keep_line_breaks_inside_text_blocks = true +ij_markdown_max_lines_around_block_elements = 1 +ij_markdown_max_lines_around_header = 1 +ij_markdown_max_lines_between_paragraphs = 1 +ij_markdown_min_lines_around_block_elements = 1 +ij_markdown_min_lines_around_header = 1 +ij_markdown_min_lines_between_paragraphs = 1 +ij_markdown_wrap_text_if_long = true +ij_markdown_wrap_text_inside_blockquotes = true + +[{*.pb,*.textproto,*.txtpb}] +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 4 +ij_prototext_keep_blank_lines_in_code = 2 +ij_prototext_keep_indents_on_empty_lines = false +ij_prototext_keep_line_breaks = true +ij_prototext_space_after_colon = true +ij_prototext_space_after_comma = true +ij_prototext_space_before_colon = false +ij_prototext_space_before_comma = false +ij_prototext_spaces_within_braces = true +ij_prototext_spaces_within_brackets = false + +[{*.properties,spring.handlers,spring.schemas}] +ij_properties_align_group_field_declarations = false +ij_properties_keep_blank_lines = false +ij_properties_key_value_delimiter = equals +ij_properties_spaces_around_key_value_delimiter = false + +[{*.py,*.pyw}] +ij_python_align_collections_and_comprehensions = true +ij_python_align_multiline_imports = true +ij_python_align_multiline_parameters = true +ij_python_align_multiline_parameters_in_calls = true +ij_python_blank_line_at_file_end = true +ij_python_blank_lines_after_imports = 1 +ij_python_blank_lines_after_local_imports = 0 +ij_python_blank_lines_around_class = 1 +ij_python_blank_lines_around_method = 1 +ij_python_blank_lines_around_top_level_classes_functions = 2 +ij_python_blank_lines_before_first_method = 0 +ij_python_call_parameters_new_line_after_left_paren = false +ij_python_call_parameters_right_paren_on_new_line = false +ij_python_call_parameters_wrap = normal +ij_python_dict_alignment = 0 +ij_python_dict_new_line_after_left_brace = false +ij_python_dict_new_line_before_right_brace = false +ij_python_dict_wrapping = 1 +ij_python_from_import_new_line_after_left_parenthesis = false +ij_python_from_import_new_line_before_right_parenthesis = false +ij_python_from_import_parentheses_force_if_multiline = false +ij_python_from_import_trailing_comma_if_multiline = false +ij_python_from_import_wrapping = 1 +ij_python_hang_closing_brackets = false +ij_python_keep_blank_lines_in_code = 1 +ij_python_keep_blank_lines_in_declarations = 1 +ij_python_keep_indents_on_empty_lines = false +ij_python_keep_line_breaks = true +ij_python_method_parameters_new_line_after_left_paren = false +ij_python_method_parameters_right_paren_on_new_line = false +ij_python_method_parameters_wrap = normal +ij_python_new_line_after_colon = false +ij_python_new_line_after_colon_multi_clause = true +ij_python_optimize_imports_always_split_from_imports = false +ij_python_optimize_imports_case_insensitive_order = false +ij_python_optimize_imports_join_from_imports_with_same_source = false +ij_python_optimize_imports_sort_by_type_first = true +ij_python_optimize_imports_sort_imports = true +ij_python_optimize_imports_sort_names_in_from_imports = false +ij_python_space_after_comma = true +ij_python_space_after_number_sign = true +ij_python_space_after_py_colon = true +ij_python_space_before_backslash = true +ij_python_space_before_comma = false +ij_python_space_before_for_semicolon = false +ij_python_space_before_lbracket = false +ij_python_space_before_method_call_parentheses = false +ij_python_space_before_method_parentheses = false +ij_python_space_before_number_sign = true +ij_python_space_before_py_colon = false +ij_python_space_within_empty_method_call_parentheses = false +ij_python_space_within_empty_method_parentheses = false +ij_python_spaces_around_additive_operators = true +ij_python_spaces_around_assignment_operators = true +ij_python_spaces_around_bitwise_operators = true +ij_python_spaces_around_eq_in_keyword_argument = false +ij_python_spaces_around_eq_in_named_parameter = false +ij_python_spaces_around_equality_operators = true +ij_python_spaces_around_multiplicative_operators = true +ij_python_spaces_around_power_operator = true +ij_python_spaces_around_relational_operators = true +ij_python_spaces_around_shift_operators = true +ij_python_spaces_within_braces = false +ij_python_spaces_within_brackets = false +ij_python_spaces_within_method_call_parentheses = false +ij_python_spaces_within_method_parentheses = false +ij_python_use_continuation_indent_for_arguments = false +ij_python_use_continuation_indent_for_collection_and_comprehensions = false +ij_python_use_continuation_indent_for_parameters = true +ij_python_wrap_long_lines = false + +[{*.qute.htm,*.qute.html,*.qute.json,*.qute.txt,*.qute.yaml,*.qute.yml}] +ij_qute_keep_indents_on_empty_lines = false + +[{*.toml,Cargo.lock,Cargo.toml.orig,Gopkg.lock,Pipfile,poetry.lock}] +ij_toml_keep_indents_on_empty_lines = false + +[{*.yaml,*.yml}] +indent_size = 2 +ij_yaml_align_values_properties = do_not_align +ij_yaml_autoinsert_sequence_marker = true +ij_yaml_block_mapping_on_new_line = false +ij_yaml_indent_sequence_value = true +ij_yaml_keep_indents_on_empty_lines = false +ij_yaml_keep_line_breaks = true +ij_yaml_sequence_on_new_line = false +ij_yaml_space_before_colon = false +ij_yaml_spaces_within_braces = true +ij_yaml_spaces_within_brackets = true diff --git a/flink-cyber/pom.xml b/flink-cyber/pom.xml index 160c8b54..703613b5 100644 --- a/flink-cyber/pom.xml +++ b/flink-cyber/pom.xml @@ -100,7 +100,8 @@ 0.38 3.0.2 4.5.13 - ${project.parent.version}-${flink.version}-cdh${cdh.version}-${timestamp} + ${project.parent.version}-${flink.version}-cdh${cdh.version}-${timestamp} + 0.0.6 7.5 ${maven.build.timestamp} @@ -115,6 +116,7 @@ 21.3.9 4.12 5.7.0 + 0.1.3 @@ -232,7 +234,7 @@ ${flink.version} - + com.github.erosb everit-json-schema @@ -572,6 +574,28 @@ + + + org.ec4j.maven + editorconfig-maven-plugin + ${editorconfig-maven-plugin.version} + + + check + verify + + check + + + + + + src/main/resources/** + src/test/resources/** + + + + From 7399fddfbcb32558b690600036da58915ce5a2d3 Mon Sep 17 00:00:00 2001 From: Stas Panasiuk Date: Tue, 28 May 2024 12:51:30 -0400 Subject: [PATCH 02/15] Code style fixes --- .../src/main/assemblies/cloudera.xml | 301 +++++++++--------- flink-cyber/cyber-parcel/pom.xml | 49 +-- .../flink-enrichment-cidr/pom.xml | 17 +- .../flink-enrichment-combined/pom.xml | 14 +- .../flink-enrichment-geocode/pom.xml | 5 +- .../src/main/assembly/assembly.xml | 80 ++--- flink-cyber/flink-logging/pom.xml | 2 +- flink-cyber/flink-profiler-java/pom.xml | 5 +- flink-cyber/flink-stellar/pom.xml | 7 +- .../src/main/assembly/assembly.xml | 30 +- .../main/assembly/stand-alone-assembly.xml | 84 ++--- .../src/main/assembly/assembly.xml | 78 ++--- flink-cyber/metron-hbase-common/pom.xml | 2 +- flink-cyber/metron-parser-chain/pom.xml | 18 +- flink-cyber/parser-chains-flink/pom.xml | 4 +- flink-cyber/pom.xml | 7 + 16 files changed, 361 insertions(+), 342 deletions(-) diff --git a/flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml b/flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml index 0e05233f..60e9628c 100644 --- a/flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml +++ b/flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml @@ -17,166 +17,171 @@ --> - - cloudera - - tar.gz - + + cloudera + + tar.gz + - true - ${cloudera.tar.name} + true + ${cloudera.tar.name} - - - lib - false - false - false - true - true - - org.slf4j:slf4j-api - org.apache.logging.log4j:log4j-api - org.apache.logging.log4j:log4j-core - org.apache.logging.log4j:log4j-slf4j-impl - - - + + + lib + false + false + false + true + true + + org.slf4j:slf4j-api + org.apache.logging.log4j:log4j-api + org.apache.logging.log4j:log4j-core + org.apache.logging.log4j:log4j-slf4j-impl + + + - - - ../caracal-generator/target/caracal-generator-${project.version}.jar - jobs/ - caracal-generator-${cybersec.full.version}.jar - 0644 - + + + ../caracal-generator/target/caracal-generator-${project.version}.jar + jobs/ + caracal-generator-${cybersec.full.version}.jar + 0644 + - - ../flink-enrichment/flink-enrichment-combined/target/flink-enrichment-combined-${project.version}.jar - jobs/ - flink-enrichment-combined-${cybersec.full.version}.jar - 0644 - + + + ../flink-enrichment/flink-enrichment-combined/target/flink-enrichment-combined-${project.version}.jar + + jobs/ + flink-enrichment-combined-${cybersec.full.version}.jar + 0644 + - - ../flink-enrichment/flink-enrichment-load/target/flink-enrichment-loading-${project.version}.jar - jobs/ - flink-enrichment-loading-${cybersec.full.version}.jar - 0644 - + + ../flink-enrichment/flink-enrichment-load/target/flink-enrichment-loading-${project.version}.jar + + jobs/ + flink-enrichment-loading-${cybersec.full.version}.jar + 0644 + - - ../flink-profiler-java/target/flink-profiler-java-${project.version}.jar - jobs/ - flink-profiler-java-${cybersec.full.version}.jar - 0644 - - - ../parser-chains-flink/target/parser-chains-flink-${project.version}.jar - jobs/ - parser-chains-flink-${cybersec.full.version}.jar - 0644 - + + ../flink-profiler-java/target/flink-profiler-java-${project.version}.jar + jobs/ + flink-profiler-java-${cybersec.full.version}.jar + 0644 + + + ../parser-chains-flink/target/parser-chains-flink-${project.version}.jar + jobs/ + parser-chains-flink-${cybersec.full.version}.jar + 0644 + - - ../flink-indexing/flink-indexing-hive/target/flink-indexing-hive-${project.version}.jar - jobs/ - flink-indexing-hive-${cybersec.full.version}.jar - 0644 - + + ../flink-indexing/flink-indexing-hive/target/flink-indexing-hive-${project.version}.jar + jobs/ + flink-indexing-hive-${cybersec.full.version}.jar + 0644 + - - ../flink-commands/scoring-commands/target/scoring-commands-${project.version}.jar - tools/ - scoring-commands-${cybersec.full.version}.jar - 0644 - + + ../flink-commands/scoring-commands/target/scoring-commands-${project.version}.jar + tools/ + scoring-commands-${cybersec.full.version}.jar + 0644 + - - ../flink-commands/scoring-commands/README.md - tools/ - scoring-commands-README.md - 0644 - + + ../flink-commands/scoring-commands/README.md + tools/ + scoring-commands-README.md + 0644 + - - ../flink-commands/kafka-commands/target/kafka-commands-${project.version}.jar - tools/ - kafka-commands-${cybersec.full.version}.jar - 0644 - + + ../flink-commands/kafka-commands/target/kafka-commands-${project.version}.jar + tools/ + kafka-commands-${cybersec.full.version}.jar + 0644 + - - ../flink-commands/json-commands/target/json-commands-${project.version}.jar - tools/ - json-commands-${cybersec.full.version}.jar - 0644 - + + ../flink-commands/json-commands/target/json-commands-${project.version}.jar + tools/ + json-commands-${cybersec.full.version}.jar + 0644 + - - ../metron-parser-chain/parser-chains-config-service/target/parser-chains-config-service-${project.version}.jar - tools/ - parser-chains-config-service-${project.version}.jar - 0644 - + + + ../metron-parser-chain/parser-chains-config-service/target/parser-chains-config-service-${project.version}.jar + + tools/ + parser-chains-config-service-${project.version}.jar + 0644 + - + - - - src/main/resources/conf - conf - 0644 - - *.json - log4j.properties - - - - src/main/resources/conf/templates - conf/templates - 0644 - - - src/main/resources/conf/templates/generate - conf/templates/generate - 0644 - - - src/main/resources/conf/templates/index - conf/templates/index - 0644 - - - src/main/resources/conf/templates/parse - conf/templates/parse - 0644 - - - src/main/resources/conf/templates/profile - conf/templates/profile - 0644 - - - src/main/resources/conf/templates/triage - conf/templates/triage - 0644 - - - target/classes/scripts - bin - 0755 - - - src/main/resources/conf - tools - 0644 - - sample-rule.json - log4j.properties - - - + + + src/main/resources/conf + conf + 0644 + + *.json + log4j.properties + + + + src/main/resources/conf/templates + conf/templates + 0644 + + + src/main/resources/conf/templates/generate + conf/templates/generate + 0644 + + + src/main/resources/conf/templates/index + conf/templates/index + 0644 + + + src/main/resources/conf/templates/parse + conf/templates/parse + 0644 + + + src/main/resources/conf/templates/profile + conf/templates/profile + 0644 + + + src/main/resources/conf/templates/triage + conf/templates/triage + 0644 + + + target/classes/scripts + bin + 0755 + + + src/main/resources/conf + tools + 0644 + + sample-rule.json + log4j.properties + + + diff --git a/flink-cyber/cyber-parcel/pom.xml b/flink-cyber/cyber-parcel/pom.xml index f4621f36..e8b3396b 100644 --- a/flink-cyber/cyber-parcel/pom.xml +++ b/flink-cyber/cyber-parcel/pom.xml @@ -31,26 +31,27 @@ under the License. cyber-parcel pom - - - - cloudera-internal - http://maven.jenkins.cloudera.com:8081/artifactory/libs-release-local - - true - - - false - - - + + + + cloudera-internal + http://maven.jenkins.cloudera.com:8081/artifactory/libs-release-local + + true + + + false + + + - ${project.parent.basedir} - CYBERSEC-${cybersec.full.version} + ${project.parent.basedir} + CYBERSEC-${cybersec.full.version} ${parcel.basename}.tar.gz ${parcel.basename}-el7.parcel - file://${cybersec.basedir}/cyber-jobs/target/cyber-jobs-${project.parent.version}.tar.gz + file://${cybersec.basedir}/cyber-jobs/target/cyber-jobs-${project.parent.version}.tar.gz + @@ -78,7 +79,7 @@ under the License. ${flink.tar.gz.url} cybersec.dist.tar.gz target/ - true + true @@ -132,8 +133,10 @@ under the License. - - + + @@ -163,10 +166,10 @@ under the License. ${project.build.directory}/${parcel.filename}.sha1 sha1 - - ${project.build.directory}/${parcel.filename}.sha - sha - + + ${project.build.directory}/${parcel.filename}.sha + sha + diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml index d21ad700..17b56286 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml @@ -126,7 +126,7 @@ test - + org.mockito mockito-core 3.3.3 @@ -185,12 +185,12 @@ hibernate-validator 8.0.0.Final - - com.cloudera.parserchains - parser-chains-core - ${project.parent.version} - compile - + + com.cloudera.parserchains + parser-chains-core + ${project.parent.version} + compile + @@ -232,7 +232,8 @@ - + com.cloudera.cyber.enrichment.cidr.IpRegionCidrJobKafka diff --git a/flink-cyber/flink-enrichment/flink-enrichment-combined/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-combined/pom.xml index 4bc90143..2ce7212f 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-combined/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-combined/pom.xml @@ -86,12 +86,12 @@ ${project.parent.version} compile - - com.cloudera.cyber - flink-alert-scoring - ${project.parent.version} - compile - + + com.cloudera.cyber + flink-alert-scoring + ${project.parent.version} + compile + org.apache.flink flink-streaming-java @@ -151,7 +151,7 @@ - + ${main.class} diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml index 2fb92f1e..8f4410fd 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml @@ -120,7 +120,7 @@ test - + org.mockito mockito-core 3.3.3 @@ -202,7 +202,8 @@ - + com.cloudera.cyber.enrichment.geocode.IpGeoJobKafka diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/assembly/assembly.xml b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/assembly/assembly.xml index e6e0f39f..e9a2913d 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/assembly/assembly.xml +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/assembly/assembly.xml @@ -24,44 +24,44 @@ --> - archive - - tar.gz - - false - - - ${project.basedir}/src/main/config - config - true - - **/*.formatted - **/*.filtered - **/*.j2 - - 0644 - unix - true - - - ${project.basedir}/src/main/scripts - bin - true - - **/*.formatted - **/*.filtered - - 0755 - unix - true - - - ${project.basedir}/target - - ${project.artifactId}-${project.version}-uber.jar - - lib - true - - + archive + + tar.gz + + false + + + ${project.basedir}/src/main/config + config + true + + **/*.formatted + **/*.filtered + **/*.j2 + + 0644 + unix + true + + + ${project.basedir}/src/main/scripts + bin + true + + **/*.formatted + **/*.filtered + + 0755 + unix + true + + + ${project.basedir}/target + + ${project.artifactId}-${project.version}-uber.jar + + lib + true + + diff --git a/flink-cyber/flink-logging/pom.xml b/flink-cyber/flink-logging/pom.xml index 47c3eec6..4cb9425e 100644 --- a/flink-cyber/flink-logging/pom.xml +++ b/flink-cyber/flink-logging/pom.xml @@ -74,7 +74,7 @@ shade - + org.apache.flink:force-shading com.google.code.findbugs:jsr305 diff --git a/flink-cyber/flink-profiler-java/pom.xml b/flink-cyber/flink-profiler-java/pom.xml index 424da0b3..9749cf04 100644 --- a/flink-cyber/flink-profiler-java/pom.xml +++ b/flink-cyber/flink-profiler-java/pom.xml @@ -123,7 +123,7 @@ slf4j-api - org.apache.logging.log4j + org.apache.logging.log4j log4j-slf4j-impl @@ -233,7 +233,8 @@ - + com.cloudera.cyber.profiler.ProfileJobKafka diff --git a/flink-cyber/flink-stellar/pom.xml b/flink-cyber/flink-stellar/pom.xml index b153e0a3..b64b11f5 100644 --- a/flink-cyber/flink-stellar/pom.xml +++ b/flink-cyber/flink-stellar/pom.xml @@ -77,7 +77,7 @@ org.adrianwalker multiline-string - + com.trendmicro tlsh 3.7.1 @@ -348,7 +348,8 @@ ${basedir}/src/main/antlr4/imports ${basedir}/src/main/antlr4 - ${basedir}/src/main/java/org/apache/metron/stellar/common/generated + ${basedir}/src/main/java/org/apache/metron/stellar/common/generated + @@ -459,7 +460,7 @@ ${project.name} + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> diff --git a/flink-cyber/flink-stellar/src/main/assembly/assembly.xml b/flink-cyber/flink-stellar/src/main/assembly/assembly.xml index b2041499..a4da8e67 100644 --- a/flink-cyber/flink-stellar/src/main/assembly/assembly.xml +++ b/flink-cyber/flink-stellar/src/main/assembly/assembly.xml @@ -24,19 +24,19 @@ --> - archive - - tar.gz - - false - - - ${project.basedir}/target - - ${project.artifactId}-${project.version}-uber.jar - - lib - true - - + archive + + tar.gz + + false + + + ${project.basedir}/target + + ${project.artifactId}-${project.version}-uber.jar + + lib + true + + diff --git a/flink-cyber/flink-stellar/src/main/assembly/stand-alone-assembly.xml b/flink-cyber/flink-stellar/src/main/assembly/stand-alone-assembly.xml index 4b649946..f2c7a864 100644 --- a/flink-cyber/flink-stellar/src/main/assembly/stand-alone-assembly.xml +++ b/flink-cyber/flink-stellar/src/main/assembly/stand-alone-assembly.xml @@ -24,46 +24,46 @@ --> - stand-alone - - tar.gz - - false - - - ${project.basedir}/src/main/scripts/stand-alone - bin - true - - **/*.formatted - **/*.filtered - - 0755 - unix - true - - - ${project.basedir}/target - - ${project.artifactId}-${project.version}-uber.jar - - lib - true - - - ${project.basedir} - - README.md - - - - - ${project.basedir}/src/main/resources/META-INF - - LICENSE - NOTICE - - - - + stand-alone + + tar.gz + + false + + + ${project.basedir}/src/main/scripts/stand-alone + bin + true + + **/*.formatted + **/*.filtered + + 0755 + unix + true + + + ${project.basedir}/target + + ${project.artifactId}-${project.version}-uber.jar + + lib + true + + + ${project.basedir} + + README.md + + + + + ${project.basedir}/src/main/resources/META-INF + + LICENSE + NOTICE + + + + \ No newline at end of file diff --git a/flink-cyber/metron-common/src/main/assembly/assembly.xml b/flink-cyber/metron-common/src/main/assembly/assembly.xml index b177a020..07d0a9e8 100644 --- a/flink-cyber/metron-common/src/main/assembly/assembly.xml +++ b/flink-cyber/metron-common/src/main/assembly/assembly.xml @@ -24,43 +24,43 @@ --> - archive - - tar.gz - - false - - - ${project.basedir}/src/main/config - config - true - - **/*.formatted - **/*.filtered - - 0644 - unix - true - - - ${project.basedir}/src/main/scripts - bin - true - - **/*.formatted - **/*.filtered - - 0755 - unix - true - - - ${project.basedir}/target - - ${project.artifactId}-${project.version}-uber.jar - - lib - true - - + archive + + tar.gz + + false + + + ${project.basedir}/src/main/config + config + true + + **/*.formatted + **/*.filtered + + 0644 + unix + true + + + ${project.basedir}/src/main/scripts + bin + true + + **/*.formatted + **/*.filtered + + 0755 + unix + true + + + ${project.basedir}/target + + ${project.artifactId}-${project.version}-uber.jar + + lib + true + + diff --git a/flink-cyber/metron-hbase-common/pom.xml b/flink-cyber/metron-hbase-common/pom.xml index bd00d038..669ce5b5 100644 --- a/flink-cyber/metron-hbase-common/pom.xml +++ b/flink-cyber/metron-hbase-common/pom.xml @@ -40,7 +40,7 @@ UTF-8 - + com.cloudera.cyber metron-common ${project.parent.version} diff --git a/flink-cyber/metron-parser-chain/pom.xml b/flink-cyber/metron-parser-chain/pom.xml index 2a8a1fb3..dd0aaa3f 100644 --- a/flink-cyber/metron-parser-chain/pom.xml +++ b/flink-cyber/metron-parser-chain/pom.xml @@ -220,14 +220,14 @@ true uber - - org.apache.flink:force-shading - com.google.code.findbugs:jsr305 - org.slf4j:* - log4j:* - org.apache.logging.log4j:* - - + + org.apache.flink:force-shading + com.google.code.findbugs:jsr305 + org.slf4j:* + log4j:* + org.apache.logging.log4j:* + + *:* @@ -240,7 +240,7 @@ + implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> .yaml LICENSE.txt diff --git a/flink-cyber/parser-chains-flink/pom.xml b/flink-cyber/parser-chains-flink/pom.xml index 366efe32..d76c6205 100644 --- a/flink-cyber/parser-chains-flink/pom.xml +++ b/flink-cyber/parser-chains-flink/pom.xml @@ -247,7 +247,7 @@ - + com.cloudera.cyber.parser.ParserJobKafka @@ -263,7 +263,7 @@ 3.4 - + diff --git a/flink-cyber/pom.xml b/flink-cyber/pom.xml index 703613b5..6191df6f 100644 --- a/flink-cyber/pom.xml +++ b/flink-cyber/pom.xml @@ -592,6 +592,13 @@ src/main/resources/** src/test/resources/** + src/test/classpath-resources/** + **/*.jj + **/node_modules/** + **/.angular/** + **/node/** + **/dist/** + **/target/** From b7d03f46f46b85a8df3f766f527453c787353049 Mon Sep 17 00:00:00 2001 From: Stas Panasiuk Date: Tue, 28 May 2024 13:20:32 -0400 Subject: [PATCH 03/15] GitHub actions codestyle check optimization --- .github/workflows/build_and_test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9b042f4e..6e1a70e7 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -20,6 +20,8 @@ jobs: container: maven:3-eclipse-temurin-8 steps: - uses: actions/checkout@v3 + - name: Check codestyle + run: mvn -B editorconfig:check --file flink-cyber/pom.xml - name: Cache local Maven repository uses: actions/cache@v3 with: @@ -28,7 +30,7 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Build and Test with Maven - run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B verify --file flink-cyber/pom.xml + run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B package --file flink-cyber/pom.xml # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive - name: Update dependency graph From 3ce6999766bc3a76f5cf566b06fafd8b87fb757e Mon Sep 17 00:00:00 2001 From: Stas Panasiuk Date: Tue, 28 May 2024 13:24:13 -0400 Subject: [PATCH 04/15] GitHub actions codestyle check optimization #2 --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6e1a70e7..ec1a06a0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -20,8 +20,6 @@ jobs: container: maven:3-eclipse-temurin-8 steps: - uses: actions/checkout@v3 - - name: Check codestyle - run: mvn -B editorconfig:check --file flink-cyber/pom.xml - name: Cache local Maven repository uses: actions/cache@v3 with: @@ -29,6 +27,8 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: Check codestyle + run: mvn -B editorconfig:check --file flink-cyber/pom.xml - name: Build and Test with Maven run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B package --file flink-cyber/pom.xml From 3fe7bdd894e45f9d0127b51da7188b7f994ca7aa Mon Sep 17 00:00:00 2001 From: Stas Panasiuk Date: Wed, 9 Oct 2024 09:26:33 -0400 Subject: [PATCH 05/15] codestyle fixes --- flink-cyber/flink-cyber-api/pom.xml | 2 +- flink-cyber/parser-chains-flink/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-cyber/flink-cyber-api/pom.xml b/flink-cyber/flink-cyber-api/pom.xml index 2d72699b..0c66409f 100644 --- a/flink-cyber/flink-cyber-api/pom.xml +++ b/flink-cyber/flink-cyber-api/pom.xml @@ -43,7 +43,7 @@ import - + diff --git a/flink-cyber/parser-chains-flink/pom.xml b/flink-cyber/parser-chains-flink/pom.xml index 91329e08..becaab2b 100644 --- a/flink-cyber/parser-chains-flink/pom.xml +++ b/flink-cyber/parser-chains-flink/pom.xml @@ -279,7 +279,7 @@ META-INF/*.SF META-INF/*.DSA META-INF/*.RSA - + From da5ef2e1aed87b2fd3b01cfc77c8de2c948f4335 Mon Sep 17 00:00:00 2001 From: Stas Panasiuk Date: Thu, 10 Oct 2024 00:50:34 -0400 Subject: [PATCH 06/15] codestyle fixes --- flink-cyber/google_checks.xml | 420 ++++++++++++++++++ .../controller/ClusterController.java | 1 + .../cloudera/cyber/parser/ParserChainMap.java | 2 + flink-cyber/pom.xml | 31 +- 4 files changed, 435 insertions(+), 19 deletions(-) create mode 100644 flink-cyber/google_checks.xml diff --git a/flink-cyber/google_checks.xml b/flink-cyber/google_checks.xml new file mode 100644 index 00000000..b2c8eec7 --- /dev/null +++ b/flink-cyber/google_checks.xml @@ -0,0 +1,420 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java index 05b1099f..e74685b1 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java @@ -27,6 +27,7 @@ public class ClusterController { private final ClusterService clusterService; + @Operation(description = "Retrieves information about all cluster services.") @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "A list of all clusters.") diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserChainMap.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserChainMap.java index 546665b9..46b4b47d 100644 --- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserChainMap.java +++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserChainMap.java @@ -17,4 +17,6 @@ import java.util.HashMap; public class ParserChainMap extends HashMap { + + } diff --git a/flink-cyber/pom.xml b/flink-cyber/pom.xml index 35541499..3598ea0e 100644 --- a/flink-cyber/pom.xml +++ b/flink-cyber/pom.xml @@ -119,7 +119,7 @@ 21.3.9 4.12 5.7.0 - 0.1.3 + 3.0.0 @@ -633,31 +633,24 @@ - org.ec4j.maven - editorconfig-maven-plugin - ${editorconfig-maven-plugin.version} + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven-checkstyle-plugin.version} + + google_checks.xml + true + true + false + - check - verify + validate + validate check - - - src/main/resources/** - src/test/resources/** - src/test/classpath-resources/** - **/*.jj - **/node_modules/** - **/.angular/** - **/node/** - **/dist/** - **/target/** - - From 6f5fc084b3c76bea2fbdcfb2030b2c3a8993dd20 Mon Sep 17 00:00:00 2001 From: Stas Panasiuk Date: Fri, 11 Oct 2024 03:59:23 -0400 Subject: [PATCH 07/15] switch to maven checkstyle --- flink-cyber/.editorconfig | 1482 --------------------------------- flink-cyber/google_checks.xml | 2 +- flink-cyber/pom.xml | 1 + 3 files changed, 2 insertions(+), 1483 deletions(-) delete mode 100644 flink-cyber/.editorconfig diff --git a/flink-cyber/.editorconfig b/flink-cyber/.editorconfig deleted file mode 100644 index 33f83bed..00000000 --- a/flink-cyber/.editorconfig +++ /dev/null @@ -1,1482 +0,0 @@ -[*] -charset = utf-8 -end_of_line = lf -indent_size = 4 -indent_style = space -insert_final_newline = false -max_line_length = 120 -tab_width = 4 -ij_continuation_indent_size = 8 -ij_formatter_off_tag = @formatter:off -ij_formatter_on_tag = @formatter:on -ij_formatter_tags_enabled = true -ij_smart_tabs = false -# ij_visual_guides = -ij_wrap_on_typing = false - -[*.css] -ij_css_align_closing_brace_with_properties = false -ij_css_blank_lines_around_nested_selector = 1 -ij_css_blank_lines_between_blocks = 1 -ij_css_block_comment_add_space = false -ij_css_brace_placement = end_of_line -ij_css_enforce_quotes_on_format = false -ij_css_hex_color_long_format = false -ij_css_hex_color_lower_case = false -ij_css_hex_color_short_format = false -ij_css_hex_color_upper_case = false -ij_css_keep_blank_lines_in_code = 2 -ij_css_keep_indents_on_empty_lines = false -ij_css_keep_single_line_blocks = false -ij_css_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow -ij_css_space_after_colon = true -ij_css_space_before_opening_brace = true -ij_css_use_double_quotes = true -ij_css_value_alignment = do_not_align - -[*.java] -ij_java_align_consecutive_assignments = false -ij_java_align_consecutive_variable_declarations = false -ij_java_align_group_field_declarations = false -ij_java_align_multiline_annotation_parameters = false -ij_java_align_multiline_array_initializer_expression = false -ij_java_align_multiline_assignment = false -ij_java_align_multiline_binary_operation = true -ij_java_align_multiline_chained_methods = false -ij_java_align_multiline_deconstruction_list_components = true -ij_java_align_multiline_extends_list = false -ij_java_align_multiline_for = true -ij_java_align_multiline_method_parentheses = false -ij_java_align_multiline_parameters = true -ij_java_align_multiline_parameters_in_calls = false -ij_java_align_multiline_parenthesized_expression = false -ij_java_align_multiline_records = true -ij_java_align_multiline_resources = true -ij_java_align_multiline_ternary_operation = false -ij_java_align_multiline_text_blocks = false -ij_java_align_multiline_throws_list = false -ij_java_align_subsequent_simple_methods = false -ij_java_align_throws_keyword = false -ij_java_align_types_in_multi_catch = true -ij_java_annotation_parameter_wrap = off -ij_java_array_initializer_new_line_after_left_brace = false -ij_java_array_initializer_right_brace_on_new_line = false -ij_java_array_initializer_wrap = off -ij_java_assert_statement_colon_on_next_line = false -ij_java_assert_statement_wrap = off -ij_java_assignment_wrap = off -ij_java_binary_operation_sign_on_next_line = false -ij_java_binary_operation_wrap = off -ij_java_blank_lines_after_anonymous_class_header = 0 -ij_java_blank_lines_after_class_header = 0 -ij_java_blank_lines_after_imports = 1 -ij_java_blank_lines_after_package = 1 -ij_java_blank_lines_around_class = 1 -ij_java_blank_lines_around_field = 0 -ij_java_blank_lines_around_field_in_interface = 0 -ij_java_blank_lines_around_initializer = 1 -ij_java_blank_lines_around_method = 1 -ij_java_blank_lines_around_method_in_interface = 1 -ij_java_blank_lines_before_class_end = 0 -ij_java_blank_lines_before_imports = 1 -ij_java_blank_lines_before_method_body = 0 -ij_java_blank_lines_before_package = 0 -ij_java_block_brace_style = end_of_line -ij_java_block_comment_add_space = false -ij_java_block_comment_at_first_column = true -# ij_java_builder_methods = -ij_java_call_parameters_new_line_after_left_paren = false -ij_java_call_parameters_right_paren_on_new_line = false -ij_java_call_parameters_wrap = off -ij_java_case_statement_on_separate_line = true -ij_java_catch_on_new_line = false -ij_java_class_annotation_wrap = split_into_lines -ij_java_class_brace_style = end_of_line -ij_java_class_count_to_use_import_on_demand = 10 -ij_java_class_names_in_javadoc = 1 -ij_java_deconstruction_list_wrap = normal -ij_java_do_not_indent_top_level_class_members = false -ij_java_do_not_wrap_after_single_annotation = false -ij_java_do_not_wrap_after_single_annotation_in_parameter = false -ij_java_do_while_brace_force = never -ij_java_doc_add_blank_line_after_description = true -ij_java_doc_add_blank_line_after_param_comments = false -ij_java_doc_add_blank_line_after_return = false -ij_java_doc_add_p_tag_on_empty_lines = true -ij_java_doc_align_exception_comments = true -ij_java_doc_align_param_comments = true -ij_java_doc_do_not_wrap_if_one_line = false -ij_java_doc_enable_formatting = true -ij_java_doc_enable_leading_asterisks = true -ij_java_doc_indent_on_continuation = false -ij_java_doc_keep_empty_lines = true -ij_java_doc_keep_empty_parameter_tag = true -ij_java_doc_keep_empty_return_tag = true -ij_java_doc_keep_empty_throws_tag = true -ij_java_doc_keep_invalid_tags = true -ij_java_doc_param_description_on_new_line = false -ij_java_doc_preserve_line_breaks = false -ij_java_doc_use_throws_not_exception_tag = true -ij_java_else_on_new_line = false -# ij_java_entity_dd_prefix = -ij_java_entity_dd_suffix = EJB -# ij_java_entity_eb_prefix = -ij_java_entity_eb_suffix = Bean -# ij_java_entity_hi_prefix = -ij_java_entity_hi_suffix = Home -ij_java_entity_lhi_prefix = Local -ij_java_entity_lhi_suffix = Home -ij_java_entity_li_prefix = Local -# ij_java_entity_li_suffix = -ij_java_entity_pk_class = java.lang.String -# ij_java_entity_ri_prefix = -# ij_java_entity_ri_suffix = -# ij_java_entity_vo_prefix = -ij_java_entity_vo_suffix = VO -ij_java_enum_constants_wrap = off -ij_java_enum_field_annotation_wrap = off -ij_java_extends_keyword_wrap = off -ij_java_extends_list_wrap = off -ij_java_field_annotation_wrap = split_into_lines -# ij_java_field_name_prefix = -# ij_java_field_name_suffix = -# ij_java_filter_class_prefix = -# ij_java_filter_class_suffix = -# ij_java_filter_dd_prefix = -# ij_java_filter_dd_suffix = -ij_java_finally_on_new_line = false -ij_java_for_brace_force = never -ij_java_for_statement_new_line_after_left_paren = false -ij_java_for_statement_right_paren_on_new_line = false -ij_java_for_statement_wrap = off -ij_java_generate_final_locals = false -ij_java_generate_final_parameters = false -ij_java_if_brace_force = never -ij_java_imports_layout = *,|,javax.**,java.**,|,$* -ij_java_indent_case_from_switch = true -ij_java_insert_inner_class_imports = false -ij_java_insert_override_annotation = true -ij_java_keep_blank_lines_before_right_brace = 2 -ij_java_keep_blank_lines_between_package_declaration_and_header = 2 -ij_java_keep_blank_lines_in_code = 2 -ij_java_keep_blank_lines_in_declarations = 2 -ij_java_keep_builder_methods_indents = false -ij_java_keep_control_statement_in_one_line = true -ij_java_keep_first_column_comment = true -ij_java_keep_indents_on_empty_lines = false -ij_java_keep_line_breaks = true -ij_java_keep_multiple_expressions_in_one_line = false -ij_java_keep_simple_blocks_in_one_line = false -ij_java_keep_simple_classes_in_one_line = false -ij_java_keep_simple_lambdas_in_one_line = false -ij_java_keep_simple_methods_in_one_line = false -ij_java_label_indent_absolute = false -ij_java_label_indent_size = 0 -ij_java_lambda_brace_style = end_of_line -ij_java_layout_static_imports_separately = true -ij_java_line_comment_add_space = false -ij_java_line_comment_add_space_on_reformat = false -ij_java_line_comment_at_first_column = true -# ij_java_listener_class_prefix = -# ij_java_listener_class_suffix = -# ij_java_local_variable_name_prefix = -# ij_java_local_variable_name_suffix = -# ij_java_message_dd_prefix = -ij_java_message_dd_suffix = EJB -# ij_java_message_eb_prefix = -ij_java_message_eb_suffix = Bean -ij_java_method_annotation_wrap = split_into_lines -ij_java_method_brace_style = end_of_line -ij_java_method_call_chain_wrap = off -ij_java_method_parameters_new_line_after_left_paren = false -ij_java_method_parameters_right_paren_on_new_line = false -ij_java_method_parameters_wrap = off -ij_java_modifier_list_wrap = false -ij_java_multi_catch_types_wrap = normal -ij_java_names_count_to_use_import_on_demand = 5 -ij_java_new_line_after_lparen_in_annotation = false -ij_java_new_line_after_lparen_in_deconstruction_pattern = true -ij_java_new_line_after_lparen_in_record_header = false -ij_java_new_line_when_body_is_presented = false -ij_java_packages_to_use_import_on_demand = java.awt.*,javax.swing.* -ij_java_parameter_annotation_wrap = off -# ij_java_parameter_name_prefix = -# ij_java_parameter_name_suffix = -ij_java_parentheses_expression_new_line_after_left_paren = false -ij_java_parentheses_expression_right_paren_on_new_line = false -ij_java_place_assignment_sign_on_next_line = false -ij_java_prefer_longer_names = true -ij_java_prefer_parameters_wrap = false -ij_java_record_components_wrap = normal -# ij_java_repeat_annotations = -ij_java_repeat_synchronized = true -ij_java_replace_instanceof_and_cast = false -ij_java_replace_null_check = true -ij_java_replace_sum_lambda_with_method_ref = true -ij_java_resource_list_new_line_after_left_paren = false -ij_java_resource_list_right_paren_on_new_line = false -ij_java_resource_list_wrap = off -ij_java_rparen_on_new_line_in_annotation = false -ij_java_rparen_on_new_line_in_deconstruction_pattern = true -ij_java_rparen_on_new_line_in_record_header = false -# ij_java_servlet_class_prefix = -# ij_java_servlet_class_suffix = -# ij_java_servlet_dd_prefix = -# ij_java_servlet_dd_suffix = -# ij_java_session_dd_prefix = -ij_java_session_dd_suffix = EJB -# ij_java_session_eb_prefix = -ij_java_session_eb_suffix = Bean -# ij_java_session_hi_prefix = -ij_java_session_hi_suffix = Home -ij_java_session_lhi_prefix = Local -ij_java_session_lhi_suffix = Home -ij_java_session_li_prefix = Local -# ij_java_session_li_suffix = -# ij_java_session_ri_prefix = -# ij_java_session_ri_suffix = -# ij_java_session_si_prefix = -ij_java_session_si_suffix = Service -ij_java_space_after_closing_angle_bracket_in_type_argument = false -ij_java_space_after_colon = true -ij_java_space_after_comma = true -ij_java_space_after_comma_in_type_arguments = true -ij_java_space_after_for_semicolon = true -ij_java_space_after_quest = true -ij_java_space_after_type_cast = true -ij_java_space_before_annotation_array_initializer_left_brace = false -ij_java_space_before_annotation_parameter_list = false -ij_java_space_before_array_initializer_left_brace = false -ij_java_space_before_catch_keyword = true -ij_java_space_before_catch_left_brace = true -ij_java_space_before_catch_parentheses = true -ij_java_space_before_class_left_brace = true -ij_java_space_before_colon = true -ij_java_space_before_colon_in_foreach = true -ij_java_space_before_comma = false -ij_java_space_before_deconstruction_list = false -ij_java_space_before_do_left_brace = true -ij_java_space_before_else_keyword = true -ij_java_space_before_else_left_brace = true -ij_java_space_before_finally_keyword = true -ij_java_space_before_finally_left_brace = true -ij_java_space_before_for_left_brace = true -ij_java_space_before_for_parentheses = true -ij_java_space_before_for_semicolon = false -ij_java_space_before_if_left_brace = true -ij_java_space_before_if_parentheses = true -ij_java_space_before_method_call_parentheses = false -ij_java_space_before_method_left_brace = true -ij_java_space_before_method_parentheses = false -ij_java_space_before_opening_angle_bracket_in_type_parameter = false -ij_java_space_before_quest = true -ij_java_space_before_switch_left_brace = true -ij_java_space_before_switch_parentheses = true -ij_java_space_before_synchronized_left_brace = true -ij_java_space_before_synchronized_parentheses = true -ij_java_space_before_try_left_brace = true -ij_java_space_before_try_parentheses = true -ij_java_space_before_type_parameter_list = false -ij_java_space_before_while_keyword = true -ij_java_space_before_while_left_brace = true -ij_java_space_before_while_parentheses = true -ij_java_space_inside_one_line_enum_braces = false -ij_java_space_within_empty_array_initializer_braces = false -ij_java_space_within_empty_method_call_parentheses = false -ij_java_space_within_empty_method_parentheses = false -ij_java_spaces_around_additive_operators = true -ij_java_spaces_around_annotation_eq = true -ij_java_spaces_around_assignment_operators = true -ij_java_spaces_around_bitwise_operators = true -ij_java_spaces_around_equality_operators = true -ij_java_spaces_around_lambda_arrow = true -ij_java_spaces_around_logical_operators = true -ij_java_spaces_around_method_ref_dbl_colon = false -ij_java_spaces_around_multiplicative_operators = true -ij_java_spaces_around_relational_operators = true -ij_java_spaces_around_shift_operators = true -ij_java_spaces_around_type_bounds_in_type_parameters = true -ij_java_spaces_around_unary_operator = false -ij_java_spaces_within_angle_brackets = false -ij_java_spaces_within_annotation_parentheses = false -ij_java_spaces_within_array_initializer_braces = false -ij_java_spaces_within_braces = false -ij_java_spaces_within_brackets = false -ij_java_spaces_within_cast_parentheses = false -ij_java_spaces_within_catch_parentheses = false -ij_java_spaces_within_deconstruction_list = false -ij_java_spaces_within_for_parentheses = false -ij_java_spaces_within_if_parentheses = false -ij_java_spaces_within_method_call_parentheses = false -ij_java_spaces_within_method_parentheses = false -ij_java_spaces_within_parentheses = false -ij_java_spaces_within_record_header = false -ij_java_spaces_within_switch_parentheses = false -ij_java_spaces_within_synchronized_parentheses = false -ij_java_spaces_within_try_parentheses = false -ij_java_spaces_within_while_parentheses = false -ij_java_special_else_if_treatment = true -# ij_java_static_field_name_prefix = -# ij_java_static_field_name_suffix = -# ij_java_subclass_name_prefix = -ij_java_subclass_name_suffix = Impl -ij_java_switch_expressions_wrap = normal -ij_java_ternary_operation_signs_on_next_line = false -ij_java_ternary_operation_wrap = off -# ij_java_test_name_prefix = -ij_java_test_name_suffix = Test -ij_java_throws_keyword_wrap = off -ij_java_throws_list_wrap = off -ij_java_use_external_annotations = false -ij_java_use_fq_class_names = false -ij_java_use_relative_indents = false -ij_java_use_single_class_imports = true -ij_java_variable_annotation_wrap = off -ij_java_visibility = public -ij_java_while_brace_force = never -ij_java_while_on_new_line = false -ij_java_wrap_comments = false -ij_java_wrap_first_method_in_call_chain = false -ij_java_wrap_long_lines = false -ij_java_wrap_semicolon_after_call_chain = false - -[*.less] -indent_size = 2 -ij_less_align_closing_brace_with_properties = false -ij_less_blank_lines_around_nested_selector = 1 -ij_less_blank_lines_between_blocks = 1 -ij_less_block_comment_add_space = false -ij_less_brace_placement = 0 -ij_less_enforce_quotes_on_format = false -ij_less_hex_color_long_format = false -ij_less_hex_color_lower_case = false -ij_less_hex_color_short_format = false -ij_less_hex_color_upper_case = false -ij_less_keep_blank_lines_in_code = 2 -ij_less_keep_indents_on_empty_lines = false -ij_less_keep_single_line_blocks = false -ij_less_line_comment_add_space = false -ij_less_line_comment_at_first_column = false -ij_less_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow -ij_less_space_after_colon = true -ij_less_space_before_opening_brace = true -ij_less_use_double_quotes = true -ij_less_value_alignment = 0 - -[*.proto] -indent_size = 2 -tab_width = 2 -ij_continuation_indent_size = 4 -ij_protobuf_keep_blank_lines_in_code = 2 -ij_protobuf_keep_indents_on_empty_lines = false -ij_protobuf_keep_line_breaks = true -ij_protobuf_space_after_comma = true -ij_protobuf_space_before_comma = false -ij_protobuf_spaces_around_assignment_operators = true -ij_protobuf_spaces_within_braces = false -ij_protobuf_spaces_within_brackets = false - -[*.sass] -indent_size = 2 -ij_sass_align_closing_brace_with_properties = false -ij_sass_blank_lines_around_nested_selector = 1 -ij_sass_blank_lines_between_blocks = 1 -ij_sass_brace_placement = 0 -ij_sass_enforce_quotes_on_format = false -ij_sass_hex_color_long_format = false -ij_sass_hex_color_lower_case = false -ij_sass_hex_color_short_format = false -ij_sass_hex_color_upper_case = false -ij_sass_keep_blank_lines_in_code = 2 -ij_sass_keep_indents_on_empty_lines = false -ij_sass_keep_single_line_blocks = false -ij_sass_line_comment_add_space = false -ij_sass_line_comment_at_first_column = false -ij_sass_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow -ij_sass_space_after_colon = true -ij_sass_space_before_opening_brace = true -ij_sass_use_double_quotes = true -ij_sass_value_alignment = 0 - -[*.scala] -indent_size = 2 -tab_width = 2 -ij_continuation_indent_size = 2 -ij_scala_align_composite_pattern = true -ij_scala_align_extends_with = 0 -ij_scala_align_group_field_declarations = false -ij_scala_align_if_else = false -ij_scala_align_in_columns_case_branch = false -ij_scala_align_multiline_binary_operation = false -ij_scala_align_multiline_chained_methods = false -ij_scala_align_multiline_for = true -ij_scala_align_multiline_parameters = true -ij_scala_align_multiline_parameters_in_calls = false -ij_scala_align_multiline_parenthesized_expression = false -ij_scala_align_parameter_types_in_multiline_declarations = 0 -ij_scala_align_tuple_elements = false -ij_scala_alternate_continuation_indent_for_params = 4 -ij_scala_binary_operation_wrap = off -ij_scala_blank_lines_after_anonymous_class_header = 0 -ij_scala_blank_lines_after_class_header = 0 -ij_scala_blank_lines_after_imports = 1 -ij_scala_blank_lines_after_package = 1 -ij_scala_blank_lines_around_class = 1 -ij_scala_blank_lines_around_class_in_inner_scopes = 0 -ij_scala_blank_lines_around_field = 0 -ij_scala_blank_lines_around_field_in_inner_scopes = 0 -ij_scala_blank_lines_around_field_in_interface = 0 -ij_scala_blank_lines_around_method = 1 -ij_scala_blank_lines_around_method_in_inner_scopes = 1 -ij_scala_blank_lines_around_method_in_interface = 1 -ij_scala_blank_lines_before_class_end = 0 -ij_scala_blank_lines_before_imports = 1 -ij_scala_blank_lines_before_method_body = 0 -ij_scala_blank_lines_before_package = 0 -ij_scala_block_brace_style = end_of_line -ij_scala_block_comment_add_space = false -ij_scala_block_comment_at_first_column = true -ij_scala_call_parameters_new_line_after_lparen = 0 -ij_scala_call_parameters_right_paren_on_new_line = false -ij_scala_call_parameters_wrap = off -ij_scala_case_clause_brace_force = never -ij_scala_catch_on_new_line = false -ij_scala_class_annotation_wrap = split_into_lines -ij_scala_class_brace_style = end_of_line -ij_scala_closure_brace_force = never -ij_scala_do_not_align_block_expr_params = true -ij_scala_do_not_indent_case_clause_body = false -ij_scala_do_not_indent_tuples_close_brace = true -ij_scala_do_while_brace_force = never -ij_scala_else_on_new_line = false -ij_scala_enable_scaladoc_formatting = true -ij_scala_enforce_functional_syntax_for_unit = true -ij_scala_extends_keyword_wrap = off -ij_scala_extends_list_wrap = off -ij_scala_field_annotation_wrap = split_into_lines -ij_scala_finally_brace_force = never -ij_scala_finally_on_new_line = false -ij_scala_for_brace_force = never -ij_scala_for_statement_wrap = off -ij_scala_formatter = 0 -ij_scala_if_brace_force = never -# ij_scala_implicit_value_class_prefix = -ij_scala_implicit_value_class_suffix = Ops -ij_scala_indent_braced_function_args = true -ij_scala_indent_case_from_switch = true -ij_scala_indent_fewer_braces_in_method_call_chains = false -ij_scala_indent_first_parameter = true -ij_scala_indent_first_parameter_clause = false -ij_scala_indent_type_arguments = true -ij_scala_indent_type_parameters = true -ij_scala_indent_yield_after_one_line_enumerators = true -ij_scala_keep_blank_lines_before_right_brace = 2 -ij_scala_keep_blank_lines_in_code = 2 -ij_scala_keep_blank_lines_in_declarations = 2 -ij_scala_keep_comments_on_same_line = true -ij_scala_keep_first_column_comment = false -ij_scala_keep_indents_on_empty_lines = false -ij_scala_keep_line_breaks = true -ij_scala_keep_one_line_lambdas_in_arg_list = false -ij_scala_keep_simple_blocks_in_one_line = false -ij_scala_keep_simple_methods_in_one_line = false -ij_scala_keep_xml_formatting = false -ij_scala_line_comment_add_space = false -ij_scala_line_comment_at_first_column = true -ij_scala_method_annotation_wrap = split_into_lines -ij_scala_method_brace_force = never -ij_scala_method_brace_style = end_of_line -ij_scala_method_call_chain_wrap = off -ij_scala_method_parameters_new_line_after_left_paren = false -ij_scala_method_parameters_right_paren_on_new_line = false -ij_scala_method_parameters_wrap = off -ij_scala_modifier_list_wrap = false -ij_scala_multiline_string_align_dangling_closing_quotes = false -ij_scala_multiline_string_closing_quotes_on_new_line = false -ij_scala_multiline_string_insert_margin_on_enter = true -ij_scala_multiline_string_margin_char = | -ij_scala_multiline_string_margin_indent = 2 -ij_scala_multiline_string_opening_quotes_on_new_line = true -ij_scala_multiline_string_process_margin_on_copy_paste = true -ij_scala_new_line_after_case_clause_arrow_when_multiline_body = false -ij_scala_newline_after_annotations = false -ij_scala_not_continuation_indent_for_params = false -ij_scala_parameter_annotation_wrap = off -ij_scala_parentheses_expression_new_line_after_left_paren = false -ij_scala_parentheses_expression_right_paren_on_new_line = false -ij_scala_place_closure_parameters_on_new_line = false -ij_scala_place_self_type_on_new_line = true -ij_scala_prefer_parameters_wrap = false -ij_scala_preserve_space_after_method_declaration_name = false -ij_scala_reformat_on_compile = false -ij_scala_replace_case_arrow_with_unicode_char = false -ij_scala_replace_for_generator_arrow_with_unicode_char = false -ij_scala_replace_lambda_with_greek_letter = false -ij_scala_replace_map_arrow_with_unicode_char = false -# ij_scala_scalafmt_config_path = -ij_scala_scalafmt_fallback_to_default_settings = false -ij_scala_scalafmt_reformat_on_files_save = false -ij_scala_scalafmt_show_invalid_code_warnings = true -ij_scala_scalafmt_use_intellij_formatter_for_range_format = true -ij_scala_sd_align_exception_comments = true -ij_scala_sd_align_list_item_content = true -ij_scala_sd_align_other_tags_comments = true -ij_scala_sd_align_parameters_comments = true -ij_scala_sd_align_return_comments = true -ij_scala_sd_blank_line_after_parameters_comments = false -ij_scala_sd_blank_line_after_return_comments = false -ij_scala_sd_blank_line_before_parameters = false -ij_scala_sd_blank_line_before_tags = true -ij_scala_sd_blank_line_between_parameters = false -ij_scala_sd_keep_blank_lines_between_tags = false -ij_scala_sd_preserve_spaces_in_tags = false -ij_scala_space_after_comma = true -ij_scala_space_after_for_semicolon = true -ij_scala_space_after_modifiers_constructor = false -ij_scala_space_after_type_colon = true -ij_scala_space_before_brace_method_call = true -ij_scala_space_before_class_left_brace = true -ij_scala_space_before_for_parentheses = true -ij_scala_space_before_if_parentheses = true -ij_scala_space_before_infix_like_method_parentheses = false -ij_scala_space_before_infix_method_call_parentheses = false -ij_scala_space_before_infix_operator_like_method_call_parentheses = true -ij_scala_space_before_method_call_parentheses = false -ij_scala_space_before_method_left_brace = true -ij_scala_space_before_method_parentheses = false -ij_scala_space_before_type_colon = false -ij_scala_space_before_type_parameter_in_def_list = false -ij_scala_space_before_type_parameter_leading_context_bound_colon = false -ij_scala_space_before_type_parameter_leading_context_bound_colon_hk = true -ij_scala_space_before_type_parameter_list = false -ij_scala_space_before_type_parameter_rest_context_bound_colons = true -ij_scala_space_before_while_parentheses = true -ij_scala_space_inside_closure_braces = true -ij_scala_space_inside_self_type_braces = true -ij_scala_space_within_empty_method_call_parentheses = false -ij_scala_spaces_around_at_in_patterns = false -ij_scala_spaces_in_imports = false -ij_scala_spaces_in_one_line_blocks = false -ij_scala_spaces_within_brackets = false -ij_scala_spaces_within_for_parentheses = false -ij_scala_spaces_within_if_parentheses = false -ij_scala_spaces_within_method_call_parentheses = false -ij_scala_spaces_within_method_parentheses = false -ij_scala_spaces_within_parentheses = false -ij_scala_spaces_within_while_parentheses = false -ij_scala_special_else_if_treatment = true -ij_scala_trailing_comma_arg_list_enabled = true -ij_scala_trailing_comma_import_selector_enabled = false -ij_scala_trailing_comma_mode = trailing_comma_keep -ij_scala_trailing_comma_params_enabled = true -ij_scala_trailing_comma_pattern_arg_list_enabled = false -ij_scala_trailing_comma_tuple_enabled = false -ij_scala_trailing_comma_tuple_type_enabled = false -ij_scala_trailing_comma_type_params_enabled = false -ij_scala_try_brace_force = never -ij_scala_type_annotation_exclude_constant = true -ij_scala_type_annotation_exclude_in_dialect_sources = true -ij_scala_type_annotation_exclude_in_test_sources = false -ij_scala_type_annotation_exclude_member_of_anonymous_class = false -ij_scala_type_annotation_exclude_member_of_private_class = false -ij_scala_type_annotation_exclude_when_type_is_stable = true -ij_scala_type_annotation_function_parameter = false -ij_scala_type_annotation_implicit_modifier = true -ij_scala_type_annotation_local_definition = false -ij_scala_type_annotation_private_member = false -ij_scala_type_annotation_protected_member = true -ij_scala_type_annotation_public_member = true -ij_scala_type_annotation_structural_type = true -ij_scala_type_annotation_underscore_parameter = false -ij_scala_type_annotation_unit_type = true -ij_scala_use_alternate_continuation_indent_for_params = false -ij_scala_use_scala3_indentation_based_syntax = true -ij_scala_use_scaladoc2_formatting = false -ij_scala_variable_annotation_wrap = off -ij_scala_while_brace_force = never -ij_scala_while_on_new_line = false -ij_scala_wrap_before_with_keyword = false -ij_scala_wrap_first_method_in_call_chain = false -ij_scala_wrap_long_lines = false - -[*.scss] -indent_size = 2 -ij_scss_align_closing_brace_with_properties = false -ij_scss_blank_lines_around_nested_selector = 1 -ij_scss_blank_lines_between_blocks = 1 -ij_scss_block_comment_add_space = false -ij_scss_brace_placement = 0 -ij_scss_enforce_quotes_on_format = false -ij_scss_hex_color_long_format = false -ij_scss_hex_color_lower_case = false -ij_scss_hex_color_short_format = false -ij_scss_hex_color_upper_case = false -ij_scss_keep_blank_lines_in_code = 2 -ij_scss_keep_indents_on_empty_lines = false -ij_scss_keep_single_line_blocks = false -ij_scss_line_comment_add_space = false -ij_scss_line_comment_at_first_column = false -ij_scss_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow -ij_scss_space_after_colon = true -ij_scss_space_before_opening_brace = true -ij_scss_use_double_quotes = true -ij_scss_value_alignment = 0 - -[*.vue] -indent_size = 2 -tab_width = 2 -ij_continuation_indent_size = 4 -ij_vue_indent_children_of_top_level = template -ij_vue_interpolation_new_line_after_start_delimiter = true -ij_vue_interpolation_new_line_before_end_delimiter = true -ij_vue_interpolation_wrap = off -ij_vue_keep_indents_on_empty_lines = false -ij_vue_spaces_within_interpolation_expressions = true - -[.editorconfig] -ij_editorconfig_align_group_field_declarations = false -ij_editorconfig_space_after_colon = false -ij_editorconfig_space_after_comma = true -ij_editorconfig_space_before_colon = false -ij_editorconfig_space_before_comma = false -ij_editorconfig_spaces_around_assignment_operators = true - -[{*.ant,*.fxml,*.jhm,*.jnlp,*.jrxml,*.pom,*.rng,*.tld,*.wadl,*.wsdl,*.xml,*.xsd,*.xsl,*.xslt,*.xul}] -ij_xml_align_attributes = true -ij_xml_align_text = false -ij_xml_attribute_wrap = normal -ij_xml_block_comment_add_space = false -ij_xml_block_comment_at_first_column = true -ij_xml_keep_blank_lines = 2 -ij_xml_keep_indents_on_empty_lines = false -ij_xml_keep_line_breaks = true -ij_xml_keep_line_breaks_in_text = true -ij_xml_keep_whitespaces = false -ij_xml_keep_whitespaces_around_cdata = preserve -ij_xml_keep_whitespaces_inside_cdata = false -ij_xml_line_comment_at_first_column = true -ij_xml_space_after_tag_name = false -ij_xml_space_around_equals_in_attribute = false -ij_xml_space_inside_empty_tag = false -ij_xml_text_wrap = normal - -[{*.ats,*.cts,*.mts,*.ts}] -ij_continuation_indent_size = 4 -ij_typescript_align_imports = false -ij_typescript_align_multiline_array_initializer_expression = false -ij_typescript_align_multiline_binary_operation = false -ij_typescript_align_multiline_chained_methods = false -ij_typescript_align_multiline_extends_list = false -ij_typescript_align_multiline_for = true -ij_typescript_align_multiline_parameters = true -ij_typescript_align_multiline_parameters_in_calls = false -ij_typescript_align_multiline_ternary_operation = false -ij_typescript_align_object_properties = 0 -ij_typescript_align_union_types = false -ij_typescript_align_var_statements = 0 -ij_typescript_array_initializer_new_line_after_left_brace = false -ij_typescript_array_initializer_right_brace_on_new_line = false -ij_typescript_array_initializer_wrap = off -ij_typescript_assignment_wrap = off -ij_typescript_binary_operation_sign_on_next_line = false -ij_typescript_binary_operation_wrap = off -ij_typescript_blacklist_imports = rxjs/Rx,node_modules/**,**/node_modules/**,@angular/material,@angular/material/typings/** -ij_typescript_blank_lines_after_imports = 1 -ij_typescript_blank_lines_around_class = 1 -ij_typescript_blank_lines_around_field = 0 -ij_typescript_blank_lines_around_field_in_interface = 0 -ij_typescript_blank_lines_around_function = 1 -ij_typescript_blank_lines_around_method = 1 -ij_typescript_blank_lines_around_method_in_interface = 1 -ij_typescript_block_brace_style = end_of_line -ij_typescript_block_comment_add_space = false -ij_typescript_block_comment_at_first_column = true -ij_typescript_call_parameters_new_line_after_left_paren = false -ij_typescript_call_parameters_right_paren_on_new_line = false -ij_typescript_call_parameters_wrap = off -ij_typescript_catch_on_new_line = false -ij_typescript_chained_call_dot_on_new_line = true -ij_typescript_class_brace_style = end_of_line -ij_typescript_comma_on_new_line = false -ij_typescript_do_while_brace_force = never -ij_typescript_else_on_new_line = false -ij_typescript_enforce_trailing_comma = keep -ij_typescript_enum_constants_wrap = on_every_item -ij_typescript_extends_keyword_wrap = off -ij_typescript_extends_list_wrap = off -ij_typescript_field_prefix = _ -ij_typescript_file_name_style = relaxed -ij_typescript_finally_on_new_line = false -ij_typescript_for_brace_force = never -ij_typescript_for_statement_new_line_after_left_paren = false -ij_typescript_for_statement_right_paren_on_new_line = false -ij_typescript_for_statement_wrap = off -ij_typescript_force_quote_style = false -ij_typescript_force_semicolon_style = false -ij_typescript_function_expression_brace_style = end_of_line -ij_typescript_if_brace_force = never -ij_typescript_import_merge_members = global -ij_typescript_import_prefer_absolute_path = global -ij_typescript_import_sort_members = true -ij_typescript_import_sort_module_name = false -ij_typescript_import_use_node_resolution = true -ij_typescript_imports_wrap = on_every_item -ij_typescript_indent_case_from_switch = true -ij_typescript_indent_chained_calls = true -ij_typescript_indent_package_children = 0 -ij_typescript_jsdoc_include_types = false -ij_typescript_jsx_attribute_value = braces -ij_typescript_keep_blank_lines_in_code = 2 -ij_typescript_keep_first_column_comment = true -ij_typescript_keep_indents_on_empty_lines = false -ij_typescript_keep_line_breaks = true -ij_typescript_keep_simple_blocks_in_one_line = false -ij_typescript_keep_simple_methods_in_one_line = false -ij_typescript_line_comment_add_space = true -ij_typescript_line_comment_at_first_column = false -ij_typescript_method_brace_style = end_of_line -ij_typescript_method_call_chain_wrap = off -ij_typescript_method_parameters_new_line_after_left_paren = false -ij_typescript_method_parameters_right_paren_on_new_line = false -ij_typescript_method_parameters_wrap = off -ij_typescript_object_literal_wrap = on_every_item -ij_typescript_object_types_wrap = on_every_item -ij_typescript_parentheses_expression_new_line_after_left_paren = false -ij_typescript_parentheses_expression_right_paren_on_new_line = false -ij_typescript_place_assignment_sign_on_next_line = false -ij_typescript_prefer_as_type_cast = false -ij_typescript_prefer_explicit_types_function_expression_returns = false -ij_typescript_prefer_explicit_types_function_returns = false -ij_typescript_prefer_explicit_types_vars_fields = false -ij_typescript_prefer_parameters_wrap = false -# ij_typescript_property_prefix = -ij_typescript_reformat_c_style_comments = false -ij_typescript_space_after_colon = true -ij_typescript_space_after_comma = true -ij_typescript_space_after_dots_in_rest_parameter = false -ij_typescript_space_after_generator_mult = true -ij_typescript_space_after_property_colon = true -ij_typescript_space_after_quest = true -ij_typescript_space_after_type_colon = true -ij_typescript_space_after_unary_not = false -ij_typescript_space_before_async_arrow_lparen = true -ij_typescript_space_before_catch_keyword = true -ij_typescript_space_before_catch_left_brace = true -ij_typescript_space_before_catch_parentheses = true -ij_typescript_space_before_class_lbrace = true -ij_typescript_space_before_class_left_brace = true -ij_typescript_space_before_colon = true -ij_typescript_space_before_comma = false -ij_typescript_space_before_do_left_brace = true -ij_typescript_space_before_else_keyword = true -ij_typescript_space_before_else_left_brace = true -ij_typescript_space_before_finally_keyword = true -ij_typescript_space_before_finally_left_brace = true -ij_typescript_space_before_for_left_brace = true -ij_typescript_space_before_for_parentheses = true -ij_typescript_space_before_for_semicolon = false -ij_typescript_space_before_function_left_parenth = true -ij_typescript_space_before_generator_mult = false -ij_typescript_space_before_if_left_brace = true -ij_typescript_space_before_if_parentheses = true -ij_typescript_space_before_method_call_parentheses = false -ij_typescript_space_before_method_left_brace = true -ij_typescript_space_before_method_parentheses = false -ij_typescript_space_before_property_colon = false -ij_typescript_space_before_quest = true -ij_typescript_space_before_switch_left_brace = true -ij_typescript_space_before_switch_parentheses = true -ij_typescript_space_before_try_left_brace = true -ij_typescript_space_before_type_colon = false -ij_typescript_space_before_unary_not = false -ij_typescript_space_before_while_keyword = true -ij_typescript_space_before_while_left_brace = true -ij_typescript_space_before_while_parentheses = true -ij_typescript_spaces_around_additive_operators = true -ij_typescript_spaces_around_arrow_function_operator = true -ij_typescript_spaces_around_assignment_operators = true -ij_typescript_spaces_around_bitwise_operators = true -ij_typescript_spaces_around_equality_operators = true -ij_typescript_spaces_around_logical_operators = true -ij_typescript_spaces_around_multiplicative_operators = true -ij_typescript_spaces_around_relational_operators = true -ij_typescript_spaces_around_shift_operators = true -ij_typescript_spaces_around_unary_operator = false -ij_typescript_spaces_within_array_initializer_brackets = false -ij_typescript_spaces_within_brackets = false -ij_typescript_spaces_within_catch_parentheses = false -ij_typescript_spaces_within_for_parentheses = false -ij_typescript_spaces_within_if_parentheses = false -ij_typescript_spaces_within_imports = false -ij_typescript_spaces_within_interpolation_expressions = false -ij_typescript_spaces_within_method_call_parentheses = false -ij_typescript_spaces_within_method_parentheses = false -ij_typescript_spaces_within_object_literal_braces = false -ij_typescript_spaces_within_object_type_braces = true -ij_typescript_spaces_within_parentheses = false -ij_typescript_spaces_within_switch_parentheses = false -ij_typescript_spaces_within_type_assertion = false -ij_typescript_spaces_within_union_types = true -ij_typescript_spaces_within_while_parentheses = false -ij_typescript_special_else_if_treatment = true -ij_typescript_ternary_operation_signs_on_next_line = false -ij_typescript_ternary_operation_wrap = off -ij_typescript_union_types_wrap = on_every_item -ij_typescript_use_chained_calls_group_indents = false -ij_typescript_use_double_quotes = true -ij_typescript_use_explicit_js_extension = auto -ij_typescript_use_import_type = auto -ij_typescript_use_path_mapping = always -ij_typescript_use_public_modifier = false -ij_typescript_use_semicolon_after_statement = true -ij_typescript_var_declaration_wrap = normal -ij_typescript_while_brace_force = never -ij_typescript_while_on_new_line = false -ij_typescript_wrap_comments = false - -[{*.bash,*.sh,*.zsh}] -indent_size = 2 -tab_width = 2 -ij_shell_binary_ops_start_line = false -ij_shell_keep_column_alignment_padding = false -ij_shell_minify_program = false -ij_shell_redirect_followed_by_space = false -ij_shell_switch_cases_indented = false -ij_shell_use_unix_line_separator = true - -[{*.cjs,*.js}] -ij_continuation_indent_size = 4 -ij_javascript_align_imports = false -ij_javascript_align_multiline_array_initializer_expression = false -ij_javascript_align_multiline_binary_operation = false -ij_javascript_align_multiline_chained_methods = false -ij_javascript_align_multiline_extends_list = false -ij_javascript_align_multiline_for = true -ij_javascript_align_multiline_parameters = true -ij_javascript_align_multiline_parameters_in_calls = false -ij_javascript_align_multiline_ternary_operation = false -ij_javascript_align_object_properties = 0 -ij_javascript_align_union_types = false -ij_javascript_align_var_statements = 0 -ij_javascript_array_initializer_new_line_after_left_brace = false -ij_javascript_array_initializer_right_brace_on_new_line = false -ij_javascript_array_initializer_wrap = off -ij_javascript_assignment_wrap = off -ij_javascript_binary_operation_sign_on_next_line = false -ij_javascript_binary_operation_wrap = off -ij_javascript_blacklist_imports = rxjs/Rx,node_modules/**,**/node_modules/**,@angular/material,@angular/material/typings/** -ij_javascript_blank_lines_after_imports = 1 -ij_javascript_blank_lines_around_class = 1 -ij_javascript_blank_lines_around_field = 0 -ij_javascript_blank_lines_around_function = 1 -ij_javascript_blank_lines_around_method = 1 -ij_javascript_block_brace_style = end_of_line -ij_javascript_block_comment_add_space = false -ij_javascript_block_comment_at_first_column = true -ij_javascript_call_parameters_new_line_after_left_paren = false -ij_javascript_call_parameters_right_paren_on_new_line = false -ij_javascript_call_parameters_wrap = off -ij_javascript_catch_on_new_line = false -ij_javascript_chained_call_dot_on_new_line = true -ij_javascript_class_brace_style = end_of_line -ij_javascript_comma_on_new_line = false -ij_javascript_do_while_brace_force = never -ij_javascript_else_on_new_line = false -ij_javascript_enforce_trailing_comma = keep -ij_javascript_extends_keyword_wrap = off -ij_javascript_extends_list_wrap = off -ij_javascript_field_prefix = _ -ij_javascript_file_name_style = relaxed -ij_javascript_finally_on_new_line = false -ij_javascript_for_brace_force = never -ij_javascript_for_statement_new_line_after_left_paren = false -ij_javascript_for_statement_right_paren_on_new_line = false -ij_javascript_for_statement_wrap = off -ij_javascript_force_quote_style = false -ij_javascript_force_semicolon_style = false -ij_javascript_function_expression_brace_style = end_of_line -ij_javascript_if_brace_force = never -ij_javascript_import_merge_members = global -ij_javascript_import_prefer_absolute_path = global -ij_javascript_import_sort_members = true -ij_javascript_import_sort_module_name = false -ij_javascript_import_use_node_resolution = true -ij_javascript_imports_wrap = on_every_item -ij_javascript_indent_case_from_switch = true -ij_javascript_indent_chained_calls = true -ij_javascript_indent_package_children = 0 -ij_javascript_jsx_attribute_value = braces -ij_javascript_keep_blank_lines_in_code = 2 -ij_javascript_keep_first_column_comment = true -ij_javascript_keep_indents_on_empty_lines = false -ij_javascript_keep_line_breaks = true -ij_javascript_keep_simple_blocks_in_one_line = false -ij_javascript_keep_simple_methods_in_one_line = false -ij_javascript_line_comment_add_space = true -ij_javascript_line_comment_at_first_column = false -ij_javascript_method_brace_style = end_of_line -ij_javascript_method_call_chain_wrap = off -ij_javascript_method_parameters_new_line_after_left_paren = false -ij_javascript_method_parameters_right_paren_on_new_line = false -ij_javascript_method_parameters_wrap = off -ij_javascript_object_literal_wrap = on_every_item -ij_javascript_object_types_wrap = on_every_item -ij_javascript_parentheses_expression_new_line_after_left_paren = false -ij_javascript_parentheses_expression_right_paren_on_new_line = false -ij_javascript_place_assignment_sign_on_next_line = false -ij_javascript_prefer_as_type_cast = false -ij_javascript_prefer_explicit_types_function_expression_returns = false -ij_javascript_prefer_explicit_types_function_returns = false -ij_javascript_prefer_explicit_types_vars_fields = false -ij_javascript_prefer_parameters_wrap = false -# ij_javascript_property_prefix = -ij_javascript_reformat_c_style_comments = false -ij_javascript_space_after_colon = true -ij_javascript_space_after_comma = true -ij_javascript_space_after_dots_in_rest_parameter = false -ij_javascript_space_after_generator_mult = true -ij_javascript_space_after_property_colon = true -ij_javascript_space_after_quest = true -ij_javascript_space_after_type_colon = true -ij_javascript_space_after_unary_not = false -ij_javascript_space_before_async_arrow_lparen = true -ij_javascript_space_before_catch_keyword = true -ij_javascript_space_before_catch_left_brace = true -ij_javascript_space_before_catch_parentheses = true -ij_javascript_space_before_class_lbrace = true -ij_javascript_space_before_class_left_brace = true -ij_javascript_space_before_colon = true -ij_javascript_space_before_comma = false -ij_javascript_space_before_do_left_brace = true -ij_javascript_space_before_else_keyword = true -ij_javascript_space_before_else_left_brace = true -ij_javascript_space_before_finally_keyword = true -ij_javascript_space_before_finally_left_brace = true -ij_javascript_space_before_for_left_brace = true -ij_javascript_space_before_for_parentheses = true -ij_javascript_space_before_for_semicolon = false -ij_javascript_space_before_function_left_parenth = true -ij_javascript_space_before_generator_mult = false -ij_javascript_space_before_if_left_brace = true -ij_javascript_space_before_if_parentheses = true -ij_javascript_space_before_method_call_parentheses = false -ij_javascript_space_before_method_left_brace = true -ij_javascript_space_before_method_parentheses = false -ij_javascript_space_before_property_colon = false -ij_javascript_space_before_quest = true -ij_javascript_space_before_switch_left_brace = true -ij_javascript_space_before_switch_parentheses = true -ij_javascript_space_before_try_left_brace = true -ij_javascript_space_before_type_colon = false -ij_javascript_space_before_unary_not = false -ij_javascript_space_before_while_keyword = true -ij_javascript_space_before_while_left_brace = true -ij_javascript_space_before_while_parentheses = true -ij_javascript_spaces_around_additive_operators = true -ij_javascript_spaces_around_arrow_function_operator = true -ij_javascript_spaces_around_assignment_operators = true -ij_javascript_spaces_around_bitwise_operators = true -ij_javascript_spaces_around_equality_operators = true -ij_javascript_spaces_around_logical_operators = true -ij_javascript_spaces_around_multiplicative_operators = true -ij_javascript_spaces_around_relational_operators = true -ij_javascript_spaces_around_shift_operators = true -ij_javascript_spaces_around_unary_operator = false -ij_javascript_spaces_within_array_initializer_brackets = false -ij_javascript_spaces_within_brackets = false -ij_javascript_spaces_within_catch_parentheses = false -ij_javascript_spaces_within_for_parentheses = false -ij_javascript_spaces_within_if_parentheses = false -ij_javascript_spaces_within_imports = false -ij_javascript_spaces_within_interpolation_expressions = false -ij_javascript_spaces_within_method_call_parentheses = false -ij_javascript_spaces_within_method_parentheses = false -ij_javascript_spaces_within_object_literal_braces = false -ij_javascript_spaces_within_object_type_braces = true -ij_javascript_spaces_within_parentheses = false -ij_javascript_spaces_within_switch_parentheses = false -ij_javascript_spaces_within_type_assertion = false -ij_javascript_spaces_within_union_types = true -ij_javascript_spaces_within_while_parentheses = false -ij_javascript_special_else_if_treatment = true -ij_javascript_ternary_operation_signs_on_next_line = false -ij_javascript_ternary_operation_wrap = off -ij_javascript_union_types_wrap = on_every_item -ij_javascript_use_chained_calls_group_indents = false -ij_javascript_use_double_quotes = true -ij_javascript_use_explicit_js_extension = auto -ij_javascript_use_import_type = auto -ij_javascript_use_path_mapping = always -ij_javascript_use_public_modifier = false -ij_javascript_use_semicolon_after_statement = true -ij_javascript_var_declaration_wrap = normal -ij_javascript_while_brace_force = never -ij_javascript_while_on_new_line = false -ij_javascript_wrap_comments = false - -[{*.ft,*.vm,*.vsl}] -ij_vtl_keep_indents_on_empty_lines = false - -[{*.gant,*.groovy,*.gy}] -ij_groovy_align_group_field_declarations = false -ij_groovy_align_multiline_array_initializer_expression = false -ij_groovy_align_multiline_assignment = false -ij_groovy_align_multiline_binary_operation = false -ij_groovy_align_multiline_chained_methods = false -ij_groovy_align_multiline_extends_list = false -ij_groovy_align_multiline_for = true -ij_groovy_align_multiline_list_or_map = true -ij_groovy_align_multiline_method_parentheses = false -ij_groovy_align_multiline_parameters = true -ij_groovy_align_multiline_parameters_in_calls = false -ij_groovy_align_multiline_resources = true -ij_groovy_align_multiline_ternary_operation = false -ij_groovy_align_multiline_throws_list = false -ij_groovy_align_named_args_in_map = true -ij_groovy_align_throws_keyword = false -ij_groovy_array_initializer_new_line_after_left_brace = false -ij_groovy_array_initializer_right_brace_on_new_line = false -ij_groovy_array_initializer_wrap = off -ij_groovy_assert_statement_wrap = off -ij_groovy_assignment_wrap = off -ij_groovy_binary_operation_wrap = off -ij_groovy_blank_lines_after_class_header = 0 -ij_groovy_blank_lines_after_imports = 1 -ij_groovy_blank_lines_after_package = 1 -ij_groovy_blank_lines_around_class = 1 -ij_groovy_blank_lines_around_field = 0 -ij_groovy_blank_lines_around_field_in_interface = 0 -ij_groovy_blank_lines_around_method = 1 -ij_groovy_blank_lines_around_method_in_interface = 1 -ij_groovy_blank_lines_before_imports = 1 -ij_groovy_blank_lines_before_method_body = 0 -ij_groovy_blank_lines_before_package = 0 -ij_groovy_block_brace_style = end_of_line -ij_groovy_block_comment_add_space = false -ij_groovy_block_comment_at_first_column = true -ij_groovy_call_parameters_new_line_after_left_paren = false -ij_groovy_call_parameters_right_paren_on_new_line = false -ij_groovy_call_parameters_wrap = off -ij_groovy_catch_on_new_line = false -ij_groovy_class_annotation_wrap = split_into_lines -ij_groovy_class_brace_style = end_of_line -ij_groovy_class_count_to_use_import_on_demand = 5 -ij_groovy_do_while_brace_force = never -ij_groovy_else_on_new_line = false -ij_groovy_enable_groovydoc_formatting = true -ij_groovy_enum_constants_wrap = off -ij_groovy_extends_keyword_wrap = off -ij_groovy_extends_list_wrap = off -ij_groovy_field_annotation_wrap = split_into_lines -ij_groovy_finally_on_new_line = false -ij_groovy_for_brace_force = never -ij_groovy_for_statement_new_line_after_left_paren = false -ij_groovy_for_statement_right_paren_on_new_line = false -ij_groovy_for_statement_wrap = off -ij_groovy_ginq_general_clause_wrap_policy = 2 -ij_groovy_ginq_having_wrap_policy = 1 -ij_groovy_ginq_indent_having_clause = true -ij_groovy_ginq_indent_on_clause = true -ij_groovy_ginq_on_wrap_policy = 1 -ij_groovy_ginq_space_after_keyword = true -ij_groovy_if_brace_force = never -ij_groovy_import_annotation_wrap = 2 -ij_groovy_imports_layout = *,|,javax.**,java.**,|,$* -ij_groovy_indent_case_from_switch = true -ij_groovy_indent_label_blocks = true -ij_groovy_insert_inner_class_imports = false -ij_groovy_keep_blank_lines_before_right_brace = 2 -ij_groovy_keep_blank_lines_in_code = 2 -ij_groovy_keep_blank_lines_in_declarations = 2 -ij_groovy_keep_control_statement_in_one_line = true -ij_groovy_keep_first_column_comment = true -ij_groovy_keep_indents_on_empty_lines = false -ij_groovy_keep_line_breaks = true -ij_groovy_keep_multiple_expressions_in_one_line = false -ij_groovy_keep_simple_blocks_in_one_line = false -ij_groovy_keep_simple_classes_in_one_line = true -ij_groovy_keep_simple_lambdas_in_one_line = true -ij_groovy_keep_simple_methods_in_one_line = true -ij_groovy_label_indent_absolute = false -ij_groovy_label_indent_size = 0 -ij_groovy_lambda_brace_style = end_of_line -ij_groovy_layout_static_imports_separately = true -ij_groovy_line_comment_add_space = false -ij_groovy_line_comment_add_space_on_reformat = false -ij_groovy_line_comment_at_first_column = true -ij_groovy_method_annotation_wrap = split_into_lines -ij_groovy_method_brace_style = end_of_line -ij_groovy_method_call_chain_wrap = off -ij_groovy_method_parameters_new_line_after_left_paren = false -ij_groovy_method_parameters_right_paren_on_new_line = false -ij_groovy_method_parameters_wrap = off -ij_groovy_modifier_list_wrap = false -ij_groovy_names_count_to_use_import_on_demand = 3 -ij_groovy_packages_to_use_import_on_demand = java.awt.*,javax.swing.* -ij_groovy_parameter_annotation_wrap = off -ij_groovy_parentheses_expression_new_line_after_left_paren = false -ij_groovy_parentheses_expression_right_paren_on_new_line = false -ij_groovy_prefer_parameters_wrap = false -ij_groovy_resource_list_new_line_after_left_paren = false -ij_groovy_resource_list_right_paren_on_new_line = false -ij_groovy_resource_list_wrap = off -ij_groovy_space_after_assert_separator = true -ij_groovy_space_after_colon = true -ij_groovy_space_after_comma = true -ij_groovy_space_after_comma_in_type_arguments = true -ij_groovy_space_after_for_semicolon = true -ij_groovy_space_after_quest = true -ij_groovy_space_after_type_cast = true -ij_groovy_space_before_annotation_parameter_list = false -ij_groovy_space_before_array_initializer_left_brace = false -ij_groovy_space_before_assert_separator = false -ij_groovy_space_before_catch_keyword = true -ij_groovy_space_before_catch_left_brace = true -ij_groovy_space_before_catch_parentheses = true -ij_groovy_space_before_class_left_brace = true -ij_groovy_space_before_closure_left_brace = true -ij_groovy_space_before_colon = true -ij_groovy_space_before_comma = false -ij_groovy_space_before_do_left_brace = true -ij_groovy_space_before_else_keyword = true -ij_groovy_space_before_else_left_brace = true -ij_groovy_space_before_finally_keyword = true -ij_groovy_space_before_finally_left_brace = true -ij_groovy_space_before_for_left_brace = true -ij_groovy_space_before_for_parentheses = true -ij_groovy_space_before_for_semicolon = false -ij_groovy_space_before_if_left_brace = true -ij_groovy_space_before_if_parentheses = true -ij_groovy_space_before_method_call_parentheses = false -ij_groovy_space_before_method_left_brace = true -ij_groovy_space_before_method_parentheses = false -ij_groovy_space_before_quest = true -ij_groovy_space_before_record_parentheses = false -ij_groovy_space_before_switch_left_brace = true -ij_groovy_space_before_switch_parentheses = true -ij_groovy_space_before_synchronized_left_brace = true -ij_groovy_space_before_synchronized_parentheses = true -ij_groovy_space_before_try_left_brace = true -ij_groovy_space_before_try_parentheses = true -ij_groovy_space_before_while_keyword = true -ij_groovy_space_before_while_left_brace = true -ij_groovy_space_before_while_parentheses = true -ij_groovy_space_in_named_argument = true -ij_groovy_space_in_named_argument_before_colon = false -ij_groovy_space_within_empty_array_initializer_braces = false -ij_groovy_space_within_empty_method_call_parentheses = false -ij_groovy_spaces_around_additive_operators = true -ij_groovy_spaces_around_assignment_operators = true -ij_groovy_spaces_around_bitwise_operators = true -ij_groovy_spaces_around_equality_operators = true -ij_groovy_spaces_around_lambda_arrow = true -ij_groovy_spaces_around_logical_operators = true -ij_groovy_spaces_around_multiplicative_operators = true -ij_groovy_spaces_around_regex_operators = true -ij_groovy_spaces_around_relational_operators = true -ij_groovy_spaces_around_shift_operators = true -ij_groovy_spaces_within_annotation_parentheses = false -ij_groovy_spaces_within_array_initializer_braces = false -ij_groovy_spaces_within_braces = true -ij_groovy_spaces_within_brackets = false -ij_groovy_spaces_within_cast_parentheses = false -ij_groovy_spaces_within_catch_parentheses = false -ij_groovy_spaces_within_for_parentheses = false -ij_groovy_spaces_within_gstring_injection_braces = false -ij_groovy_spaces_within_if_parentheses = false -ij_groovy_spaces_within_list_or_map = false -ij_groovy_spaces_within_method_call_parentheses = false -ij_groovy_spaces_within_method_parentheses = false -ij_groovy_spaces_within_parentheses = false -ij_groovy_spaces_within_switch_parentheses = false -ij_groovy_spaces_within_synchronized_parentheses = false -ij_groovy_spaces_within_try_parentheses = false -ij_groovy_spaces_within_tuple_expression = false -ij_groovy_spaces_within_while_parentheses = false -ij_groovy_special_else_if_treatment = true -ij_groovy_ternary_operation_wrap = off -ij_groovy_throws_keyword_wrap = off -ij_groovy_throws_list_wrap = off -ij_groovy_use_flying_geese_braces = false -ij_groovy_use_fq_class_names = false -ij_groovy_use_fq_class_names_in_javadoc = true -ij_groovy_use_relative_indents = false -ij_groovy_use_single_class_imports = true -ij_groovy_variable_annotation_wrap = off -ij_groovy_while_brace_force = never -ij_groovy_while_on_new_line = false -ij_groovy_wrap_chain_calls_after_dot = false -ij_groovy_wrap_long_lines = false - -[{*.har,*.jsb2,*.jsb3,*.json,*.jsonc,*.postman_collection,*.postman_collection.json,*.postman_environment,*.postman_environment.json,.babelrc,.eslintrc,.prettierrc,.stylelintrc,.ws-context,bowerrc,jest.config}] -indent_size = 2 -ij_json_array_wrapping = split_into_lines -ij_json_keep_blank_lines_in_code = 0 -ij_json_keep_indents_on_empty_lines = false -ij_json_keep_line_breaks = true -ij_json_keep_trailing_comma = false -ij_json_object_wrapping = split_into_lines -ij_json_property_alignment = do_not_align -ij_json_space_after_colon = true -ij_json_space_after_comma = true -ij_json_space_before_colon = false -ij_json_space_before_comma = false -ij_json_spaces_within_braces = false -ij_json_spaces_within_brackets = false -ij_json_wrap_long_lines = false - -[{*.htm,*.html,*.sht,*.shtm,*.shtml}] -ij_html_add_new_line_before_tags = body,div,p,form,h1,h2,h3 -ij_html_align_attributes = true -ij_html_align_text = false -ij_html_attribute_wrap = normal -ij_html_block_comment_add_space = false -ij_html_block_comment_at_first_column = true -ij_html_do_not_align_children_of_min_lines = 0 -ij_html_do_not_break_if_inline_tags = title,h1,h2,h3,h4,h5,h6,p -ij_html_do_not_indent_children_of_tags = html,body,thead,tbody,tfoot -ij_html_enforce_quotes = false -ij_html_inline_tags = a,abbr,acronym,b,basefont,bdo,big,br,cite,cite,code,dfn,em,font,i,img,input,kbd,label,q,s,samp,select,small,span,strike,strong,sub,sup,textarea,tt,u,var -ij_html_keep_blank_lines = 2 -ij_html_keep_indents_on_empty_lines = false -ij_html_keep_line_breaks = true -ij_html_keep_line_breaks_in_text = true -ij_html_keep_whitespaces = false -ij_html_keep_whitespaces_inside = span,pre,textarea -ij_html_line_comment_at_first_column = true -ij_html_new_line_after_last_attribute = never -ij_html_new_line_before_first_attribute = never -ij_html_quote_style = double -ij_html_remove_new_line_before_tags = br -ij_html_space_after_tag_name = false -ij_html_space_around_equality_in_attribute = false -ij_html_space_inside_empty_tag = false -ij_html_text_wrap = normal - -[{*.http,*.rest}] -indent_size = 0 -ij_continuation_indent_size = 4 -ij_http-request_call_parameters_wrap = normal -ij_http-request_method_parameters_wrap = split_into_lines -ij_http-request_space_before_comma = true -ij_http-request_spaces_around_assignment_operators = true - -[{*.jsf,*.jsp,*.jspf,*.tag,*.tagf,*.xjsp}] -ij_jsp_jsp_prefer_comma_separated_import_list = false -ij_jsp_keep_indents_on_empty_lines = false - -[{*.jspx,*.tagx}] -ij_jspx_keep_indents_on_empty_lines = false - -[{*.kt,*.kts}] -ij_kotlin_align_in_columns_case_branch = false -ij_kotlin_align_multiline_binary_operation = false -ij_kotlin_align_multiline_extends_list = false -ij_kotlin_align_multiline_method_parentheses = false -ij_kotlin_align_multiline_parameters = true -ij_kotlin_align_multiline_parameters_in_calls = false -ij_kotlin_allow_trailing_comma = false -ij_kotlin_allow_trailing_comma_on_call_site = false -ij_kotlin_assignment_wrap = normal -ij_kotlin_blank_lines_after_class_header = 0 -ij_kotlin_blank_lines_around_block_when_branches = 0 -ij_kotlin_blank_lines_before_declaration_with_comment_or_annotation_on_separate_line = 1 -ij_kotlin_block_comment_add_space = false -ij_kotlin_block_comment_at_first_column = true -ij_kotlin_call_parameters_new_line_after_left_paren = true -ij_kotlin_call_parameters_right_paren_on_new_line = true -ij_kotlin_call_parameters_wrap = on_every_item -ij_kotlin_catch_on_new_line = false -ij_kotlin_class_annotation_wrap = split_into_lines -ij_kotlin_continuation_indent_for_chained_calls = false -ij_kotlin_continuation_indent_for_expression_bodies = false -ij_kotlin_continuation_indent_in_argument_lists = false -ij_kotlin_continuation_indent_in_elvis = false -ij_kotlin_continuation_indent_in_if_conditions = false -ij_kotlin_continuation_indent_in_parameter_lists = false -ij_kotlin_continuation_indent_in_supertype_lists = false -ij_kotlin_else_on_new_line = false -ij_kotlin_enum_constants_wrap = off -ij_kotlin_extends_list_wrap = normal -ij_kotlin_field_annotation_wrap = split_into_lines -ij_kotlin_finally_on_new_line = false -ij_kotlin_if_rparen_on_new_line = true -ij_kotlin_import_nested_classes = false -ij_kotlin_imports_layout = *,java.**,javax.**,kotlin.**,^ -ij_kotlin_insert_whitespaces_in_simple_one_line_method = true -ij_kotlin_keep_blank_lines_before_right_brace = 2 -ij_kotlin_keep_blank_lines_in_code = 2 -ij_kotlin_keep_blank_lines_in_declarations = 2 -ij_kotlin_keep_first_column_comment = true -ij_kotlin_keep_indents_on_empty_lines = false -ij_kotlin_keep_line_breaks = true -ij_kotlin_lbrace_on_next_line = false -ij_kotlin_line_break_after_multiline_when_entry = true -ij_kotlin_line_comment_add_space = false -ij_kotlin_line_comment_add_space_on_reformat = false -ij_kotlin_line_comment_at_first_column = true -ij_kotlin_method_annotation_wrap = split_into_lines -ij_kotlin_method_call_chain_wrap = normal -ij_kotlin_method_parameters_new_line_after_left_paren = true -ij_kotlin_method_parameters_right_paren_on_new_line = true -ij_kotlin_method_parameters_wrap = on_every_item -ij_kotlin_name_count_to_use_star_import = 5 -ij_kotlin_name_count_to_use_star_import_for_members = 3 -ij_kotlin_packages_to_use_import_on_demand = java.util.*,kotlinx.android.synthetic.**,io.ktor.** -ij_kotlin_parameter_annotation_wrap = off -ij_kotlin_space_after_comma = true -ij_kotlin_space_after_extend_colon = true -ij_kotlin_space_after_type_colon = true -ij_kotlin_space_before_catch_parentheses = true -ij_kotlin_space_before_comma = false -ij_kotlin_space_before_extend_colon = true -ij_kotlin_space_before_for_parentheses = true -ij_kotlin_space_before_if_parentheses = true -ij_kotlin_space_before_lambda_arrow = true -ij_kotlin_space_before_type_colon = false -ij_kotlin_space_before_when_parentheses = true -ij_kotlin_space_before_while_parentheses = true -ij_kotlin_spaces_around_additive_operators = true -ij_kotlin_spaces_around_assignment_operators = true -ij_kotlin_spaces_around_equality_operators = true -ij_kotlin_spaces_around_function_type_arrow = true -ij_kotlin_spaces_around_logical_operators = true -ij_kotlin_spaces_around_multiplicative_operators = true -ij_kotlin_spaces_around_range = false -ij_kotlin_spaces_around_relational_operators = true -ij_kotlin_spaces_around_unary_operator = false -ij_kotlin_spaces_around_when_arrow = true -ij_kotlin_variable_annotation_wrap = off -ij_kotlin_while_on_new_line = false -ij_kotlin_wrap_elvis_expressions = 1 -ij_kotlin_wrap_expression_body_functions = 1 -ij_kotlin_wrap_first_method_in_call_chain = false - -[{*.markdown,*.md}] -ij_markdown_force_one_space_after_blockquote_symbol = true -ij_markdown_force_one_space_after_header_symbol = true -ij_markdown_force_one_space_after_list_bullet = true -ij_markdown_force_one_space_between_words = true -ij_markdown_format_tables = true -ij_markdown_insert_quote_arrows_on_wrap = true -ij_markdown_keep_indents_on_empty_lines = false -ij_markdown_keep_line_breaks_inside_text_blocks = true -ij_markdown_max_lines_around_block_elements = 1 -ij_markdown_max_lines_around_header = 1 -ij_markdown_max_lines_between_paragraphs = 1 -ij_markdown_min_lines_around_block_elements = 1 -ij_markdown_min_lines_around_header = 1 -ij_markdown_min_lines_between_paragraphs = 1 -ij_markdown_wrap_text_if_long = true -ij_markdown_wrap_text_inside_blockquotes = true - -[{*.pb,*.textproto,*.txtpb}] -indent_size = 2 -tab_width = 2 -ij_continuation_indent_size = 4 -ij_prototext_keep_blank_lines_in_code = 2 -ij_prototext_keep_indents_on_empty_lines = false -ij_prototext_keep_line_breaks = true -ij_prototext_space_after_colon = true -ij_prototext_space_after_comma = true -ij_prototext_space_before_colon = false -ij_prototext_space_before_comma = false -ij_prototext_spaces_within_braces = true -ij_prototext_spaces_within_brackets = false - -[{*.properties,spring.handlers,spring.schemas}] -ij_properties_align_group_field_declarations = false -ij_properties_keep_blank_lines = false -ij_properties_key_value_delimiter = equals -ij_properties_spaces_around_key_value_delimiter = false - -[{*.py,*.pyw}] -ij_python_align_collections_and_comprehensions = true -ij_python_align_multiline_imports = true -ij_python_align_multiline_parameters = true -ij_python_align_multiline_parameters_in_calls = true -ij_python_blank_line_at_file_end = true -ij_python_blank_lines_after_imports = 1 -ij_python_blank_lines_after_local_imports = 0 -ij_python_blank_lines_around_class = 1 -ij_python_blank_lines_around_method = 1 -ij_python_blank_lines_around_top_level_classes_functions = 2 -ij_python_blank_lines_before_first_method = 0 -ij_python_call_parameters_new_line_after_left_paren = false -ij_python_call_parameters_right_paren_on_new_line = false -ij_python_call_parameters_wrap = normal -ij_python_dict_alignment = 0 -ij_python_dict_new_line_after_left_brace = false -ij_python_dict_new_line_before_right_brace = false -ij_python_dict_wrapping = 1 -ij_python_from_import_new_line_after_left_parenthesis = false -ij_python_from_import_new_line_before_right_parenthesis = false -ij_python_from_import_parentheses_force_if_multiline = false -ij_python_from_import_trailing_comma_if_multiline = false -ij_python_from_import_wrapping = 1 -ij_python_hang_closing_brackets = false -ij_python_keep_blank_lines_in_code = 1 -ij_python_keep_blank_lines_in_declarations = 1 -ij_python_keep_indents_on_empty_lines = false -ij_python_keep_line_breaks = true -ij_python_method_parameters_new_line_after_left_paren = false -ij_python_method_parameters_right_paren_on_new_line = false -ij_python_method_parameters_wrap = normal -ij_python_new_line_after_colon = false -ij_python_new_line_after_colon_multi_clause = true -ij_python_optimize_imports_always_split_from_imports = false -ij_python_optimize_imports_case_insensitive_order = false -ij_python_optimize_imports_join_from_imports_with_same_source = false -ij_python_optimize_imports_sort_by_type_first = true -ij_python_optimize_imports_sort_imports = true -ij_python_optimize_imports_sort_names_in_from_imports = false -ij_python_space_after_comma = true -ij_python_space_after_number_sign = true -ij_python_space_after_py_colon = true -ij_python_space_before_backslash = true -ij_python_space_before_comma = false -ij_python_space_before_for_semicolon = false -ij_python_space_before_lbracket = false -ij_python_space_before_method_call_parentheses = false -ij_python_space_before_method_parentheses = false -ij_python_space_before_number_sign = true -ij_python_space_before_py_colon = false -ij_python_space_within_empty_method_call_parentheses = false -ij_python_space_within_empty_method_parentheses = false -ij_python_spaces_around_additive_operators = true -ij_python_spaces_around_assignment_operators = true -ij_python_spaces_around_bitwise_operators = true -ij_python_spaces_around_eq_in_keyword_argument = false -ij_python_spaces_around_eq_in_named_parameter = false -ij_python_spaces_around_equality_operators = true -ij_python_spaces_around_multiplicative_operators = true -ij_python_spaces_around_power_operator = true -ij_python_spaces_around_relational_operators = true -ij_python_spaces_around_shift_operators = true -ij_python_spaces_within_braces = false -ij_python_spaces_within_brackets = false -ij_python_spaces_within_method_call_parentheses = false -ij_python_spaces_within_method_parentheses = false -ij_python_use_continuation_indent_for_arguments = false -ij_python_use_continuation_indent_for_collection_and_comprehensions = false -ij_python_use_continuation_indent_for_parameters = true -ij_python_wrap_long_lines = false - -[{*.qute.htm,*.qute.html,*.qute.json,*.qute.txt,*.qute.yaml,*.qute.yml}] -ij_qute_keep_indents_on_empty_lines = false - -[{*.toml,Cargo.lock,Cargo.toml.orig,Gopkg.lock,Pipfile,poetry.lock}] -ij_toml_keep_indents_on_empty_lines = false - -[{*.yaml,*.yml}] -indent_size = 2 -ij_yaml_align_values_properties = do_not_align -ij_yaml_autoinsert_sequence_marker = true -ij_yaml_block_mapping_on_new_line = false -ij_yaml_indent_sequence_value = true -ij_yaml_keep_indents_on_empty_lines = false -ij_yaml_keep_line_breaks = true -ij_yaml_sequence_on_new_line = false -ij_yaml_space_before_colon = false -ij_yaml_spaces_within_braces = true -ij_yaml_spaces_within_brackets = true diff --git a/flink-cyber/google_checks.xml b/flink-cyber/google_checks.xml index b2c8eec7..609f61fa 100644 --- a/flink-cyber/google_checks.xml +++ b/flink-cyber/google_checks.xml @@ -54,7 +54,7 @@ - + diff --git a/flink-cyber/pom.xml b/flink-cyber/pom.xml index 3598ea0e..2dbcd607 100644 --- a/flink-cyber/pom.xml +++ b/flink-cyber/pom.xml @@ -639,6 +639,7 @@ google_checks.xml true + warning true false From 81a4038f76a0d6f766e8fb1fb1f5bca63e8db49a Mon Sep 17 00:00:00 2001 From: Stas Panasiuk Date: Thu, 24 Oct 2024 01:33:24 -0400 Subject: [PATCH 08/15] code updates according to the new codestyle --- .github/workflows/build_and_test.yml | 4 +- flink-cyber/.editorconfig | 1460 +++++++++++++++ .../generator/CaracalGeneratorFlinkJob.java | 104 +- .../CaracalGeneratorFlinkJobKafka.java | 54 +- .../cyber/test/generator/GetIpMap.java | 8 +- .../cyber/test/generator/RandomSampler.java | 6 +- .../cyber/test/resources/log4j2.properties | 14 +- .../cyber/caracal/ParserChainMapFunction.java | 53 +- .../SplitBroadcastProcessFunction.java | 36 +- .../cloudera/cyber/caracal/SplitConfig.java | 12 +- .../com/cloudera/cyber/caracal/SplitJob.java | 65 +- .../cloudera/cyber/caracal/SplitJobKafka.java | 123 +- .../caracal/SplittingFlatMapFunction.java | 76 +- .../caracal/SplittingFlatMapFunctionTest.java | 19 +- .../{google_checks.xml => checkstyle.xml} | 158 +- .../libs/AbstractBooleanScalarFunction.java | 2 +- .../cyber/libs/hostnames/ExtractHostname.java | 34 +- .../hostnames/ExtractHostnameFeatures.java | 9 +- .../cyber/libs/hostnames/NsLookup.java | 83 +- .../cyber/libs/networking/IPLocal.java | 1 - .../cyber/libs/networking/InSubnet.java | 23 +- .../libs/hostnames/TestExtractHostname.java | 28 +- .../TestExtractHostnameFeatures.java | 58 +- .../cyber/libs/hostnames/TestNsLookup.java | 7 +- .../cyber/libs/networking/TestIPLocal.java | 4 +- .../cyber/libs/networking/TestInSubnet.java | 4 +- .../com/cloudera/service/common/Utils.java | 33 +- .../config/kafka/ClouderaKafkaProperties.java | 6 +- .../ClouderaManagementKafkaProperties.java | 2 +- .../service/common/request/RequestType.java | 3 +- .../cloudera/service/common/response/Job.java | 8 +- .../service/common/response/ResponseBody.java | 5 +- .../service/common/response/ResponseType.java | 3 +- .../service/common/utils/ArchiveUtil.java | 46 +- .../service/common/utils/FileUtil.java | 9 +- .../cyber/restcli/CliApplication.java | 6 +- .../restcli/configuration/KafkaConfig.java | 9 +- .../controller/KafkaListenerController.java | 54 +- .../cyber/restcli/service/JobService.java | 86 +- .../cyber/pruner/AbstractDateFilter.java | 5 +- .../cyber/pruner/DateDirectoryFilter.java | 15 +- .../cloudera/cyber/pruner/DateFileFilter.java | 4 +- .../com/cloudera/cyber/pruner/Pruner.java | 55 +- .../cloudera/cyber/pruner/PrunerConfig.java | 11 +- .../cloudera/cyber/scoring/ScoredMessage.java | 33 +- .../ScoredMessageWatermarkedStream.java | 15 +- .../com/cloudera/cyber/scoring/Scores.java | 16 +- .../cloudera/cyber/scoring/ScoringRule.java | 130 +- .../cyber/scoring/ScoringRuleCommand.java | 36 +- .../scoring/ScoringRuleCommandResult.java | 25 +- .../cyber/scoring/ScoringRuleTypeFactory.java | 13 +- .../scoring/ScoringSummarizationMode.java | 5 +- .../scoring/ScoringSummarizationModeTest.java | 19 +- .../cyber/scoring/SerializationTests.java | 45 +- flink-cyber/flink-alert-scoring/pom.xml | 3 +- .../cloudera/cyber/scoring/ScoringJob.java | 38 +- .../cyber/scoring/ScoringJobKafka.java | 30 +- .../cyber/scoring/ScoringProcessFunction.java | 83 +- .../cyber/scoring/TestScoringJob.java | 212 ++- .../TestScoringRulesProcessFunction.java | 291 ++- .../com/cloudera/cyber/scoring/TypeTests.java | 6 +- .../cyber/json/ReplaceJsonProperties.java | 13 +- .../cyber/json/ReplaceJsonPropertiesTest.java | 45 +- .../java/com/cloudera/cyber/kafka/Config.java | 18 +- .../cloudera/cyber/kafka/TopicAvroToJson.java | 12 +- .../cyber/scoring/UpsertScoringRule.java | 61 +- .../java/com/cloudera/cyber/MessageUtils.java | 68 +- .../com/cloudera/cyber/ValidateUtils.java | 17 +- .../cloudera/cyber/enrichment/Enrichment.java | 17 +- .../cyber/enrichment/MetronGeoEnrichment.java | 3 +- .../cloudera/cyber/flink/CacheMetrics.java | 8 +- .../cloudera/cyber/flink/ConfigConstants.java | 3 +- .../cyber/flink/EventTimeAndCountTrigger.java | 25 +- .../com/cloudera/cyber/flink/FlinkUtils.java | 95 +- .../cyber/flink/SourcesWithHeaders.java | 55 +- ...undedOutOfOrdernessTimestampExtractor.java | 3 +- .../java/com/cloudera/cyber/flink/Utils.java | 108 +- .../flink/operators/MessageConcatenate.java | 3 +- .../generator/FlinkFileTemplateLoader.java | 15 +- .../FreemarkerImmediateGenerator.java | 18 +- .../generator/FreemarkerTemplateSource.java | 51 +- .../cyber/generator/GenerationSource.java | 13 +- .../cyber/generator/GeneratorConfig.java | 5 +- .../cyber/generator/RandomGenerators.java | 6 +- .../cyber/generator/SyntheticEntry.java | 5 +- .../cyber/generator/SyntheticThreatEntry.java | 1 - .../cyber/generator/ThreatGenerator.java | 17 +- .../cyber/generator/ThreatGeneratorMap.java | 3 +- .../com/cloudera/cyber/generator/Utils.java | 56 +- .../generator/scenario/GeneratorScenario.java | 28 +- .../cyber/libs/CyberFunctionDefinition.java | 32 +- .../parser/MessageToParseDeserializer.java | 18 +- .../com/cloudera/cyber/MessageUtilsTest.java | 43 +- .../cloudera/cyber/TestMessageSerializer.java | 77 +- .../cyber/TestMessageTypeInformation.java | 6 +- .../java/com/cloudera/cyber/TestSigning.java | 23 +- .../java/com/cloudera/cyber/TestUtils.java | 44 +- .../com/cloudera/cyber/ValidateUtilsTest.java | 43 +- .../cyber/generator/GenerationSourceTest.java | 16 +- .../cyber/generator/GeneratorUtilsTest.java | 12 +- .../scenario/GeneratorScenarioTest.java | 36 +- .../cyber/libs/TestCyberFunctionUtils.java | 9 +- .../flink-cyber-api/enrichment_config.md | 22 +- .../java/com/cloudera/cyber/AvroTypes.java | 44 +- .../com/cloudera/cyber/CyberFunction.java | 6 +- .../cloudera/cyber/DataQualityMessage.java | 62 +- .../cyber/DataQualityMessageLevel.java | 11 +- .../cyber/DataQualityMessageTypeInfo.java | 5 +- .../com/cloudera/cyber/DedupeMessage.java | 81 +- .../com/cloudera/cyber/EnrichmentEntry.java | 61 +- .../cyber/EnrichmentEntryTypeFactory.java | 24 +- .../com/cloudera/cyber/FieldRegistry.java | 10 +- .../com/cloudera/cyber/GroupedMessage.java | 47 +- .../com/cloudera/cyber/IdentifiedMessage.java | 3 +- .../main/java/com/cloudera/cyber/Message.java | 172 +- .../cloudera/cyber/MessageTypeFactory.java | 37 +- .../com/cloudera/cyber/SignedSourceKey.java | 80 +- .../cyber/SignedSourceKeyTypeFactory.java | 21 +- .../cloudera/cyber/ThreatIntelligence.java | 77 +- .../cyber/ThreatIntelligenceTypeFactory.java | 23 +- .../com/cloudera/cyber/avro/AvroSchemas.java | 14 +- .../com/cloudera/cyber/commands/Command.java | 41 +- .../cyber/commands/CommandResponse.java | 52 +- .../cyber/commands/EnrichmentCommand.java | 39 +- .../commands/EnrichmentCommandResponse.java | 26 +- .../lookup/config/EnrichmentConfig.java | 5 +- .../lookup/config/EnrichmentField.java | 5 +- .../com/cloudera/cyber/flink/HasHeaders.java | 4 +- .../cyber/indexing/MappingColumnDto.java | 2 +- .../cloudera/cyber/indexing/MappingDto.java | 19 +- .../cloudera/cyber/parser/MessageToParse.java | 71 +- .../cloudera/cyber/SerializationTests.java | 165 +- .../java/com/cloudera/cyber/TypeTests.java | 12 +- .../com/cloudera/cyber/dedupe/Dedupe.java | 32 +- .../com/cloudera/cyber/dedupe/DedupeJob.java | 26 +- .../cloudera/cyber/dedupe/DedupeJobKafka.java | 40 +- .../dedupe/impl/CreateKeyFromMessage.java | 19 +- .../cloudera/cyber/dedupe/impl/SumAndMax.java | 13 +- .../cyber/dedupe/impl/SumAndMaxTs.java | 28 +- .../cyber/enrichment/cidr/IpRegionCidr.java | 37 +- .../enrichment/cidr/IpRegionCidrJob.java | 8 +- .../enrichment/cidr/IpRegionCidrJobKafka.java | 32 +- .../cyber/enrichment/cidr/IpRegionMap.java | 7 +- .../cidr/impl/IpRegionCidrEnrichment.java | 18 +- .../cyber/enrichment/EnrichmentJob.java | 132 +- .../cyber/enrichment/EnrichmentJobKafka.java | 62 +- .../enrichment/FilterEnrichmentType.java | 7 +- .../cyber/enrichment/geocode/IpAsnMap.java | 9 +- .../cyber/enrichment/geocode/IpGeo.java | 18 +- .../cyber/enrichment/geocode/IpGeoJob.java | 10 +- .../enrichment/geocode/IpGeoJobKafka.java | 32 +- .../cyber/enrichment/geocode/IpGeoMap.java | 16 +- .../geocode/impl/IpAsnEnrichment.java | 14 +- .../geocode/impl/IpGeoEnrichment.java | 32 +- .../enrichment/geocode/impl/MaxMindBase.java | 46 +- .../impl/types/GeoEnrichmentFields.java | 3 +- .../geocode/impl/types/GeoFields.java | 8 +- .../impl/types/MetronGeoEnrichmentFields.java | 19 +- .../enrichment/geocode/IpAsnMapTest.java | 25 +- .../load/BatchEnrichmentLoader.java | 4 +- .../load/BatchEnrichmentLoaderCSV.java | 54 +- .../BatchEnrichmentLoaderCSVHbaseKafka.java | 14 +- .../CsvToEnrichmentCommandDeserializer.java | 58 +- ...svToEnrichmentCommandDeserializerTest.java | 77 +- .../HbaseBatchEnrichmentCSVLoaderTest.java | 101 +- .../cyber/enrichment/ConfigUtils.java | 40 +- .../hbase/EnrichmentLookupBuilder.java | 1 - .../hbase/HbaseEnrichmentFunction.java | 32 +- .../hbase/HbaseEnrichmentMapFunction.java | 46 +- .../cyber/enrichment/hbase/HbaseJob.java | 33 +- .../enrichment/hbase/HbaseJobRawKafka.java | 70 +- .../enrichment/hbase/MapMergeFunction.java | 20 +- .../enrichment/hbase/MetronLookupKey.java | 13 +- .../hbase/MetronLookupKeyBuilder.java | 6 +- .../enrichment/hbase/PrefixMapFunction.java | 9 +- .../enrichment/hbase/SimpleLookupKey.java | 14 +- .../hbase/SimpleLookupKeyBuilder.java | 7 +- .../hbase/config/EnrichmentConfig.java | 15 +- .../hbase/config/EnrichmentFieldsConfig.java | 54 +- .../hbase/config/EnrichmentStorageConfig.java | 24 +- .../hbase/config/EnrichmentStorageFormat.java | 7 +- .../hbase/config/EnrichmentsConfig.java | 48 +- .../EnrichmentCommandMutationConverter.java | 3 +- .../HbaseEnrichmentMutationConverter.java | 3 +- ...etronHbaseEnrichmentMutationConverter.java | 8 +- ...impleHbaseEnrichmentMutationConverter.java | 12 +- .../writer/HbaseEnrichmentCommandSink.java | 7 +- .../hbase/config/EnrichmentsConfigTest.java | 21 +- .../EnrichmentBroadcastProcessFunction.java | 117 +- .../enrichment/lookup/EnrichmentKey.java | 5 +- .../lookup/EnrichmentLookupResult.java | 3 +- .../cyber/enrichment/lookup/LookupJob.java | 95 +- .../enrichment/lookup/LookupJobKafka.java | 38 +- .../enrichment/rest/AsyncHttpRequest.java | 55 +- .../rest/BasicAuthorizationConfig.java | 20 +- .../rest/BearerTokenAuthorizationConfig.java | 7 +- .../rest/EndpointAuthorizationConfig.java | 15 +- .../cyber/enrichment/rest/GetRestRequest.java | 5 +- .../enrichment/rest/PostRestRequest.java | 12 +- .../enrichment/rest/RestEnrichmentConfig.java | 75 +- .../cyber/enrichment/rest/RestLookupJob.java | 62 +- .../enrichment/rest/RestLookupJobKafka.java | 18 +- .../cyber/enrichment/rest/RestRequest.java | 73 +- .../rest/RestRequestCacheExpiry.java | 15 +- .../cyber/enrichment/rest/RestRequestKey.java | 17 +- .../enrichment/rest/RestRequestResult.java | 9 +- .../cyber/enrichment/rest/TlsConfig.java | 10 +- .../enrichment/rest/AsyncHttpRequestTest.java | 38 +- .../enrichment/rest/GetRestRequestTest.java | 9 +- .../enrichment/rest/impl/MockRestServer.java | 136 +- .../stellar/StellarEnrichmentJob.java | 33 +- .../stellar/StellarEnrichmentJobKafka.java | 15 +- .../adapter/MetronGeoEnrichmentAdapter.java | 13 +- .../functions/GeoEnrichmentFunctions.java | 79 +- .../flink/StellarEnrichMapFunction.java | 76 +- .../enrichment/threatq/ThreatQConfig.java | 3 +- .../enrichment/threatq/ThreatQEntry.java | 210 ++- .../enrichment/threatq/ThreatQHBaseMap.java | 38 +- .../cyber/enrichment/threatq/ThreatQJob.java | 45 +- .../enrichment/threatq/ThreatQJobKafka.java | 40 +- .../enrichment/threatq/ThreatQParser.java | 33 +- .../threatq/ThreatQParserFlatMap.java | 11 +- .../adapters/cif/AbstractCIFAdapter.java | 44 +- .../adapters/cif/CIFHbaseAdapter.java | 169 +- .../enrichment/adapters/geo/GeoAdapter.java | 67 +- .../adapters/host/AbstractHostAdapter.java | 49 +- .../host/HostFromJSONListAdapter.java | 92 +- .../host/HostFromPropertiesFileAdapter.java | 96 +- .../adapters/jdbc/BaseJdbcConfig.java | 77 +- .../enrichment/adapters/jdbc/JdbcAdapter.java | 152 +- .../enrichment/adapters/jdbc/JdbcConfig.java | 17 +- .../enrichment/adapters/jdbc/MySqlConfig.java | 39 +- .../adapters/maxmind/MaxMindDatabase.java | 182 +- .../adapters/maxmind/MaxMindDbUtilities.java | 174 +- .../maxmind/asn/GeoLiteAsnDatabase.java | 220 +-- .../maxmind/geo/GeoLiteCityDatabase.java | 352 ++-- .../maxmind/geo/hash/DistanceStrategies.java | 45 +- .../maxmind/geo/hash/DistanceStrategy.java | 3 +- .../maxmind/geo/hash/GeoHashUtil.java | 292 ++- .../simplehbase/SimpleHBaseAdapter.java | 165 +- .../simplehbase/SimpleHBaseConfig.java | 75 +- .../adapters/stellar/StellarAdapter.java | 329 ++-- .../threatintel/ThreatIntelAdapter.java | 208 +-- .../threatintel/ThreatIntelConfig.java | 195 +- .../metron/enrichment/cache/CacheKey.java | 111 +- .../metron/enrichment/cache/ObjectCache.java | 157 +- .../enrichment/cache/ObjectCacheConfig.java | 188 +- .../enrichment/cli/LatencySummarizer.java | 327 ++-- .../enrichment/configuration/Enrichment.java | 65 +- .../converter/AbstractConverter.java | 53 +- .../converter/EnrichmentConverter.java | 28 +- .../converter/EnrichmentHelper.java | 14 +- .../enrichment/converter/EnrichmentKey.java | 178 +- .../enrichment/converter/EnrichmentValue.java | 57 +- .../enrichment/converter/HbaseConverter.java | 12 +- .../interfaces/EnrichmentAdapter.java | 36 +- .../enrichment/lookup/EnrichmentLookup.java | 192 +- .../metron/enrichment/lookup/Lookup.java | 122 +- .../metron/enrichment/lookup/LookupKV.java | 36 +- .../metron/enrichment/lookup/LookupKey.java | 16 +- .../metron/enrichment/lookup/LookupValue.java | 6 +- .../lookup/accesstracker/AccessTracker.java | 17 +- .../accesstracker/AccessTrackerCreator.java | 14 +- .../accesstracker/AccessTrackerUtil.java | 39 +- .../lookup/accesstracker/AccessTrackers.java | 28 +- .../accesstracker/BloomAccessTracker.java | 56 +- .../accesstracker/NoopAccessTracker.java | 68 +- .../PersistentAccessTracker.java | 51 +- .../PersistentBloomTrackerCreator.java | 142 +- .../enrichment/lookup/handler/Handler.java | 24 +- .../lookup/handler/KeyWithContext.java | 27 +- .../parallel/EnrichmentCallable.java | 71 +- .../parallel/EnrichmentContext.java | 33 +- .../parallel/EnrichmentStrategies.java | 122 +- .../parallel/EnrichmentStrategy.java | 59 +- .../parallel/WorkerPoolStrategies.java | 37 +- .../stellar/AsnEnrichmentFunctions.java | 124 +- .../stellar/EnrichmentObjectGet.java | 143 +- .../stellar/GeoEnrichmentFunctions.java | 113 +- .../enrichment/stellar/GeoHashFunctions.java | 505 +++-- .../metron/enrichment/stellar/ObjectGet.java | 117 +- .../SimpleHBaseEnrichmentFunctions.java | 435 ++--- .../enrichment/utils/EnrichmentUtils.java | 172 +- .../enrichment/utils/ThreatIntelUtils.java | 241 +-- .../triage/ThreatTriageProcessor.java | 142 +- .../maxmind/asn/GeoLiteAsnDatabaseTest.java | 21 +- .../maxmind/geo/GeoLiteCityDatabaseTest.java | 25 +- .../SimpleHBaseEnrichmentFunctionsTest.java | 18 +- .../cyber/hbase/AbstractHbaseMapFunction.java | 47 +- .../hbase/AbstractHbaseSinkFunction.java | 10 +- .../cyber/hbase/HbaseConfiguration.java | 8 +- .../com/cloudera/cyber/hbase/LookupKey.java | 8 +- .../cyber/indexing/hive/FieldExtractor.java | 4 +- .../indexing/hive/FilterMapFunction.java | 9 +- .../cloudera/cyber/indexing/hive/HiveJob.java | 6 +- .../cyber/indexing/hive/HiveJobKafka.java | 14 +- .../hive/HiveStreamingMessageWriter.java | 228 ++- .../indexing/hive/StreamingHiveSink.java | 2 +- .../indexing/hive/TimestampNormalizer.java | 19 +- .../hive/tableapi/TableApiAbstractJob.java | 839 +++++---- .../hive/tableapi/TableApiJobFactory.java | 10 +- .../hive/tableapi/impl/MapRowToAvro.java | 3 +- .../hive/tableapi/impl/TableApiHiveJob.java | 8 +- .../hive/tableapi/impl/TableApiKafkaJob.java | 156 +- .../indexing/hive/util/AvroSchemaUtil.java | 26 +- .../indexing/hive/util/FlinkSchemaUtil.java | 19 +- .../cloudera/cyber/indexing/ParquetJob.java | 3 + .../cyber/indexing/ParquetJobKafka.java | 15 +- .../cyber/indexing/CollectionField.java | 23 +- .../indexing/FilterStreamFieldsByConfig.java | 74 +- .../cloudera/cyber/indexing/IndexEntry.java | 3 +- .../cyber/indexing/SearchIndexJob.java | 38 +- .../cyber/indexing/SolrClientBuilder.java | 3 +- .../indexing/SolrCollectionFieldsSource.java | 40 +- .../cloudera/cyber/indexing/SolrIndexer.java | 30 +- .../com/cloudera/cyber/indexing/SolrJob.java | 11 +- .../cloudera/cyber/indexing/SolrJobKafka.java | 26 +- .../stellar/MetronCompatibilityParser.java | 71 +- .../cyber/stellar/MetronRawDataExtractor.java | 37 +- .../org/apache/metron/common/Constants.java | 171 +- .../apache/metron/parsers/BasicParser.java | 107 +- .../parsers/DefaultMessageParserResult.java | 75 +- .../org/apache/metron/parsers/GrokParser.java | 495 ++--- .../apache/metron/parsers/ParseException.java | 5 +- .../metron/parsers/asa/BasicAsaParser.java | 409 +++-- .../metron/parsers/bro/BasicBroParser.java | 263 +-- .../metron/parsers/bro/JSONCleaner.java | 123 +- .../apache/metron/parsers/cef/CEFParser.java | 478 ++--- .../apache/metron/parsers/csv/CSVParser.java | 106 +- .../filters/AbstractMessageFilter.java | 19 +- .../parsers/filters/BroMessageFilter.java | 88 +- .../metron/parsers/filters/Filters.java | 67 +- .../metron/parsers/filters/StellarFilter.java | 56 +- .../parsers/fireeye/BasicFireEyeParser.java | 257 +-- .../parsers/interfaces/Configurable.java | 10 +- .../parsers/interfaces/MessageFilter.java | 9 +- .../parsers/interfaces/MessageParser.java | 123 +- .../interfaces/MessageParserResult.java | 37 +- .../metron/parsers/ise/BasicIseParser.java | 143 +- .../apache/metron/parsers/ise/ISEParser.java | 1460 ++++++++------- .../parsers/ise/ISEParserConstants.java | 109 +- .../parsers/ise/ISEParserTokenManager.java | 1334 +++++++------- .../metron/parsers/ise/JavaCharStream.java | 1214 +++++++------ .../metron/parsers/ise/ParseException.java | 323 ++-- .../org/apache/metron/parsers/ise/Token.java | 245 +-- .../metron/parsers/ise/TokenMgrError.java | 299 +-- .../metron/parsers/json/JSONMapParser.java | 347 ++-- .../parsers/lancope/BasicLancopeParser.java | 98 +- .../metron/parsers/leef/LEEFParser.java | 445 ++--- .../parsers/logstash/BasicLogstashParser.java | 131 +- .../paloalto/BasicPaloAltoFirewallParser.java | 1048 +++++++---- .../apache/metron/parsers/paloalto/README.md | 18 +- .../regex/RegularExpressionsParser.java | 210 ++- .../parsers/snort/BasicSnortParser.java | 272 +-- .../sourcefire/BasicSourcefireParser.java | 192 +- .../parsers/syslog/BaseSyslogParser.java | 206 +-- .../parsers/syslog/Syslog3164Parser.java | 15 +- .../parsers/syslog/Syslog5424Parser.java | 27 +- .../metron/parsers/utils/DateUtils.java | 197 +- .../metron/parsers/utils/GrokUtils.java | 46 +- .../metron/parsers/utils/ParserUtils.java | 98 +- .../metron/parsers/utils/SyslogUtils.java | 141 +- .../websphere/GrokWebSphereParser.java | 194 +- .../parsers/MultiLineGrokParserTest.java | 24 +- .../MultiLineWithErrorsGrokParserTest.java | 24 +- .../parsers/bro/BasicBroParserTest.java | 33 +- .../metron/parsers/cef/CEFParserTest.java | 58 +- .../fireeye/BasicFireEyeParserTest.java | 25 +- .../regex/RegularExpressionsParserTest.java | 21 +- .../connector/jdbc/JdbcConnectionOptions.java | 28 +- .../connector/jdbc/JdbcExecutionOptions.java | 7 +- .../cyber/jdbc/connector/jdbc/JdbcSink.java | 31 +- .../internal/GenericJdbcSinkFunction.java | 7 +- .../jdbc/internal/JdbcOutputFormat.java | 82 +- .../jdbc/internal/JdbcStatementBuilder.java | 6 +- .../connection/JdbcConnectionProvider.java | 2 +- .../PhoenixJdbcConnectionProvider.java | 13 +- .../executor/JdbcBatchStatementExecutor.java | 3 +- .../SimpleBatchStatementExecutor.java | 7 +- .../options/JdbcConnectorOptions.java | 63 +- .../FieldValueProfileAggregateFunction.java | 9 +- .../cyber/profiler/FirstSeenHBase.java | 14 +- .../cyber/profiler/FirstSeenHbaseLookup.java | 64 +- .../cyber/profiler/MessageKeySelector.java | 13 +- .../profiler/ProfileAggregateFunction.java | 21 +- .../cloudera/cyber/profiler/ProfileJob.java | 309 ++-- .../cyber/profiler/ProfileJobKafka.java | 191 +- .../cyber/profiler/ProfileMessage.java | 12 +- .../cyber/profiler/ProfileMessageFilter.java | 29 +- ...ileMessageToMeasurementDataDtoMapping.java | 96 +- .../profiler/ProfileMessageToMessageMap.java | 28 +- .../cyber/profiler/ProfileStatsJoin.java | 8 +- .../cloudera/cyber/profiler/ProfileUtils.java | 22 +- .../ScoredMessageToProfileMessageMap.java | 23 +- .../StatsProfileAggregateFunction.java | 6 +- .../profiler/StatsProfileKeySelector.java | 5 +- .../FieldValueProfileGroupAcc.java | 127 +- .../profiler/accumulator/ProfileGroupAcc.java | 13 +- .../accumulator/SerializableUnion.java | 17 +- .../cyber/profiler/accumulator/StatsAcc.java | 7 +- .../accumulator/StatsProfileGroupAcc.java | 44 +- .../profiler/dto/MeasurementDataDto.java | 8 +- .../cyber/profiler/dto/MeasurementDto.java | 16 +- .../cyber/profiler/dto/ProfileDto.java | 18 +- .../phoenix/PhoenixAuthenticationType.java | 2 +- .../profiler/phoenix/PhoenixThickClient.java | 13 +- .../profiler/phoenix/PhoenixThinClient.java | 37 +- .../profiler/FirstSeenHbaseLookupTest.java | 18 +- .../cyber/profiler/FirstSeenHbaseTest.java | 21 +- .../cyber/profiler/ProfileGroupTest.java | 9 +- .../cyber/profiler/ProfileJobTest.java | 62 +- .../profiler/ProfileMessageFilterTest.java | 35 +- .../cyber/profiler/ProfileStatsJoinTest.java | 20 +- .../ScoredMessageToProfileMessageMapTest.java | 31 +- .../FieldValueProfileGroupAccTest.java | 31 +- .../ProfileGroupConfigTestUtils.java | 10 +- .../accumulator/StatsProfileGroupAccTest.java | 24 +- .../profiler/ProfileAggregationMethod.java | 35 +- .../cyber/profiler/ProfileConfig.java | 5 +- .../cyber/profiler/ProfileGroupConfig.java | 77 +- .../profiler/ProfileMeasurementConfig.java | 32 +- .../profiler/ProfileGroupConfigTest.java | 211 +-- .../ProfileMeasurementConfigTest.java | 141 +- .../cloudera/cyber/rules/BaseDynamicRule.java | 14 +- .../com/cloudera/cyber/rules/DynamicRule.java | 5 +- .../cyber/rules/DynamicRuleCommand.java | 83 +- .../cyber/rules/DynamicRuleCommandResult.java | 3 +- .../rules/DynamicRuleProcessFunction.java | 90 +- .../com/cloudera/cyber/rules/RuleType.java | 29 +- .../cyber/rules/engines/JavaScriptEngine.java | 24 +- .../engines/JavaScriptGraaljsEngine.java | 17 +- .../engines/JavaScriptNashornEngine.java | 14 +- .../cyber/rules/engines/RuleEngine.java | 7 +- .../cyber/rules/engines/StellarEngine.java | 4 +- .../rules/engines/UnimplementedEngine.java | 2 - .../rules/engines/TestJavascriptEngine.java | 99 +- .../com/cloudera/cyber/sessions/Session.java | 31 +- .../cloudera/cyber/sessions/SessionJob.java | 13 +- .../cyber/sessions/SessionJobKafka.java | 36 +- .../stellar/common/BaseStellarProcessor.java | 385 ++-- .../metron/stellar/common/BooleanOp.java | 6 +- .../common/CachingStellarProcessor.java | 369 ++-- .../metron/stellar/common/Constants.java | 156 +- .../DefaultStellarStatefulExecutor.java | 246 +-- .../metron/stellar/common/FrameContext.java | 44 +- .../stellar/common/LambdaExpression.java | 73 +- .../stellar/common/StellarAssignment.java | 207 +-- .../stellar/common/StellarCompiler.java | 1617 +++++++++-------- .../common/StellarPredicateProcessor.java | 74 +- .../stellar/common/StellarProcessor.java | 33 +- .../common/StellarStatefulExecutor.java | 92 +- .../common/benchmark/Microbenchmark.java | 71 +- .../benchmark/StellarMicrobenchmark.java | 418 ++--- .../configuration/ConfigurationType.java | 78 +- .../configuration/ConfigurationsUtils.java | 291 +-- .../stellar/common/encoding/Encodings.java | 218 +-- .../evaluators/ArithmeticEvaluator.java | 133 +- .../ComparisonExpressionEvaluator.java | 23 +- ...arisonExpressionWithOperatorEvaluator.java | 113 +- .../ComparisonOperatorsEvaluator.java | 282 +-- .../evaluators/DoubleLiteralEvaluator.java | 16 +- .../EqualityOperatorsEvaluator.java | 88 +- .../evaluators/FloatLiteralEvaluator.java | 16 +- .../evaluators/IntLiteralEvaluator.java | 16 +- .../evaluators/LongLiteralEvaluator.java | 32 +- .../common/evaluators/NumberEvaluator.java | 4 +- .../evaluators/NumberLiteralEvaluator.java | 73 +- .../shell/DefaultStellarAutoCompleter.java | 358 ++-- .../shell/DefaultStellarShellExecutor.java | 634 +++---- .../common/shell/StellarAutoCompleter.java | 36 +- .../shell/StellarExecutionListeners.java | 37 +- .../shell/StellarExecutionNotifier.java | 34 +- .../stellar/common/shell/StellarResult.java | 348 ++-- .../common/shell/StellarShellExecutor.java | 84 +- .../stellar/common/shell/VariableResult.java | 122 +- .../common/shell/cli/PausableInput.java | 646 +++---- .../common/shell/cli/StellarShell.java | 655 +++---- .../cli/StellarShellOptionsValidator.java | 166 +- .../shell/specials/AssignmentCommand.java | 86 +- .../common/shell/specials/Comment.java | 39 +- .../common/shell/specials/DocCommand.java | 124 +- .../shell/specials/MagicDefineGlobal.java | 89 +- .../shell/specials/MagicListFunctions.java | 66 +- .../shell/specials/MagicListGlobals.java | 41 +- .../shell/specials/MagicListVariables.java | 93 +- .../shell/specials/MagicUndefineGlobal.java | 65 +- .../common/shell/specials/QuitCommand.java | 37 +- .../common/shell/specials/SpecialCommand.java | 44 +- .../metron/stellar/common/system/Clock.java | 27 +- .../stellar/common/system/Environment.java | 17 +- .../stellar/common/utils/BloomFilter.java | 108 +- .../stellar/common/utils/ConcatMap.java | 365 ++-- .../stellar/common/utils/ConversionUtils.java | 86 +- .../stellar/common/utils/JSONUtils.java | 142 +- .../stellar/common/utils/PatternCache.java | 19 +- .../stellar/common/utils/SerDeUtils.java | 348 ++-- .../common/utils/StellarProcessorUtils.java | 526 +++--- .../common/utils/UniqueFileReplicator.java | 20 +- .../common/utils/VFSClassloaderUtil.java | 285 +-- .../common/utils/cli/OptionHandler.java | 21 +- .../common/utils/hashing/DefaultHasher.java | 229 +-- .../utils/hashing/EnumConfigurable.java | 11 +- .../common/utils/hashing/HashStrategy.java | 94 +- .../stellar/common/utils/hashing/Hasher.java | 34 +- .../utils/hashing/tlsh/SlidingWindow.java | 6 +- .../common/utils/hashing/tlsh/TLSH.java | 7 +- .../utils/hashing/tlsh/TLSHBuilder.java | 11 +- .../utils/hashing/tlsh/TLSHConstants.java | 210 ++- .../common/utils/hashing/tlsh/TLSHHasher.java | 79 +- .../common/utils/hashing/tlsh/TLSHScorer.java | 9 +- .../common/utils/hashing/tlsh/TLSHUtil.java | 11 +- .../common/utils/math/MathOperation.java | 36 +- .../common/utils/math/MathOperations.java | 103 +- .../utils/math/StellarMathFunction.java | 64 +- .../stellar/dsl/BaseStellarFunction.java | 31 +- .../apache/metron/stellar/dsl/Context.java | 227 ++- .../stellar/dsl/DefaultVariableResolver.java | 42 +- .../metron/stellar/dsl/ErrorListener.java | 37 +- .../metron/stellar/dsl/FunctionMarker.java | 7 +- .../metron/stellar/dsl/GrammarUtils.java | 180 +- .../stellar/dsl/MapVariableResolver.java | 65 +- .../metron/stellar/dsl/ParseException.java | 17 +- .../dsl/Predicate2StellarFunction.java | 22 +- .../apache/metron/stellar/dsl/Stellar.java | 22 +- .../metron/stellar/dsl/StellarFunction.java | 19 +- .../stellar/dsl/StellarFunctionInfo.java | 197 +- .../metron/stellar/dsl/StellarFunctions.java | 23 +- .../org/apache/metron/stellar/dsl/Token.java | 87 +- .../metron/stellar/dsl/VariableResolver.java | 12 +- .../dsl/functions/ConversionFunctions.java | 102 +- .../dsl/functions/DataStructureFunctions.java | 377 ++-- .../stellar/dsl/functions/DateFunctions.java | 706 +++---- .../dsl/functions/EncodingFunctions.java | 242 +-- .../dsl/functions/FunctionalFunctions.java | 355 ++-- .../stellar/dsl/functions/HashFunctions.java | 82 +- .../stellar/dsl/functions/MapFunctions.java | 228 +-- .../stellar/dsl/functions/MathFunctions.java | 305 ++-- .../dsl/functions/NetworkFunctions.java | 465 +++-- .../metron/stellar/dsl/functions/Ordinal.java | 11 +- .../dsl/functions/OrdinalFunctions.java | 204 ++- .../stellar/dsl/functions/RegExFunctions.java | 239 +-- .../stellar/dsl/functions/RestConfig.java | 340 ++-- .../stellar/dsl/functions/RestFunctions.java | 965 +++++----- .../stellar/dsl/functions/SetFunctions.java | 470 ++--- .../stellar/dsl/functions/ShellFunctions.java | 446 ++--- .../dsl/functions/StringFunctions.java | 1108 +++++------ .../dsl/functions/SystemFunctions.java | 93 +- .../stellar/dsl/functions/TextFunctions.java | 138 +- .../resolver/BaseFunctionResolver.java | 378 ++-- .../resolver/ClasspathFunctionResolver.java | 401 ++-- .../functions/resolver/FunctionResolver.java | 53 +- .../resolver/SimpleFunctionResolver.java | 44 +- .../resolver/SingletonFunctionResolver.java | 20 +- .../RestFunctionsIntegrationTest.java | 42 +- .../org/apache/metron/common/Creator.java | 7 +- .../metron/common/aggregator/Aggregator.java | 6 +- .../metron/common/aggregator/Aggregators.java | 143 +- .../common/configuration/ConfigOption.java | 210 +-- .../common/configuration/Configuration.java | 11 +- .../ConfigurationOperations.java | 34 +- .../configuration/ConfigurationType.java | 78 +- .../common/configuration/Configurations.java | 192 +- .../configuration/ConfigurationsUtils.java | 50 +- .../EnrichmentConfigurationOperations.java | 32 +- .../EnrichmentConfigurations.java | 130 +- .../configuration/FieldTransformer.java | 336 ++-- .../common/configuration/FieldValidator.java | 283 +-- .../GlobalConfigurationOperations.java | 49 +- .../IndexingConfigurationOperations.java | 35 +- .../configuration/IndexingConfigurations.java | 628 +++---- .../ParserConfigurationOperations.java | 31 +- .../configuration/ParserConfigurations.java | 104 +- .../configuration/SensorParserConfig.java | 1033 +++++------ .../configuration/SensorParserGroup.java | 92 +- .../enrichment/EnrichmentConfig.java | 174 +- .../enrichment/SensorEnrichmentConfig.java | 159 +- .../SensorEnrichmentUpdateConfig.java | 302 +-- .../common/configuration/enrichment/Type.java | 11 +- .../enrichment/handler/Config.java | 92 +- .../enrichment/handler/ConfigHandler.java | 109 +- .../enrichment/handler/Configs.java | 60 +- .../enrichment/handler/ListConfig.java | 60 +- .../enrichment/handler/StellarConfig.java | 224 ++- .../enrichment/threatintel/RiskLevelRule.java | 242 +-- .../enrichment/threatintel/RuleScore.java | 127 +- .../threatintel/ThreatIntelConfig.java | 69 +- .../enrichment/threatintel/ThreatScore.java | 106 +- .../threatintel/ThreatTriageConfig.java | 192 +- .../configuration/profiler/ProfileConfig.java | 563 +++--- .../configuration/profiler/ProfileResult.java | 174 +- .../profiler/ProfileResultExpressions.java | 79 +- .../profiler/ProfileTriageExpressions.java | 112 +- .../profiler/ProfilerConfig.java | 252 +-- .../writer/ConfigurationStrategy.java | 39 +- .../writer/ConfigurationsStrategies.java | 156 +- .../writer/EnrichmentWriterConfiguration.java | 144 +- .../writer/IndexingWriterConfiguration.java | 105 +- .../writer/ParserWriterConfiguration.java | 154 +- .../SingleBatchConfigurationFacade.java | 93 +- .../writer/WriterConfiguration.java | 150 +- .../metron/common/csv/CSVConverter.java | 231 ++- .../metron/common/error/MetronError.java | 416 +++-- .../common/field/DeDotFieldNameConverter.java | 30 +- .../common/field/FieldNameConverter.java | 19 +- .../common/field/FieldNameConverters.java | 146 +- .../common/field/NoopFieldNameConverter.java | 15 +- .../transformation/FieldTransformation.java | 23 +- .../transformation/FieldTransformations.java | 52 +- .../IPProtocolTransformation.java | 367 ++-- .../RegexSelectTransformation.java | 124 +- .../transformation/RemoveTransformation.java | 106 +- .../transformation/RenameTransformation.java | 56 +- .../transformation/SelectTransformation.java | 54 +- .../SimpleFieldTransformation.java | 35 +- .../transformation/StellarTransformation.java | 103 +- .../field/validation/FieldValidation.java | 19 +- .../field/validation/FieldValidations.java | 53 +- .../field/validation/QueryValidation.java | 88 +- .../field/validation/SimpleValidation.java | 85 +- .../validation/network/DomainValidation.java | 40 +- .../validation/network/EmailValidation.java | 38 +- .../validation/network/IPValidation.java | 221 +-- .../validation/network/URLValidation.java | 40 +- .../validation/primitive/DateValidation.java | 199 +- .../primitive/IntegerValidation.java | 41 +- .../primitive/NotEmptyValidation.java | 27 +- .../validation/primitive/RegexValidation.java | 86 +- .../common/hadoop/SequenceFileIterable.java | 180 +- .../metadata/DefaultRawMessageStrategy.java | 104 +- .../metadata/EnvelopedRawMessageStrategy.java | 213 +-- .../common/message/metadata/MetadataUtil.java | 83 +- .../common/message/metadata/RawMessage.java | 111 +- .../metadata/RawMessageStrategies.java | 102 +- .../message/metadata/RawMessageStrategy.java | 92 +- .../common/performance/PerformanceLogger.java | 319 ++-- .../performance/ThresholdCalculator.java | 31 +- .../metron/common/performance/Timing.java | 70 +- .../apache/metron/common/system/Clock.java | 35 +- .../metron/common/system/Environment.java | 11 +- .../metron/common/system/FakeClock.java | 225 +-- .../common/typosquat/AdditionStrategy.java | 29 +- .../typosquat/BitsquattingStrategy.java | 64 +- .../common/typosquat/HomoglyphStrategy.java | 168 +- .../common/typosquat/HyphenationStrategy.java | 27 +- .../common/typosquat/InsertionStrategy.java | 57 +- .../metron/common/typosquat/Keyboards.java | 257 +-- .../common/typosquat/OmissionStrategy.java | 27 +- .../common/typosquat/RepetitionStrategy.java | 41 +- .../common/typosquat/ReplacementStrategy.java | 45 +- .../common/typosquat/SubdomainStrategy.java | 42 +- .../typosquat/TranspositionStrategy.java | 43 +- .../typosquat/TyposquattingStrategies.java | 165 +- .../typosquat/TyposquattingStrategy.java | 10 +- .../common/typosquat/VowelSwapStrategy.java | 57 +- .../common/utils/CompressionStrategies.java | 110 +- .../common/utils/CompressionStrategy.java | 50 +- .../apache/metron/common/utils/HDFSUtils.java | 132 +- .../apache/metron/common/utils/HashUtils.java | 48 +- .../apache/metron/common/utils/JSONUtils.java | 393 ++-- .../apache/metron/common/utils/KeyUtil.java | 76 +- .../metron/common/utils/LazyLogger.java | 69 +- .../common/utils/LazyLoggerFactory.java | 77 +- .../metron/common/utils/LazyLoggerImpl.java | 1086 +++++------ .../metron/common/utils/MessageUtils.java | 17 +- .../metron/common/utils/ReflectionUtils.java | 183 +- .../metron/common/utils/RuntimeErrors.java | 93 +- .../metron/common/utils/StringUtils.java | 55 +- .../metron/common/utils/cli/CLIOptions.java | 14 +- .../common/utils/cli/OptionHandler.java | 142 +- .../common/utils/file/ReaderSpliterator.java | 187 +- .../utils/timestamp/TimestampConverter.java | 10 +- .../utils/timestamp/TimestampConverters.java | 57 +- .../metron/common/writer/BulkMessage.java | 63 +- .../common/writer/BulkMessageWriter.java | 38 +- .../common/writer/BulkWriterResponse.java | 53 +- .../metron/common/writer/MessageId.java | 68 +- .../metron/common/writer/MessageWriter.java | 16 +- .../common/zookeeper/ConfigurationsCache.java | 38 +- .../configurations/ConfigurationsUpdater.java | 224 +-- .../configurations/EnrichmentUpdater.java | 64 +- .../configurations/IndexingUpdater.java | 74 +- .../configurations/ParserUpdater.java | 8 +- .../zookeeper/configurations/Reloadable.java | 10 +- .../metron/common/utils/HashUtilsTest.java | 11 +- .../common/utils/LazyLoggerImplTest.java | 15 +- .../metron/common/utils/TestConstants.java | 16 +- .../org/apache/metron/hbase/ColumnList.java | 229 +-- .../metron/hbase/HBaseProjectionCriteria.java | 88 +- .../apache/metron/hbase/HTableProvider.java | 60 +- .../java/org/apache/metron/hbase/IColumn.java | 12 +- .../org/apache/metron/hbase/ICounter.java | 9 +- .../org/apache/metron/hbase/TableConfig.java | 59 +- .../apache/metron/hbase/TableProvider.java | 35 +- .../metron/hbase/client/HBaseClient.java | 539 +++--- .../apache/metron/hbase/mock/MockHTable.java | 38 +- .../queryservice/Application.java | 34 +- .../queryservice/StartupComponent.java | 35 +- .../common/ApplicationConstants.java | 74 +- .../FailedAllClusterReponseException.java | 3 +- .../common/exception/FailedJobAction.java | 17 +- .../exception/JobValidationException.java | 17 +- .../exception/KafkaClusterNotFound.java | 17 +- .../common/exception/KafkaException.java | 17 +- .../common/utils/CollectionsUtils.java | 50 +- .../common/utils/IDGenerator.java | 2 +- .../common/utils/UniqueIDGenerator.java | 70 +- .../config/AngularForwardingConfig.java | 7 +- .../queryservice/config/AppProperties.java | 86 +- .../queryservice/config/CachingConfig.java | 22 +- .../queryservice/config/ConfigOption.java | 2 +- .../config/MainConfiguration.java | 68 +- .../config/kafka/ClouderaKafkaProperties.java | 4 +- .../kafka/ClouderaReplyingKafkaTemplate.java | 43 +- .../config/kafka/KafkaConfig.java | 175 +- .../controller/ChainController.java | 172 +- .../controller/ClusterController.java | 13 +- .../GlobalExceptionAdviceController.java | 3 +- .../controller/JobController.java | 58 +- .../controller/ParserController.java | 37 +- .../controller/ParserSampleController.java | 55 +- .../controller/PipelineController.java | 21 +- ...ocumentationPluginsManagerBootAdapter.java | 8 +- .../hack/TypeNameExtractorBootAdapter.java | 8 +- .../model/describe/ConfigParamDescriptor.java | 402 ++-- .../model/describe/ParserDescriptor.java | 173 +- .../describe/SampleFolderDescriptor.java | 3 +- .../queryservice/model/enums/JobActions.java | 4 +- .../model/enums/KafkaMessageType.java | 44 +- .../model/exec/ChainTestRequest.java | 4 +- .../model/exec/ChainTestResponse.java | 19 +- .../queryservice/model/exec/ParserResult.java | 32 +- .../model/exec/PipelineResult.java | 4 +- .../queryservice/model/exec/ResultLog.java | 34 +- .../queryservice/model/exec/SampleData.java | 4 +- .../model/sample/ParserSample.java | 22 +- .../model/summary/ObjectMapper.java | 2 +- .../model/summary/ParserChainSummary.java | 93 +- .../model/summary/ParserSummary.java | 111 +- .../model/summary/ParserSummaryMapper.java | 18 +- .../service/ChainBuilderService.java | 1 + .../service/ChainExecutorService.java | 3 +- .../service/ChainPersistenceService.java | 74 +- .../queryservice/service/ClusterService.java | 19 +- .../queryservice/service/IndexingService.java | 12 +- .../queryservice/service/JobService.java | 14 +- .../queryservice/service/KafkaService.java | 58 +- .../service/ParserDiscoveryService.java | 40 +- .../service/ParserSampleService.java | 27 +- .../queryservice/service/PipelineService.java | 201 +- .../service/ResultLogBuilder.java | 45 +- .../impl/DefaultChainBuilderService.java | 2 +- .../impl/DefaultChainExecutorService.java | 115 +- .../impl/DefaultParserDiscoveryService.java | 241 +-- .../FileBasedChainPersistenceService.java | 239 +-- .../FileBasedChainPersistenceServiceTest.java | 27 +- .../service/ResultLogBuilderTest.java | 14 +- .../parserchains/core/ChainBuilder.java | 1 + .../cloudera/parserchains/core/ChainLink.java | 4 + .../parserchains/core/ChainRunner.java | 8 +- .../cloudera/parserchains/core/Constants.java | 1 + .../core/DefaultChainBuilder.java | 36 +- .../parserchains/core/DefaultChainRunner.java | 43 +- .../cloudera/parserchains/core/FieldName.java | 10 +- .../parserchains/core/FieldValue.java | 8 +- .../core/InvalidParserException.java | 16 +- .../cloudera/parserchains/core/LinkName.java | 14 +- .../cloudera/parserchains/core/Message.java | 69 +- .../core/MessageToParseFieldValue.java | 3 +- .../parserchains/core/NextChainLink.java | 21 +- .../cloudera/parserchains/core/Parser.java | 1 - .../parserchains/core/ParserBuilder.java | 4 +- .../core/ReflectiveParserBuilder.java | 26 +- .../com/cloudera/parserchains/core/Regex.java | 20 +- .../parserchains/core/RouterLink.java | 14 +- .../parserchains/core/StringFieldValue.java | 9 +- .../cloudera/parserchains/core/Validator.java | 7 +- .../AnnotationBasedParserInfoBuilder.java | 19 +- .../core/catalog/ClassIndexParserCatalog.java | 20 +- .../core/catalog/Configurable.java | 3 + .../core/catalog/MessageParser.java | 4 +- .../parserchains/core/catalog/Parameter.java | 8 + .../parserchains/core/catalog/ParserInfo.java | 19 +- .../core/catalog/ParserInfoBuilder.java | 1 + .../core/model/config/ConfigDescription.java | 6 +- .../core/model/config/ConfigDescriptor.java | 28 +- .../core/model/config/ConfigKey.java | 30 +- .../core/model/config/ConfigName.java | 22 +- .../core/model/config/ConfigValue.java | 24 +- .../core/model/define/ConfigValueSchema.java | 19 +- .../core/model/define/ParserChainSchema.java | 193 +- .../core/model/define/ParserID.java | 23 +- .../core/model/define/ParserName.java | 20 +- .../core/model/define/ParserSchema.java | 291 +-- .../core/model/define/RouteSchema.java | 45 +- .../core/model/define/RoutingSchema.java | 25 +- .../core/utils/AnnotationUtils.java | 17 +- .../parserchains/core/utils/JSONUtils.java | 383 ++-- .../parserchains/core/utils/StringUtils.java | 23 +- .../parserchains/core/RouterLinkTest.java | 25 +- .../parsers/AlwaysFailParser.java | 22 +- .../parserchains/parsers/AvroParser.java | 43 +- .../parserchains/parsers/CsvTextParser.java | 79 +- .../parsers/DelimitedKeyValueParser.java | 73 +- .../parsers/DelimitedTextParser.java | 43 +- .../parserchains/parsers/GrokParser.java | 53 +- .../parserchains/parsers/JSONParser.java | 82 +- .../parserchains/parsers/JSONPathParser.java | 18 +- .../parsers/RemoveFieldParser.java | 39 +- .../parsers/RenameFieldParser.java | 51 +- .../parsers/SimpleStellarParser.java | 22 +- .../parserchains/parsers/StellarParser.java | 41 +- .../parserchains/parsers/SyslogParser.java | 48 +- .../parsers/TimestampFormatParser.java | 48 +- .../parserchains/parsers/TimestampParser.java | 24 +- .../parserchains/parsers/XMLFlattener.java | 42 +- .../parserchains/parsers/XPathParser.java | 72 +- .../parsers/DelimitedKeyValueParserTest.java | 28 +- .../parserchains/parsers/GrokParserTest.java | 17 +- .../parsers/StellarParserTest.java | 19 +- .../parsers/SyslogParserTest.java | 27 +- .../nifi/CDPDCCyberConnectionService.java | 3 +- .../cyber/parser/ChainParserMapFunction.java | 122 +- .../MessageToEnrichmentCommandFunction.java | 103 +- .../cloudera/cyber/parser/ParserChainMap.java | 1 - .../com/cloudera/cyber/parser/ParserJob.java | 90 +- .../cloudera/cyber/parser/ParserJobKafka.java | 92 +- .../StreamingEnrichmentsSourceFilter.java | 3 +- .../cyber/parser/TopicParserConfig.java | 3 +- .../cyber/parser/TopicPatternToChainMap.java | 11 +- .../parser/ChainParserMapFunctionTest.java | 33 +- ...essageToEnrichmentCommandFunctionTest.java | 91 +- .../cyber/parser/TestNetflowBParser.java | 116 +- .../parser/TestParserJobChainDirectory.java | 27 +- flink-cyber/pom.xml | 32 +- 834 files changed, 41146 insertions(+), 36126 deletions(-) create mode 100644 flink-cyber/.editorconfig rename flink-cyber/{google_checks.xml => checkstyle.xml} (78%) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ec1a06a0..4d77151f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -28,9 +28,9 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Check codestyle - run: mvn -B editorconfig:check --file flink-cyber/pom.xml + run: mvn -B validate --file flink-cyber/pom.xml - name: Build and Test with Maven - run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B package --file flink-cyber/pom.xml + run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B verify --file flink-cyber/pom.xml # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive - name: Update dependency graph diff --git a/flink-cyber/.editorconfig b/flink-cyber/.editorconfig new file mode 100644 index 00000000..1da0dda1 --- /dev/null +++ b/flink-cyber/.editorconfig @@ -0,0 +1,1460 @@ +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = false +max_line_length = 120 +tab_width = 4 +ij_continuation_indent_size = 8 +ij_formatter_off_tag = @formatter:off +ij_formatter_on_tag = @formatter:on +ij_formatter_tags_enabled = true +ij_smart_tabs = false +ij_wrap_on_typing = false + +[*.css] +ij_css_align_closing_brace_with_properties = false +ij_css_blank_lines_around_nested_selector = 1 +ij_css_blank_lines_between_blocks = 1 +ij_css_block_comment_add_space = false +ij_css_brace_placement = end_of_line +ij_css_enforce_quotes_on_format = false +ij_css_hex_color_long_format = false +ij_css_hex_color_lower_case = false +ij_css_hex_color_short_format = false +ij_css_hex_color_upper_case = false +ij_css_keep_blank_lines_in_code = 2 +ij_css_keep_indents_on_empty_lines = false +ij_css_keep_single_line_blocks = false +ij_css_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow +ij_css_space_after_colon = true +ij_css_space_before_opening_brace = true +ij_css_use_double_quotes = true +ij_css_value_alignment = do_not_align + +[*.dcl] +ij_declarative_keep_indents_on_empty_lines = false + +[*.java] +ij_continuation_indent_size = 6 +ij_java_align_consecutive_assignments = false +ij_java_align_consecutive_variable_declarations = false +ij_java_align_group_field_declarations = false +ij_java_align_multiline_annotation_parameters = false +ij_java_align_multiline_array_initializer_expression = false +ij_java_align_multiline_assignment = false +ij_java_align_multiline_binary_operation = true +ij_java_align_multiline_chained_methods = true +ij_java_align_multiline_deconstruction_list_components = true +ij_java_align_multiline_extends_list = false +ij_java_align_multiline_for = true +ij_java_align_multiline_method_parentheses = false +ij_java_align_multiline_parameters = true +ij_java_align_multiline_parameters_in_calls = false +ij_java_align_multiline_parenthesized_expression = false +ij_java_align_multiline_records = true +ij_java_align_multiline_resources = true +ij_java_align_multiline_ternary_operation = false +ij_java_align_multiline_text_blocks = false +ij_java_align_multiline_throws_list = false +ij_java_align_subsequent_simple_methods = false +ij_java_align_throws_keyword = false +ij_java_align_types_in_multi_catch = true +ij_java_annotation_parameter_wrap = off +ij_java_array_initializer_new_line_after_left_brace = false +ij_java_array_initializer_right_brace_on_new_line = false +ij_java_array_initializer_wrap = normal +ij_java_assert_statement_colon_on_next_line = true +ij_java_assert_statement_wrap = normal +ij_java_assignment_wrap = normal +ij_java_binary_operation_sign_on_next_line = false +ij_java_binary_operation_wrap = normal +ij_java_blank_lines_after_anonymous_class_header = 0 +ij_java_blank_lines_after_class_header = 0 +ij_java_blank_lines_after_imports = 1 +ij_java_blank_lines_after_package = 1 +ij_java_blank_lines_around_class = 1 +ij_java_blank_lines_around_field = 0 +ij_java_blank_lines_around_field_in_interface = 0 +ij_java_blank_lines_around_initializer = 1 +ij_java_blank_lines_around_method = 1 +ij_java_blank_lines_around_method_in_interface = 1 +ij_java_blank_lines_before_class_end = 0 +ij_java_blank_lines_before_imports = 1 +ij_java_blank_lines_before_method_body = 0 +ij_java_blank_lines_before_package = 1 +ij_java_block_brace_style = end_of_line +ij_java_block_comment_add_space = false +ij_java_block_comment_at_first_column = true +ij_java_call_parameters_new_line_after_left_paren = false +ij_java_call_parameters_right_paren_on_new_line = false +ij_java_call_parameters_wrap = normal +ij_java_case_statement_on_separate_line = true +ij_java_catch_on_new_line = false +ij_java_class_annotation_wrap = split_into_lines +ij_java_class_brace_style = end_of_line +ij_java_class_count_to_use_import_on_demand = 999 +ij_java_class_names_in_javadoc = 1 +ij_java_deconstruction_list_wrap = normal +ij_java_do_not_indent_top_level_class_members = false +ij_java_do_not_wrap_after_single_annotation = false +ij_java_do_not_wrap_after_single_annotation_in_parameter = false +ij_java_do_while_brace_force = always +ij_java_doc_add_blank_line_after_description = true +ij_java_doc_add_blank_line_after_param_comments = false +ij_java_doc_add_blank_line_after_return = false +ij_java_doc_add_p_tag_on_empty_lines = true +ij_java_doc_align_exception_comments = true +ij_java_doc_align_param_comments = true +ij_java_doc_do_not_wrap_if_one_line = false +ij_java_doc_enable_formatting = true +ij_java_doc_enable_leading_asterisks = true +ij_java_doc_indent_on_continuation = true +ij_java_doc_keep_empty_lines = true +ij_java_doc_keep_empty_parameter_tag = true +ij_java_doc_keep_empty_return_tag = true +ij_java_doc_keep_empty_throws_tag = true +ij_java_doc_keep_invalid_tags = true +ij_java_doc_param_description_on_new_line = false +ij_java_doc_preserve_line_breaks = false +ij_java_doc_use_throws_not_exception_tag = true +ij_java_else_on_new_line = false +ij_java_entity_dd_suffix = EJB +ij_java_entity_eb_suffix = Bean +ij_java_entity_hi_suffix = Home +ij_java_entity_lhi_prefix = Local +ij_java_entity_lhi_suffix = Home +ij_java_entity_li_prefix = Local +ij_java_entity_pk_class = java.lang.String +ij_java_entity_vo_suffix = VO +ij_java_enum_constants_wrap = normal +ij_java_enum_field_annotation_wrap = off +ij_java_extends_keyword_wrap = normal +ij_java_extends_list_wrap = normal +ij_java_field_annotation_wrap = split_into_lines +ij_java_finally_on_new_line = false +ij_java_for_brace_force = always +ij_java_for_statement_new_line_after_left_paren = false +ij_java_for_statement_right_paren_on_new_line = false +ij_java_for_statement_wrap = normal +ij_java_generate_final_locals = false +ij_java_generate_final_parameters = false +ij_java_generate_use_type_annotation_before_type = true +ij_java_if_brace_force = always +ij_java_imports_layout = $*,|,* +ij_java_indent_case_from_switch = true +ij_java_insert_inner_class_imports = false +ij_java_insert_override_annotation = true +ij_java_keep_blank_lines_before_right_brace = 2 +ij_java_keep_blank_lines_between_package_declaration_and_header = 2 +ij_java_keep_blank_lines_in_code = 2 +ij_java_keep_blank_lines_in_declarations = 2 +ij_java_keep_builder_methods_indents = false +ij_java_keep_control_statement_in_one_line = true +ij_java_keep_first_column_comment = true +ij_java_keep_indents_on_empty_lines = false +ij_java_keep_line_breaks = true +ij_java_keep_multiple_expressions_in_one_line = false +ij_java_keep_simple_blocks_in_one_line = false +ij_java_keep_simple_classes_in_one_line = false +ij_java_keep_simple_lambdas_in_one_line = false +ij_java_keep_simple_methods_in_one_line = false +ij_java_label_indent_absolute = false +ij_java_label_indent_size = 0 +ij_java_lambda_brace_style = end_of_line +ij_java_layout_static_imports_separately = true +ij_java_line_comment_add_space = false +ij_java_line_comment_add_space_on_reformat = false +ij_java_line_comment_at_first_column = true +ij_java_message_dd_suffix = EJB +ij_java_message_eb_suffix = Bean +ij_java_method_annotation_wrap = split_into_lines +ij_java_method_brace_style = end_of_line +ij_java_method_call_chain_wrap = normal +ij_java_method_parameters_new_line_after_left_paren = false +ij_java_method_parameters_right_paren_on_new_line = false +ij_java_method_parameters_wrap = normal +ij_java_modifier_list_wrap = false +ij_java_multi_catch_types_wrap = normal +ij_java_names_count_to_use_import_on_demand = 999 +ij_java_new_line_after_lparen_in_annotation = false +ij_java_new_line_after_lparen_in_deconstruction_pattern = true +ij_java_new_line_after_lparen_in_record_header = false +ij_java_new_line_when_body_is_presented = false +ij_java_parameter_annotation_wrap = normal +ij_java_parentheses_expression_new_line_after_left_paren = false +ij_java_parentheses_expression_right_paren_on_new_line = false +ij_java_place_assignment_sign_on_next_line = false +ij_java_prefer_longer_names = true +ij_java_prefer_parameters_wrap = false +ij_java_record_components_wrap = normal +ij_java_repeat_synchronized = true +ij_java_replace_instanceof_and_cast = false +ij_java_replace_null_check = true +ij_java_replace_sum_lambda_with_method_ref = true +ij_java_resource_list_new_line_after_left_paren = false +ij_java_resource_list_right_paren_on_new_line = false +ij_java_resource_list_wrap = normal +ij_java_rparen_on_new_line_in_annotation = false +ij_java_rparen_on_new_line_in_deconstruction_pattern = true +ij_java_rparen_on_new_line_in_record_header = false +ij_java_session_dd_suffix = EJB +ij_java_session_eb_suffix = Bean +ij_java_session_hi_suffix = Home +ij_java_session_lhi_prefix = Local +ij_java_session_lhi_suffix = Home +ij_java_session_li_prefix = Local +ij_java_session_si_suffix = Service +ij_java_space_after_closing_angle_bracket_in_type_argument = false +ij_java_space_after_colon = true +ij_java_space_after_comma = true +ij_java_space_after_comma_in_type_arguments = true +ij_java_space_after_for_semicolon = true +ij_java_space_after_quest = true +ij_java_space_after_type_cast = true +ij_java_space_before_annotation_array_initializer_left_brace = false +ij_java_space_before_annotation_parameter_list = false +ij_java_space_before_array_initializer_left_brace = true +ij_java_space_before_catch_keyword = true +ij_java_space_before_catch_left_brace = true +ij_java_space_before_catch_parentheses = true +ij_java_space_before_class_left_brace = true +ij_java_space_before_colon = true +ij_java_space_before_colon_in_foreach = true +ij_java_space_before_comma = false +ij_java_space_before_deconstruction_list = false +ij_java_space_before_do_left_brace = true +ij_java_space_before_else_keyword = true +ij_java_space_before_else_left_brace = true +ij_java_space_before_finally_keyword = true +ij_java_space_before_finally_left_brace = true +ij_java_space_before_for_left_brace = true +ij_java_space_before_for_parentheses = true +ij_java_space_before_for_semicolon = false +ij_java_space_before_if_left_brace = true +ij_java_space_before_if_parentheses = true +ij_java_space_before_method_call_parentheses = false +ij_java_space_before_method_left_brace = true +ij_java_space_before_method_parentheses = false +ij_java_space_before_opening_angle_bracket_in_type_parameter = false +ij_java_space_before_quest = true +ij_java_space_before_switch_left_brace = true +ij_java_space_before_switch_parentheses = true +ij_java_space_before_synchronized_left_brace = true +ij_java_space_before_synchronized_parentheses = true +ij_java_space_before_try_left_brace = true +ij_java_space_before_try_parentheses = true +ij_java_space_before_type_parameter_list = false +ij_java_space_before_while_keyword = true +ij_java_space_before_while_left_brace = true +ij_java_space_before_while_parentheses = true +ij_java_space_inside_one_line_enum_braces = true +ij_java_space_within_empty_array_initializer_braces = false +ij_java_space_within_empty_method_call_parentheses = false +ij_java_space_within_empty_method_parentheses = false +ij_java_spaces_around_additive_operators = true +ij_java_spaces_around_annotation_eq = true +ij_java_spaces_around_assignment_operators = true +ij_java_spaces_around_bitwise_operators = true +ij_java_spaces_around_equality_operators = true +ij_java_spaces_around_lambda_arrow = true +ij_java_spaces_around_logical_operators = true +ij_java_spaces_around_method_ref_dbl_colon = false +ij_java_spaces_around_multiplicative_operators = true +ij_java_spaces_around_relational_operators = true +ij_java_spaces_around_shift_operators = true +ij_java_spaces_around_type_bounds_in_type_parameters = true +ij_java_spaces_around_unary_operator = false +ij_java_spaces_inside_block_braces_when_body_is_present = false +ij_java_spaces_within_angle_brackets = false +ij_java_spaces_within_annotation_parentheses = false +ij_java_spaces_within_array_initializer_braces = false +ij_java_spaces_within_braces = false +ij_java_spaces_within_brackets = false +ij_java_spaces_within_cast_parentheses = false +ij_java_spaces_within_catch_parentheses = false +ij_java_spaces_within_deconstruction_list = false +ij_java_spaces_within_for_parentheses = false +ij_java_spaces_within_if_parentheses = false +ij_java_spaces_within_method_call_parentheses = false +ij_java_spaces_within_method_parentheses = false +ij_java_spaces_within_parentheses = false +ij_java_spaces_within_record_header = false +ij_java_spaces_within_switch_parentheses = false +ij_java_spaces_within_synchronized_parentheses = false +ij_java_spaces_within_try_parentheses = false +ij_java_spaces_within_while_parentheses = false +ij_java_special_else_if_treatment = true +ij_java_subclass_name_suffix = Impl +ij_java_switch_expressions_wrap = normal +ij_java_ternary_operation_signs_on_next_line = true +ij_java_ternary_operation_wrap = normal +ij_java_test_name_suffix = Test +ij_java_throws_keyword_wrap = normal +ij_java_throws_list_wrap = normal +ij_java_use_external_annotations = false +ij_java_use_fq_class_names = false +ij_java_use_relative_indents = false +ij_java_use_single_class_imports = true +ij_java_variable_annotation_wrap = normal +ij_java_visibility = public +ij_java_while_brace_force = always +ij_java_while_on_new_line = false +ij_java_wrap_comments = false +ij_java_wrap_first_method_in_call_chain = false +ij_java_wrap_long_lines = false +ij_java_wrap_semicolon_after_call_chain = false + +[*.less] +indent_size = 2 +ij_less_align_closing_brace_with_properties = false +ij_less_blank_lines_around_nested_selector = 1 +ij_less_blank_lines_between_blocks = 1 +ij_less_block_comment_add_space = false +ij_less_brace_placement = 0 +ij_less_enforce_quotes_on_format = false +ij_less_hex_color_long_format = false +ij_less_hex_color_lower_case = false +ij_less_hex_color_short_format = false +ij_less_hex_color_upper_case = false +ij_less_keep_blank_lines_in_code = 2 +ij_less_keep_indents_on_empty_lines = false +ij_less_keep_single_line_blocks = false +ij_less_line_comment_add_space = false +ij_less_line_comment_at_first_column = false +ij_less_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow +ij_less_space_after_colon = true +ij_less_space_before_opening_brace = true +ij_less_use_double_quotes = true +ij_less_value_alignment = 0 + +[*.proto] +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 4 +ij_protobuf_keep_blank_lines_in_code = 2 +ij_protobuf_keep_indents_on_empty_lines = false +ij_protobuf_keep_line_breaks = true +ij_protobuf_space_after_comma = true +ij_protobuf_space_before_comma = false +ij_protobuf_spaces_around_assignment_operators = true +ij_protobuf_spaces_within_braces = false +ij_protobuf_spaces_within_brackets = false + +[*.sass] +indent_size = 2 +ij_sass_align_closing_brace_with_properties = false +ij_sass_blank_lines_around_nested_selector = 1 +ij_sass_blank_lines_between_blocks = 1 +ij_sass_brace_placement = 0 +ij_sass_enforce_quotes_on_format = false +ij_sass_hex_color_long_format = false +ij_sass_hex_color_lower_case = false +ij_sass_hex_color_short_format = false +ij_sass_hex_color_upper_case = false +ij_sass_keep_blank_lines_in_code = 2 +ij_sass_keep_indents_on_empty_lines = false +ij_sass_keep_single_line_blocks = false +ij_sass_line_comment_add_space = false +ij_sass_line_comment_at_first_column = false +ij_sass_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow +ij_sass_space_after_colon = true +ij_sass_space_before_opening_brace = true +ij_sass_use_double_quotes = true +ij_sass_value_alignment = 0 + +[*.scala] +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 2 +ij_scala_align_composite_pattern = true +ij_scala_align_extends_with = 0 +ij_scala_align_group_field_declarations = false +ij_scala_align_if_else = false +ij_scala_align_in_columns_case_branch = false +ij_scala_align_multiline_binary_operation = false +ij_scala_align_multiline_chained_methods = false +ij_scala_align_multiline_for = true +ij_scala_align_multiline_parameters = true +ij_scala_align_multiline_parameters_in_calls = false +ij_scala_align_multiline_parenthesized_expression = false +ij_scala_align_parameter_types_in_multiline_declarations = 0 +ij_scala_align_tuple_elements = false +ij_scala_alternate_continuation_indent_for_params = 4 +ij_scala_binary_operation_wrap = off +ij_scala_blank_lines_after_anonymous_class_header = 0 +ij_scala_blank_lines_after_class_header = 0 +ij_scala_blank_lines_after_imports = 1 +ij_scala_blank_lines_after_package = 1 +ij_scala_blank_lines_around_class = 1 +ij_scala_blank_lines_around_class_in_inner_scopes = 0 +ij_scala_blank_lines_around_field = 0 +ij_scala_blank_lines_around_field_in_inner_scopes = 0 +ij_scala_blank_lines_around_field_in_interface = 0 +ij_scala_blank_lines_around_method = 1 +ij_scala_blank_lines_around_method_in_inner_scopes = 1 +ij_scala_blank_lines_around_method_in_interface = 1 +ij_scala_blank_lines_before_class_end = 0 +ij_scala_blank_lines_before_imports = 1 +ij_scala_blank_lines_before_method_body = 0 +ij_scala_blank_lines_before_package = 0 +ij_scala_block_brace_style = end_of_line +ij_scala_block_comment_add_space = false +ij_scala_block_comment_at_first_column = true +ij_scala_call_parameters_new_line_after_lparen = 0 +ij_scala_call_parameters_right_paren_on_new_line = false +ij_scala_call_parameters_wrap = off +ij_scala_case_clause_brace_force = never +ij_scala_catch_on_new_line = false +ij_scala_class_annotation_wrap = split_into_lines +ij_scala_class_brace_style = end_of_line +ij_scala_closure_brace_force = never +ij_scala_do_not_align_block_expr_params = true +ij_scala_do_not_indent_case_clause_body = false +ij_scala_do_not_indent_tuples_close_brace = true +ij_scala_do_while_brace_force = never +ij_scala_else_on_new_line = false +ij_scala_enable_scaladoc_formatting = true +ij_scala_enforce_functional_syntax_for_unit = true +ij_scala_extends_keyword_wrap = off +ij_scala_extends_list_wrap = off +ij_scala_field_annotation_wrap = split_into_lines +ij_scala_finally_brace_force = never +ij_scala_finally_on_new_line = false +ij_scala_for_brace_force = never +ij_scala_for_statement_wrap = off +ij_scala_formatter = 0 +ij_scala_if_brace_force = never +ij_scala_implicit_value_class_suffix = Ops +ij_scala_indent_braced_function_args = true +ij_scala_indent_case_from_switch = true +ij_scala_indent_fewer_braces_in_method_call_chains = false +ij_scala_indent_first_parameter = true +ij_scala_indent_first_parameter_clause = false +ij_scala_indent_type_arguments = true +ij_scala_indent_type_parameters = true +ij_scala_indent_yield_after_one_line_enumerators = true +ij_scala_keep_blank_lines_before_right_brace = 2 +ij_scala_keep_blank_lines_in_code = 2 +ij_scala_keep_blank_lines_in_declarations = 2 +ij_scala_keep_comments_on_same_line = true +ij_scala_keep_first_column_comment = false +ij_scala_keep_indents_on_empty_lines = false +ij_scala_keep_line_breaks = true +ij_scala_keep_one_line_lambdas_in_arg_list = false +ij_scala_keep_simple_blocks_in_one_line = false +ij_scala_keep_simple_methods_in_one_line = false +ij_scala_keep_xml_formatting = false +ij_scala_line_comment_add_space = false +ij_scala_line_comment_at_first_column = true +ij_scala_method_annotation_wrap = split_into_lines +ij_scala_method_brace_force = never +ij_scala_method_brace_style = end_of_line +ij_scala_method_call_chain_wrap = off +ij_scala_method_parameters_new_line_after_left_paren = false +ij_scala_method_parameters_right_paren_on_new_line = false +ij_scala_method_parameters_wrap = off +ij_scala_modifier_list_wrap = false +ij_scala_multiline_string_align_dangling_closing_quotes = false +ij_scala_multiline_string_closing_quotes_on_new_line = false +ij_scala_multiline_string_insert_margin_on_enter = true +ij_scala_multiline_string_margin_char = | +ij_scala_multiline_string_margin_indent = 2 +ij_scala_multiline_string_opening_quotes_on_new_line = true +ij_scala_multiline_string_process_margin_on_copy_paste = true +ij_scala_new_line_after_case_clause_arrow_when_multiline_body = false +ij_scala_newline_after_annotations = false +ij_scala_not_continuation_indent_for_params = false +ij_scala_parameter_annotation_wrap = off +ij_scala_parentheses_expression_new_line_after_left_paren = false +ij_scala_parentheses_expression_right_paren_on_new_line = false +ij_scala_place_closure_parameters_on_new_line = false +ij_scala_place_self_type_on_new_line = true +ij_scala_prefer_parameters_wrap = false +ij_scala_preserve_space_after_method_declaration_name = false +ij_scala_reformat_on_compile = false +ij_scala_replace_case_arrow_with_unicode_char = false +ij_scala_replace_for_generator_arrow_with_unicode_char = false +ij_scala_replace_lambda_with_greek_letter = false +ij_scala_replace_map_arrow_with_unicode_char = false +ij_scala_scalafmt_fallback_to_default_settings = false +ij_scala_scalafmt_reformat_on_files_save = false +ij_scala_scalafmt_show_invalid_code_warnings = true +ij_scala_scalafmt_use_intellij_formatter_for_range_format = true +ij_scala_sd_align_exception_comments = true +ij_scala_sd_align_list_item_content = true +ij_scala_sd_align_other_tags_comments = true +ij_scala_sd_align_parameters_comments = true +ij_scala_sd_align_return_comments = true +ij_scala_sd_blank_line_after_parameters_comments = false +ij_scala_sd_blank_line_after_return_comments = false +ij_scala_sd_blank_line_before_parameters = false +ij_scala_sd_blank_line_before_tags = true +ij_scala_sd_blank_line_between_parameters = false +ij_scala_sd_keep_blank_lines_between_tags = false +ij_scala_sd_preserve_spaces_in_tags = false +ij_scala_space_after_comma = true +ij_scala_space_after_for_semicolon = true +ij_scala_space_after_modifiers_constructor = false +ij_scala_space_after_type_colon = true +ij_scala_space_before_brace_method_call = true +ij_scala_space_before_class_left_brace = true +ij_scala_space_before_for_parentheses = true +ij_scala_space_before_if_parentheses = true +ij_scala_space_before_infix_like_method_parentheses = false +ij_scala_space_before_infix_method_call_parentheses = false +ij_scala_space_before_infix_operator_like_method_call_parentheses = true +ij_scala_space_before_method_call_parentheses = false +ij_scala_space_before_method_left_brace = true +ij_scala_space_before_method_parentheses = false +ij_scala_space_before_type_colon = false +ij_scala_space_before_type_parameter_in_def_list = false +ij_scala_space_before_type_parameter_leading_context_bound_colon = false +ij_scala_space_before_type_parameter_leading_context_bound_colon_hk = true +ij_scala_space_before_type_parameter_list = false +ij_scala_space_before_type_parameter_rest_context_bound_colons = true +ij_scala_space_before_while_parentheses = true +ij_scala_space_inside_closure_braces = true +ij_scala_space_inside_self_type_braces = true +ij_scala_space_within_empty_method_call_parentheses = false +ij_scala_spaces_around_at_in_patterns = false +ij_scala_spaces_in_imports = false +ij_scala_spaces_in_one_line_blocks = false +ij_scala_spaces_within_brackets = false +ij_scala_spaces_within_for_parentheses = false +ij_scala_spaces_within_if_parentheses = false +ij_scala_spaces_within_method_call_parentheses = false +ij_scala_spaces_within_method_parentheses = false +ij_scala_spaces_within_parentheses = false +ij_scala_spaces_within_while_parentheses = false +ij_scala_special_else_if_treatment = true +ij_scala_trailing_comma_arg_list_enabled = true +ij_scala_trailing_comma_import_selector_enabled = false +ij_scala_trailing_comma_mode = trailing_comma_keep +ij_scala_trailing_comma_params_enabled = true +ij_scala_trailing_comma_pattern_arg_list_enabled = false +ij_scala_trailing_comma_tuple_enabled = false +ij_scala_trailing_comma_tuple_type_enabled = false +ij_scala_trailing_comma_type_params_enabled = false +ij_scala_try_brace_force = never +ij_scala_type_annotation_exclude_constant = true +ij_scala_type_annotation_exclude_in_dialect_sources = true +ij_scala_type_annotation_exclude_in_test_sources = false +ij_scala_type_annotation_exclude_member_of_anonymous_class = false +ij_scala_type_annotation_exclude_member_of_private_class = false +ij_scala_type_annotation_exclude_when_type_is_stable = true +ij_scala_type_annotation_function_parameter = false +ij_scala_type_annotation_implicit_modifier = true +ij_scala_type_annotation_local_definition = false +ij_scala_type_annotation_private_member = false +ij_scala_type_annotation_protected_member = true +ij_scala_type_annotation_public_member = true +ij_scala_type_annotation_structural_type = true +ij_scala_type_annotation_underscore_parameter = false +ij_scala_type_annotation_unit_type = true +ij_scala_use_alternate_continuation_indent_for_params = false +ij_scala_use_scala3_indentation_based_syntax = true +ij_scala_use_scaladoc2_formatting = false +ij_scala_variable_annotation_wrap = off +ij_scala_while_brace_force = never +ij_scala_while_on_new_line = false +ij_scala_wrap_before_with_keyword = false +ij_scala_wrap_first_method_in_call_chain = false +ij_scala_wrap_long_lines = false + +[*.scss] +indent_size = 2 +ij_scss_align_closing_brace_with_properties = false +ij_scss_blank_lines_around_nested_selector = 1 +ij_scss_blank_lines_between_blocks = 1 +ij_scss_block_comment_add_space = false +ij_scss_brace_placement = 0 +ij_scss_enforce_quotes_on_format = false +ij_scss_hex_color_long_format = false +ij_scss_hex_color_lower_case = false +ij_scss_hex_color_short_format = false +ij_scss_hex_color_upper_case = false +ij_scss_keep_blank_lines_in_code = 2 +ij_scss_keep_indents_on_empty_lines = false +ij_scss_keep_single_line_blocks = false +ij_scss_line_comment_add_space = false +ij_scss_line_comment_at_first_column = false +ij_scss_properties_order = font,font-family,font-size,font-weight,font-style,font-variant,font-size-adjust,font-stretch,line-height,position,z-index,top,right,bottom,left,display,visibility,float,clear,overflow,overflow-x,overflow-y,clip,zoom,align-content,align-items,align-self,flex,flex-flow,flex-basis,flex-direction,flex-grow,flex-shrink,flex-wrap,justify-content,order,box-sizing,width,min-width,max-width,height,min-height,max-height,margin,margin-top,margin-right,margin-bottom,margin-left,padding,padding-top,padding-right,padding-bottom,padding-left,table-layout,empty-cells,caption-side,border-spacing,border-collapse,list-style,list-style-position,list-style-type,list-style-image,content,quotes,counter-reset,counter-increment,resize,cursor,user-select,nav-index,nav-up,nav-right,nav-down,nav-left,transition,transition-delay,transition-timing-function,transition-duration,transition-property,transform,transform-origin,animation,animation-name,animation-duration,animation-play-state,animation-timing-function,animation-delay,animation-iteration-count,animation-direction,text-align,text-align-last,vertical-align,white-space,text-decoration,text-emphasis,text-emphasis-color,text-emphasis-style,text-emphasis-position,text-indent,text-justify,letter-spacing,word-spacing,text-outline,text-transform,text-wrap,text-overflow,text-overflow-ellipsis,text-overflow-mode,word-wrap,word-break,tab-size,hyphens,pointer-events,opacity,color,border,border-width,border-style,border-color,border-top,border-top-width,border-top-style,border-top-color,border-right,border-right-width,border-right-style,border-right-color,border-bottom,border-bottom-width,border-bottom-style,border-bottom-color,border-left,border-left-width,border-left-style,border-left-color,border-radius,border-top-left-radius,border-top-right-radius,border-bottom-right-radius,border-bottom-left-radius,border-image,border-image-source,border-image-slice,border-image-width,border-image-outset,border-image-repeat,outline,outline-width,outline-style,outline-color,outline-offset,background,background-color,background-image,background-repeat,background-attachment,background-position,background-position-x,background-position-y,background-clip,background-origin,background-size,box-decoration-break,box-shadow,text-shadow +ij_scss_space_after_colon = true +ij_scss_space_before_opening_brace = true +ij_scss_use_double_quotes = true +ij_scss_value_alignment = 0 + +[*.vue] +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 4 +ij_vue_indent_children_of_top_level = template +ij_vue_interpolation_new_line_after_start_delimiter = true +ij_vue_interpolation_new_line_before_end_delimiter = true +ij_vue_interpolation_wrap = off +ij_vue_keep_indents_on_empty_lines = false +ij_vue_spaces_within_interpolation_expressions = true + +[.editorconfig] +ij_editorconfig_align_group_field_declarations = false +ij_editorconfig_space_after_colon = false +ij_editorconfig_space_after_comma = true +ij_editorconfig_space_before_colon = false +ij_editorconfig_space_before_comma = false +ij_editorconfig_spaces_around_assignment_operators = true + +[{*.ant,*.fxml,*.jhm,*.jnlp,*.jrxml,*.pom,*.rng,*.tld,*.wadl,*.wsdl,*.xml,*.xsd,*.xsl,*.xslt,*.xul}] +ij_xml_align_attributes = true +ij_xml_align_text = false +ij_xml_attribute_wrap = normal +ij_xml_block_comment_add_space = false +ij_xml_block_comment_at_first_column = true +ij_xml_keep_blank_lines = 2 +ij_xml_keep_indents_on_empty_lines = false +ij_xml_keep_line_breaks = true +ij_xml_keep_line_breaks_in_text = true +ij_xml_keep_whitespaces = false +ij_xml_keep_whitespaces_around_cdata = preserve +ij_xml_keep_whitespaces_inside_cdata = false +ij_xml_line_comment_at_first_column = true +ij_xml_space_after_tag_name = false +ij_xml_space_around_equals_in_attribute = false +ij_xml_space_inside_empty_tag = false +ij_xml_text_wrap = normal + +[{*.ats,*.cts,*.mts,*.ts}] +ij_continuation_indent_size = 4 +ij_typescript_align_imports = false +ij_typescript_align_multiline_array_initializer_expression = false +ij_typescript_align_multiline_binary_operation = false +ij_typescript_align_multiline_chained_methods = false +ij_typescript_align_multiline_extends_list = false +ij_typescript_align_multiline_for = true +ij_typescript_align_multiline_parameters = true +ij_typescript_align_multiline_parameters_in_calls = false +ij_typescript_align_multiline_ternary_operation = false +ij_typescript_align_object_properties = 0 +ij_typescript_align_union_types = false +ij_typescript_align_var_statements = 0 +ij_typescript_array_initializer_new_line_after_left_brace = false +ij_typescript_array_initializer_right_brace_on_new_line = false +ij_typescript_array_initializer_wrap = off +ij_typescript_assignment_wrap = off +ij_typescript_binary_operation_sign_on_next_line = false +ij_typescript_binary_operation_wrap = off +ij_typescript_blacklist_imports = rxjs/Rx,node_modules/**,**/node_modules/**,@angular/material,@angular/material/typings/** +ij_typescript_blank_lines_after_imports = 1 +ij_typescript_blank_lines_around_class = 1 +ij_typescript_blank_lines_around_field = 0 +ij_typescript_blank_lines_around_field_in_interface = 0 +ij_typescript_blank_lines_around_function = 1 +ij_typescript_blank_lines_around_method = 1 +ij_typescript_blank_lines_around_method_in_interface = 1 +ij_typescript_block_brace_style = end_of_line +ij_typescript_block_comment_add_space = false +ij_typescript_block_comment_at_first_column = true +ij_typescript_call_parameters_new_line_after_left_paren = false +ij_typescript_call_parameters_right_paren_on_new_line = false +ij_typescript_call_parameters_wrap = off +ij_typescript_catch_on_new_line = false +ij_typescript_chained_call_dot_on_new_line = true +ij_typescript_class_brace_style = end_of_line +ij_typescript_comma_on_new_line = false +ij_typescript_do_while_brace_force = never +ij_typescript_else_on_new_line = false +ij_typescript_enforce_trailing_comma = keep +ij_typescript_enum_constants_wrap = on_every_item +ij_typescript_extends_keyword_wrap = off +ij_typescript_extends_list_wrap = off +ij_typescript_field_prefix = _ +ij_typescript_file_name_style = relaxed +ij_typescript_finally_on_new_line = false +ij_typescript_for_brace_force = never +ij_typescript_for_statement_new_line_after_left_paren = false +ij_typescript_for_statement_right_paren_on_new_line = false +ij_typescript_for_statement_wrap = off +ij_typescript_force_quote_style = false +ij_typescript_force_semicolon_style = false +ij_typescript_function_expression_brace_style = end_of_line +ij_typescript_if_brace_force = never +ij_typescript_import_merge_members = global +ij_typescript_import_prefer_absolute_path = global +ij_typescript_import_sort_members = true +ij_typescript_import_sort_module_name = false +ij_typescript_import_use_node_resolution = true +ij_typescript_imports_wrap = on_every_item +ij_typescript_indent_case_from_switch = true +ij_typescript_indent_chained_calls = true +ij_typescript_indent_package_children = 0 +ij_typescript_jsdoc_include_types = false +ij_typescript_jsx_attribute_value = braces +ij_typescript_keep_blank_lines_in_code = 2 +ij_typescript_keep_first_column_comment = true +ij_typescript_keep_indents_on_empty_lines = false +ij_typescript_keep_line_breaks = true +ij_typescript_keep_simple_blocks_in_one_line = false +ij_typescript_keep_simple_methods_in_one_line = false +ij_typescript_line_comment_add_space = true +ij_typescript_line_comment_at_first_column = false +ij_typescript_method_brace_style = end_of_line +ij_typescript_method_call_chain_wrap = off +ij_typescript_method_parameters_new_line_after_left_paren = false +ij_typescript_method_parameters_right_paren_on_new_line = false +ij_typescript_method_parameters_wrap = off +ij_typescript_object_literal_wrap = on_every_item +ij_typescript_object_types_wrap = on_every_item +ij_typescript_parentheses_expression_new_line_after_left_paren = false +ij_typescript_parentheses_expression_right_paren_on_new_line = false +ij_typescript_place_assignment_sign_on_next_line = false +ij_typescript_prefer_as_type_cast = false +ij_typescript_prefer_explicit_types_function_expression_returns = false +ij_typescript_prefer_explicit_types_function_returns = false +ij_typescript_prefer_explicit_types_vars_fields = false +ij_typescript_prefer_parameters_wrap = false +ij_typescript_reformat_c_style_comments = false +ij_typescript_space_after_colon = true +ij_typescript_space_after_comma = true +ij_typescript_space_after_dots_in_rest_parameter = false +ij_typescript_space_after_generator_mult = true +ij_typescript_space_after_property_colon = true +ij_typescript_space_after_quest = true +ij_typescript_space_after_type_colon = true +ij_typescript_space_after_unary_not = false +ij_typescript_space_before_async_arrow_lparen = true +ij_typescript_space_before_catch_keyword = true +ij_typescript_space_before_catch_left_brace = true +ij_typescript_space_before_catch_parentheses = true +ij_typescript_space_before_class_lbrace = true +ij_typescript_space_before_class_left_brace = true +ij_typescript_space_before_colon = true +ij_typescript_space_before_comma = false +ij_typescript_space_before_do_left_brace = true +ij_typescript_space_before_else_keyword = true +ij_typescript_space_before_else_left_brace = true +ij_typescript_space_before_finally_keyword = true +ij_typescript_space_before_finally_left_brace = true +ij_typescript_space_before_for_left_brace = true +ij_typescript_space_before_for_parentheses = true +ij_typescript_space_before_for_semicolon = false +ij_typescript_space_before_function_left_parenth = true +ij_typescript_space_before_generator_mult = false +ij_typescript_space_before_if_left_brace = true +ij_typescript_space_before_if_parentheses = true +ij_typescript_space_before_method_call_parentheses = false +ij_typescript_space_before_method_left_brace = true +ij_typescript_space_before_method_parentheses = false +ij_typescript_space_before_property_colon = false +ij_typescript_space_before_quest = true +ij_typescript_space_before_switch_left_brace = true +ij_typescript_space_before_switch_parentheses = true +ij_typescript_space_before_try_left_brace = true +ij_typescript_space_before_type_colon = false +ij_typescript_space_before_unary_not = false +ij_typescript_space_before_while_keyword = true +ij_typescript_space_before_while_left_brace = true +ij_typescript_space_before_while_parentheses = true +ij_typescript_spaces_around_additive_operators = true +ij_typescript_spaces_around_arrow_function_operator = true +ij_typescript_spaces_around_assignment_operators = true +ij_typescript_spaces_around_bitwise_operators = true +ij_typescript_spaces_around_equality_operators = true +ij_typescript_spaces_around_logical_operators = true +ij_typescript_spaces_around_multiplicative_operators = true +ij_typescript_spaces_around_relational_operators = true +ij_typescript_spaces_around_shift_operators = true +ij_typescript_spaces_around_unary_operator = false +ij_typescript_spaces_within_array_initializer_brackets = false +ij_typescript_spaces_within_brackets = false +ij_typescript_spaces_within_catch_parentheses = false +ij_typescript_spaces_within_for_parentheses = false +ij_typescript_spaces_within_if_parentheses = false +ij_typescript_spaces_within_imports = false +ij_typescript_spaces_within_interpolation_expressions = false +ij_typescript_spaces_within_method_call_parentheses = false +ij_typescript_spaces_within_method_parentheses = false +ij_typescript_spaces_within_object_literal_braces = false +ij_typescript_spaces_within_object_type_braces = true +ij_typescript_spaces_within_parentheses = false +ij_typescript_spaces_within_switch_parentheses = false +ij_typescript_spaces_within_type_assertion = false +ij_typescript_spaces_within_union_types = true +ij_typescript_spaces_within_while_parentheses = false +ij_typescript_special_else_if_treatment = true +ij_typescript_ternary_operation_signs_on_next_line = false +ij_typescript_ternary_operation_wrap = off +ij_typescript_union_types_wrap = on_every_item +ij_typescript_use_chained_calls_group_indents = false +ij_typescript_use_double_quotes = true +ij_typescript_use_explicit_js_extension = auto +ij_typescript_use_import_type = auto +ij_typescript_use_path_mapping = always +ij_typescript_use_public_modifier = false +ij_typescript_use_semicolon_after_statement = true +ij_typescript_var_declaration_wrap = normal +ij_typescript_while_brace_force = never +ij_typescript_while_on_new_line = false +ij_typescript_wrap_comments = false + +[{*.bash,*.sh,*.zsh}] +indent_size = 2 +tab_width = 2 +ij_shell_binary_ops_start_line = false +ij_shell_keep_column_alignment_padding = false +ij_shell_minify_program = false +ij_shell_redirect_followed_by_space = false +ij_shell_switch_cases_indented = false +ij_shell_use_unix_line_separator = true + +[{*.cjs,*.js}] +ij_continuation_indent_size = 4 +ij_javascript_align_imports = false +ij_javascript_align_multiline_array_initializer_expression = false +ij_javascript_align_multiline_binary_operation = false +ij_javascript_align_multiline_chained_methods = false +ij_javascript_align_multiline_extends_list = false +ij_javascript_align_multiline_for = true +ij_javascript_align_multiline_parameters = true +ij_javascript_align_multiline_parameters_in_calls = false +ij_javascript_align_multiline_ternary_operation = false +ij_javascript_align_object_properties = 0 +ij_javascript_align_union_types = false +ij_javascript_align_var_statements = 0 +ij_javascript_array_initializer_new_line_after_left_brace = false +ij_javascript_array_initializer_right_brace_on_new_line = false +ij_javascript_array_initializer_wrap = off +ij_javascript_assignment_wrap = off +ij_javascript_binary_operation_sign_on_next_line = false +ij_javascript_binary_operation_wrap = off +ij_javascript_blacklist_imports = rxjs/Rx,node_modules/**,**/node_modules/**,@angular/material,@angular/material/typings/** +ij_javascript_blank_lines_after_imports = 1 +ij_javascript_blank_lines_around_class = 1 +ij_javascript_blank_lines_around_field = 0 +ij_javascript_blank_lines_around_function = 1 +ij_javascript_blank_lines_around_method = 1 +ij_javascript_block_brace_style = end_of_line +ij_javascript_block_comment_add_space = false +ij_javascript_block_comment_at_first_column = true +ij_javascript_call_parameters_new_line_after_left_paren = false +ij_javascript_call_parameters_right_paren_on_new_line = false +ij_javascript_call_parameters_wrap = off +ij_javascript_catch_on_new_line = false +ij_javascript_chained_call_dot_on_new_line = true +ij_javascript_class_brace_style = end_of_line +ij_javascript_comma_on_new_line = false +ij_javascript_do_while_brace_force = never +ij_javascript_else_on_new_line = false +ij_javascript_enforce_trailing_comma = keep +ij_javascript_extends_keyword_wrap = off +ij_javascript_extends_list_wrap = off +ij_javascript_field_prefix = _ +ij_javascript_file_name_style = relaxed +ij_javascript_finally_on_new_line = false +ij_javascript_for_brace_force = never +ij_javascript_for_statement_new_line_after_left_paren = false +ij_javascript_for_statement_right_paren_on_new_line = false +ij_javascript_for_statement_wrap = off +ij_javascript_force_quote_style = false +ij_javascript_force_semicolon_style = false +ij_javascript_function_expression_brace_style = end_of_line +ij_javascript_if_brace_force = never +ij_javascript_import_merge_members = global +ij_javascript_import_prefer_absolute_path = global +ij_javascript_import_sort_members = true +ij_javascript_import_sort_module_name = false +ij_javascript_import_use_node_resolution = true +ij_javascript_imports_wrap = on_every_item +ij_javascript_indent_case_from_switch = true +ij_javascript_indent_chained_calls = true +ij_javascript_indent_package_children = 0 +ij_javascript_jsx_attribute_value = braces +ij_javascript_keep_blank_lines_in_code = 2 +ij_javascript_keep_first_column_comment = true +ij_javascript_keep_indents_on_empty_lines = false +ij_javascript_keep_line_breaks = true +ij_javascript_keep_simple_blocks_in_one_line = false +ij_javascript_keep_simple_methods_in_one_line = false +ij_javascript_line_comment_add_space = true +ij_javascript_line_comment_at_first_column = false +ij_javascript_method_brace_style = end_of_line +ij_javascript_method_call_chain_wrap = off +ij_javascript_method_parameters_new_line_after_left_paren = false +ij_javascript_method_parameters_right_paren_on_new_line = false +ij_javascript_method_parameters_wrap = off +ij_javascript_object_literal_wrap = on_every_item +ij_javascript_object_types_wrap = on_every_item +ij_javascript_parentheses_expression_new_line_after_left_paren = false +ij_javascript_parentheses_expression_right_paren_on_new_line = false +ij_javascript_place_assignment_sign_on_next_line = false +ij_javascript_prefer_as_type_cast = false +ij_javascript_prefer_explicit_types_function_expression_returns = false +ij_javascript_prefer_explicit_types_function_returns = false +ij_javascript_prefer_explicit_types_vars_fields = false +ij_javascript_prefer_parameters_wrap = false +ij_javascript_reformat_c_style_comments = false +ij_javascript_space_after_colon = true +ij_javascript_space_after_comma = true +ij_javascript_space_after_dots_in_rest_parameter = false +ij_javascript_space_after_generator_mult = true +ij_javascript_space_after_property_colon = true +ij_javascript_space_after_quest = true +ij_javascript_space_after_type_colon = true +ij_javascript_space_after_unary_not = false +ij_javascript_space_before_async_arrow_lparen = true +ij_javascript_space_before_catch_keyword = true +ij_javascript_space_before_catch_left_brace = true +ij_javascript_space_before_catch_parentheses = true +ij_javascript_space_before_class_lbrace = true +ij_javascript_space_before_class_left_brace = true +ij_javascript_space_before_colon = true +ij_javascript_space_before_comma = false +ij_javascript_space_before_do_left_brace = true +ij_javascript_space_before_else_keyword = true +ij_javascript_space_before_else_left_brace = true +ij_javascript_space_before_finally_keyword = true +ij_javascript_space_before_finally_left_brace = true +ij_javascript_space_before_for_left_brace = true +ij_javascript_space_before_for_parentheses = true +ij_javascript_space_before_for_semicolon = false +ij_javascript_space_before_function_left_parenth = true +ij_javascript_space_before_generator_mult = false +ij_javascript_space_before_if_left_brace = true +ij_javascript_space_before_if_parentheses = true +ij_javascript_space_before_method_call_parentheses = false +ij_javascript_space_before_method_left_brace = true +ij_javascript_space_before_method_parentheses = false +ij_javascript_space_before_property_colon = false +ij_javascript_space_before_quest = true +ij_javascript_space_before_switch_left_brace = true +ij_javascript_space_before_switch_parentheses = true +ij_javascript_space_before_try_left_brace = true +ij_javascript_space_before_type_colon = false +ij_javascript_space_before_unary_not = false +ij_javascript_space_before_while_keyword = true +ij_javascript_space_before_while_left_brace = true +ij_javascript_space_before_while_parentheses = true +ij_javascript_spaces_around_additive_operators = true +ij_javascript_spaces_around_arrow_function_operator = true +ij_javascript_spaces_around_assignment_operators = true +ij_javascript_spaces_around_bitwise_operators = true +ij_javascript_spaces_around_equality_operators = true +ij_javascript_spaces_around_logical_operators = true +ij_javascript_spaces_around_multiplicative_operators = true +ij_javascript_spaces_around_relational_operators = true +ij_javascript_spaces_around_shift_operators = true +ij_javascript_spaces_around_unary_operator = false +ij_javascript_spaces_within_array_initializer_brackets = false +ij_javascript_spaces_within_brackets = false +ij_javascript_spaces_within_catch_parentheses = false +ij_javascript_spaces_within_for_parentheses = false +ij_javascript_spaces_within_if_parentheses = false +ij_javascript_spaces_within_imports = false +ij_javascript_spaces_within_interpolation_expressions = false +ij_javascript_spaces_within_method_call_parentheses = false +ij_javascript_spaces_within_method_parentheses = false +ij_javascript_spaces_within_object_literal_braces = false +ij_javascript_spaces_within_object_type_braces = true +ij_javascript_spaces_within_parentheses = false +ij_javascript_spaces_within_switch_parentheses = false +ij_javascript_spaces_within_type_assertion = false +ij_javascript_spaces_within_union_types = true +ij_javascript_spaces_within_while_parentheses = false +ij_javascript_special_else_if_treatment = true +ij_javascript_ternary_operation_signs_on_next_line = false +ij_javascript_ternary_operation_wrap = off +ij_javascript_union_types_wrap = on_every_item +ij_javascript_use_chained_calls_group_indents = false +ij_javascript_use_double_quotes = true +ij_javascript_use_explicit_js_extension = auto +ij_javascript_use_import_type = auto +ij_javascript_use_path_mapping = always +ij_javascript_use_public_modifier = false +ij_javascript_use_semicolon_after_statement = true +ij_javascript_var_declaration_wrap = normal +ij_javascript_while_brace_force = never +ij_javascript_while_on_new_line = false +ij_javascript_wrap_comments = false + +[{*.ft,*.vm,*.vsl}] +ij_vtl_keep_indents_on_empty_lines = false + +[{*.gant,*.groovy,*.gy}] +ij_groovy_align_group_field_declarations = false +ij_groovy_align_multiline_array_initializer_expression = false +ij_groovy_align_multiline_assignment = false +ij_groovy_align_multiline_binary_operation = false +ij_groovy_align_multiline_chained_methods = false +ij_groovy_align_multiline_extends_list = false +ij_groovy_align_multiline_for = true +ij_groovy_align_multiline_list_or_map = true +ij_groovy_align_multiline_method_parentheses = false +ij_groovy_align_multiline_parameters = true +ij_groovy_align_multiline_parameters_in_calls = false +ij_groovy_align_multiline_resources = true +ij_groovy_align_multiline_ternary_operation = false +ij_groovy_align_multiline_throws_list = false +ij_groovy_align_named_args_in_map = true +ij_groovy_align_throws_keyword = false +ij_groovy_array_initializer_new_line_after_left_brace = false +ij_groovy_array_initializer_right_brace_on_new_line = false +ij_groovy_array_initializer_wrap = off +ij_groovy_assert_statement_wrap = off +ij_groovy_assignment_wrap = off +ij_groovy_binary_operation_wrap = off +ij_groovy_blank_lines_after_class_header = 0 +ij_groovy_blank_lines_after_imports = 1 +ij_groovy_blank_lines_after_package = 1 +ij_groovy_blank_lines_around_class = 1 +ij_groovy_blank_lines_around_field = 0 +ij_groovy_blank_lines_around_field_in_interface = 0 +ij_groovy_blank_lines_around_method = 1 +ij_groovy_blank_lines_around_method_in_interface = 1 +ij_groovy_blank_lines_before_imports = 1 +ij_groovy_blank_lines_before_method_body = 0 +ij_groovy_blank_lines_before_package = 0 +ij_groovy_block_brace_style = end_of_line +ij_groovy_block_comment_add_space = false +ij_groovy_block_comment_at_first_column = true +ij_groovy_call_parameters_new_line_after_left_paren = false +ij_groovy_call_parameters_right_paren_on_new_line = false +ij_groovy_call_parameters_wrap = off +ij_groovy_catch_on_new_line = false +ij_groovy_class_annotation_wrap = split_into_lines +ij_groovy_class_brace_style = end_of_line +ij_groovy_class_count_to_use_import_on_demand = 5 +ij_groovy_do_while_brace_force = never +ij_groovy_else_on_new_line = false +ij_groovy_enable_groovydoc_formatting = true +ij_groovy_enum_constants_wrap = off +ij_groovy_extends_keyword_wrap = off +ij_groovy_extends_list_wrap = off +ij_groovy_field_annotation_wrap = split_into_lines +ij_groovy_finally_on_new_line = false +ij_groovy_for_brace_force = never +ij_groovy_for_statement_new_line_after_left_paren = false +ij_groovy_for_statement_right_paren_on_new_line = false +ij_groovy_for_statement_wrap = off +ij_groovy_ginq_general_clause_wrap_policy = 2 +ij_groovy_ginq_having_wrap_policy = 1 +ij_groovy_ginq_indent_having_clause = true +ij_groovy_ginq_indent_on_clause = true +ij_groovy_ginq_on_wrap_policy = 1 +ij_groovy_ginq_space_after_keyword = true +ij_groovy_if_brace_force = never +ij_groovy_import_annotation_wrap = 2 +ij_groovy_imports_layout = *,|,javax.**,java.**,|,$* +ij_groovy_indent_case_from_switch = true +ij_groovy_indent_label_blocks = true +ij_groovy_insert_inner_class_imports = false +ij_groovy_keep_blank_lines_before_right_brace = 2 +ij_groovy_keep_blank_lines_in_code = 2 +ij_groovy_keep_blank_lines_in_declarations = 2 +ij_groovy_keep_control_statement_in_one_line = true +ij_groovy_keep_first_column_comment = true +ij_groovy_keep_indents_on_empty_lines = false +ij_groovy_keep_line_breaks = true +ij_groovy_keep_multiple_expressions_in_one_line = false +ij_groovy_keep_simple_blocks_in_one_line = false +ij_groovy_keep_simple_classes_in_one_line = true +ij_groovy_keep_simple_lambdas_in_one_line = true +ij_groovy_keep_simple_methods_in_one_line = true +ij_groovy_label_indent_absolute = false +ij_groovy_label_indent_size = 0 +ij_groovy_lambda_brace_style = end_of_line +ij_groovy_layout_static_imports_separately = true +ij_groovy_line_comment_add_space = false +ij_groovy_line_comment_add_space_on_reformat = false +ij_groovy_line_comment_at_first_column = true +ij_groovy_method_annotation_wrap = split_into_lines +ij_groovy_method_brace_style = end_of_line +ij_groovy_method_call_chain_wrap = off +ij_groovy_method_parameters_new_line_after_left_paren = false +ij_groovy_method_parameters_right_paren_on_new_line = false +ij_groovy_method_parameters_wrap = off +ij_groovy_modifier_list_wrap = false +ij_groovy_names_count_to_use_import_on_demand = 3 +ij_groovy_packages_to_use_import_on_demand = java.awt.*,javax.swing.* +ij_groovy_parameter_annotation_wrap = off +ij_groovy_parentheses_expression_new_line_after_left_paren = false +ij_groovy_parentheses_expression_right_paren_on_new_line = false +ij_groovy_prefer_parameters_wrap = false +ij_groovy_resource_list_new_line_after_left_paren = false +ij_groovy_resource_list_right_paren_on_new_line = false +ij_groovy_resource_list_wrap = off +ij_groovy_space_after_assert_separator = true +ij_groovy_space_after_colon = true +ij_groovy_space_after_comma = true +ij_groovy_space_after_comma_in_type_arguments = true +ij_groovy_space_after_for_semicolon = true +ij_groovy_space_after_quest = true +ij_groovy_space_after_type_cast = true +ij_groovy_space_before_annotation_parameter_list = false +ij_groovy_space_before_array_initializer_left_brace = false +ij_groovy_space_before_assert_separator = false +ij_groovy_space_before_catch_keyword = true +ij_groovy_space_before_catch_left_brace = true +ij_groovy_space_before_catch_parentheses = true +ij_groovy_space_before_class_left_brace = true +ij_groovy_space_before_closure_left_brace = true +ij_groovy_space_before_colon = true +ij_groovy_space_before_comma = false +ij_groovy_space_before_do_left_brace = true +ij_groovy_space_before_else_keyword = true +ij_groovy_space_before_else_left_brace = true +ij_groovy_space_before_finally_keyword = true +ij_groovy_space_before_finally_left_brace = true +ij_groovy_space_before_for_left_brace = true +ij_groovy_space_before_for_parentheses = true +ij_groovy_space_before_for_semicolon = false +ij_groovy_space_before_if_left_brace = true +ij_groovy_space_before_if_parentheses = true +ij_groovy_space_before_method_call_parentheses = false +ij_groovy_space_before_method_left_brace = true +ij_groovy_space_before_method_parentheses = false +ij_groovy_space_before_quest = true +ij_groovy_space_before_record_parentheses = false +ij_groovy_space_before_switch_left_brace = true +ij_groovy_space_before_switch_parentheses = true +ij_groovy_space_before_synchronized_left_brace = true +ij_groovy_space_before_synchronized_parentheses = true +ij_groovy_space_before_try_left_brace = true +ij_groovy_space_before_try_parentheses = true +ij_groovy_space_before_while_keyword = true +ij_groovy_space_before_while_left_brace = true +ij_groovy_space_before_while_parentheses = true +ij_groovy_space_in_named_argument = true +ij_groovy_space_in_named_argument_before_colon = false +ij_groovy_space_within_empty_array_initializer_braces = false +ij_groovy_space_within_empty_method_call_parentheses = false +ij_groovy_spaces_around_additive_operators = true +ij_groovy_spaces_around_assignment_operators = true +ij_groovy_spaces_around_bitwise_operators = true +ij_groovy_spaces_around_equality_operators = true +ij_groovy_spaces_around_lambda_arrow = true +ij_groovy_spaces_around_logical_operators = true +ij_groovy_spaces_around_multiplicative_operators = true +ij_groovy_spaces_around_regex_operators = true +ij_groovy_spaces_around_relational_operators = true +ij_groovy_spaces_around_shift_operators = true +ij_groovy_spaces_within_annotation_parentheses = false +ij_groovy_spaces_within_array_initializer_braces = false +ij_groovy_spaces_within_braces = true +ij_groovy_spaces_within_brackets = false +ij_groovy_spaces_within_cast_parentheses = false +ij_groovy_spaces_within_catch_parentheses = false +ij_groovy_spaces_within_for_parentheses = false +ij_groovy_spaces_within_gstring_injection_braces = false +ij_groovy_spaces_within_if_parentheses = false +ij_groovy_spaces_within_list_or_map = false +ij_groovy_spaces_within_method_call_parentheses = false +ij_groovy_spaces_within_method_parentheses = false +ij_groovy_spaces_within_parentheses = false +ij_groovy_spaces_within_switch_parentheses = false +ij_groovy_spaces_within_synchronized_parentheses = false +ij_groovy_spaces_within_try_parentheses = false +ij_groovy_spaces_within_tuple_expression = false +ij_groovy_spaces_within_while_parentheses = false +ij_groovy_special_else_if_treatment = true +ij_groovy_ternary_operation_wrap = off +ij_groovy_throws_keyword_wrap = off +ij_groovy_throws_list_wrap = off +ij_groovy_use_flying_geese_braces = false +ij_groovy_use_fq_class_names = false +ij_groovy_use_fq_class_names_in_javadoc = true +ij_groovy_use_relative_indents = false +ij_groovy_use_single_class_imports = true +ij_groovy_variable_annotation_wrap = off +ij_groovy_while_brace_force = never +ij_groovy_while_on_new_line = false +ij_groovy_wrap_chain_calls_after_dot = false +ij_groovy_wrap_long_lines = false + +[{*.har,*.jsb2,*.jsb3,*.json,*.jsonc,*.postman_collection,*.postman_collection.json,*.postman_environment,*.postman_environment.json,.babelrc,.eslintrc,.prettierrc,.stylelintrc,.ws-context,jest.config}] +indent_size = 2 +ij_json_array_wrapping = split_into_lines +ij_json_keep_blank_lines_in_code = 0 +ij_json_keep_indents_on_empty_lines = false +ij_json_keep_line_breaks = true +ij_json_keep_trailing_comma = false +ij_json_object_wrapping = split_into_lines +ij_json_property_alignment = do_not_align +ij_json_space_after_colon = true +ij_json_space_after_comma = true +ij_json_space_before_colon = false +ij_json_space_before_comma = false +ij_json_spaces_within_braces = false +ij_json_spaces_within_brackets = false +ij_json_wrap_long_lines = false + +[{*.htm,*.html,*.sht,*.shtm,*.shtml}] +ij_html_add_new_line_before_tags = body,div,p,form,h1,h2,h3 +ij_html_align_attributes = true +ij_html_align_text = false +ij_html_attribute_wrap = normal +ij_html_block_comment_add_space = false +ij_html_block_comment_at_first_column = true +ij_html_do_not_align_children_of_min_lines = 0 +ij_html_do_not_break_if_inline_tags = title,h1,h2,h3,h4,h5,h6,p +ij_html_do_not_indent_children_of_tags = html,body,thead,tbody,tfoot +ij_html_enforce_quotes = false +ij_html_inline_tags = a,abbr,acronym,b,basefont,bdo,big,br,cite,cite,code,dfn,em,font,i,img,input,kbd,label,q,s,samp,select,small,span,strike,strong,sub,sup,textarea,tt,u,var +ij_html_keep_blank_lines = 2 +ij_html_keep_indents_on_empty_lines = false +ij_html_keep_line_breaks = true +ij_html_keep_line_breaks_in_text = true +ij_html_keep_whitespaces = false +ij_html_keep_whitespaces_inside = span,pre,textarea +ij_html_line_comment_at_first_column = true +ij_html_new_line_after_last_attribute = never +ij_html_new_line_before_first_attribute = never +ij_html_quote_style = double +ij_html_remove_new_line_before_tags = br +ij_html_space_after_tag_name = false +ij_html_space_around_equality_in_attribute = false +ij_html_space_inside_empty_tag = false +ij_html_text_wrap = normal + +[{*.http,*.rest}] +indent_size = 0 +ij_continuation_indent_size = 4 +ij_http-request_call_parameters_wrap = normal +ij_http-request_method_parameters_wrap = split_into_lines +ij_http-request_space_before_comma = true +ij_http-request_spaces_around_assignment_operators = true + +[{*.jsf,*.jsp,*.jspf,*.tag,*.tagf,*.xjsp}] +ij_jsp_jsp_prefer_comma_separated_import_list = false +ij_jsp_keep_indents_on_empty_lines = false + +[{*.jspx,*.tagx}] +ij_jspx_keep_indents_on_empty_lines = false + +[{*.kt,*.kts}] +ij_kotlin_align_in_columns_case_branch = false +ij_kotlin_align_multiline_binary_operation = false +ij_kotlin_align_multiline_extends_list = false +ij_kotlin_align_multiline_method_parentheses = false +ij_kotlin_align_multiline_parameters = true +ij_kotlin_align_multiline_parameters_in_calls = false +ij_kotlin_allow_trailing_comma = false +ij_kotlin_allow_trailing_comma_on_call_site = false +ij_kotlin_assignment_wrap = normal +ij_kotlin_blank_lines_after_class_header = 0 +ij_kotlin_blank_lines_around_block_when_branches = 0 +ij_kotlin_blank_lines_before_declaration_with_comment_or_annotation_on_separate_line = 1 +ij_kotlin_block_comment_add_space = false +ij_kotlin_block_comment_at_first_column = true +ij_kotlin_call_parameters_new_line_after_left_paren = true +ij_kotlin_call_parameters_right_paren_on_new_line = true +ij_kotlin_call_parameters_wrap = on_every_item +ij_kotlin_catch_on_new_line = false +ij_kotlin_class_annotation_wrap = split_into_lines +ij_kotlin_continuation_indent_for_chained_calls = false +ij_kotlin_continuation_indent_for_expression_bodies = false +ij_kotlin_continuation_indent_in_argument_lists = false +ij_kotlin_continuation_indent_in_elvis = false +ij_kotlin_continuation_indent_in_if_conditions = false +ij_kotlin_continuation_indent_in_parameter_lists = false +ij_kotlin_continuation_indent_in_supertype_lists = false +ij_kotlin_else_on_new_line = false +ij_kotlin_enum_constants_wrap = off +ij_kotlin_extends_list_wrap = normal +ij_kotlin_field_annotation_wrap = split_into_lines +ij_kotlin_finally_on_new_line = false +ij_kotlin_if_rparen_on_new_line = true +ij_kotlin_import_nested_classes = false +ij_kotlin_imports_layout = *,java.**,javax.**,kotlin.**,^ +ij_kotlin_indent_before_arrow_on_new_line = true +ij_kotlin_insert_whitespaces_in_simple_one_line_method = true +ij_kotlin_keep_blank_lines_before_right_brace = 2 +ij_kotlin_keep_blank_lines_in_code = 2 +ij_kotlin_keep_blank_lines_in_declarations = 2 +ij_kotlin_keep_first_column_comment = true +ij_kotlin_keep_indents_on_empty_lines = false +ij_kotlin_keep_line_breaks = true +ij_kotlin_lbrace_on_next_line = false +ij_kotlin_line_break_after_multiline_when_entry = true +ij_kotlin_line_comment_add_space = false +ij_kotlin_line_comment_add_space_on_reformat = false +ij_kotlin_line_comment_at_first_column = true +ij_kotlin_method_annotation_wrap = split_into_lines +ij_kotlin_method_call_chain_wrap = normal +ij_kotlin_method_parameters_new_line_after_left_paren = true +ij_kotlin_method_parameters_right_paren_on_new_line = true +ij_kotlin_method_parameters_wrap = on_every_item +ij_kotlin_name_count_to_use_star_import = 5 +ij_kotlin_name_count_to_use_star_import_for_members = 3 +ij_kotlin_packages_to_use_import_on_demand = java.util.*,kotlinx.android.synthetic.**,io.ktor.** +ij_kotlin_parameter_annotation_wrap = off +ij_kotlin_space_after_comma = true +ij_kotlin_space_after_extend_colon = true +ij_kotlin_space_after_type_colon = true +ij_kotlin_space_before_catch_parentheses = true +ij_kotlin_space_before_comma = false +ij_kotlin_space_before_extend_colon = true +ij_kotlin_space_before_for_parentheses = true +ij_kotlin_space_before_if_parentheses = true +ij_kotlin_space_before_lambda_arrow = true +ij_kotlin_space_before_type_colon = false +ij_kotlin_space_before_when_parentheses = true +ij_kotlin_space_before_while_parentheses = true +ij_kotlin_spaces_around_additive_operators = true +ij_kotlin_spaces_around_assignment_operators = true +ij_kotlin_spaces_around_equality_operators = true +ij_kotlin_spaces_around_function_type_arrow = true +ij_kotlin_spaces_around_logical_operators = true +ij_kotlin_spaces_around_multiplicative_operators = true +ij_kotlin_spaces_around_range = false +ij_kotlin_spaces_around_relational_operators = true +ij_kotlin_spaces_around_unary_operator = false +ij_kotlin_spaces_around_when_arrow = true +ij_kotlin_variable_annotation_wrap = off +ij_kotlin_while_on_new_line = false +ij_kotlin_wrap_elvis_expressions = 1 +ij_kotlin_wrap_expression_body_functions = 1 +ij_kotlin_wrap_first_method_in_call_chain = false + +[{*.lua,*.lua.txt}] +ij_lua_align_consecutive_variable_declarations = false +ij_lua_align_multiline_parameters = false +ij_lua_align_multiline_parameters_in_calls = false +ij_lua_call_parameters_wrap = off +ij_lua_keep_indents_on_empty_lines = false +ij_lua_keep_simple_blocks_in_one_line = true +ij_lua_method_parameters_wrap = off +ij_lua_space_after_comma = true +ij_lua_space_before_comma = false +ij_lua_spaces_around_assignment_operators = true + +[{*.markdown,*.md}] +ij_markdown_force_one_space_after_blockquote_symbol = true +ij_markdown_force_one_space_after_header_symbol = true +ij_markdown_force_one_space_after_list_bullet = true +ij_markdown_force_one_space_between_words = true +ij_markdown_format_tables = true +ij_markdown_insert_quote_arrows_on_wrap = true +ij_markdown_keep_indents_on_empty_lines = false +ij_markdown_keep_line_breaks_inside_text_blocks = true +ij_markdown_max_lines_around_block_elements = 1 +ij_markdown_max_lines_around_header = 1 +ij_markdown_max_lines_between_paragraphs = 1 +ij_markdown_min_lines_around_block_elements = 1 +ij_markdown_min_lines_around_header = 1 +ij_markdown_min_lines_between_paragraphs = 1 +ij_markdown_wrap_text_if_long = true +ij_markdown_wrap_text_inside_blockquotes = true + +[{*.pb,*.textproto,*.txtpb}] +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 4 +ij_prototext_keep_blank_lines_in_code = 2 +ij_prototext_keep_indents_on_empty_lines = false +ij_prototext_keep_line_breaks = true +ij_prototext_space_after_colon = true +ij_prototext_space_after_comma = true +ij_prototext_space_before_colon = false +ij_prototext_space_before_comma = false +ij_prototext_spaces_within_braces = true +ij_prototext_spaces_within_brackets = false + +[{*.properties,spring.handlers,spring.schemas}] +ij_properties_align_group_field_declarations = false +ij_properties_keep_blank_lines = false +ij_properties_key_value_delimiter = equals +ij_properties_spaces_around_key_value_delimiter = false + +[{*.py,*.pyw}] +ij_python_align_collections_and_comprehensions = true +ij_python_align_multiline_imports = true +ij_python_align_multiline_parameters = true +ij_python_align_multiline_parameters_in_calls = true +ij_python_blank_line_at_file_end = true +ij_python_blank_lines_after_imports = 1 +ij_python_blank_lines_after_local_imports = 0 +ij_python_blank_lines_around_class = 1 +ij_python_blank_lines_around_method = 1 +ij_python_blank_lines_around_top_level_classes_functions = 2 +ij_python_blank_lines_before_first_method = 0 +ij_python_call_parameters_new_line_after_left_paren = false +ij_python_call_parameters_right_paren_on_new_line = false +ij_python_call_parameters_wrap = normal +ij_python_dict_alignment = 0 +ij_python_dict_new_line_after_left_brace = false +ij_python_dict_new_line_before_right_brace = false +ij_python_dict_wrapping = 1 +ij_python_from_import_new_line_after_left_parenthesis = false +ij_python_from_import_new_line_before_right_parenthesis = false +ij_python_from_import_parentheses_force_if_multiline = false +ij_python_from_import_trailing_comma_if_multiline = false +ij_python_from_import_wrapping = 1 +ij_python_hang_closing_brackets = false +ij_python_keep_blank_lines_in_code = 1 +ij_python_keep_blank_lines_in_declarations = 1 +ij_python_keep_indents_on_empty_lines = false +ij_python_keep_line_breaks = true +ij_python_method_parameters_new_line_after_left_paren = false +ij_python_method_parameters_right_paren_on_new_line = false +ij_python_method_parameters_wrap = normal +ij_python_new_line_after_colon = false +ij_python_new_line_after_colon_multi_clause = true +ij_python_optimize_imports_always_split_from_imports = false +ij_python_optimize_imports_case_insensitive_order = false +ij_python_optimize_imports_join_from_imports_with_same_source = false +ij_python_optimize_imports_sort_by_type_first = true +ij_python_optimize_imports_sort_imports = true +ij_python_optimize_imports_sort_names_in_from_imports = false +ij_python_space_after_comma = true +ij_python_space_after_number_sign = true +ij_python_space_after_py_colon = true +ij_python_space_before_backslash = true +ij_python_space_before_comma = false +ij_python_space_before_for_semicolon = false +ij_python_space_before_lbracket = false +ij_python_space_before_method_call_parentheses = false +ij_python_space_before_method_parentheses = false +ij_python_space_before_number_sign = true +ij_python_space_before_py_colon = false +ij_python_space_within_empty_method_call_parentheses = false +ij_python_space_within_empty_method_parentheses = false +ij_python_spaces_around_additive_operators = true +ij_python_spaces_around_assignment_operators = true +ij_python_spaces_around_bitwise_operators = true +ij_python_spaces_around_eq_in_keyword_argument = false +ij_python_spaces_around_eq_in_named_parameter = false +ij_python_spaces_around_equality_operators = true +ij_python_spaces_around_multiplicative_operators = true +ij_python_spaces_around_power_operator = true +ij_python_spaces_around_relational_operators = true +ij_python_spaces_around_shift_operators = true +ij_python_spaces_within_braces = false +ij_python_spaces_within_brackets = false +ij_python_spaces_within_method_call_parentheses = false +ij_python_spaces_within_method_parentheses = false +ij_python_use_continuation_indent_for_arguments = false +ij_python_use_continuation_indent_for_collection_and_comprehensions = false +ij_python_use_continuation_indent_for_parameters = true +ij_python_wrap_long_lines = false + +[{*.qute.htm,*.qute.html,*.qute.json,*.qute.txt,*.qute.yaml,*.qute.yml}] +ij_qute_keep_indents_on_empty_lines = false + +[{*.toml,Cargo.lock,Cargo.toml.orig,Gopkg.lock,Pipfile,poetry.lock}] +ij_toml_keep_indents_on_empty_lines = false + +[{*.yaml,*.yml}] +indent_size = 2 +ij_yaml_align_values_properties = do_not_align +ij_yaml_autoinsert_sequence_marker = true +ij_yaml_block_mapping_on_new_line = false +ij_yaml_indent_sequence_value = true +ij_yaml_keep_indents_on_empty_lines = false +ij_yaml_keep_line_breaks = true +ij_yaml_line_comment_add_space = false +ij_yaml_line_comment_add_space_on_reformat = false +ij_yaml_line_comment_at_first_column = true +ij_yaml_sequence_on_new_line = false +ij_yaml_space_before_colon = false +ij_yaml_spaces_within_braces = true +ij_yaml_spaces_within_brackets = true diff --git a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/CaracalGeneratorFlinkJob.java b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/CaracalGeneratorFlinkJob.java index d2710418..51eaf666 100644 --- a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/CaracalGeneratorFlinkJob.java +++ b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/CaracalGeneratorFlinkJob.java @@ -20,6 +20,13 @@ import com.cloudera.cyber.libs.networking.IPLocal; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.utils.ParameterTool; @@ -30,21 +37,6 @@ import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.util.Preconditions; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - public abstract class CaracalGeneratorFlinkJob { public static final String PARAMS_RECORDS_LIMIT = "generator.count"; @@ -61,7 +53,9 @@ public abstract class CaracalGeneratorFlinkJob { protected static final String DPI_SMTP_TOPIC = "dpi_smtp"; protected static final String GENERATOR_AVRO_TOPIC = "generator.avro"; protected static final String THREAT_TOPIC_NAME = "threats"; - protected static final String AVRO_WITH_CUSTOM_CONFIG_ERROR = String.format("'%s' should not be specified when '%s' is true. Select either a custom generation config file or generate the default avro.", PARAMS_GENERATOR_CONFIG, PARAMS_SCHEMA); + protected static final String AVRO_WITH_CUSTOM_CONFIG_ERROR = String.format( + "'%s' should not be specified when '%s' is true. Select either a custom generation config file or generate the default avro.", + PARAMS_GENERATOR_CONFIG, PARAMS_SCHEMA); public StreamExecutionEnvironment createPipeline(ParameterTool params) throws IOException { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -84,9 +78,10 @@ private GeneratorConfig getGeneratorConfig(ParameterTool params) throws IOExcept GeneratorConfig generatorConfig = new GeneratorConfig(); if (avroGeneratorFlag && generatorConfigFile == null) { - generatorConfig.setGenerationSources(Collections - .singletonList( - new GenerationSource("Netflow/netflow_avro_sample1.json", GENERATOR_AVRO_TOPIC, SCHEMA_PATH, 1.0))); + generatorConfig.setGenerationSources(Collections + .singletonList( + new GenerationSource("Netflow/netflow_avro_sample1.json", GENERATOR_AVRO_TOPIC, SCHEMA_PATH, + 1.0))); } else if (generatorConfigFile == null) { generatorConfig.setGenerationSources(getNetflowSampleMap()); } else { @@ -94,9 +89,9 @@ private GeneratorConfig getGeneratorConfig(ParameterTool params) throws IOExcept Path configPath = new Path(generatorConfigFile); try (InputStream configStream = configPath.getFileSystem().open(configPath)) { generatorConfig = new ObjectMapper().readValue( - configStream, - new TypeReference() { - }); + configStream, + new TypeReference() { + }); } } generatorConfig.open(); @@ -105,33 +100,36 @@ private GeneratorConfig getGeneratorConfig(ParameterTool params) throws IOExcept } private SingleOutputStreamOperator> generateMetrics( - SingleOutputStreamOperator> generatedInput) { + SingleOutputStreamOperator> generatedInput) { return generatedInput - .map(new MapFunction, Tuple2>() { - @Override - public Tuple2 map(Tuple2 stringStringTuple2) { - return Tuple2.of(stringStringTuple2.f0, 1); - } - }) - .keyBy(value -> value.f0) - .window(TumblingProcessingTimeWindows.of(Time.seconds(30))) - .sum(1); + .map(new MapFunction, Tuple2>() { + @Override + public Tuple2 map(Tuple2 stringStringTuple2) { + return Tuple2.of(stringStringTuple2.f0, 1); + } + }) + .keyBy(value -> value.f0) + .window(TumblingProcessingTimeWindows.of(Time.seconds(30))) + .sum(1); } private void generateRandomThreatResults(ParameterTool params, - SingleOutputStreamOperator> generatedInput, GeneratorConfig generatorConfig) { + SingleOutputStreamOperator> generatedInput, + GeneratorConfig generatorConfig) { if (params.get(PARAMS_GENERATOR_CONFIG) == null) { // add random threat intelligence for a sample of the generated IPs IPLocal localIp = new IPLocal(); - List characterTopics = generatorConfig.getGenerationSources().stream().filter(gs -> Objects.isNull(gs.getOutputAvroSchema())). - map(GenerationSource::getTopic).collect(Collectors.toList()); - SingleOutputStreamOperator> threats = generatedInput.filter(t -> characterTopics.contains(t.f0)) - .map(new GetIpMap()) - .filter(f -> f != null && !localIp.eval(f)) - .filter(new RandomSampler<>(THREAT_PROBABILITY)) - .map(new ThreatGeneratorMap(THREAT_TOPIC_NAME)); + List characterTopics = generatorConfig.getGenerationSources().stream() + .filter(gs -> Objects.isNull(gs.getOutputAvroSchema())) + .map(GenerationSource::getTopic).collect(Collectors.toList()); + SingleOutputStreamOperator> threats = + generatedInput.filter(t -> characterTopics.contains(t.f0)) + .map(new GetIpMap()) + .filter(f -> f != null && !localIp.eval(f)) + .filter(new RandomSampler<>(THREAT_PROBABILITY)) + .map(new ThreatGeneratorMap(THREAT_TOPIC_NAME)); writeResults(params, threats); } } @@ -145,30 +143,36 @@ private List getNetflowSampleMap() { outputs.add(new GenerationSource("Netflow/netflow_sample_b.json", NETFLOW_B_TOPIC, null, 1.0)); outputs.add(new GenerationSource("Netflow/netflow_sample_b_error.json", NETFLOW_B_TOPIC, null, 1.0)); - outputs.add(new GenerationSource("DPI_Logs/Metadata_Module/http/http_sample_1.json", DPI_HTTP_TOPIC, null, 1.5)); - outputs.add(new GenerationSource("DPI_Logs/Metadata_Module/http/http_sample_2.json", DPI_HTTP_TOPIC, null, 1.0)); - outputs.add(new GenerationSource("DPI_Logs/Metadata_Module/http/http_sample_3.json", DPI_HTTP_TOPIC, null, 1.0)); - outputs.add(new GenerationSource("DPI_Logs/Metadata_Module/http/http_sample_4.json", DPI_HTTP_TOPIC, null, 1.0)); + outputs.add( + new GenerationSource("DPI_Logs/Metadata_Module/http/http_sample_1.json", DPI_HTTP_TOPIC, null, 1.5)); + outputs.add( + new GenerationSource("DPI_Logs/Metadata_Module/http/http_sample_2.json", DPI_HTTP_TOPIC, null, 1.0)); + outputs.add( + new GenerationSource("DPI_Logs/Metadata_Module/http/http_sample_3.json", DPI_HTTP_TOPIC, null, 1.0)); + outputs.add( + new GenerationSource("DPI_Logs/Metadata_Module/http/http_sample_4.json", DPI_HTTP_TOPIC, null, 1.0)); outputs.add(new GenerationSource("DPI_Logs/Metadata_Module/DNS/dns_sample_1.json", DPI_DNS_TOPIC, null, 1.0)); outputs.add(new GenerationSource("DPI_Logs/Metadata_Module/DNS/dns_sample_2.json", DPI_DNS_TOPIC, null, 1.0)); outputs.add(new GenerationSource("DPI_Logs/Metadata_Module/DNS/dns_sample_3.json", DPI_DNS_TOPIC, null, 1.0)); - outputs.add(new GenerationSource("DPI_Logs/Metadata_Module/SMTP/smtp_sample_1.json", DPI_SMTP_TOPIC, null, 1.0)); + outputs.add( + new GenerationSource("DPI_Logs/Metadata_Module/SMTP/smtp_sample_1.json", DPI_SMTP_TOPIC, null, 1.0)); return outputs; } private SingleOutputStreamOperator> createSourceFromTemplateSource(ParameterTool params, - StreamExecutionEnvironment env, GeneratorConfig outputs) { + StreamExecutionEnvironment env, + GeneratorConfig outputs) { return env.addSource(new FreemarkerTemplateSource( - outputs, params.getLong(PARAMS_RECORDS_LIMIT, -1), params.getInt(PARAMS_EPS, DEFAULT_EPS))) - .name("Weighted Data Source"); + outputs, params.getLong(PARAMS_RECORDS_LIMIT, -1), params.getInt(PARAMS_EPS, DEFAULT_EPS))) + .name("Weighted Data Source"); } protected abstract void writeMetrics(ParameterTool params, - SingleOutputStreamOperator> metrics); + SingleOutputStreamOperator> metrics); protected abstract void writeResults(ParameterTool params, - SingleOutputStreamOperator> generatedInput); + SingleOutputStreamOperator> generatedInput); } diff --git a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/CaracalGeneratorFlinkJobKafka.java b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/CaracalGeneratorFlinkJobKafka.java index 68003dea..0b0aee6d 100644 --- a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/CaracalGeneratorFlinkJobKafka.java +++ b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/CaracalGeneratorFlinkJobKafka.java @@ -12,7 +12,10 @@ package com.cloudera.cyber.test.generator; +import static com.cloudera.cyber.flink.Utils.readKafkaProperties; + import com.cloudera.cyber.flink.FlinkUtils; +import java.nio.charset.StandardCharsets; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.base.DeliveryGuarantee; @@ -21,11 +24,6 @@ import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.util.Preconditions; import org.apache.kafka.clients.producer.ProducerRecord; -import static com.cloudera.cyber.flink.Utils.readKafkaProperties; - -import java.nio.charset.StandardCharsets; - -import java.nio.charset.StandardCharsets; public class CaracalGeneratorFlinkJobKafka extends CaracalGeneratorFlinkJob { @@ -35,34 +33,42 @@ public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); ParameterTool params = com.cloudera.cyber.flink.Utils.getParamToolsFromProperties(args); FlinkUtils.executeEnv(new CaracalGeneratorFlinkJobKafka() - .createPipeline(params), "Caracal Data generator", params); + .createPipeline(params), "Caracal Data generator", params); } @Override protected void writeMetrics(ParameterTool params, SingleOutputStreamOperator> metrics) { - KafkaSink> metricsSink = KafkaSink.>builder().setRecordSerializer( - (KafkaRecordSerializationSchema>) (stringIntegerTuple2, kafkaSinkContext, timestamp) -> new ProducerRecord<>( - params.get("generator.metrics", "generator.metrics"), - null, - timestamp, - stringIntegerTuple2.f0.getBytes(StandardCharsets.UTF_8), - stringIntegerTuple2.f1.toString().getBytes(StandardCharsets.UTF_8) - )).setKafkaProducerConfig( - readKafkaProperties(params, PRODUCER_ID_PREFIX.concat("generator.metrics"), false)).setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE).build(); + KafkaSink> metricsSink = + KafkaSink.>builder().setRecordSerializer( + (KafkaRecordSerializationSchema>) + (stringIntegerTuple2, kafkaSinkContext, timestamp) -> new ProducerRecord<>( + params.get("generator.metrics", "generator.metrics"), + null, + timestamp, + stringIntegerTuple2.f0.getBytes(StandardCharsets.UTF_8), + stringIntegerTuple2.f1.toString().getBytes(StandardCharsets.UTF_8) + )).setKafkaProducerConfig( + readKafkaProperties(params, PRODUCER_ID_PREFIX.concat("generator.metrics"), false)) + .setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE).build(); metrics.sinkTo(metricsSink).name("Metrics Sink"); } @Override protected void writeResults(ParameterTool params, - SingleOutputStreamOperator> generatedInput) { - KafkaSink> kafkaSink = KafkaSink.>builder().setRecordSerializer( - (KafkaRecordSerializationSchema>) (stringStringTuple2, kafkaSinkContext, aLong) -> new ProducerRecord<>( - stringStringTuple2.f0, - stringStringTuple2.f1 - )). - setKafkaProducerConfig(readKafkaProperties(params, PRODUCER_ID_PREFIX.concat("generator.output"), false)). - setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE). - build(); + SingleOutputStreamOperator> generatedInput) { + KafkaSink> kafkaSink = + KafkaSink.>builder() + .setRecordSerializer( + (KafkaRecordSerializationSchema>) + (stringStringTuple2, kafkaSinkContext, longValue) -> new ProducerRecord<>( + stringStringTuple2.f0, + stringStringTuple2.f1 + )) + .setKafkaProducerConfig(readKafkaProperties(params, + PRODUCER_ID_PREFIX.concat("generator.output"), + false)) + .setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE) + .build(); generatedInput.sinkTo(kafkaSink).name("Text Generator Sink"); } diff --git a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/GetIpMap.java b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/GetIpMap.java index 6c323240..935ef701 100644 --- a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/GetIpMap.java +++ b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/GetIpMap.java @@ -12,16 +12,16 @@ package com.cloudera.cyber.test.generator; -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.java.tuple.Tuple2; - import java.nio.charset.Charset; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.java.tuple.Tuple2; public class GetIpMap implements MapFunction, String> { - private static final Pattern ipExtractPattern = Pattern.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})"); + private static final Pattern ipExtractPattern = + Pattern.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})"); @Override public String map(Tuple2 s) { diff --git a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/RandomSampler.java b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/RandomSampler.java index 14a87a12..22916c8a 100644 --- a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/RandomSampler.java +++ b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/generator/RandomSampler.java @@ -12,12 +12,12 @@ package com.cloudera.cyber.test.generator; -import org.apache.flink.api.common.functions.FilterFunction; - import java.util.concurrent.ThreadLocalRandom; +import org.apache.flink.api.common.functions.FilterFunction; public class RandomSampler implements FilterFunction { - private double threatProbability; + private final double threatProbability; + public RandomSampler(double threatProbability) { this.threatProbability = threatProbability; } diff --git a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/resources/log4j2.properties b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/resources/log4j2.properties index 2552e629..554818f6 100644 --- a/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/resources/log4j2.properties +++ b/flink-cyber/caracal-generator/src/main/java/com/cloudera/cyber/test/resources/log4j2.properties @@ -9,11 +9,9 @@ # either express or implied. Refer to the License for the specific permissions and # limitations governing your use of the file. # - -appender.console.type = Console -appender.console.name = STDOUT -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n - -rootLogger.level = warn -rootLogger.appenderRef.stdout.ref = STDOUT \ No newline at end of file +appender.console.type=Console +appender.console.name=STDOUT +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n +rootLogger.level=warn +rootLogger.appenderRef.stdout.ref=STDOUT \ No newline at end of file diff --git a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/ParserChainMapFunction.java b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/ParserChainMapFunction.java index 805c28d5..ffeb7ee4 100644 --- a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/ParserChainMapFunction.java +++ b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/ParserChainMapFunction.java @@ -13,23 +13,26 @@ package com.cloudera.cyber.caracal; import com.cloudera.cyber.Message; -import com.cloudera.cyber.parser.ParserJob; -import com.cloudera.parserchains.core.*; +import com.cloudera.parserchains.core.ChainBuilder; +import com.cloudera.parserchains.core.ChainLink; +import com.cloudera.parserchains.core.ChainRunner; +import com.cloudera.parserchains.core.DefaultChainBuilder; +import com.cloudera.parserchains.core.DefaultChainRunner; +import com.cloudera.parserchains.core.InvalidParserException; +import com.cloudera.parserchains.core.ReflectiveParserBuilder; import com.cloudera.parserchains.core.catalog.ClassIndexParserCatalog; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.OutputTag; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; /** - * Map function to apply the TMO parser chains to Messages + * Map function to apply the TMO parser chains to Messages. */ @RequiredArgsConstructor @Slf4j @@ -45,16 +48,16 @@ public class ParserChainMapFunction extends RichMapFunction { public void open(Configuration parameters) throws Exception { super.open(parameters); ChainBuilder chainBuilder = new DefaultChainBuilder(new ReflectiveParserBuilder(), - new ClassIndexParserCatalog()); - chains = chainConfig.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, v -> - { - try { - return chainBuilder.build(v.getValue().getChainSchema()); - } catch (InvalidParserException e) { - log.error("Cannot build parser chain", e); - return null; - } - })); + new ClassIndexParserCatalog()); + chains = chainConfig.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, + v -> { + try { + return chainBuilder.build(v.getValue().getChainSchema()); + } catch (InvalidParserException e) { + log.error("Cannot build parser chain", e); + return null; + } + })); chainRunner = new DefaultChainRunner(); } @@ -71,14 +74,16 @@ public Message map(Message message) { com.cloudera.parserchains.core.Message.Builder builder = com.cloudera.parserchains.core.Message.builder(); message.getExtensions().entrySet().forEach(e -> { builder - .addField(e.getKey(), e.getValue().toString()).build(); - }); + .addField(e.getKey(), e.getValue()).build(); + }); - List out = chainRunner.run(builder.build(), chains.get(source), results); + List out = + chainRunner.run(builder.build(), chains.get(source), results); com.cloudera.parserchains.core.Message lastMessage = out.get(out.size() - 1); return message.toBuilder().extensions(lastMessage.getFields().entrySet().stream() - .collect(Collectors.toMap(k -> k.getKey().get(), v->v.getValue().get()))) - .build(); + .collect(Collectors.toMap(k -> k.getKey().get(), + v -> v.getValue().get()))) + .build(); } } diff --git a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitBroadcastProcessFunction.java b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitBroadcastProcessFunction.java index db55972c..4ac1145f 100644 --- a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitBroadcastProcessFunction.java +++ b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitBroadcastProcessFunction.java @@ -14,6 +14,11 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.parser.MessageToParse; +import java.io.Serializable; +import java.security.PrivateKey; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.flink.configuration.Configuration; @@ -21,18 +26,15 @@ import org.apache.flink.util.Collector; import org.apache.flink.util.Preconditions; -import java.io.Serializable; -import java.security.PrivateKey; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; - @Slf4j -public class SplitBroadcastProcessFunction extends KeyedBroadcastProcessFunction implements Serializable { +public class SplitBroadcastProcessFunction + extends KeyedBroadcastProcessFunction implements Serializable { - @NonNull private final Map configs; + @NonNull + private final Map configs; private Map splitters = new HashMap<>(); - @NonNull private final PrivateKey signKey; + @NonNull + private final PrivateKey signKey; public SplitBroadcastProcessFunction(Map configs, PrivateKey signKey) { super(); @@ -43,11 +45,11 @@ public SplitBroadcastProcessFunction(Map configs, PrivateKe this.configs = configs; this.signKey = signKey; - splitters = configs.entrySet().stream(). - collect(Collectors.toMap( - k -> k.getKey(), - v -> new SplittingFlatMapFunction(v.getValue(), signKey) - )); + splitters = configs.entrySet().stream() + .collect(Collectors.toMap( + k -> k.getKey(), + v -> new SplittingFlatMapFunction(v.getValue(), signKey) + )); } @Override @@ -63,7 +65,8 @@ public void open(Configuration parameters) throws Exception { } @Override - public void processElement(MessageToParse messageToParse, ReadOnlyContext readOnlyContext, Collector collector) throws Exception { + public void processElement(MessageToParse messageToParse, ReadOnlyContext readOnlyContext, + Collector collector) throws Exception { SplittingFlatMapFunction splitter = splitters.get(messageToParse.getTopic()); if (splitter == null) { throw new RuntimeException(String.format("Splitter not found for topic %s", messageToParse.getTopic())); @@ -72,7 +75,8 @@ public void processElement(MessageToParse messageToParse, ReadOnlyContext readOn } @Override - public void processBroadcastElement(SplitConfig splitConfig, Context context, Collector collector) throws Exception { + public void processBroadcastElement(SplitConfig splitConfig, Context context, Collector collector) + throws Exception { log.info(String.format("Adding splitter %s on thread %d ", splitConfig, Thread.currentThread().getId())); context.getBroadcastState(SplitJob.Descriptors.broadcastState).put(splitConfig.getTopic(), splitConfig); SplittingFlatMapFunction f = new SplittingFlatMapFunction(splitConfig, signKey); diff --git a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitConfig.java b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitConfig.java index a6821114..94fe55fd 100644 --- a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitConfig.java +++ b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitConfig.java @@ -13,9 +13,12 @@ package com.cloudera.cyber.caracal; import com.cloudera.parserchains.core.model.define.ParserChainSchema; -import lombok.*; - import java.io.Serializable; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @Builder @@ -27,10 +30,11 @@ public class SplitConfig implements Serializable { private String splitPath; private String headerPath; private String timestampField; - @Builder.Default private ParserChainSchema chainSchema = new ParserChainSchema(); + @Builder.Default + private ParserChainSchema chainSchema = new ParserChainSchema(); /** - * An optional function (javascript to apply to the timestamp) + * An optional function (javascript to apply to the timestamp). */ @Builder.Default private String timestampFunction = ""; diff --git a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java index 63afe76e..13cca1eb 100644 --- a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java +++ b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java @@ -12,10 +12,23 @@ package com.cloudera.cyber.caracal; +import static com.cloudera.cyber.parser.ParserJob.PARAM_PRIVATE_KEY; +import static com.cloudera.cyber.parser.ParserJob.PARAM_PRIVATE_KEY_FILE; + import com.cloudera.cyber.Message; import com.cloudera.cyber.parser.MessageToParse; import com.cloudera.parserchains.core.utils.JSONUtils; import com.fasterxml.jackson.core.type.TypeReference; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.state.MapStateDescriptor; @@ -28,20 +41,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.windowing.time.Time; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.KeyFactory; -import java.security.PrivateKey; -import java.security.spec.PKCS8EncodedKeySpec; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.cloudera.cyber.parser.ParserJob.PARAM_PRIVATE_KEY; -import static com.cloudera.cyber.parser.ParserJob.PARAM_PRIVATE_KEY_FILE; - @Slf4j public abstract class SplitJob { @@ -60,36 +59,34 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws DataStream source = createSource(env, params, configMap.keySet()); BroadcastStream configStream = createConfigSource(env, params) - .broadcast(Descriptors.broadcastState); + .broadcast(Descriptors.broadcastState); - byte[] privKeyBytes = params.has(PARAM_PRIVATE_KEY_FILE) ? - Files.readAllBytes(Paths.get(params.get(PARAM_PRIVATE_KEY_FILE))) : - Base64.getDecoder().decode(params.getRequired(PARAM_PRIVATE_KEY)); + byte[] privKeyBytes = params.has(PARAM_PRIVATE_KEY_FILE) + ? Files.readAllBytes(Paths.get(params.get(PARAM_PRIVATE_KEY_FILE))) + : Base64.getDecoder().decode(params.getRequired(PARAM_PRIVATE_KEY)); KeyFactory keyFactory = KeyFactory.getInstance("RSA"); PKCS8EncodedKeySpec privSpec = new PKCS8EncodedKeySpec(privKeyBytes); PrivateKey signKey = keyFactory.generatePrivate(privSpec); SingleOutputStreamOperator results = source - .keyBy(k -> k.getTopic()) - .connect(configStream) - .process(new SplitBroadcastProcessFunction(configMap, signKey)); + .keyBy(k -> k.getTopic()) + .connect(configStream) + .process(new SplitBroadcastProcessFunction(configMap, signKey)); writeOriginalsResults(params, source); SingleOutputStreamOperator parsed = results.map(new ParserChainMapFunction(configMap)); - DataStream> counts = parsed.map(new MapFunction>() { - @Override - public Tuple2 map(Message message) throws Exception { - return Tuple2.of(message.getSource(), 1L); - } - }) - .keyBy(0) - .timeWindow(Time.milliseconds(params.getLong(PARAM_COUNT_INTERVAL, DEFAULT_COUNT_INTERVAL))) - .allowedLateness(Time.milliseconds(0)) - .sum(1); + DataStream> counts = parsed + .map((MapFunction>) message -> Tuple2.of(message.getSource(), 1L)) + .keyBy(0) + .timeWindow(Time.milliseconds( + params.getLong(PARAM_COUNT_INTERVAL, + DEFAULT_COUNT_INTERVAL))) + .allowedLateness(Time.milliseconds(0)) + .sum(1); writeCounts(params, counts); @@ -99,7 +96,8 @@ public Tuple2 map(Message message) throws Exception { } protected static class Descriptors { - public static MapStateDescriptor broadcastState = new MapStateDescriptor("configs", String.class, SplitConfig.class); + public static MapStateDescriptor broadcastState = + new MapStateDescriptor("configs", String.class, SplitConfig.class); } protected List parseConfig() throws IllegalArgumentException, IOException { @@ -120,5 +118,6 @@ protected List parseConfig() throws IllegalArgumentException, IOExc protected abstract void writeCounts(ParameterTool params, DataStream> sums); - protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, Iterable topics); + protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, + Iterable topics); } diff --git a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJobKafka.java b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJobKafka.java index ee7f9255..ba1af579 100644 --- a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJobKafka.java +++ b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJobKafka.java @@ -12,10 +12,24 @@ package com.cloudera.cyber.caracal; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_PATTERN; +import static com.cloudera.cyber.flink.FlinkUtils.createRawKafkaSource; +import static com.cloudera.cyber.flink.Utils.readKafkaProperties; +import static com.cloudera.cyber.parser.ParserJobKafka.PARAMS_CONFIG_TOPIC; +import static com.cloudera.cyber.parser.ParserJobKafka.PARAMS_ORIGINAL_ENABLED; +import static com.cloudera.cyber.parser.ParserJobKafka.PARAMS_ORIGINAL_LOGS_PATH; + import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; import com.cloudera.cyber.parser.MessageToParse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Properties; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.serialization.SerializationSchema; @@ -37,17 +51,6 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.util.DigestUtils; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Collections; -import java.util.Properties; - -import static com.cloudera.cyber.flink.ConfigConstants.*; -import static com.cloudera.cyber.flink.FlinkUtils.createRawKafkaSource; -import static com.cloudera.cyber.flink.Utils.readKafkaProperties; -import static com.cloudera.cyber.parser.ParserJobKafka.*; - @Slf4j public class SplitJobKafka extends SplitJob { @@ -65,12 +68,14 @@ public static void main(String[] args) throws Exception { ParameterTool params = Utils.getParamToolsFromProperties(args); // need to load the config file locally and put in a property - String configJson = new String(Files.readAllBytes(Paths.get(params.get(PARAMS_CONFIG_FILE, DEFAULT_CONFIG_FILE))), StandardCharsets.UTF_8); + String configJson = + new String(Files.readAllBytes(Paths.get(params.get(PARAMS_CONFIG_FILE, DEFAULT_CONFIG_FILE))), + StandardCharsets.UTF_8); log.info(String.format("Splits configuration: %s", configJson)); StreamExecutionEnvironment env = new SplitJobKafka(configJson) - .createPipeline(params); + .createPipeline(params); FlinkUtils.setupEnv(env, params); env.execute("Caracal Split Parser"); @@ -78,30 +83,35 @@ public static void main(String[] args) throws Exception { @Override protected DataStream createConfigSource(StreamExecutionEnvironment env, ParameterTool params) { - String groupId = createGroupId(params.get(PARAMS_TOPIC_INPUT, "") + params.get(PARAMS_TOPIC_PATTERN, ""), "cyber-split-parser-config-"); + String groupId = createGroupId(params.get(PARAMS_TOPIC_INPUT, "") + params.get(PARAMS_TOPIC_PATTERN, ""), + "cyber-split-parser-config-"); Properties kafkaProperties = readKafkaProperties(params, groupId, true); KafkaSource source = - KafkaSource.builder().setTopics(params.getRequired(PARAMS_CONFIG_TOPIC)).setValueOnlyDeserializer(new SimpleStringSchema()).setProperties(kafkaProperties).build(); + KafkaSource.builder().setTopics(params.getRequired(PARAMS_CONFIG_TOPIC)) + .setValueOnlyDeserializer(new SimpleStringSchema()).setProperties(kafkaProperties).build(); - return env.fromSource(source, WatermarkStrategy.noWatermarks(), "Config Kafka Feed").uid("config.source.kafka").setParallelism(1).setMaxParallelism(1) - .map(new SplitConfigJsonParserMap()) - .name("Config Source").uid("config.source").setMaxParallelism(1).setParallelism(1); + return env.fromSource(source, WatermarkStrategy.noWatermarks(), "Config Kafka Feed").uid("config.source.kafka") + .setParallelism(1).setMaxParallelism(1) + .map(new SplitConfigJsonParserMap()) + .name("Config Source").uid("config.source").setMaxParallelism(1).setParallelism(1); } @Override protected void writeResults(ParameterTool params, DataStream results) { KafkaSink sink = new FlinkUtils<>(Message.class).createKafkaSink( - params.getRequired(PARAMS_TOPIC_OUTPUT), - "splits-parser", - params); + params.getRequired(PARAMS_TOPIC_OUTPUT), + "splits-parser", + params); results.sinkTo(sink).name("Kafka Results").uid("kafka.results"); } @Override protected void writeOriginalsResults(ParameterTool params, DataStream results) { - if (!params.getBoolean(PARAMS_ORIGINAL_ENABLED, true)) return; + if (!params.getBoolean(PARAMS_ORIGINAL_ENABLED, true)) { + return; + } // write the original sources to HDFS files Path path = new Path(params.getRequired(PARAMS_ORIGINAL_LOGS_PATH)); @@ -110,14 +120,14 @@ protected void writeOriginalsResults(ParameterTool params, DataStream sink = StreamingFileSink - .forBulkFormat(path, ParquetAvroWriters.forReflectRecord(MessageToParse.class)) - .withRollingPolicy(OnCheckpointRollingPolicy.build()) - .withOutputFileConfig(OutputFileConfig - .builder() - .withPartPrefix("logs") - .withPartSuffix(".parquet") - .build()) - .build(); + .forBulkFormat(path, ParquetAvroWriters.forReflectRecord(MessageToParse.class)) + .withRollingPolicy(OnCheckpointRollingPolicy.build()) + .withOutputFileConfig(OutputFileConfig + .builder() + .withPartPrefix("logs") + .withPartSuffix(".parquet") + .build()) + .build(); results.addSink(sink).name("Original Archiver").uid("original.archiver"); } @@ -127,18 +137,19 @@ protected void writeCounts(ParameterTool params, DataStream Properties kafkaProperties = readKafkaProperties(params, "splits-parser", false); String topic = params.get(PARAM_COUNT_TOPIC, DEFAULT_COUNT_TOPIC); - SerializationSchema> keySerializationSchema = new SerializationSchema>() { + SerializationSchema> keySerializationSchema = + new SerializationSchema>() { - @Override - public void open(InitializationContext context) { + @Override + public void open(InitializationContext context) { - } + } - @Override - public byte[] serialize(Tuple2 metric) { - return metric.f0.getBytes(); - } - }; + @Override + public byte[] serialize(Tuple2 metric) { + return metric.f0.getBytes(); + } + }; SerializationSchema> valueSerial = new SerializationSchema>() { @Override @@ -152,25 +163,33 @@ public byte[] serialize(Tuple2 metric) { } }; KafkaSink> sink = KafkaSink.>builder() - .setBootstrapServers(kafkaProperties.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) - .setRecordSerializer(KafkaRecordSerializationSchema.builder() - .setTopic(topic) - .setKeySerializationSchema(keySerializationSchema) - .setValueSerializationSchema(valueSerial) - .build()) - .setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE) - .setKafkaProducerConfig(kafkaProperties) - .build(); + .setBootstrapServers(kafkaProperties.getProperty( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) + .setRecordSerializer(KafkaRecordSerializationSchema.builder() + .setTopic( + topic) + .setKeySerializationSchema( + keySerializationSchema) + .setValueSerializationSchema( + valueSerial) + .build()) + .setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE) + .setKafkaProducerConfig(kafkaProperties) + .build(); sums.sinkTo(sink).name("Count Results").uid("count.results"); } @Override - protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, Iterable topics) { - log.info(params.mergeWith(ParameterTool.fromMap(Collections.singletonMap(PARAMS_TOPIC_PATTERN, String.join("|", topics)))).toMap().toString()); + protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, + Iterable topics) { + log.info(params.mergeWith( + ParameterTool.fromMap(Collections.singletonMap(PARAMS_TOPIC_PATTERN, String.join("|", topics)))).toMap() + .toString()); return createRawKafkaSource(env, - params.mergeWith(ParameterTool.fromMap(Collections.singletonMap(PARAMS_TOPIC_PATTERN, String.join("|", topics)))), - createGroupId(params.get("topic.input", "") + params.get("topic.pattern", ""), "cyber-split-parser-")); + params.mergeWith( + ParameterTool.fromMap(Collections.singletonMap(PARAMS_TOPIC_PATTERN, String.join("|", topics)))), + createGroupId(params.get("topic.input", "") + params.get("topic.pattern", ""), "cyber-split-parser-")); } private String createGroupId(String inputTopic, String prefix) { diff --git a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplittingFlatMapFunction.java b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplittingFlatMapFunction.java index c985538f..aafb5d2a 100644 --- a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplittingFlatMapFunction.java +++ b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplittingFlatMapFunction.java @@ -12,6 +12,9 @@ package com.cloudera.cyber.caracal; +import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.toMap; + import com.cloudera.cyber.Message; import com.cloudera.cyber.SignedSourceKey; import com.cloudera.cyber.parser.MessageToParse; @@ -21,15 +24,6 @@ import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.spi.json.JacksonJsonProvider; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.common.functions.RichFlatMapFunction; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.Collector; -import org.apache.flink.util.Preconditions; - -import javax.script.ScriptException; import java.nio.charset.StandardCharsets; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -39,12 +33,17 @@ import java.util.List; import java.util.Map; import java.util.UUID; - -import static java.util.Collections.emptyMap; -import static java.util.stream.Collectors.toMap; +import javax.script.ScriptException; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.functions.RichFlatMapFunction; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.Collector; +import org.apache.flink.util.Preconditions; /** - * Splits up JSON array based on a configured path and dupes 'header' in to multiple messages + * Splits up JSON array based on a configured path and dupes 'header' in to multiple messages. */ @Slf4j public class SplittingFlatMapFunction extends RichFlatMapFunction { @@ -66,10 +65,11 @@ public class SplittingFlatMapFunction extends RichFlatMapFunction collector) throws E signature.update(input.getOriginalBytes()); final byte[] sig = signature.sign(); - DocumentContext documentContext = JsonPath.using(STRICT_PROVIDER_CONFIGURATION).parse(new String(input.getOriginalBytes(), StandardCharsets.UTF_8)); + DocumentContext documentContext = JsonPath.using(STRICT_PROVIDER_CONFIGURATION) + .parse(new String(input.getOriginalBytes(), StandardCharsets.UTF_8)); // header is all top level simple type fields. Object header = documentContext.read(headerPath); new HashMap<>(); - Map headerFields = header instanceof Map ? - ((Map) header).entrySet().stream() - .filter(e -> isScalar(e.getValue())) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)) : - emptyMap(); + Map headerFields = header instanceof Map + ? ((Map) header).entrySet().stream() + .filter(e -> isScalar(e.getValue())) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)) + : emptyMap(); Object jsonPathResult = documentContext.read(jsonPath); ((List) jsonPathResult).forEach(part -> { Map fields = (Map) part; - String tsString = (String) (timestampSource == TimestampSource.SPLIT ? - fields.get(tsField) : - headerFields.get(tsField)); + String tsString = (String) (timestampSource == TimestampSource.SPLIT + ? fields.get(tsField) + : headerFields.get(tsField)); long ts; try { ts = Long.valueOf(tsString); @@ -162,19 +163,19 @@ public void flatMap(MessageToParse input, Collector collector) throws E metricGroup.addGroup("parser." + input.getTopic()).counter(1);*/ collector.collect(Message.builder() - .originalSource( - SignedSourceKey.builder() - .topic(input.getTopic()) - .partition(input.getPartition()) - .offset(input.getOffset()) - .signature(sig) - .build() - ) - .source(input.getTopic()) - .ts(ts) - .extensions(childPlusHeader(fields, headerFields)).build()); + .originalSource( + SignedSourceKey.builder() + .topic(input.getTopic()) + .partition(input.getPartition()) + .offset(input.getOffset()) + .signature(sig) + .build() + ) + .source(input.getTopic()) + .ts(ts) + .extensions(childPlusHeader(fields, headerFields)).build()); }); - + } private boolean isScalar(Object value) { @@ -183,7 +184,8 @@ private boolean isScalar(Object value) { private Map childPlusHeader(Map part, Map header) { Map result = Streams.concat(header.entrySet().stream(), part.entrySet().stream()) - .collect(toMap(Map.Entry::getKey, v -> v.getValue().toString(), (a, b) -> b)); + .collect( + toMap(Map.Entry::getKey, v -> v.getValue().toString(), (a, b) -> b)); return result; } diff --git a/flink-cyber/caracal-parser/src/test/java/com/cloudera/cyber/caracal/SplittingFlatMapFunctionTest.java b/flink-cyber/caracal-parser/src/test/java/com/cloudera/cyber/caracal/SplittingFlatMapFunctionTest.java index a609bafb..46898841 100644 --- a/flink-cyber/caracal-parser/src/test/java/com/cloudera/cyber/caracal/SplittingFlatMapFunctionTest.java +++ b/flink-cyber/caracal-parser/src/test/java/com/cloudera/cyber/caracal/SplittingFlatMapFunctionTest.java @@ -12,25 +12,24 @@ package com.cloudera.cyber.caracal; +import static com.cloudera.cyber.flink.Utils.getResourceAsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.collection.IsMapContaining.hasKey; + import com.cloudera.cyber.Message; import com.cloudera.cyber.TestUtils; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.Collector; -import org.junit.Test; - import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; - -import static com.cloudera.cyber.flink.Utils.getResourceAsString; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsMapContaining.hasKey; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.Collector; +import org.junit.Test; public class SplittingFlatMapFunctionTest { - private final static String testInput = getResourceAsString("dpi.json"); + private static final String testInput = getResourceAsString("dpi.json"); @Test public void testSplittingWithHeader() throws Exception { diff --git a/flink-cyber/google_checks.xml b/flink-cyber/checkstyle.xml similarity index 78% rename from flink-cyber/google_checks.xml rename to flink-cyber/checkstyle.xml index 609f61fa..2b73f445 100644 --- a/flink-cyber/google_checks.xml +++ b/flink-cyber/checkstyle.xml @@ -38,14 +38,6 @@ - - - - - - - @@ -54,7 +46,7 @@ - + @@ -83,26 +75,30 @@ value="LITERAL_DO, LITERAL_ELSE, LITERAL_FOR, LITERAL_IF, LITERAL_WHILE"/> + + LITERAL_DO, LITERAL_ELSE, LITERAL_FOR, LITERAL_IF, + LITERAL_WHILE, METHOD_DEF, + OBJBLOCK, STATIC_INIT"/> + + + + + + value="LITERAL_IF, LITERAL_ELSE, LITERAL_DO"/> + INSTANCE_INIT, ANNOTATION_DEF, ENUM_DEF"/> @@ -112,10 +108,8 @@ + value="COMMA, SEMI, TYPECAST, LITERAL_IF, LITERAL_ELSE, + LITERAL_WHILE, LITERAL_DO, LITERAL_FOR, DO_WHILE"/> @@ -126,13 +120,13 @@ + TYPE_EXTENSION_AND"/> @@ -141,9 +135,9 @@ - @@ -161,9 +155,8 @@ + value="PACKAGE_DEF, IMPORT, STATIC_IMPORT, CLASS_DEF, ENUM_DEF, + STATIC_INIT, INSTANCE_INIT, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/> @@ -199,8 +192,7 @@ value="Package name ''{0}'' must match pattern ''{1}''."/> - + @@ -210,7 +202,7 @@ value="Member name ''{0}'' must match pattern ''{1}''."/> - + @@ -235,7 +227,7 @@ value="Pattern variable name ''{0}'' must match pattern ''{1}''."/> - + @@ -271,24 +263,22 @@ value="GenericWhitespace ''{0}'' is not preceded with whitespace."/> - - - - - - - - - - - + + + + + + + + + + + - @@ -298,8 +288,8 @@ + value="CTOR_DEF, LITERAL_NEW, METHOD_CALL, METHOD_DEF, + ENUM_CONSTANT_DEF"/> + value="ANNOTATION, ANNOTATION_FIELD_DEF, CTOR_DEF, DOT, ENUM_CONSTANT_DEF, + EXPR, LITERAL_DO, LITERAL_FOR, LITERAL_IF, LITERAL_NEW, + LITERAL_WHILE, METHOD_CALL, + METHOD_DEF, QUESTION, RESOURCE_SPECIFICATION"/> @@ -325,8 +314,7 @@ + value="CLASS_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/> @@ -347,39 +335,37 @@ + value="CLASS_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/> - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - + diff --git a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/AbstractBooleanScalarFunction.java b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/AbstractBooleanScalarFunction.java index f4b2f60a..038f9221 100644 --- a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/AbstractBooleanScalarFunction.java +++ b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/AbstractBooleanScalarFunction.java @@ -16,7 +16,7 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.table.functions.ScalarFunction; -public class AbstractBooleanScalarFunction extends ScalarFunction { +public class AbstractBooleanScalarFunction extends ScalarFunction { public TypeInformation getResultType(Class[] signature) { return Types.BOOLEAN; diff --git a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/ExtractHostname.java b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/ExtractHostname.java index d8e537be..9363b94c 100644 --- a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/ExtractHostname.java +++ b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/ExtractHostname.java @@ -15,13 +15,13 @@ import com.cloudera.cyber.CyberFunction; import com.cloudera.cyber.libs.AbstractStringScalarFunction; import com.google.common.net.InternetDomainName; -import org.apache.commons.lang3.StringUtils; - import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; /** * Extracts parts of domain names * + *

* Uses https://guava.dev/releases/24.0-jre/api/docs/com/google/common/net/InternetDomainName.html * which relies on https://publicsuffix.org/list/ for TLDs */ @@ -30,33 +30,33 @@ public class ExtractHostname extends AbstractStringScalarFunction { public static final String REVERSE_IP_SUFFIX = "in-addr.arpa"; + public static String reverseResult(String hostname, HostnameFeature feature) { + return feature.equals(HostnameFeature.TLD) ? REVERSE_IP_SUFFIX : + StringUtils.reverseDelimited(StringUtils.removeEnd(hostname, REVERSE_IP_SUFFIX), '.'); + } + + public String eval(String hostname, HostnameFeature feature) { + if (hostname.endsWith(REVERSE_IP_SUFFIX) || hostname.endsWith(REVERSE_IP_SUFFIX + ".")) { + return reverseResult(hostname, feature); + } + return feature.process.apply(InternetDomainName.from(hostname)).toString(); + } + public enum HostnameFeature { TLD(dns -> dns.publicSuffix()), NO_TLD(dns -> removeTld(dns)), - NO_SUBS(dns ->dns.topPrivateDomain()), + NO_SUBS(dns -> dns.topPrivateDomain()), NO_SUBS_NO_TLD(dns -> removeTld(dns.topPrivateDomain())); protected Function process; - private static InternetDomainName removeTld(InternetDomainName dns) { - return InternetDomainName.from(StringUtils.removeEnd(dns.toString(), dns.publicSuffix().toString())); - } - private HostnameFeature(Function process) { this.process = process; } - } - public String eval(String hostname, HostnameFeature feature) { - if (hostname.endsWith(REVERSE_IP_SUFFIX) || hostname.endsWith(REVERSE_IP_SUFFIX + ".")) { - return reverseResult(hostname, feature); + private static InternetDomainName removeTld(InternetDomainName dns) { + return InternetDomainName.from(StringUtils.removeEnd(dns.toString(), dns.publicSuffix().toString())); } - return feature.process.apply(InternetDomainName.from(hostname)).toString(); - } - - public static String reverseResult(String hostname, HostnameFeature feature) { - return feature.equals(HostnameFeature.TLD) ? REVERSE_IP_SUFFIX : - StringUtils.reverseDelimited(StringUtils.removeEnd(hostname, REVERSE_IP_SUFFIX),'.'); } } diff --git a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/ExtractHostnameFeatures.java b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/ExtractHostnameFeatures.java index b266493e..40fe173f 100644 --- a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/ExtractHostnameFeatures.java +++ b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/ExtractHostnameFeatures.java @@ -12,18 +12,17 @@ package com.cloudera.cyber.libs.hostnames; +import static com.cloudera.cyber.libs.hostnames.ExtractHostname.REVERSE_IP_SUFFIX; +import static com.cloudera.cyber.libs.hostnames.ExtractHostname.reverseResult; + import com.cloudera.cyber.libs.AbstractMapScalarFunction; import com.google.common.net.InternetDomainName; - import java.util.HashMap; import java.util.Map; -import static com.cloudera.cyber.libs.hostnames.ExtractHostname.REVERSE_IP_SUFFIX; -import static com.cloudera.cyber.libs.hostnames.ExtractHostname.reverseResult; - public class ExtractHostnameFeatures extends AbstractMapScalarFunction { - public Map eval(String hostname) { + public Map eval(String hostname) { HashMap results = new HashMap(ExtractHostname.HostnameFeature.values().length); if (hostname.endsWith(REVERSE_IP_SUFFIX) || hostname.endsWith(REVERSE_IP_SUFFIX + ".")) { // optimisation, all the features except TLD are the same, the duplication is for consistency for users diff --git a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/NsLookup.java b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/NsLookup.java index 638adaa2..a76f0ee6 100644 --- a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/NsLookup.java +++ b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/hostnames/NsLookup.java @@ -17,6 +17,12 @@ import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeinfo.Types; @@ -25,51 +31,48 @@ import org.xbill.DNS.TextParseException; import org.xbill.DNS.Type; -import javax.annotation.CheckForNull; -import javax.annotation.Nonnull; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @CyberFunction("nslookup") @Slf4j public class NsLookup extends AbstractStringScalarFunction { + @SuppressWarnings("checkstyle:Indentation") private LoadingCache> cache = Caffeine.newBuilder() - .expireAfterAccess(60, TimeUnit.SECONDS) - .maximumSize(1000) - .build(new CacheLoader>() { - @CheckForNull - @Override - public List load(@Nonnull NsLookupQuery key) throws Exception { - try { - Lookup lookup = new Lookup(key.getQuery(), key.getType()); - Record[] records = lookup.run(); - if (lookup.getResult() == Lookup.SUCCESSFUL) { - if (records != null) { - return Stream.of(records).map(r -> NsLookupRecord.builder() - .type(Type.string(r.getType())) - .name(r.getName().toString(true)) - .result(r.rdataToString() - ) - .build() - ).collect(Collectors.toList()); - } else { - log.warn("Dns Lookup query='{}' type='{}' returned null records", key.getQuery(), key.getType()); - return null; - } - } else { - log.error("Dns Lookup query='{}' type='{}' failed with error {}", key.getQuery(), key.getType(), lookup.getErrorString()); - return null; - } - } catch (TextParseException e) { - log.error(String.format("Dns Lookup query='%s' type='%s' failed.", key.getQuery(), key.getType()), e); - return null; - } - } - } - ); + .expireAfterAccess(60, TimeUnit.SECONDS) + .maximumSize(1000) + .build(new CacheLoader>() { + @CheckForNull + @Override + public List load(@Nonnull NsLookupQuery key) throws Exception { + try { + Lookup lookup = new Lookup(key.getQuery(), key.getType()); + Record[] records = lookup.run(); + if (lookup.getResult() == Lookup.SUCCESSFUL) { + if (records != null) { + return Stream.of(records).map(r -> NsLookupRecord.builder() + .type(Type.string(r.getType())) + .name(r.getName().toString(true)) + .result(r.rdataToString() + ) + .build() + ).collect(Collectors.toList()); + } else { + log.warn("Dns Lookup query='{}' type='{}' returned null records", key.getQuery(), + key.getType()); + return null; + } + } else { + log.error("Dns Lookup query='{}' type='{}' failed with error {}", key.getQuery(), + key.getType(), lookup.getErrorString()); + return null; + } + } catch (TextParseException e) { + log.error(String.format("Dns Lookup query='%s' type='%s' failed.", key.getQuery(), key.getType()), + e); + return null; + } + } + } + ); public List eval(String q, String type) { return cache.get(NsLookupQuery.builder().query(q).type(Type.value(type)).build()); diff --git a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/networking/IPLocal.java b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/networking/IPLocal.java index eae6c3a1..6914e594 100644 --- a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/networking/IPLocal.java +++ b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/networking/IPLocal.java @@ -14,7 +14,6 @@ import com.cloudera.cyber.CyberFunction; import com.cloudera.cyber.libs.AbstractBooleanScalarFunction; - import java.net.InetAddress; import java.net.UnknownHostException; diff --git a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/networking/InSubnet.java b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/networking/InSubnet.java index 313f5ab5..faafd4e7 100644 --- a/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/networking/InSubnet.java +++ b/flink-cyber/cyber-functions/src/main/java/com/cloudera/cyber/libs/networking/InSubnet.java @@ -17,25 +17,24 @@ import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; -import org.apache.commons.net.util.SubnetUtils; - +import java.util.concurrent.TimeUnit; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; -import java.util.concurrent.TimeUnit; +import org.apache.commons.net.util.SubnetUtils; @CyberFunction("in_subnet") public class InSubnet extends AbstractBooleanScalarFunction { private LoadingCache cache = Caffeine.newBuilder() - .expireAfterAccess(60, TimeUnit.MINUTES) - .maximumSize(1000) - .build(new CacheLoader() { - @CheckForNull - @Override - public SubnetUtils load(@Nonnull String key) throws Exception { - return new SubnetUtils(key); - } - }); + .expireAfterAccess(60, TimeUnit.MINUTES) + .maximumSize(1000) + .build(new CacheLoader() { + @CheckForNull + @Override + public SubnetUtils load(@Nonnull String key) throws Exception { + return new SubnetUtils(key); + } + }); public Boolean eval(String ip, String subnet) { return cache.get(subnet).getInfo().isInRange(ip); diff --git a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestExtractHostname.java b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestExtractHostname.java index bf9a6210..e1a9221d 100644 --- a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestExtractHostname.java +++ b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestExtractHostname.java @@ -12,35 +12,43 @@ package com.cloudera.cyber.libs.hostnames; -import org.junit.Test; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import org.junit.Test; + public class TestExtractHostname { @Test public void testExtractHostnameTLD() { - assertThat(new ExtractHostname().eval("www.google.co.uk", ExtractHostname.HostnameFeature.TLD), equalTo("co.uk")); - assertThat(new ExtractHostname().eval("1.0.0.10.in-addr.arpa", ExtractHostname.HostnameFeature.TLD), equalTo("in-addr.arpa")); + assertThat(new ExtractHostname().eval("www.google.co.uk", ExtractHostname.HostnameFeature.TLD), + equalTo("co.uk")); + assertThat(new ExtractHostname().eval("1.0.0.10.in-addr.arpa", ExtractHostname.HostnameFeature.TLD), + equalTo("in-addr.arpa")); } @Test public void testExtractHostnameNO_TLD() { - assertThat(new ExtractHostname().eval("www.google.co.uk", ExtractHostname.HostnameFeature.NO_TLD), equalTo("www.google")); - assertThat(new ExtractHostname().eval("1.0.0.10.in-addr.arpa", ExtractHostname.HostnameFeature.NO_TLD), equalTo("10.0.0.1")); + assertThat(new ExtractHostname().eval("www.google.co.uk", ExtractHostname.HostnameFeature.NO_TLD), + equalTo("www.google")); + assertThat(new ExtractHostname().eval("1.0.0.10.in-addr.arpa", ExtractHostname.HostnameFeature.NO_TLD), + equalTo("10.0.0.1")); } @Test public void testExtractHostnameNO_SUBS_NO_TLD() { - assertThat(new ExtractHostname().eval("www.google.co.uk", ExtractHostname.HostnameFeature.NO_SUBS_NO_TLD), equalTo("google")); - assertThat(new ExtractHostname().eval("1.0.0.10.in-addr.arpa", ExtractHostname.HostnameFeature.NO_SUBS_NO_TLD), equalTo("10.0.0.1")); + assertThat(new ExtractHostname().eval("www.google.co.uk", ExtractHostname.HostnameFeature.NO_SUBS_NO_TLD), + equalTo("google")); + assertThat(new ExtractHostname().eval("1.0.0.10.in-addr.arpa", ExtractHostname.HostnameFeature.NO_SUBS_NO_TLD), + equalTo("10.0.0.1")); } @Test public void testExtractHostnameNO_SUBS() { - assertThat(new ExtractHostname().eval("www.google.co.uk", ExtractHostname.HostnameFeature.NO_SUBS), equalTo("google.co.uk")); - assertThat(new ExtractHostname().eval("1.0.0.10.in-addr.arpa", ExtractHostname.HostnameFeature.NO_SUBS), equalTo("10.0.0.1")); + assertThat(new ExtractHostname().eval("www.google.co.uk", ExtractHostname.HostnameFeature.NO_SUBS), + equalTo("google.co.uk")); + assertThat(new ExtractHostname().eval("1.0.0.10.in-addr.arpa", ExtractHostname.HostnameFeature.NO_SUBS), + equalTo("10.0.0.1")); } } diff --git a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestExtractHostnameFeatures.java b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestExtractHostnameFeatures.java index 26c95754..6b03b8ec 100644 --- a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestExtractHostnameFeatures.java +++ b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestExtractHostnameFeatures.java @@ -12,46 +12,46 @@ package com.cloudera.cyber.libs.hostnames; -import org.junit.Test; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.hasEntry; +import org.junit.Test; + public class TestExtractHostnameFeatures { @Test public void testExtractHostnameFeatures() { assertThat("Extracts from www.google.co.uk", new ExtractHostnameFeatures().eval("www.google.co.uk"), - allOf( - hasEntry("TLD", "co.uk"), - hasEntry("NO_TLD", "www.google"), - hasEntry("NO_SUBS_NO_TLD", "google"), - hasEntry("NO_SUBS", "google.co.uk") - )); + allOf( + hasEntry("TLD", "co.uk"), + hasEntry("NO_TLD", "www.google"), + hasEntry("NO_SUBS_NO_TLD", "google"), + hasEntry("NO_SUBS", "google.co.uk") + )); assertThat("Extracts from test.aero", new ExtractHostnameFeatures().eval("test.aero"), - allOf( - hasEntry("TLD", "aero"), - hasEntry("NO_TLD", "test"), - hasEntry("NO_SUBS_NO_TLD", "test"), - hasEntry("NO_SUBS", "test.aero") - )); + allOf( + hasEntry("TLD", "aero"), + hasEntry("NO_TLD", "test"), + hasEntry("NO_SUBS_NO_TLD", "test"), + hasEntry("NO_SUBS", "test.aero") + )); assertThat("Extracts from test.aero", new ExtractHostnameFeatures().eval("sub.test.aero"), - allOf( - hasEntry("TLD", "aero"), - hasEntry("NO_TLD", "sub.test"), - hasEntry("NO_SUBS_NO_TLD", "test"), - hasEntry("NO_SUBS", "test.aero") - )); - - - assertThat("Works with reverse IP", new ExtractHostnameFeatures().eval("1.0.0.10.in-addr.arpa"), - allOf( - hasEntry("TLD", "in-addr.arpa"), - hasEntry("NO_TLD", "10.0.0.1"), - hasEntry("NO_SUBS_NO_TLD", "10.0.0.1"), - hasEntry("NO_SUBS", "10.0.0.1") - )); + allOf( + hasEntry("TLD", "aero"), + hasEntry("NO_TLD", "sub.test"), + hasEntry("NO_SUBS_NO_TLD", "test"), + hasEntry("NO_SUBS", "test.aero") + )); + + + assertThat("Works with reverse IP", new ExtractHostnameFeatures().eval("1.0.0.10.in-addr.arpa"), + allOf( + hasEntry("TLD", "in-addr.arpa"), + hasEntry("NO_TLD", "10.0.0.1"), + hasEntry("NO_SUBS_NO_TLD", "10.0.0.1"), + hasEntry("NO_SUBS", "10.0.0.1") + )); } } diff --git a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestNsLookup.java b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestNsLookup.java index b7f69db2..cb4c2c96 100644 --- a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestNsLookup.java +++ b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/hostnames/TestNsLookup.java @@ -12,14 +12,13 @@ package com.cloudera.cyber.libs.hostnames; -import org.junit.Test; - -import java.util.List; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; +import java.util.List; +import org.junit.Test; + public class TestNsLookup { @Test diff --git a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/networking/TestIPLocal.java b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/networking/TestIPLocal.java index 2d237a93..8e2c50f0 100644 --- a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/networking/TestIPLocal.java +++ b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/networking/TestIPLocal.java @@ -12,11 +12,11 @@ package com.cloudera.cyber.libs.networking; -import org.junit.Test; - import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import org.junit.Test; + public class TestIPLocal { @Test diff --git a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/networking/TestInSubnet.java b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/networking/TestInSubnet.java index 08c2bb0a..c939d18f 100644 --- a/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/networking/TestInSubnet.java +++ b/flink-cyber/cyber-functions/src/test/java/com/cloudera/cyber/libs/networking/TestInSubnet.java @@ -12,11 +12,11 @@ package com.cloudera.cyber.libs.networking; -import org.junit.Test; - import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import org.junit.Test; + public class TestInSubnet { @Test diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/Utils.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/Utils.java index be7aa8cf..9bd44a64 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/Utils.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/Utils.java @@ -1,10 +1,9 @@ package com.cloudera.service.common; -import lombok.experimental.UtilityClass; -import org.apache.commons.lang3.StringUtils; - import java.util.Arrays; import java.util.function.Function; +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.StringUtils; @UtilityClass public class Utils { @@ -13,24 +12,26 @@ public class Utils { * Retrieves an enum constant of the specified enum type based on a case-insensitive search of a string representation, * using a custom mapping function to extract the string representation from the enum constant. * - * @param the enum type - * @param name the string representation to search for - * @param enumClass the Class object representing the enum type - * @param nameFunction a function that extracts the string representation from an enum constant - * @return the enum constant matching the provided string representation, or null if not found + * @param the enum type + * @param name the string representation to search for + * @param enumClass the Class object representing the enum type + * @param nameFunction a function that extracts the string representation from an enum constant + * @return the enum constant matching the provided string representation, or null if not found * @throws NullPointerException if {@code name}, {@code enumClass}, or {@code nameFunction} is null */ - public static > T getEnumFromString(String name, Class enumClass, Function nameFunction) { + public static > T getEnumFromString(String name, Class enumClass, + Function nameFunction) { return Arrays.stream(enumClass.getEnumConstants()) - .filter(type -> StringUtils.equalsIgnoreCase(name, nameFunction.apply(type))) - .findFirst() - .orElse(null); + .filter(type -> StringUtils.equalsIgnoreCase(name, nameFunction.apply(type))) + .findFirst() + .orElse(null); } - public static > T getEnumFromStringContains(String name, Class enumClass, Function nameFunction) { + public static > T getEnumFromStringContains(String name, Class enumClass, + Function nameFunction) { return Arrays.stream(enumClass.getEnumConstants()) - .filter(type -> StringUtils.containsIgnoreCase(name, nameFunction.apply(type))) - .findFirst() - .orElse(null); + .filter(type -> StringUtils.containsIgnoreCase(name, nameFunction.apply(type))) + .findFirst() + .orElse(null); } } diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaKafkaProperties.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaKafkaProperties.java index ed89a9ab..0b722aee 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaKafkaProperties.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaKafkaProperties.java @@ -3,12 +3,10 @@ import lombok.Getter; import lombok.Setter; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; @Getter @Setter public class ClouderaKafkaProperties extends KafkaProperties { - private String replyTopic; - private String requestTopic; + private String replyTopic; + private String requestTopic; } diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaManagementKafkaProperties.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaManagementKafkaProperties.java index 9fe98901..578dd177 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaManagementKafkaProperties.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/config/kafka/ClouderaManagementKafkaProperties.java @@ -1,4 +1,4 @@ package com.cloudera.service.common.config.kafka; -public class ClouderaManagementKafkaProperties extends ClouderaKafkaProperties{ +public class ClouderaManagementKafkaProperties extends ClouderaKafkaProperties { } diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java index f6a247be..fdb28262 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java @@ -1,5 +1,6 @@ package com.cloudera.service.common.request; public enum RequestType { - GET_ALL_CLUSTERS_SERVICE_REQUEST, GET_CLUSTER_SERVICE_REQUEST, START_JOB_REQUEST, RESTART_JOB_REQUEST, STOP_JOB_REQUEST, GET_JOB_CONFIG_REQUEST, UPDATE_JOB_CONFIG_REQUEST + GET_ALL_CLUSTERS_SERVICE_REQUEST, GET_CLUSTER_SERVICE_REQUEST, START_JOB_REQUEST, RESTART_JOB_REQUEST, + STOP_JOB_REQUEST, GET_JOB_CONFIG_REQUEST, UPDATE_JOB_CONFIG_REQUEST } diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/Job.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/Job.java index 6b0d5bff..750c7a63 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/Job.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/Job.java @@ -49,12 +49,14 @@ public String[] getScript(Job job) { case GENERATOR: case PROFILE: case PARSER: - return new String[]{scriptName, job.getJobBranch(), job.getJobPipeline(), job.getJobName()}; + return new String[] {scriptName, job.getJobBranch(), job.getJobPipeline(), job.getJobName()}; case INDEX: case TRIAGE: - return new String[]{scriptName, job.getJobBranch(), job.getJobPipeline()}; + return new String[] {scriptName, job.getJobBranch(), job.getJobPipeline()}; + default: + break; } - return new String[]{}; + return new String[] {}; } } diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseBody.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseBody.java index 777464d2..2b20eac8 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseBody.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseBody.java @@ -1,13 +1,12 @@ package com.cloudera.service.common.response; +import java.util.List; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; -import java.util.Map; - @Data @Builder @NoArgsConstructor diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java index f67d81e9..f05858d9 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java @@ -1,5 +1,6 @@ package com.cloudera.service.common.response; public enum ResponseType { - GET_ALL_CLUSTERS_SERVICE_RESPONSE, GET_CLUSTER_SERVICE_RESPONSE, START_JOB_RESPONSE, RESTART_JOB_RESPONSE, STOP_JOB_RESPONSE, GET_JOB_CONFIG_RESPONSE, UPDATE_JOB_CONFIG_RESPONSE, ERROR_RESPONSE + GET_ALL_CLUSTERS_SERVICE_RESPONSE, GET_CLUSTER_SERVICE_RESPONSE, START_JOB_RESPONSE, RESTART_JOB_RESPONSE, + STOP_JOB_RESPONSE, GET_JOB_CONFIG_RESPONSE, UPDATE_JOB_CONFIG_RESPONSE, ERROR_RESPONSE } diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/ArchiveUtil.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/ArchiveUtil.java index 0bffca6b..35fd9af3 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/ArchiveUtil.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/ArchiveUtil.java @@ -1,15 +1,5 @@ package com.cloudera.service.common.utils; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; -import org.apache.flink.core.fs.FileStatus; - import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -24,6 +14,15 @@ import java.nio.file.StandardCopyOption; import java.util.Base64; import java.util.List; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.flink.core.fs.FileStatus; @Slf4j @UtilityClass @@ -37,7 +36,7 @@ public static void compressToTarGzFile(String inputPath, String outputPath) thro public static byte[] compressToTarGzInMemory(String inputPath, boolean base64) throws IOException { final byte[] bytes = compressToTarGzInMemory(inputPath); - if (base64){ + if (base64) { return Base64.getEncoder().encode(bytes); } else { return bytes; @@ -58,8 +57,8 @@ private static void compressToTarGz(String inputPath, OutputStream outputStream) } try (BufferedOutputStream buffOut = new BufferedOutputStream(outputStream); - GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(buffOut); - TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut)) { + GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(buffOut); + TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut)) { try { for (FileStatus file : fileList) { @@ -71,7 +70,7 @@ private static void compressToTarGz(String inputPath, OutputStream outputStream) } } - private static void addFileToTar(TarArchiveOutputStream tOut, FileStatus file, String rootPath) throws IOException { + private static void addFileToTar(TarArchiveOutputStream tarOut, FileStatus file, String rootPath) throws IOException { final Path filePath = Paths.get(file.getPath().getPath()); String pathInsideTar; if (filePath.startsWith(rootPath)) { @@ -84,12 +83,12 @@ private static void addFileToTar(TarArchiveOutputStream tOut, FileStatus file, S } TarArchiveEntry tarEntry = new TarArchiveEntry( - filePath.toFile(), - pathInsideTar); + filePath.toFile(), + pathInsideTar); - tOut.putArchiveEntry(tarEntry); - Files.copy(filePath, tOut); - tOut.closeArchiveEntry(); + tarOut.putArchiveEntry(tarEntry); + Files.copy(filePath, tarOut); + tarOut.closeArchiveEntry(); } public static void decompressFromTarGzFile(String pathToTar, String outputPath) throws IOException { @@ -106,12 +105,13 @@ public static void decompressFromTarGzInMemory(byte[] rawData, String outputPath decompressFromTarGzInMemory(rawData, outputPath, false); } - public static void decompressFromTarGzInMemory(byte[] rawData, String outputPath, boolean base64) throws IOException { + public static void decompressFromTarGzInMemory(byte[] rawData, String outputPath, boolean base64) + throws IOException { if (rawData == null) { throw new IOException("Provided null as .tar.gz data which is not allowed!"); } final byte[] data; - if (base64){ + if (base64) { data = Base64.getDecoder().decode(rawData); } else { data = rawData; @@ -124,8 +124,8 @@ public static void decompressFromTarGzInMemory(byte[] rawData, String outputPath private static void decompressFromTarGz(InputStream inputStream, String outputPath) throws IOException { try (BufferedInputStream bi = new BufferedInputStream(inputStream); - GzipCompressorInputStream gzi = new GzipCompressorInputStream(bi); - TarArchiveInputStream ti = new TarArchiveInputStream(gzi)) { + GzipCompressorInputStream gzi = new GzipCompressorInputStream(bi); + TarArchiveInputStream ti = new TarArchiveInputStream(gzi)) { ArchiveEntry entry; while ((entry = ti.getNextEntry()) != null) { diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/FileUtil.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/FileUtil.java index fe84a637..73787d84 100644 --- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/FileUtil.java +++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/utils/FileUtil.java @@ -1,14 +1,13 @@ package com.cloudera.service.common.utils; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import lombok.experimental.UtilityClass; import org.apache.flink.core.fs.FileStatus; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - @UtilityClass public class FileUtil { @@ -31,7 +30,7 @@ public static List listFiles(Path path, boolean recursive) throws IO for (FileStatus fileStatus : statusList) { if (fileStatus.isDir() && recursive) { final List recursiveStatusList = listFiles(fileStatus.getPath(), recursive); - if (recursiveStatusList != null){ + if (recursiveStatusList != null) { result.addAll(recursiveStatusList); } } else { diff --git a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/CliApplication.java b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/CliApplication.java index c58f0cb5..8f2ac440 100644 --- a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/CliApplication.java +++ b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/CliApplication.java @@ -7,8 +7,8 @@ @SpringBootApplication(exclude = KafkaAutoConfiguration.class) public class CliApplication { - public static void main(String[] args) { - SpringApplication.run(CliApplication.class, args); - } + public static void main(String[] args) { + SpringApplication.run(CliApplication.class, args); + } } diff --git a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/configuration/KafkaConfig.java b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/configuration/KafkaConfig.java index 249a8d3a..15e750cf 100644 --- a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/configuration/KafkaConfig.java +++ b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/configuration/KafkaConfig.java @@ -32,7 +32,8 @@ public ClouderaKafkaProperties kafkaProperties() { @Bean public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { - ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); factory.setConsumerFactory(kafkaConsumerFactory()); factory.setReplyTemplate(kafkaTemplate()); return factory; @@ -44,10 +45,12 @@ public KafkaTemplate kafkaTemplate() { } private ProducerFactory kafkaProducerFactory() { - return new DefaultKafkaProducerFactory<>(kafkaProperties().buildConsumerProperties(), new StringSerializer(), new JsonSerializer<>()); + return new DefaultKafkaProducerFactory<>(kafkaProperties().buildConsumerProperties(), new StringSerializer(), + new JsonSerializer<>()); } private ConsumerFactory kafkaConsumerFactory() { - return new DefaultKafkaConsumerFactory<>(kafkaProperties().buildConsumerProperties(), new StringDeserializer(), new JsonDeserializer<>(RequestBody.class)); + return new DefaultKafkaConsumerFactory<>(kafkaProperties().buildConsumerProperties(), new StringDeserializer(), + new JsonDeserializer<>(RequestBody.class)); } } diff --git a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java index a6e0e56c..0e0fbfee 100644 --- a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java +++ b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java @@ -8,6 +8,9 @@ import com.cloudera.service.common.response.Job; import com.cloudera.service.common.response.ResponseBody; import com.cloudera.service.common.response.ResponseType; +import java.io.IOException; +import java.util.Collections; +import java.util.List; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -21,10 +24,6 @@ import org.springframework.messaging.support.MessageHeaderAccessor; import org.springframework.stereotype.Component; -import java.io.IOException; -import java.util.Collections; -import java.util.List; - @Component @RequiredArgsConstructor @Slf4j @@ -43,7 +42,9 @@ public class KafkaListenerController { @KafkaListener(topics = "#{kafkaProperties.getRequestTopic()}", containerFactory = "kafkaListenerContainerFactory") @SendTo({"#{kafkaProperties.getReplyTopic()}"}) - public Message handleMessage(RequestBody requestBody, @Header(KafkaHeaders.RECEIVED_MESSAGE_KEY) String key, @Header(KafkaHeaders.REPLY_TOPIC) byte[] replyTo, + public Message handleMessage(RequestBody requestBody, + @Header(KafkaHeaders.RECEIVED_MESSAGE_KEY) String key, + @Header(KafkaHeaders.REPLY_TOPIC) byte[] replyTo, @Header(KafkaHeaders.CORRELATION_ID) byte[] correlationId) { log.info("Start processing message\n Message key: '{}' \n value: '{}'", key, requestBody); @@ -59,9 +60,10 @@ public Message handleMessage(RequestBody requestBody, @Header(Kafk try { Job job = jobService.restartJob(requestBody.getJobIdHex()); ResponseBody responseBody = ResponseBody.builder() - .jobs(Collections.singletonList(job)) - .build(); - return buildResponseMessage(responseBody, ResponseType.RESTART_JOB_RESPONSE, replyTo, correlationId); + .jobs(Collections.singletonList(job)) + .build(); + return buildResponseMessage(responseBody, ResponseType.RESTART_JOB_RESPONSE, replyTo, + correlationId); } catch (IOException e) { return handleErrorResponse(e, replyTo, correlationId); } @@ -69,8 +71,8 @@ public Message handleMessage(RequestBody requestBody, @Header(Kafk try { Job job = jobService.stopJob(requestBody.getJobIdHex()); ResponseBody responseBody = ResponseBody.builder() - .jobs(Collections.singletonList(job)) - .build(); + .jobs(Collections.singletonList(job)) + .build(); return buildResponseMessage(responseBody, ResponseType.STOP_JOB_RESPONSE, replyTo, correlationId); } catch (IOException e) { return handleErrorResponse(e, replyTo, correlationId); @@ -81,26 +83,30 @@ public Message handleMessage(RequestBody requestBody, @Header(Kafk try { jobService.updateConfig(requestBody.getPayload()); final ResponseBody responseBody = ResponseBody.builder().build(); - return buildResponseMessage(responseBody, ResponseType.UPDATE_JOB_CONFIG_RESPONSE, replyTo, correlationId); + return buildResponseMessage(responseBody, ResponseType.UPDATE_JOB_CONFIG_RESPONSE, replyTo, + correlationId); } catch (IOException e) { return handleErrorResponse(e, replyTo, correlationId); } + default: + break; } return null; } - private Message getResponseBodyMessage(byte[] replyTo, byte[] correlationId, ResponseType responseType) { + private Message getResponseBodyMessage(byte[] replyTo, byte[] correlationId, + ResponseType responseType) { try { List jobs = jobService.getJobs(); ResponseBody responseBody = ResponseBody.builder() - .jobs(jobs) - .clusterMeta(ClusterMeta.builder() - .name(clusterName) - .clusterId(clusterId) - .clusterStatus(clusterStatus) - .version(clusterVersion) - .build()) - .build(); + .jobs(jobs) + .clusterMeta(ClusterMeta.builder() + .name(clusterName) + .clusterId(clusterId) + .clusterStatus(clusterStatus) + .version(clusterVersion) + .build()) + .build(); return buildResponseMessage(responseBody, responseType, replyTo, correlationId); } catch (IOException e) { return handleErrorResponse(e, replyTo, correlationId); @@ -109,12 +115,14 @@ private Message getResponseBodyMessage(byte[] replyTo, byte[] corr private Message handleErrorResponse(Exception e, byte[] replyTo, byte[] correlationId) { ResponseBody responseBody = ResponseBody.builder() - .errorMessage(Collections.singletonMap(e.getClass().toString(), e.getMessage())) - .build(); + .errorMessage( + Collections.singletonMap(e.getClass().toString(), e.getMessage())) + .build(); return buildResponseMessage(responseBody, ResponseType.ERROR_RESPONSE, replyTo, correlationId); } - private Message buildResponseMessage(ResponseBody body, ResponseType responseType, byte[] replyTo, byte[] correlationId) { + private Message buildResponseMessage(ResponseBody body, ResponseType responseType, byte[] replyTo, + byte[] correlationId) { MessageHeaderAccessor accessor = new MessageHeaderAccessor(); accessor.setHeader(KafkaHeaders.MESSAGE_KEY, responseType.name()); accessor.setHeader(KafkaHeaders.CORRELATION_ID, correlationId); diff --git a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java index 835bef27..dbf43adf 100644 --- a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java +++ b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java @@ -3,14 +3,6 @@ import com.cloudera.service.common.Utils; import com.cloudera.service.common.response.Job; import com.cloudera.service.common.utils.ArchiveUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.JobID; -import org.apache.flink.api.common.JobStatus; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -24,29 +16,39 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobStatus; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; @Slf4j @Service public class JobService { @Value("${cluster.pipeline.dir}") private String pipelineDir; - public static final String LOG_CLI_JOB_INFO = "Successfully read jobs from cli with exit code {}. job count '{}' jobs data '[{}]'"; - private final Pattern pattern = Pattern.compile("^(?[\\d.:\\s]+)\\s:\\s(?[a-fA-F0-9]+)\\s:\\s(?[\\w.-]+)\\s\\((?\\w+)\\)$"); + public static final String LOG_CLI_JOB_INFO = + "Successfully read jobs from cli with exit code {}. job count '{}' jobs data '[{}]'"; + private final Pattern pattern = Pattern.compile( + "^(?[\\d.:\\s]+)\\s:\\s(?[a-fA-F0-9]+)\\s:\\s(?[\\w.-]+)\\s\\((?\\w+)\\)$"); public List getJobs() throws IOException { List jobs = new ArrayList<>(); int exitValue = fillJobList(jobs); - log.info(LOG_CLI_JOB_INFO, exitValue, jobs.size(), jobs.stream().map(Objects::toString).collect(Collectors.joining(","))); + log.info(LOG_CLI_JOB_INFO, exitValue, jobs.size(), + jobs.stream().map(Objects::toString).collect(Collectors.joining(","))); return jobs; } public Job getJob(String id) throws IOException { List jobs = getJobs(); return jobs.stream() - .filter(job -> StringUtils.equalsIgnoreCase(job.getJobId().toHexString(), id)) - .findFirst() - .orElse(null); + .filter(job -> StringUtils.equalsIgnoreCase(job.getJobId().toHexString(), id)) + .findFirst() + .orElse(null); } public Job restartJob(String id) throws IOException { @@ -60,18 +62,26 @@ public Job restartJob(String id) throws IOException { processBuilder.directory(new File(pipelineDir)); } Process process = processBuilder.start(); - log.debug("Command input stream '{}' \n Command error stream '{}'", IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); + log.debug("Command input stream '{}' \n Command error stream '{}'", + IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), + IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); int waitForCode = process.waitFor(); if (process.exitValue() != 0) { - log.error("Failed to run job with exit code '{}' wait fore code '{}'. Command input stream '{}' \n Command error stream '{}'", process.exitValue(), waitForCode, - IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); + log.error( + "Failed to run job with exit code '{}' wait fore code '{}'." + + "Command input stream '{}' \n Command error stream '{}'", + process.exitValue(), waitForCode, + IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), + IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); return job; } } catch (IOException ioe) { - log.error("There was an error when starting the restart operation for the job {}. {}", job, ioe.getMessage()); + log.error("There was an error when starting the restart operation for the job {}. {}", job, + ioe.getMessage()); throw ioe; } catch (InterruptedException ie) { - log.error("An unexpected event occurred while waiting for the restart to complete for the job {}. {}", job, ie.getMessage()); + log.error("An unexpected event occurred while waiting for the restart to complete for the job {}. {}", + job, ie.getMessage()); Thread.currentThread().interrupt(); } } @@ -84,18 +94,25 @@ public Job stopJob(String id) throws IOException { try { ProcessBuilder processBuilder = new ProcessBuilder("flink", "cancel", id); Process process = processBuilder.start(); - log.info("Command input stream '{}' \n Command error stream '{}'", IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); + log.info("Command input stream '{}' \n Command error stream '{}'", + IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), + IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); process.waitFor(); if (process.exitValue() != 0) { - log.error("Failed to run job with exit code {}. Command output stream '{}' \n Command error stream '{}'", process.exitValue(), - IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); + log.error( + "Failed to run job with exit code {}. Command output stream '{}' \n Command error stream '{}'", + process.exitValue(), + IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), + IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); return job; } } catch (IOException ioe) { - log.error("There was an error when starting the restart operation for the job {}. {}", job, ioe.getMessage()); + log.error("There was an error when starting the restart operation for the job {}. {}", job, + ioe.getMessage()); throw ioe; } catch (InterruptedException ie) { - log.error("An unexpected event occurred while waiting for the restart to complete for the job {}. {}", job, ie.getMessage()); + log.error("An unexpected event occurred while waiting for the restart to complete for the job {}. {}", + job, ie.getMessage()); Thread.currentThread().interrupt(); } } @@ -129,16 +146,18 @@ private List readJobFromInputStream(InputStream inputStream) throws IOExcep Matcher stringMatcher = pattern.matcher(stringLine); if (stringMatcher.matches()) { String jobFullName = stringMatcher.group("jobFullName"); - Job.JobType jobType = Utils.getEnumFromStringContains(jobFullName, Job.JobType.class, Job.JobType::getName); + Job.JobType jobType = + Utils.getEnumFromStringContains(jobFullName, Job.JobType.class, Job.JobType::getName); if (jobType != null) { Job job = Job.builder() - .jobId(JobID.fromHexString(stringMatcher.group("jobId"))) - .jobIdString(stringMatcher.group("jobId")) - .jobFullName(jobFullName) - .jobType(jobType) - .jobState(JobStatus.valueOf(StringUtils.toRootUpperCase(stringMatcher.group("jobStatus")))) - .startTime(stringMatcher.group("date")) - .build(); + .jobId(JobID.fromHexString(stringMatcher.group("jobId"))) + .jobIdString(stringMatcher.group("jobId")) + .jobFullName(jobFullName) + .jobType(jobType) + .jobState(JobStatus.valueOf( + StringUtils.toRootUpperCase(stringMatcher.group("jobStatus")))) + .startTime(stringMatcher.group("date")) + .build(); setJobParameters(job, jobFullName); jobs.add(job); } @@ -155,7 +174,8 @@ private void setJobParameters(Job job, String fullJobName) { String[] jobParameters = fullJobName.split("\\."); job.setJobBranch(jobParameters[0]); job.setJobPipeline(jobParameters[1]); - if (job.getJobType() == Job.JobType.PROFILE || job.getJobType() == Job.JobType.GENERATOR || job.getJobType() == Job.JobType.PARSER) { + if (job.getJobType() == Job.JobType.PROFILE || job.getJobType() == Job.JobType.GENERATOR + || job.getJobType() == Job.JobType.PARSER) { job.setJobName(jobParameters[jobParameters.length - 1]); } } diff --git a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/AbstractDateFilter.java b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/AbstractDateFilter.java index 0dae0d1c..43f7c103 100644 --- a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/AbstractDateFilter.java +++ b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/AbstractDateFilter.java @@ -12,6 +12,8 @@ package com.cloudera.cyber.pruner; +import java.io.IOException; +import java.util.Date; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; @@ -19,9 +21,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import java.io.IOException; -import java.util.Date; - @Slf4j abstract class AbstractDateFilter extends Configured implements PathFilter { protected final FileSystem fileSystem; diff --git a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/DateDirectoryFilter.java b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/DateDirectoryFilter.java index 9a787f8f..b37f498a 100644 --- a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/DateDirectoryFilter.java +++ b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/DateDirectoryFilter.java @@ -12,15 +12,14 @@ package com.cloudera.cyber.pruner; +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import java.io.IOException; -import java.text.DateFormat; -import java.text.ParseException; - @Slf4j public class DateDirectoryFilter extends AbstractDateFilter { @@ -33,15 +32,15 @@ protected DateDirectoryFilter(FileSystem fileSystem, long maxAge, DateFormat df) @Override protected boolean accept(FileStatus status) { - return status.isDirectory() && - getDirectoryDate(status.getPath()) < earliestAllowed && - directoryEmpty(status); + return status.isDirectory() + && getDirectoryDate(status.getPath()) < earliestAllowed + && directoryEmpty(status); } private boolean directoryEmpty(FileStatus status) { try { return fileSystem.listStatus(status.getPath()).length == 0; - } catch(IOException e) { + } catch (IOException e) { log.warn(String.format("Directory empty test failed on %s", status.getPath(), e)); return true; } diff --git a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/DateFileFilter.java b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/DateFileFilter.java index f06c8ac8..159b3934 100644 --- a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/DateFileFilter.java +++ b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/DateFileFilter.java @@ -25,7 +25,7 @@ protected DateFileFilter(FileSystem fileSystem, long maxAge) { @Override protected boolean accept(FileStatus status) { - return !status.isDirectory() && - status.getModificationTime() < earliestAllowed; + return !status.isDirectory() + && status.getModificationTime() < earliestAllowed; } } diff --git a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/Pruner.java b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/Pruner.java index b1a6d55c..72c292a6 100644 --- a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/Pruner.java +++ b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/Pruner.java @@ -12,26 +12,32 @@ package com.cloudera.cyber.pruner; +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Date; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.cli.*; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.codehaus.jackson.map.ObjectMapper; -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.Date; - /** * Prune data from the cloudera cyber reference implementation + * *

* Heavily inspired by https://github.com/apache/metron/blob/master/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/HDFSDataPruner.java */ @@ -50,6 +56,7 @@ public class Pruner { @NonNull private PrunerConfig config; + @SuppressWarnings("checkstyle:MemberName") transient FileSystem _fileSystem; public FileSystem getFileSystem() throws IOException { @@ -73,14 +80,18 @@ long prune() throws IOException, SQLException { pruneDirs(config.getLogsLocation() + "/**", config.getLogsMaxMs()); // prune hive tables - if (!hiveUri.isEmpty()) deleteFromHive(hiveUri, hiveUser, hivePassword, config.getEventsTable()); + if (!hiveUri.isEmpty()) { + deleteFromHive(hiveUri, hiveUser, hivePassword, config.getEventsTable()); + } return filesPruned; } - private int deleteFromHive(String hiveUri, String hiveUser, String hivePassword, String eventsTable) throws SQLException { + private int deleteFromHive(String hiveUri, String hiveUser, String hivePassword, String eventsTable) + throws SQLException { int rows; - String sql = String.format("DELETE FROM %s WHERE ts > %d", eventsTable, new Date().getTime() - config.getEventsMaxMs()); + String sql = String.format("DELETE FROM %s WHERE ts > %d", eventsTable, + new Date().getTime() - config.getEventsMaxMs()); try (Connection connection = DriverManager.getConnection(hiveUri, hiveUser, hivePassword)) { PreparedStatement preparedStatement = connection.prepareStatement(sql); rows = preparedStatement.executeUpdate(); @@ -90,10 +101,13 @@ private int deleteFromHive(String hiveUri, String hiveUser, String hivePassword, private void pruneDirs(String globPath, long maxAge) throws IOException { // object store filesystems don't need to remove paths - if (globPath.startsWith("s3") || globPath.startsWith("adls:")) return; + if (globPath.startsWith("s3") || globPath.startsWith("adls:")) { + return; + } // remove directories within the date range which are empty - FileStatus[] filesToDelete = getFileSystem().globStatus(new Path(globPath), new DateDirectoryFilter(getFileSystem(), maxAge, config.getDateFormatter())); + FileStatus[] filesToDelete = getFileSystem().globStatus(new Path(globPath), + new DateDirectoryFilter(getFileSystem(), maxAge, config.getDateFormatter())); for (FileStatus fileStatus : filesToDelete) { log.debug("Deleting File: {}", fileStatus.getPath()); getFileSystem().delete(fileStatus.getPath(), false); @@ -102,7 +116,8 @@ private void pruneDirs(String globPath, long maxAge) throws IOException { private long pruneFiles(String globPath, long maxAge) throws IOException { long filesPruned = 0; - FileStatus[] filesToDelete = getFileSystem().globStatus(new Path(globPath), new DateFileFilter(getFileSystem(), maxAge)); + FileStatus[] filesToDelete = + getFileSystem().globStatus(new Path(globPath), new DateFileFilter(getFileSystem(), maxAge)); for (FileStatus fileStatus : filesToDelete) { log.info("Deleting File: {}", fileStatus.getPath()); getFileSystem().delete(fileStatus.getPath(), false); @@ -172,11 +187,11 @@ public static void main(String[] args) { PrunerConfig config = new ObjectMapper().readValue(new File(cmd.getOptionValue("c")), PrunerConfig.class); Pruner pruner = new Pruner( - cmd.getOptionValue("f", new Configuration().get("fs.defaultFS")), - cmd.getOptionValue("i", ""), - cmd.getOptionValue("u", ""), - cmd.getOptionValue("p", ""), - config); + cmd.getOptionValue("f", new Configuration().get("fs.defaultFS")), + cmd.getOptionValue("i", ""), + cmd.getOptionValue("u", ""), + cmd.getOptionValue("p", ""), + config); Long pruned = pruner.prune(); log.info("Pruned {} files using {}", pruned, config); diff --git a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/PrunerConfig.java b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/PrunerConfig.java index 01d097a7..a0639263 100644 --- a/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/PrunerConfig.java +++ b/flink-cyber/data-management/retention-cleaner/src/main/java/com/cloudera/cyber/pruner/PrunerConfig.java @@ -12,14 +12,13 @@ package com.cloudera.cyber.pruner; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.text.DateFormat; -import java.text.SimpleDateFormat; - @Data @Builder @AllArgsConstructor @@ -32,7 +31,7 @@ public class PrunerConfig { private String logsLocation = "/data/logs/"; /** - * Hive Table containing all the events + * Hive Table containing all the events. */ @Builder.Default private String eventsTable = "events"; @@ -50,8 +49,9 @@ public class PrunerConfig { private transient DateFormat dateFormat; public DateFormat getDateFormatter() { - if (dateFormat == null) + if (dateFormat == null) { this.dateFormat = new SimpleDateFormat(datePattern); + } return dateFormat; } @@ -62,6 +62,7 @@ public long getOriginalsMaxMs() { public long getLogsMaxMs() { return logsMaxAge * 3600000; } + public long getEventsMaxMs() { return eventsMaxAge * 3600000; } diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java index 144aafab..e8139ba6 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java @@ -12,9 +12,12 @@ package com.cloudera.cyber.scoring; +import static com.cloudera.cyber.AvroTypes.toListOf; + import com.cloudera.cyber.IdentifiedMessage; import com.cloudera.cyber.Message; import com.cloudera.cyber.avro.AvroSchemas; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -29,10 +32,6 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.types.Row; -import java.util.List; - -import static com.cloudera.cyber.AvroTypes.toListOf; - @Data @EqualsAndHashCode @ToString @@ -54,23 +53,25 @@ public long getTs() { return message.getTs(); } - public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName(), null) - .fields() - .name("message").type(Message.SCHEMA$).noDefault() - .name("cyberScoresDetails").type(Schema.createArray(Scores.SCHEMA$)).noDefault() - .optionalDouble("cyberScore") - .endRecord(); + public static final Schema SCHEMA$ = + AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName(), + null) + .fields() + .name("message").type(Message.SCHEMA$).noDefault() + .name("cyberScoresDetails").type(Schema.createArray(Scores.SCHEMA$)).noDefault() + .optionalDouble("cyberScore") + .endRecord(); public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( - new String[]{"message", "cyberScoresDetails", "cyberScore"}, - Message.FLINK_TYPE_INFO, Types.OBJECT_ARRAY(Scores.FLINK_TYPE_INFO), Types.DOUBLE); + new String[] {"message", "cyberScoresDetails", "cyberScore"}, + Message.FLINK_TYPE_INFO, Types.OBJECT_ARRAY(Scores.FLINK_TYPE_INFO), Types.DOUBLE); public Row toRow() { return Row.of(message == null ? null : message.toRow(), - cyberScoresDetails == null ? null : cyberScoresDetails.stream() - .map(Scores::toRow) - .toArray(Row[]::new), - cyberScore); + cyberScoresDetails == null ? null : cyberScoresDetails.stream() + .map(Scores::toRow) + .toArray(Row[]::new), + cyberScore); } @Override diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessageWatermarkedStream.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessageWatermarkedStream.java index 0eb592b4..2cd5ac08 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessageWatermarkedStream.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessageWatermarkedStream.java @@ -1,27 +1,26 @@ /* * Copyright 2020 - 2022 Cloudera. All Rights Reserved. * - * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file + * except in compliance with the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0. * - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. Refer to the License for the specific permissions and + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. Refer to the License for the specific permissions and * limitations governing your use of the file. */ package com.cloudera.cyber.scoring; +import java.time.Duration; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.streaming.api.datastream.DataStream; -import java.time.Duration; - public class ScoredMessageWatermarkedStream { public static DataStream of(DataStream inputStream, long maximumLatenessMillis) { WatermarkStrategy watermarkStrategy = WatermarkStrategy - .forBoundedOutOfOrderness(Duration.ofMillis(maximumLatenessMillis)) - .withTimestampAssigner((scoredMessage, timestamp) -> scoredMessage.getMessage().getTs()); + .forBoundedOutOfOrderness(Duration.ofMillis(maximumLatenessMillis)) + .withTimestampAssigner((scoredMessage, timestamp) -> scoredMessage.getMessage().getTs()); return inputStream.assignTimestampsAndWatermarks(watermarkStrategy); } } diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/Scores.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/Scores.java index 1a98db43..291e1032 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/Scores.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/Scores.java @@ -36,16 +36,16 @@ public class Scores extends SpecificRecordBase implements SpecificRecord { private String reason; public static final Schema SCHEMA$ = SchemaBuilder.record(Scores.class.getName()) - .namespace(Scores.class.getPackage().getName()) - .fields() - .name("ruleId").type(AvroTypes.uuidType).noDefault() - .requiredDouble("score") - .requiredString("reason") - .endRecord(); + .namespace(Scores.class.getPackage().getName()) + .fields() + .name("ruleId").type(AvroTypes.uuidType).noDefault() + .requiredDouble("score") + .requiredString("reason") + .endRecord(); public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( - new String[]{"ruleId", "score", "reason"}, - Types.STRING, Types.DOUBLE, Types.STRING); + new String[] {"ruleId", "score", "reason"}, + Types.STRING, Types.DOUBLE, Types.STRING); public Row toRow() { return Row.of(ruleId, score, reason); diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRule.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRule.java index f8c62abe..ba54e169 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRule.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRule.java @@ -1,20 +1,26 @@ /* * Copyright 2020 - 2022 Cloudera. All Rights Reserved. * - * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file + * except in compliance with the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0. * - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. Refer to the License for the specific permissions and + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. Refer to the License for the specific permissions and * limitations governing your use of the file. */ package com.cloudera.cyber.scoring; +import static java.util.stream.Collectors.toList; + import com.cloudera.cyber.Message; import com.cloudera.cyber.rules.BaseDynamicRule; import com.cloudera.cyber.rules.RuleType; +import java.time.Instant; +import java.util.Arrays; +import java.util.Map; +import java.util.UUID; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -24,13 +30,6 @@ import org.apache.avro.SchemaBuilder; import org.apache.flink.api.common.typeinfo.TypeInfo; -import java.time.Instant; -import java.util.Arrays; -import java.util.Map; -import java.util.UUID; - -import static java.util.stream.Collectors.toList; - @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @@ -75,23 +74,23 @@ public Map apply(Message message) { } public static final Schema SCHEMA$ = SchemaBuilder - .record(ScoringRule.class.getName()) - .namespace(ScoringRule.class.getPackage().getName()) - .fields() - .requiredString("name") - .requiredInt("order") - .requiredLong("tsStart") - .requiredLong("tsEnd") - .name("type").type(Schema.createEnum( - RuleType.class.getName(), - "", - RuleType.class.getPackage().getName(), - Arrays.stream(RuleType.values()).map(Enum::name).collect(toList()))).noDefault() - .requiredString("ruleScript") - .requiredString("id") - .requiredBoolean("enabled") - .nullableInt("version", 0) - .endRecord(); + .record(ScoringRule.class.getName()) + .namespace(ScoringRule.class.getPackage().getName()) + .fields() + .requiredString("name") + .requiredInt("order") + .requiredLong("tsStart") + .requiredLong("tsEnd") + .name("type").type(Schema.createEnum( + RuleType.class.getName(), + "", + RuleType.class.getPackage().getName(), + Arrays.stream(RuleType.values()).map(Enum::name).collect(toList()))).noDefault() + .requiredString("ruleScript") + .requiredString("id") + .requiredBoolean("enabled") + .nullableInt("version", 0) + .endRecord(); @Override public Schema getSchema() { @@ -101,32 +100,61 @@ public Schema getSchema() { @Override public Object get(int field$) { switch (field$) { - case 0: return getName(); - case 1: return getOrder(); - case 2: return getTsStart().toEpochMilli(); - case 3: return getTsEnd().toEpochMilli(); - case 4: return getType(); - case 5: return getRuleScript(); - case 6: return id; - case 7: return enabled; - case 8: return getVersion(); - default: throw new AvroRuntimeException("Bad index"); + case 0: + return getName(); + case 1: + return getOrder(); + case 2: + return getTsStart().toEpochMilli(); + case 3: + return getTsEnd().toEpochMilli(); + case 4: + return getType(); + case 5: + return getRuleScript(); + case 6: + return id; + case 7: + return enabled; + case 8: + return getVersion(); + default: + throw new AvroRuntimeException("Bad index"); } } @Override public void put(int field$, Object value$) { switch (field$) { - case 0: this.setName(value$.toString()); break; - case 1: this.setOrder((int) value$); break; - case 2: this.setTsStart(Instant.ofEpochMilli((long)value$)); break; - case 3: this.setTsEnd(Instant.ofEpochMilli((long)value$)); break; - case 4: this.setType(RuleType.valueOf(value$.toString())); break; - case 5: this.setRuleScript(value$.toString()); break; - case 6: this.setId(value$.toString()); break; - case 7: this.setEnabled((boolean) value$); break; - case 8 : this.setVersion((int) value$); break; - default: throw new AvroRuntimeException("Bad index"); + case 0: + this.setName(value$.toString()); + break; + case 1: + this.setOrder((int) value$); + break; + case 2: + this.setTsStart(Instant.ofEpochMilli((long) value$)); + break; + case 3: + this.setTsEnd(Instant.ofEpochMilli((long) value$)); + break; + case 4: + this.setType(RuleType.valueOf(value$.toString())); + break; + case 5: + this.setRuleScript(value$.toString()); + break; + case 6: + this.setId(value$.toString()); + break; + case 7: + this.setEnabled((boolean) value$); + break; + case 8: + this.setVersion((int) value$); + break; + default: + throw new AvroRuntimeException("Bad index"); } } @@ -134,7 +162,8 @@ public void put(int field$, Object value$) { return new ScoringRuleBuilderImpl().$fillValuesFrom(this); } - public static abstract class ScoringRuleBuilder> extends BaseDynamicRuleBuilder { + public abstract static class ScoringRuleBuilder> + extends BaseDynamicRuleBuilder { private String id; private boolean enabled; @@ -164,7 +193,8 @@ public B enabled(boolean enabled) { public abstract C build(); public String toString() { - return "ScoringRule.ScoringRuleBuilder(super=" + super.toString() + ", id=" + this.id + ", enabled=" + this.enabled + ")"; + return "ScoringRule.ScoringRuleBuilder(super=" + super.toString() + ", id=" + this.id + ", enabled=" + + this.enabled + ")"; } } diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleCommand.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleCommand.java index 27eb2282..8ec09cfd 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleCommand.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleCommand.java @@ -1,12 +1,12 @@ /* * Copyright 2020 - 2022 Cloudera. All Rights Reserved. * - * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file + * except in compliance with the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0. * - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. Refer to the License for the specific permissions and + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. Refer to the License for the specific permissions and * limitations governing your use of the file. */ @@ -14,6 +14,8 @@ import com.cloudera.cyber.rules.DynamicRuleCommand; import com.cloudera.cyber.rules.DynamicRuleCommandType; +import java.util.Arrays; +import java.util.stream.Collectors; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -21,10 +23,6 @@ import lombok.experimental.SuperBuilder; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; -import org.apache.flink.api.common.typeinfo.TypeInfo; - -import java.util.Arrays; -import java.util.stream.Collectors; @Data @EqualsAndHashCode(callSuper = true) @@ -35,16 +33,18 @@ public class ScoringRuleCommand extends DynamicRuleCommand { public static Schema SCHEMA$ = SchemaBuilder.record(ScoringRuleCommand.class.getName()) - .namespace(ScoringRuleCommand.class.getPackage().getName()) - .fields() - .requiredString("id") - .name("type").type(Schema.createEnum(DynamicRuleCommandType.class.getName(), "", DynamicRuleCommandType.class.getPackage().getName(), - Arrays.stream(DynamicRuleCommandType.values()).map(v -> v.name()).collect(Collectors.toList()))).noDefault() - .requiredLong("ts") - .optionalString("ruleId") - .name("rule").type().optional().type(ScoringRule.SCHEMA$) - .name("headers").type(Schema.createMap(Schema.create(Schema.Type.STRING))).noDefault() - .endRecord(); + .namespace(ScoringRuleCommand.class.getPackage().getName()) + .fields() + .requiredString("id") + .name("type").type(Schema.createEnum(DynamicRuleCommandType.class.getName(), "", + DynamicRuleCommandType.class.getPackage().getName(), + Arrays.stream(DynamicRuleCommandType.values()).map(v -> v.name()).collect(Collectors.toList()))) + .noDefault() + .requiredLong("ts") + .optionalString("ruleId") + .name("rule").type().optional().type(ScoringRule.SCHEMA$) + .name("headers").type(Schema.createMap(Schema.create(Schema.Type.STRING))).noDefault() + .endRecord(); @Override public Schema getSchema() { diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleCommandResult.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleCommandResult.java index f8b3994a..f193d601 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleCommandResult.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleCommandResult.java @@ -1,12 +1,12 @@ /* * Copyright 2020 - 2022 Cloudera. All Rights Reserved. * - * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file + * except in compliance with the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0. * - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. Refer to the License for the specific permissions and + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. Refer to the License for the specific permissions and * limitations governing your use of the file. */ @@ -32,13 +32,13 @@ private ScoringRuleCommandResult(String cmdId, boolean success, ScoringRule rule } public static final Schema SCHEMA$ = SchemaBuilder.record(ScoringRuleCommandResult.class.getName()) - .namespace(ScoringRuleCommandResult.class.getPackage().getName()) - .fields() - .requiredString("id") - .requiredBoolean("result") - .name("rule").type().optional().type(ScoringRule.SCHEMA$) - .optionalInt("parallelSubtaskNumber") - .endRecord(); + .namespace(ScoringRuleCommandResult.class.getPackage().getName()) + .fields() + .requiredString("id") + .requiredBoolean("result") + .name("rule").type().optional().type(ScoringRule.SCHEMA$) + .optionalInt("parallelSubtaskNumber") + .endRecord(); @Override public Schema getSchema() { @@ -49,7 +49,8 @@ public static ScoringRuleCommandResultBuilder builder() { return new ScoringRuleCommandResultBuilder(); } - public static class ScoringRuleCommandResultBuilder extends DynamicRuleCommandResultBuilder { + public static class ScoringRuleCommandResultBuilder + extends DynamicRuleCommandResultBuilder { @Override protected ScoringRuleCommandResultBuilder self() { diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleTypeFactory.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleTypeFactory.java index 06924515..f50274f5 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleTypeFactory.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringRuleTypeFactory.java @@ -1,24 +1,23 @@ /* * Copyright 2020 - 2022 Cloudera. All Rights Reserved. * - * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file + * except in compliance with the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0. * - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. Refer to the License for the specific permissions and + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. Refer to the License for the specific permissions and * limitations governing your use of the file. */ package com.cloudera.cyber.scoring; +import java.lang.reflect.Type; +import java.util.Map; import org.apache.flink.api.common.typeinfo.TypeInfoFactory; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; -import java.lang.reflect.Type; -import java.util.Map; - public class ScoringRuleTypeFactory extends TypeInfoFactory { @Override diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringSummarizationMode.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringSummarizationMode.java index 96bddebe..37b0487d 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringSummarizationMode.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoringSummarizationMode.java @@ -1,12 +1,11 @@ package com.cloudera.cyber.scoring; -import org.apache.commons.collections.CollectionUtils; - import java.util.DoubleSummaryStatistics; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.collections.CollectionUtils; public enum ScoringSummarizationMode { SUM(ScoringSummarizationMode::sum), @@ -20,7 +19,7 @@ public enum ScoringSummarizationMode { this.scoreFunction = scoreFunction; } - public static ScoringSummarizationMode DEFAULT() { + public static ScoringSummarizationMode defaultValue() { return POSITIVE_MEAN; } diff --git a/flink-cyber/flink-alert-scoring-api/src/test/java/com/cloudera/cyber/scoring/ScoringSummarizationModeTest.java b/flink-cyber/flink-alert-scoring-api/src/test/java/com/cloudera/cyber/scoring/ScoringSummarizationModeTest.java index 6636316b..5bf379df 100644 --- a/flink-cyber/flink-alert-scoring-api/src/test/java/com/cloudera/cyber/scoring/ScoringSummarizationModeTest.java +++ b/flink-cyber/flink-alert-scoring-api/src/test/java/com/cloudera/cyber/scoring/ScoringSummarizationModeTest.java @@ -1,22 +1,21 @@ package com.cloudera.cyber.scoring; -import org.junit.Test; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; import java.util.Arrays; import java.util.List; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; +import org.junit.Test; public class ScoringSummarizationModeTest { @Test - public void testDefault(){ - assertThat(ScoringSummarizationMode.DEFAULT(), equalTo(ScoringSummarizationMode.POSITIVE_MEAN)); + public void testDefault() { + assertThat(ScoringSummarizationMode.defaultValue(), equalTo(ScoringSummarizationMode.POSITIVE_MEAN)); } @Test - public void testSum(){ + public void testSum() { List scoreList = Arrays.asList(-1.0, 1.0, 2.0, 6.0); final ScoringSummarizationMode summarizationMode = ScoringSummarizationMode.SUM; @@ -24,7 +23,7 @@ public void testSum(){ } @Test - public void testMax(){ + public void testMax() { List scoreList = Arrays.asList(-1.0, 1.0, 2.0, 6.0); final ScoringSummarizationMode summarizationMode = ScoringSummarizationMode.MAX; @@ -32,7 +31,7 @@ public void testMax(){ } @Test - public void testMean(){ + public void testMean() { List scoreList = Arrays.asList(-1.0, 1.0, 2.0, 6.0); final ScoringSummarizationMode summarizationMode = ScoringSummarizationMode.MEAN; @@ -40,7 +39,7 @@ public void testMean(){ } @Test - public void testPositiveMean(){ + public void testPositiveMean() { List scoreList = Arrays.asList(-1.0, 1.0, 2.0, 6.0); final ScoringSummarizationMode summarizationMode = ScoringSummarizationMode.POSITIVE_MEAN; diff --git a/flink-cyber/flink-alert-scoring-api/src/test/java/com/cloudera/cyber/scoring/SerializationTests.java b/flink-cyber/flink-alert-scoring-api/src/test/java/com/cloudera/cyber/scoring/SerializationTests.java index e1c29b8f..b5254d7d 100644 --- a/flink-cyber/flink-alert-scoring-api/src/test/java/com/cloudera/cyber/scoring/SerializationTests.java +++ b/flink-cyber/flink-alert-scoring-api/src/test/java/com/cloudera/cyber/scoring/SerializationTests.java @@ -1,18 +1,26 @@ /* * Copyright 2020 - 2022 Cloudera. All Rights Reserved. * - * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file + * except in compliance with the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0. * - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. Refer to the License for the specific permissions and + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. Refer to the License for the specific permissions and * limitations governing your use of the file. */ package com.cloudera.cyber.scoring; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + import com.cloudera.cyber.rules.RuleType; +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; import org.apache.avro.Schema; import org.apache.avro.reflect.ReflectData; import org.apache.avro.specific.SpecificRecordBase; @@ -23,15 +31,6 @@ import org.junit.Ignore; import org.junit.Test; -import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.UUID; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertThat; - public class SerializationTests { @Test @@ -45,16 +44,16 @@ public void testScore() throws IOException { @Ignore public void testScoringRule() throws IOException { ScoringRule test = ScoringRule.builder() - .id(UUID.randomUUID().toString()) - .enabled(true) - .name("test") - .order(1) - .ruleScript("test()") - .tsStart(Instant.now()) - .tsEnd(Instant.now().plus(5, ChronoUnit.MINUTES)) - .type(RuleType.JS) - .version(0) - .build(); + .id(UUID.randomUUID().toString()) + .enabled(true) + .name("test") + .order(1) + .ruleScript("test()") + .tsStart(Instant.now()) + .tsEnd(Instant.now().plus(5, ChronoUnit.MINUTES)) + .type(RuleType.JS) + .version(0) + .build(); ScoringRule out = test(test); assertThat(out, equalTo(test)); diff --git a/flink-cyber/flink-alert-scoring/pom.xml b/flink-cyber/flink-alert-scoring/pom.xml index 043b44cf..be0b9b7a 100644 --- a/flink-cyber/flink-alert-scoring/pom.xml +++ b/flink-cyber/flink-alert-scoring/pom.xml @@ -140,7 +140,8 @@ - + com.cloudera.cyber.scoring.ScoringJobKafka diff --git a/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringJob.java b/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringJob.java index def97f28..ec49dbf1 100644 --- a/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringJob.java +++ b/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringJob.java @@ -15,6 +15,7 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.rules.RulesForm; +import java.util.List; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.state.MapStateDescriptor; @@ -29,16 +30,15 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.OutputTag; -import java.util.List; - /** - * A job that will score alerts based on a set of dynamic rules stored in the state engine + * A job that will score alerts based on a set of dynamic rules stored in the state engine. */ public abstract class ScoringJob { private static final String RULE_STATE_TAG = "rules"; private static final String RULE_COMMANDS_TAG = "rules-sink"; - public static final OutputTag COMMAND_RESULT_OUTPUT_TAG = new OutputTag<>(RULE_COMMANDS_TAG, TypeInformation.of(ScoringRuleCommandResult.class)); + public static final OutputTag COMMAND_RESULT_OUTPUT_TAG = + new OutputTag<>(RULE_COMMANDS_TAG, TypeInformation.of(ScoringRuleCommandResult.class)); protected StreamExecutionEnvironment createPipeline(final ParameterTool params) { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -48,9 +48,10 @@ protected StreamExecutionEnvironment createPipeline(final ParameterTool params) final DataStream ruleCommands = createRulesSource(env, params); final DataStream data = createSource(env, params); - final SingleOutputStreamOperator results = score(data, ruleCommands, Descriptors.rulesResultSink, Descriptors.rulesState, params) - .name("Process Rules") - .uid("process-rules"); + final SingleOutputStreamOperator results = + score(data, ruleCommands, Descriptors.rulesResultSink, Descriptors.rulesState, params) + .name("Process Rules") + .uid("process-rules"); writeResults(params, results); // output the results of the rules commands @@ -59,20 +60,24 @@ protected StreamExecutionEnvironment createPipeline(final ParameterTool params) return env; } - public static SingleOutputStreamOperator enrich(DataStream source, DataStream ruleCommands, ParameterTool params) { - return score(source, ruleCommands, Descriptors.rulesResultSink, Descriptors.rulesState, params).name("Process Rules") - .uid("process-rules"); + public static SingleOutputStreamOperator enrich(DataStream source, + DataStream ruleCommands, + ParameterTool params) { + return score(source, ruleCommands, Descriptors.rulesResultSink, Descriptors.rulesState, params).name( + "Process Rules") + .uid("process-rules"); } @VisibleForTesting static class Descriptors { public static final MapStateDescriptor> rulesState = - new MapStateDescriptor<>( - RULE_STATE_TAG, TypeInformation.of(RulesForm.class), Types.LIST(new AvroTypeInfo<>(ScoringRule.class))); + new MapStateDescriptor<>( + RULE_STATE_TAG, TypeInformation.of(RulesForm.class), + Types.LIST(new AvroTypeInfo<>(ScoringRule.class))); public static final OutputTag rulesResultSink = - new OutputTag(RULE_COMMANDS_TAG) { - }; + new OutputTag(RULE_COMMANDS_TAG) { + }; public static ListStateDescriptor activeOrderedRules; } @@ -82,14 +87,15 @@ public static SingleOutputStreamOperator score(DataStream> rulesState, ParameterTool params) { BroadcastStream rulesStream = ruleCommands.broadcast(rulesState); - return data.connect(rulesStream).process(new ScoringProcessFunction(rulesResultSink, rulesState,params)); + return data.connect(rulesStream).process(new ScoringProcessFunction(rulesResultSink, rulesState, params)); } protected abstract void writeResults(ParameterTool params, DataStream results); protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); - protected abstract DataStream createRulesSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract DataStream createRulesSource(StreamExecutionEnvironment env, + ParameterTool params); protected abstract void writeQueryResult(ParameterTool params, DataStream results); } diff --git a/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringJobKafka.java b/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringJobKafka.java index 1112d5fa..619c431c 100644 --- a/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringJobKafka.java +++ b/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringJobKafka.java @@ -12,9 +12,12 @@ package com.cloudera.cyber.scoring; +import static com.cloudera.cyber.flink.FlinkUtils.createKafkaSource; + import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; +import java.util.regex.Pattern; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.kafka.sink.KafkaSink; @@ -23,10 +26,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.regex.Pattern; - -import static com.cloudera.cyber.flink.FlinkUtils.createKafkaSource; - public class ScoringJobKafka extends ScoringJob { private static final String SCORING_GROUP_ID = "scoring"; @@ -37,14 +36,14 @@ public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); ParameterTool params = Utils.getParamToolsFromProperties(args); FlinkUtils.executeEnv(new ScoringJobKafka() - .createPipeline(params), "Flink Scoring", params); + .createPipeline(params), "Flink Scoring", params); } @Override protected void writeResults(ParameterTool params, DataStream results) { KafkaSink sink = new FlinkUtils<>(ScoredMessage.class).createKafkaSink( - params.getRequired("topic.output"), SCORING_GROUP_ID, - params); + params.getRequired("topic.output"), SCORING_GROUP_ID, + params); results.sinkTo(sink).name("Kafka Results").uid("kafka.results"); } @@ -54,29 +53,32 @@ protected DataStream createSource(StreamExecutionEnvironment env, Param Pattern inputTopic = Pattern.compile(params.getRequired("topic.pattern")); return env.fromSource(createKafkaSource(inputTopic, - params, - SCORING_GROUP_ID), WatermarkStrategy.noWatermarks(), "Kafka Source") - .uid("kafka.input"); + params, + SCORING_GROUP_ID), WatermarkStrategy.noWatermarks(), "Kafka Source") + .uid("kafka.input"); } @Override protected DataStream createRulesSource(StreamExecutionEnvironment env, ParameterTool params) { String topic = params.getRequired("query.input.topic"); - KafkaSource source = new FlinkUtils<>(ScoringRuleCommand.class).createKafkaGenericSource(topic, params, SCORING_RULES_GROUP_ID); + KafkaSource source = + new FlinkUtils<>(ScoringRuleCommand.class).createKafkaGenericSource(topic, params, + SCORING_RULES_GROUP_ID); return env.fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Rule Source") - .uid("kafka.input.rule.command"); + .uid("kafka.input.rule.command"); } @Override protected void writeQueryResult(ParameterTool params, DataStream results) { String topic = params.getRequired("query.output.topic"); - KafkaSink sink = new FlinkUtils<>(ScoringRuleCommandResult.class).createKafkaSink(topic, SCORING_RULES_GROUP_ID, params); + KafkaSink sink = + new FlinkUtils<>(ScoringRuleCommandResult.class).createKafkaSink(topic, SCORING_RULES_GROUP_ID, params); results.sinkTo(sink).name("Kafka Rule Command Results").uid("kafka.output.rule.command.results"); } - public static String SCORING_SUMMATION_NAME(){ + public static String scoringSummationName() { return SCORING_SUMMARIZATION; } } diff --git a/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringProcessFunction.java b/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringProcessFunction.java index d0f0a015..9c25f1d1 100644 --- a/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringProcessFunction.java +++ b/flink-cyber/flink-alert-scoring/src/main/java/com/cloudera/cyber/scoring/ScoringProcessFunction.java @@ -12,9 +12,17 @@ package com.cloudera.cyber.scoring; +import static com.cloudera.cyber.scoring.ScoringRule.RESULT_REASON; +import static com.cloudera.cyber.scoring.ScoringRule.RESULT_SCORE; + import com.cloudera.cyber.Message; import com.cloudera.cyber.rules.DynamicRuleProcessFunction; import com.cloudera.cyber.rules.RulesForm; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.state.MapStateDescriptor; import org.apache.flink.api.java.utils.ParameterTool; @@ -23,28 +31,26 @@ import org.apache.flink.util.OutputTag; import org.apache.flink.util.Preconditions; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -import static com.cloudera.cyber.scoring.ScoringRule.RESULT_REASON; -import static com.cloudera.cyber.scoring.ScoringRule.RESULT_SCORE; - @Slf4j -public class ScoringProcessFunction extends DynamicRuleProcessFunction { +public class ScoringProcessFunction + extends DynamicRuleProcessFunction { private final String scoringSummarizationModeName; private transient ScoringSummarizationMode scoringSummarizationMode; - public static final String ILLEGAL_SUMMARIZATION_METHOD_ERROR_MESSAGE = "Summarization method '%s' is not a legal value '%s'"; + public static final String ILLEGAL_SUMMARIZATION_METHOD_ERROR_MESSAGE = + "Summarization method '%s' is not a legal value '%s'"; - public ScoringProcessFunction(OutputTag rulesResultSink, MapStateDescriptor> rulesDescriptor, ParameterTool params) { + public ScoringProcessFunction(OutputTag rulesResultSink, + MapStateDescriptor> rulesDescriptor, + ParameterTool params) { super(rulesResultSink, rulesDescriptor); - this.scoringSummarizationModeName = params.get(ScoringJobKafka.SCORING_SUMMATION_NAME(), ScoringSummarizationMode.DEFAULT().name()); + this.scoringSummarizationModeName = + params.get(ScoringJobKafka.scoringSummationName(), ScoringSummarizationMode.defaultValue().name()); - Preconditions.checkArgument(ScoringSummarizationMode.isLegalSummarizationMode(this.scoringSummarizationModeName), - String.format(ILLEGAL_SUMMARIZATION_METHOD_ERROR_MESSAGE, this.scoringSummarizationModeName, ScoringSummarizationMode.legalSummarizationModes())); + Preconditions.checkArgument( + ScoringSummarizationMode.isLegalSummarizationMode(this.scoringSummarizationModeName), + String.format(ILLEGAL_SUMMARIZATION_METHOD_ERROR_MESSAGE, this.scoringSummarizationModeName, + ScoringSummarizationMode.legalSummarizationModes())); } @@ -60,24 +66,24 @@ protected void processMessages(Message message, Collector collect List scores = Collections.emptyList(); if (rules != null && !rules.isEmpty()) { scores = rules.stream() - .map(r -> { - Map results = r.apply(message); - if (results != null) { - Object score = results.get(RESULT_SCORE); - if (!(score instanceof Double)){ - score = new Double(String.valueOf(score)); - } - return Scores.builder() - .ruleId(r.getId()) - .score((Double) score) - .reason((String) results.get(RESULT_REASON)) - .build(); - } else { - return null; - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .map(r -> { + Map results = r.apply(message); + if (results != null) { + Object score = results.get(RESULT_SCORE); + if (!(score instanceof Double)) { + score = new Double(String.valueOf(score)); + } + return Scores.builder() + .ruleId(r.getId()) + .score((Double) score) + .reason((String) results.get(RESULT_REASON)) + .build(); + } else { + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); //TODO remove if not needed. Was unused, which resulted to additional calculations for each message //DoubleSummaryStatistics scoreSummary = scores.stream().collect(Collectors.summarizingDouble(s -> s.getScore())); @@ -89,13 +95,14 @@ protected void processMessages(Message message, Collector collect collector.collect(scoreMessage(message, scores, scoringSummarizationMode)); } - public static ScoredMessage scoreMessage(Message message, List scores, ScoringSummarizationMode summarizationMode) { + public static ScoredMessage scoreMessage(Message message, List scores, + ScoringSummarizationMode summarizationMode) { final List scoreValues = scores.stream().map(Scores::getScore).collect(Collectors.toList()); return ScoredMessage.builder() - .message(message) - .cyberScoresDetails(scores) - .cyberScore(summarizationMode.calculateScore(scoreValues)) - .build(); + .message(message) + .cyberScoresDetails(scores) + .cyberScore(summarizationMode.calculateScore(scoreValues)) + .build(); } } diff --git a/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TestScoringJob.java b/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TestScoringJob.java index adef0a12..8cde97c4 100644 --- a/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TestScoringJob.java +++ b/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TestScoringJob.java @@ -12,9 +12,28 @@ package com.cloudera.cyber.scoring; +import static com.cloudera.cyber.flink.FlinkUtils.PARAMS_PARALLELISM; +import static com.cloudera.cyber.rules.DynamicRuleCommandType.DELETE; +import static com.cloudera.cyber.rules.DynamicRuleCommandType.LIST; +import static com.cloudera.cyber.rules.DynamicRuleCommandType.UPSERT; +import static com.cloudera.cyber.rules.RuleType.JS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + import com.cloudera.cyber.Message; import com.cloudera.cyber.TestUtils; import com.cloudera.cyber.rules.DynamicRuleCommandResult; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeoutException; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.utils.ParameterTool; @@ -25,17 +44,6 @@ import org.apache.flink.test.util.ManualSource; import org.junit.Test; -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.TimeoutException; - -import static com.cloudera.cyber.flink.FlinkUtils.PARAMS_PARALLELISM; -import static com.cloudera.cyber.rules.DynamicRuleCommandType.*; -import static com.cloudera.cyber.rules.RuleType.JS; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - @Slf4j public class TestScoringJob extends ScoringJob { @@ -57,23 +65,23 @@ public void testPipeline() throws Exception { String ruleId = UUID.randomUUID().toString(); String ruleName = "test-rule"; ScoringRule rule = ScoringRule.builder() - .id(ruleId) - .name(ruleName) - .tsStart(Instant.now()) - .tsEnd(Instant.now().plus(Duration.ofMinutes(5))) - .order(0) - .type(JS) - .ruleScript("return { score: 1.0, reason: message.test }") - .enabled(true) - .build(); + .id(ruleId) + .name(ruleName) + .tsStart(Instant.now()) + .tsEnd(Instant.now().plus(Duration.ofMinutes(5))) + .order(0) + .type(JS) + .ruleScript("return { score: 1.0, reason: message.test }") + .enabled(true) + .build(); sendRule(ScoringRuleCommand.builder() - .type(UPSERT) - .rule(rule) - .ts(1L) - .id(UUID.randomUUID().toString()) - .headers(Collections.emptyMap()) - .build()); + .type(UPSERT) + .rule(rule) + .ts(1L) + .id(UUID.randomUUID().toString()) + .headers(Collections.emptyMap()) + .build()); verifySuccessfulResponse(rule); @@ -81,17 +89,17 @@ public void testPipeline() throws Exception { long scoredMessageTs = 100L; String expectedReason = "test-value"; sendRecord(Message.builder() - .ts(scoredMessageTs) - .extensions(Collections.singletonMap("test", expectedReason)) - .originalSource(TestUtils.source("test", 0, 0)) - .source("test") - .build()); + .ts(scoredMessageTs) + .extensions(Collections.singletonMap("test", expectedReason)) + .originalSource(TestUtils.source("test", 0, 0)) + .source("test") + .build()); source.sendWatermark(100L); querySource.sendWatermark(100L); sendRule(ScoringRuleCommand.builder() - .type(LIST).ts(900).id(UUID.randomUUID().toString()).headers(Collections.emptyMap()).build()); + .type(LIST).ts(900).id(UUID.randomUUID().toString()).headers(Collections.emptyMap()).build()); source.sendWatermark(1000L); querySource.sendWatermark(1000L); @@ -99,10 +107,10 @@ public void testPipeline() throws Exception { verifyListResult(Collections.singletonList(rule)); sendRule(ScoringRuleCommand.builder() - .type(DELETE) - .ts(1000L) - .id(UUID.randomUUID().toString()) - .ruleId(ruleId).headers(Collections.emptyMap()).build()); + .type(DELETE) + .ts(1000L) + .id(UUID.randomUUID().toString()) + .ruleId(ruleId).headers(Collections.emptyMap()).build()); source.sendWatermark(1500L); querySource.sendWatermark(1500L); @@ -111,11 +119,11 @@ public void testPipeline() throws Exception { long unscoredMessageTs = 2000L; sendRecord(Message.builder() - .extensions(Collections.singletonMap("test", expectedReason)) - .ts(unscoredMessageTs) - .originalSource(TestUtils.source("test", 0, 0)) - .source("test") - .build()); + .extensions(Collections.singletonMap("test", expectedReason)) + .ts(unscoredMessageTs) + .originalSource(TestUtils.source("test", 0, 0)) + .source("test") + .build()); source.sendWatermark(3000L); querySource.sendWatermark(3000L); @@ -123,7 +131,8 @@ public void testPipeline() throws Exception { JobTester.stopTest(); List scoredMessages = collectScoredMessages(); - verifyMessageScores(scoredMessages, scoredMessageTs, 1.0, Collections.singletonList(Scores.builder().ruleId(ruleId).score(1.0).reason(expectedReason).build())); + verifyMessageScores(scoredMessages, scoredMessageTs, 1.0, + Collections.singletonList(Scores.builder().ruleId(ruleId).score(1.0).reason(expectedReason).build())); verifyMessageScores(scoredMessages, unscoredMessageTs, 0.0, Collections.emptyList()); } @@ -131,7 +140,7 @@ public void testPipeline() throws Exception { public void testPipelineCyberScore() throws Exception { Map props = new HashMap() {{ put(PARAMS_PARALLELISM, "1"); - put(ScoringJobKafka.SCORING_SUMMATION_NAME(), ScoringSummarizationMode.SUM.name()); + put(ScoringJobKafka.scoringSummationName(), ScoringSummarizationMode.SUM.name()); }}; JobTester.startTest(createPipeline(ParameterTool.fromMap(props))); @@ -140,41 +149,41 @@ public void testPipelineCyberScore() throws Exception { String ruleId2 = UUID.randomUUID().toString(); String ruleName2 = "test-rule2"; ScoringRule rule1 = ScoringRule.builder() - .id(ruleId1) - .name(ruleName1) - .tsStart(Instant.now()) - .tsEnd(Instant.now().plus(Duration.ofMinutes(5))) - .order(0) - .type(JS) - .ruleScript("return { score: 1.0, reason: message.test }") - .enabled(true) - .build(); + .id(ruleId1) + .name(ruleName1) + .tsStart(Instant.now()) + .tsEnd(Instant.now().plus(Duration.ofMinutes(5))) + .order(0) + .type(JS) + .ruleScript("return { score: 1.0, reason: message.test }") + .enabled(true) + .build(); ScoringRule rule2 = ScoringRule.builder() - .id(ruleId2) - .name(ruleName2) - .tsStart(Instant.now()) - .tsEnd(Instant.now().plus(Duration.ofMinutes(5))) - .order(1) - .type(JS) - .ruleScript("return { score: 2.0, reason: message.test }") - .enabled(true) - .build(); + .id(ruleId2) + .name(ruleName2) + .tsStart(Instant.now()) + .tsEnd(Instant.now().plus(Duration.ofMinutes(5))) + .order(1) + .type(JS) + .ruleScript("return { score: 2.0, reason: message.test }") + .enabled(true) + .build(); sendRule(ScoringRuleCommand.builder() - .type(UPSERT) - .rule(rule1) - .ts(1L) - .id(UUID.randomUUID().toString()) - .headers(Collections.emptyMap()) - .build()); + .type(UPSERT) + .rule(rule1) + .ts(1L) + .id(UUID.randomUUID().toString()) + .headers(Collections.emptyMap()) + .build()); sendRule(ScoringRuleCommand.builder() - .type(UPSERT) - .rule(rule2) - .ts(2L) - .id(UUID.randomUUID().toString()) - .headers(Collections.emptyMap()) - .build()); + .type(UPSERT) + .rule(rule2) + .ts(2L) + .id(UUID.randomUUID().toString()) + .headers(Collections.emptyMap()) + .build()); verifySuccessfulResponse(rule1); verifySuccessfulResponse(rule2); @@ -183,17 +192,17 @@ public void testPipelineCyberScore() throws Exception { long scoredMessageTs = 100L; String expectedReason = "test-value"; sendRecord(Message.builder() - .ts(scoredMessageTs) - .extensions(Collections.singletonMap("test", expectedReason)) - .originalSource(TestUtils.source("test", 0, 0)) - .source("test") - .build()); + .ts(scoredMessageTs) + .extensions(Collections.singletonMap("test", expectedReason)) + .originalSource(TestUtils.source("test", 0, 0)) + .source("test") + .build()); source.sendWatermark(100L); querySource.sendWatermark(100L); sendRule(ScoringRuleCommand.builder() - .type(LIST).ts(900).id(UUID.randomUUID().toString()).headers(Collections.emptyMap()).build()); + .type(LIST).ts(900).id(UUID.randomUUID().toString()).headers(Collections.emptyMap()).build()); source.sendWatermark(1000L); querySource.sendWatermark(1000L); @@ -201,15 +210,15 @@ public void testPipelineCyberScore() throws Exception { verifyListResult(Arrays.asList(rule1, rule2)); sendRule(ScoringRuleCommand.builder() - .type(DELETE) - .ts(1000L) - .id(UUID.randomUUID().toString()) - .ruleId(ruleId1).headers(Collections.emptyMap()).build()); + .type(DELETE) + .ts(1000L) + .id(UUID.randomUUID().toString()) + .ruleId(ruleId1).headers(Collections.emptyMap()).build()); sendRule(ScoringRuleCommand.builder() - .type(DELETE) - .ts(1000L) - .id(UUID.randomUUID().toString()) - .ruleId(ruleId2).headers(Collections.emptyMap()).build()); + .type(DELETE) + .ts(1000L) + .id(UUID.randomUUID().toString()) + .ruleId(ruleId2).headers(Collections.emptyMap()).build()); source.sendWatermark(1500L); querySource.sendWatermark(1500L); @@ -219,11 +228,11 @@ public void testPipelineCyberScore() throws Exception { long unscoredMessageTs = 2000L; sendRecord(Message.builder() - .extensions(Collections.singletonMap("test", expectedReason)) - .ts(unscoredMessageTs) - .originalSource(TestUtils.source("test", 0, 0)) - .source("test") - .build()); + .extensions(Collections.singletonMap("test", expectedReason)) + .ts(unscoredMessageTs) + .originalSource(TestUtils.source("test", 0, 0)) + .source("test") + .build()); source.sendWatermark(3000L); querySource.sendWatermark(3000L); @@ -232,19 +241,20 @@ public void testPipelineCyberScore() throws Exception { List scoredMessages = collectScoredMessages(); verifyMessageScores(scoredMessages, scoredMessageTs, 3.0, - Arrays.asList( - Scores.builder().ruleId(ruleId1).score(1.0).reason(expectedReason).build(), - Scores.builder().ruleId(ruleId2).score(2.0).reason(expectedReason).build())); + Arrays.asList( + Scores.builder().ruleId(ruleId1).score(1.0).reason(expectedReason).build(), + Scores.builder().ruleId(ruleId2).score(2.0).reason(expectedReason).build())); verifyMessageScores(scoredMessages, unscoredMessageTs, 0.0, Collections.emptyList()); } - void verifyMessageScores(List scoredMessages, long originalMessageTimestamp, double expectedCyberScore, List expectedScores) { - final Optional scoredMessage = scoredMessages.stream(). - filter((m) -> (m.getTs() == originalMessageTimestamp)) - .findFirst(); + void verifyMessageScores(List scoredMessages, long originalMessageTimestamp, + double expectedCyberScore, List expectedScores) { + final Optional scoredMessage = scoredMessages.stream() + .filter((m) -> (m.getTs() == originalMessageTimestamp)) + .findFirst(); List actualScores = scoredMessage - .map(ScoredMessage::getCyberScoresDetails) - .orElse(Collections.emptyList()); + .map(ScoredMessage::getCyberScoresDetails) + .orElse(Collections.emptyList()); assertThat("message scores match", expectedScores, equalTo(actualScores)); assertThat(scoredMessage.isPresent(), equalTo(true)); assertThat("message scores match", scoredMessage.get().getCyberScore(), equalTo(expectedCyberScore)); diff --git a/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TestScoringRulesProcessFunction.java b/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TestScoringRulesProcessFunction.java index 8b2626fd..8236cbce 100644 --- a/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TestScoringRulesProcessFunction.java +++ b/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TestScoringRulesProcessFunction.java @@ -12,11 +12,25 @@ package com.cloudera.cyber.scoring; +import static com.cloudera.cyber.flink.FlinkUtils.PARAMS_PARALLELISM; +import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.cloudera.cyber.Message; import com.cloudera.cyber.TestUtils; import com.cloudera.cyber.rules.DynamicRuleCommandType; import com.cloudera.cyber.rules.RuleType; import com.google.common.collect.ImmutableMap; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.BroadcastOperatorTestHarness; @@ -24,31 +38,25 @@ import org.junit.Assert; import org.junit.Test; -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; - -import static com.cloudera.cyber.flink.FlinkUtils.PARAMS_PARALLELISM; -import static org.junit.Assert.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class TestScoringRulesProcessFunction { private static final String FIRST_RULE_EXTENSION_KEY = "test.key.1"; private static final String SECOND_RULE_EXTENSION_KEY = "test.key.2"; private static final String MATCH_REASON = "key match"; private static final String NO_MATCH_REASON = "no match"; - private static final String RULE_SCRIPT_FORMAT = "if (message.containsKey(\"%s\")) {\n" + - " return {score: %f, reason: '"+ MATCH_REASON + "'};\n" + - " } else {\n" + - " return {score: 0.0, reason: '" + NO_MATCH_REASON + "'};\n" + - " }"; - - private static final String RULE_SCRIPT_FORMAT_BROKEN = "if 111(message.containsKey(\"%s\")) {\n" + - " return {score: %f, reason: '"+ MATCH_REASON + "'};\n" + - " } else {\n" + - " return {score: 0.0, reason: '" + NO_MATCH_REASON + "'};\n" + - " }"; + private static final String RULE_SCRIPT_FORMAT = "if (message.containsKey(\"%s\")) {\n" + + " return {score: %f, reason: '" + MATCH_REASON + "'};\n" + + " } else {\n" + + " return {score: 0.0, reason: '" + NO_MATCH_REASON + + "'};\n" + + " }"; + + private static final String RULE_SCRIPT_FORMAT_BROKEN = "if 111(message.containsKey(\"%s\")) {\n" + + " return {score: %f, reason: '" + MATCH_REASON + + "'};\n" + + " } else {\n" + + " return {score: 0.0, reason: '" + NO_MATCH_REASON + + "'};\n" + + " }"; @Test public void testUpsertRule() throws Exception { @@ -56,26 +64,39 @@ public void testUpsertRule() throws Exception { // insert a new rule double expectedScore = 90.0; - ScoringRule ruleV0 = buildScoringRule(true, 1, "first ruleV0", String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, expectedScore)); - ConcurrentLinkedQueue> scoringRuleCommandResults = verifyUpsert(ruleV0, false, harness, null); + ScoringRule ruleV0 = buildScoringRule(true, 1, "first ruleV0", + String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, expectedScore)); + ConcurrentLinkedQueue> scoringRuleCommandResults = + verifyUpsert(ruleV0, false, harness, null); // send some messages to trigger the rule List expectedScoredMessages = new ArrayList<>(); - sendMessage(expectedScoredMessages, new HashMap() {{ put(FIRST_RULE_EXTENSION_KEY, "key value");}}, - Collections.singletonList(Scores.builder().ruleId(ruleV0.getId()).score(expectedScore).reason(MATCH_REASON).build()), harness); + sendMessage(expectedScoredMessages, new HashMap() {{ + put(FIRST_RULE_EXTENSION_KEY, "key value"); + }}, + Collections.singletonList( + Scores.builder().ruleId(ruleV0.getId()).score(expectedScore).reason(MATCH_REASON).build()), + harness); expectedScoredMessages.get(0).setCyberScore(expectedScore); sendMessage(expectedScoredMessages, Collections.emptyMap(), - Collections.singletonList(Scores.builder().ruleId(ruleV0.getId()).score(0.0).reason(NO_MATCH_REASON).build()), harness); + Collections.singletonList( + Scores.builder().ruleId(ruleV0.getId()).score(0.0).reason(NO_MATCH_REASON).build()), harness); expectedScoredMessages.get(1).setCyberScore(0.0); // update the rule script expectedScore = 70.0; - ScoringRule ruleV1 = ruleV0.toBuilder().ruleScript(String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, expectedScore)).build(); + ScoringRule ruleV1 = + ruleV0.toBuilder().ruleScript(String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, expectedScore)) + .build(); verifyUpsert(ruleV1, true, harness, scoringRuleCommandResults); // trigger the rule again and make sure the results match the new version - sendMessage(expectedScoredMessages, new HashMap() {{ put(FIRST_RULE_EXTENSION_KEY, "key value");}}, - Collections.singletonList(Scores.builder().ruleId(ruleV0.getId()).score(expectedScore).reason(MATCH_REASON).build()), harness); + sendMessage(expectedScoredMessages, new HashMap() {{ + put(FIRST_RULE_EXTENSION_KEY, "key value"); + }}, + Collections.singletonList( + Scores.builder().ruleId(ruleV0.getId()).score(expectedScore).reason(MATCH_REASON).build()), + harness); expectedScoredMessages.get(2).setCyberScore(expectedScore); assertEquals(expectedScoredMessages, harness.extractOutputValues()); @@ -88,7 +109,8 @@ public void testUpsertRuleFailed() throws Exception { // insert a new rule double expectedScore = 90.0; - ScoringRule ruleV0 = buildScoringRule(true, 1, "first ruleV0", String.format(RULE_SCRIPT_FORMAT_BROKEN, FIRST_RULE_EXTENSION_KEY, expectedScore)); + ScoringRule ruleV0 = buildScoringRule(true, 1, "first ruleV0", + String.format(RULE_SCRIPT_FORMAT_BROKEN, FIRST_RULE_EXTENSION_KEY, expectedScore)); verifyUpsert(ruleV0, false, harness, null, false); } @@ -99,19 +121,25 @@ public void testEnableDisableRule() throws Exception { // insert a new rule double firstExpectedScore = 90.0; - ScoringRule firstRule = buildScoringRule(true, 1, "first ruleV0", String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, firstExpectedScore)); - ConcurrentLinkedQueue> scoringRuleCommandResults = verifyUpsert(firstRule, false, harness, null); + ScoringRule firstRule = buildScoringRule(true, 1, "first ruleV0", + String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, firstExpectedScore)); + ConcurrentLinkedQueue> scoringRuleCommandResults = + verifyUpsert(firstRule, false, harness, null); double secondExpectedScore = 50.0; - ScoringRule secondRule = buildScoringRule(true, 2, "second ruleV0", String.format(RULE_SCRIPT_FORMAT,SECOND_RULE_EXTENSION_KEY, secondExpectedScore )); + ScoringRule secondRule = buildScoringRule(true, 2, "second ruleV0", + String.format(RULE_SCRIPT_FORMAT, SECOND_RULE_EXTENSION_KEY, secondExpectedScore)); verifyUpsert(secondRule, false, harness, scoringRuleCommandResults); // send a message to // trigger the rule - Map extensionsToTriggerRule = new HashMap() {{ put(FIRST_RULE_EXTENSION_KEY, "key value");}}; + Map extensionsToTriggerRule = new HashMap() {{ + put(FIRST_RULE_EXTENSION_KEY, "key value"); + }}; List expectedScoredMessages = new ArrayList<>(); - List allScores=Arrays.asList(Scores.builder().ruleId(firstRule.getId()).score(firstExpectedScore).reason(MATCH_REASON).build(), - Scores.builder().ruleId(secondRule.getId()).score(0.0).reason(NO_MATCH_REASON).build()); + List allScores = Arrays.asList( + Scores.builder().ruleId(firstRule.getId()).score(firstExpectedScore).reason(MATCH_REASON).build(), + Scores.builder().ruleId(secondRule.getId()).score(0.0).reason(NO_MATCH_REASON).build()); sendMessage(expectedScoredMessages, extensionsToTriggerRule, allScores, harness); // disable the rule @@ -119,7 +147,7 @@ public void testEnableDisableRule() throws Exception { // trigger the rule again - no scores should be returned sendMessage(expectedScoredMessages, extensionsToTriggerRule, - Collections.singletonList(allScores.get(1)), harness); + Collections.singletonList(allScores.get(1)), harness); // enable the rule verifyEnabledDisabled(firstRule, DynamicRuleCommandType.ENABLE, harness, scoringRuleCommandResults); @@ -135,29 +163,36 @@ public void testEnableDisableRule() throws Exception { @Test public void testIllegalScoreSummarization() { - String badScoringMethod = "BAD_SCORING"; - Map props = ImmutableMap.of(PARAMS_PARALLELISM, "1", - ScoringJobKafka.SCORING_SUMMATION_NAME(), badScoringMethod); - - assertThrows(IllegalArgumentException.class, () -> new ScoringProcessFunction(ScoringJob.Descriptors.rulesResultSink, ScoringJob.Descriptors.rulesState, ParameterTool.fromMap(props)), - String.format(ScoringProcessFunction.ILLEGAL_SUMMARIZATION_METHOD_ERROR_MESSAGE, badScoringMethod, ScoringSummarizationMode.legalSummarizationModes())); - } + String badScoringMethod = "BAD_SCORING"; + Map props = ImmutableMap.of(PARAMS_PARALLELISM, "1", + ScoringJobKafka.scoringSummationName(), badScoringMethod); + + assertThrows(IllegalArgumentException.class, + () -> new ScoringProcessFunction(ScoringJob.Descriptors.rulesResultSink, + ScoringJob.Descriptors.rulesState, ParameterTool.fromMap(props)), + String.format(ScoringProcessFunction.ILLEGAL_SUMMARIZATION_METHOD_ERROR_MESSAGE, badScoringMethod, + ScoringSummarizationMode.legalSummarizationModes())); + } @Test public void testScoreMessages() { - testScoreMessage(TestUtils.createMessage(), Collections.emptyList(), 0.0, ScoringSummarizationMode.DEFAULT()); + testScoreMessage(TestUtils.createMessage(), Collections.emptyList(), 0.0, + ScoringSummarizationMode.defaultValue()); List scores = Arrays.asList(Scores.builder().ruleId("1").reason("reason 1").score(50.0).build(), - Scores.builder().ruleId("2").reason("reason 2").score(50.0).build()); + Scores.builder().ruleId("2").reason("reason 2").score(50.0).build()); testScoreMessage(TestUtils.createMessage(), scores, 100.00, ScoringSummarizationMode.SUM); testScoreMessage(TestUtils.createMessage(), scores, 50.00, ScoringSummarizationMode.MEAN); - testScoreMessage(TestUtils.createMessage(), Collections.singletonList(Scores.builder().ruleId("1").reason("reason 1").score(50.0).build()), 50.0, ScoringSummarizationMode.MAX); + testScoreMessage(TestUtils.createMessage(), + Collections.singletonList(Scores.builder().ruleId("1").reason("reason 1").score(50.0).build()), 50.0, + ScoringSummarizationMode.MAX); } - private void testScoreMessage(Message baseMessage, List scoreDetails, double expectedScore, ScoringSummarizationMode mode) { - ScoredMessage scoredMessage = ScoringProcessFunction.scoreMessage(baseMessage, scoreDetails, mode); + private void testScoreMessage(Message baseMessage, List scoreDetails, double expectedScore, + ScoringSummarizationMode mode) { + ScoredMessage scoredMessage = ScoringProcessFunction.scoreMessage(baseMessage, scoreDetails, mode); Assert.assertEquals(baseMessage, scoredMessage.getMessage()); Assert.assertEquals(scoreDetails, scoredMessage.getCyberScoresDetails()); Assert.assertEquals(expectedScore, scoredMessage.getCyberScore(), 0.01); @@ -169,27 +204,32 @@ public void testDeleteRule() throws Exception { // insert a new rule double expectedScore = 90.0; - ScoringRule rule = buildScoringRule(true, 1, "first ruleV0", String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, expectedScore)); - ConcurrentLinkedQueue> scoringRuleCommandResults = verifyUpsert(rule, false, harness, null); + ScoringRule rule = buildScoringRule(true, 1, "first ruleV0", + String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, expectedScore)); + ConcurrentLinkedQueue> scoringRuleCommandResults = + verifyUpsert(rule, false, harness, null); // send a message to trigger the rule - Map extensionsToTriggerRule = new HashMap() {{ put(FIRST_RULE_EXTENSION_KEY, "key value");}}; + Map extensionsToTriggerRule = new HashMap() {{ + put(FIRST_RULE_EXTENSION_KEY, "key value"); + }}; List expectedScoredMessages = new ArrayList<>(); sendMessage(expectedScoredMessages, extensionsToTriggerRule, - Collections.singletonList(Scores.builder().ruleId(rule.getId()).score(expectedScore).reason(MATCH_REASON).build()), harness); + Collections.singletonList( + Scores.builder().ruleId(rule.getId()).score(expectedScore).reason(MATCH_REASON).build()), harness); verifyDelete(rule, rule.getId(), harness, scoringRuleCommandResults); // trigger the rule again and make sure the deleted rule is not triggered sendMessage(expectedScoredMessages, extensionsToTriggerRule, - Collections.emptyList(), harness); + Collections.emptyList(), harness); // delete the rule again should be no op verifyDelete(null, rule.getId(), harness, scoringRuleCommandResults); // trigger the rule again and make sure the deleted rule is not triggered sendMessage(expectedScoredMessages, extensionsToTriggerRule, - Collections.emptyList(), harness); + Collections.emptyList(), harness); expectedScoredMessages.get(0).setCyberScore(expectedScore); expectedScoredMessages.get(1).setCyberScore(0.0); @@ -203,41 +243,54 @@ public void testGetListRule() throws Exception { // insert a new rule double expectedScore = 90.0; - ScoringRule rule = buildScoringRule(true, 1, "first ruleV0", String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, expectedScore)); + ScoringRule rule = buildScoringRule(true, 1, "first ruleV0", + String.format(RULE_SCRIPT_FORMAT, FIRST_RULE_EXTENSION_KEY, expectedScore)); // getting the rule before inserting returns null - ConcurrentLinkedQueue> scoringRuleCommandResults = verifyGet(null, rule.getId(), harness, null); + ConcurrentLinkedQueue> scoringRuleCommandResults = + verifyGet(null, rule.getId(), harness, null); verifyList(null, harness, scoringRuleCommandResults); // insert the rule verifyUpsert(rule, false, harness, scoringRuleCommandResults); // send a message to trigger the rule - Map extensionsToTriggerRule = new HashMap() {{ put(FIRST_RULE_EXTENSION_KEY, "key value");}}; + Map extensionsToTriggerRule = new HashMap() {{ + put(FIRST_RULE_EXTENSION_KEY, "key value"); + }}; List expectedScoredMessages = new ArrayList<>(); sendMessage(expectedScoredMessages, extensionsToTriggerRule, - Collections.singletonList(Scores.builder().ruleId(rule.getId()).score(expectedScore).reason(MATCH_REASON).build()), harness); + Collections.singletonList( + Scores.builder().ruleId(rule.getId()).score(expectedScore).reason(MATCH_REASON).build()), harness); verifyGet(rule, rule.getId(), harness, scoringRuleCommandResults); verifyList(rule, harness, scoringRuleCommandResults); // trigger the rule again - get should not change the rule set sendMessage(expectedScoredMessages, extensionsToTriggerRule, - Collections.singletonList(Scores.builder().ruleId(rule.getId()).score(expectedScore).reason(MATCH_REASON).build()), harness); + Collections.singletonList( + Scores.builder().ruleId(rule.getId()).score(expectedScore).reason(MATCH_REASON).build()), harness); expectedScoredMessages.forEach(sm -> sm.setCyberScore(expectedScore)); assertEquals(expectedScoredMessages, harness.extractOutputValues()); } - private void sendMessage(List expectedScoredMessages, Map extensions, List expectedScores, BroadcastOperatorTestHarness harness) throws Exception { + private void sendMessage(List expectedScoredMessages, Map extensions, + List expectedScores, + BroadcastOperatorTestHarness harness) + throws Exception { Message sentMessage = TestUtils.createMessage(extensions); harness.processElement(new StreamRecord<>(sentMessage)); - expectedScoredMessages.add(ScoringProcessFunction.scoreMessage(sentMessage, expectedScores, ScoringSummarizationMode.DEFAULT())); + expectedScoredMessages.add(ScoringProcessFunction.scoreMessage(sentMessage, expectedScores, + ScoringSummarizationMode.defaultValue())); } - private BroadcastOperatorTestHarness createTestHarness() throws Exception { + private BroadcastOperatorTestHarness createTestHarness() + throws Exception { BroadcastOperatorTestHarness harness = ProcessFunctionTestHarnesses - .forBroadcastProcessFunction(new ScoringProcessFunction(ScoringJob.Descriptors.rulesResultSink, ScoringJob.Descriptors.rulesState, ParameterTool.fromMap(new HashMap<>())), ScoringJob.Descriptors.rulesState); + .forBroadcastProcessFunction(new ScoringProcessFunction(ScoringJob.Descriptors.rulesResultSink, + ScoringJob.Descriptors.rulesState, ParameterTool.fromMap(new HashMap<>())), + ScoringJob.Descriptors.rulesState); harness.open(); return harness; @@ -246,65 +299,101 @@ private BroadcastOperatorTestHarness private ScoringRule buildScoringRule(boolean enabled, int order, String name, String ruleScript) { Instant now = Instant.now(); - return ScoringRule.builder(). - id(UUID.randomUUID().toString()). - order(order). - enabled(enabled). - name(name). - tsStart(now.minus(Duration.ofMinutes(5))). - tsEnd(now.plus(Duration.ofMinutes(5))). - type(RuleType.JS). - ruleScript(ruleScript).build(); + return ScoringRule.builder() + .id(UUID.randomUUID().toString()) + .order(order) + .enabled(enabled) + .name(name) + .tsStart(now.minus(Duration.ofMinutes(5))) + .tsEnd(now.plus(Duration.ofMinutes(5))) + .type(RuleType.JS) + .ruleScript(ruleScript).build(); } - private ConcurrentLinkedQueue> verifyUpsert(ScoringRule rule, boolean isUpdate, BroadcastOperatorTestHarness harness, ConcurrentLinkedQueue> scoringRuleCommandResults) throws Exception { + private ConcurrentLinkedQueue> verifyUpsert(ScoringRule rule, + boolean isUpdate, + BroadcastOperatorTestHarness harness, + ConcurrentLinkedQueue> scoringRuleCommandResults) + throws Exception { return verifyUpsert(rule, isUpdate, harness, scoringRuleCommandResults, true); } - private ConcurrentLinkedQueue> verifyUpsert(ScoringRule rule, boolean isUpdate, BroadcastOperatorTestHarness harness, ConcurrentLinkedQueue> scoringRuleCommandResults, Boolean success) throws Exception { - ScoringRuleCommand upsertCommand = buildScoringRuleCommand(DynamicRuleCommandType.UPSERT, rule, rule.getId()); + private ConcurrentLinkedQueue> verifyUpsert(ScoringRule rule, + boolean isUpdate, + BroadcastOperatorTestHarness harness, + ConcurrentLinkedQueue> scoringRuleCommandResults, + Boolean success) + throws Exception { + ScoringRuleCommand upsertCommand = buildScoringRuleCommand(DynamicRuleCommandType.UPSERT, rule, rule.getId()); int version = rule.getVersion(); if (isUpdate) { version += 1; } - ScoringRuleCommandResult expectedResult = ScoringRuleCommandResult.builder().cmdId(upsertCommand.getId()).success(success).rule(rule.withVersion(version)).build(); + ScoringRuleCommandResult expectedResult = + ScoringRuleCommandResult.builder().cmdId(upsertCommand.getId()).success(success) + .rule(rule.withVersion(version)).build(); return verifyBroadcast(upsertCommand, harness, scoringRuleCommandResults, expectedResult); } - private ConcurrentLinkedQueue> verifyEnabledDisabled(ScoringRule rule, DynamicRuleCommandType type, BroadcastOperatorTestHarness harness, ConcurrentLinkedQueue> scoringRuleCommandResults) throws Exception { - ScoringRuleCommand disableCommand = buildScoringRuleCommand(type, rule, rule.getId()); + private ConcurrentLinkedQueue> verifyEnabledDisabled(ScoringRule rule, + DynamicRuleCommandType type, + BroadcastOperatorTestHarness harness, + ConcurrentLinkedQueue> scoringRuleCommandResults) + throws Exception { + ScoringRuleCommand disableCommand = buildScoringRuleCommand(type, rule, rule.getId()); boolean expectedEnabled = (type == DynamicRuleCommandType.ENABLE); - ScoringRuleCommandResult expectedResult = ScoringRuleCommandResult.builder().cmdId(disableCommand.getId()).success(true).rule(rule.withEnabled(expectedEnabled)).build(); + ScoringRuleCommandResult expectedResult = + ScoringRuleCommandResult.builder().cmdId(disableCommand.getId()).success(true) + .rule(rule.withEnabled(expectedEnabled)).build(); return verifyBroadcast(disableCommand, harness, scoringRuleCommandResults, expectedResult); } - private ConcurrentLinkedQueue> verifyDelete(ScoringRule rule, String ruleIdToDelete, BroadcastOperatorTestHarness harness, ConcurrentLinkedQueue> scoringRuleCommandResults) throws Exception { - ScoringRuleCommand deleteCommand = buildScoringRuleCommand(DynamicRuleCommandType.DELETE, null, ruleIdToDelete); - return verifyBroadcast(deleteCommand, harness, scoringRuleCommandResults, ScoringRuleCommandResult.builder().cmdId(deleteCommand.getId()).success(true).rule(rule).build()); + private ConcurrentLinkedQueue> verifyDelete(ScoringRule rule, + String ruleIdToDelete, + BroadcastOperatorTestHarness harness, + ConcurrentLinkedQueue> scoringRuleCommandResults) + throws Exception { + ScoringRuleCommand deleteCommand = buildScoringRuleCommand(DynamicRuleCommandType.DELETE, null, ruleIdToDelete); + return verifyBroadcast(deleteCommand, harness, scoringRuleCommandResults, + ScoringRuleCommandResult.builder().cmdId(deleteCommand.getId()).success(true).rule(rule).build()); } - private ConcurrentLinkedQueue> verifyGet(ScoringRule rule, String ruleId, BroadcastOperatorTestHarness harness, ConcurrentLinkedQueue> scoringRuleCommandResults) throws Exception { - ScoringRuleCommand getCommand = buildScoringRuleCommand(DynamicRuleCommandType.GET, null, ruleId); - return verifyBroadcast(getCommand, harness, scoringRuleCommandResults, ScoringRuleCommandResult.builder().cmdId(getCommand.getId()).success(true).rule(rule).build()); + private ConcurrentLinkedQueue> verifyGet(ScoringRule rule, String ruleId, + BroadcastOperatorTestHarness harness, + ConcurrentLinkedQueue> scoringRuleCommandResults) + throws Exception { + ScoringRuleCommand getCommand = buildScoringRuleCommand(DynamicRuleCommandType.GET, null, ruleId); + return verifyBroadcast(getCommand, harness, scoringRuleCommandResults, + ScoringRuleCommandResult.builder().cmdId(getCommand.getId()).success(true).rule(rule).build()); } - private ConcurrentLinkedQueue> verifyList(ScoringRule rule, BroadcastOperatorTestHarness harness, ConcurrentLinkedQueue> scoringRuleCommandResults) throws Exception { - ScoringRuleCommand listCommand = buildScoringRuleCommand(DynamicRuleCommandType.LIST, null, null); - return verifyBroadcast(listCommand, harness, scoringRuleCommandResults, ScoringRuleCommandResult.builder().cmdId(listCommand.getId()).success(true).rule(rule).build()); + private ConcurrentLinkedQueue> verifyList(ScoringRule rule, + BroadcastOperatorTestHarness harness, + ConcurrentLinkedQueue> scoringRuleCommandResults) + throws Exception { + ScoringRuleCommand listCommand = buildScoringRuleCommand(DynamicRuleCommandType.LIST, null, null); + return verifyBroadcast(listCommand, harness, scoringRuleCommandResults, + ScoringRuleCommandResult.builder().cmdId(listCommand.getId()).success(true).rule(rule).build()); } - private ConcurrentLinkedQueue> verifyBroadcast(ScoringRuleCommand command, BroadcastOperatorTestHarness harness, - ConcurrentLinkedQueue> scoringRuleCommandResults, ScoringRuleCommandResult expectedReply) throws Exception { + private ConcurrentLinkedQueue> verifyBroadcast(ScoringRuleCommand command, + BroadcastOperatorTestHarness harness, + ConcurrentLinkedQueue> scoringRuleCommandResults, + ScoringRuleCommandResult expectedReply) + throws Exception { return verifyBroadcast(command, harness, scoringRuleCommandResults, Collections.singletonList(expectedReply)); } - private ConcurrentLinkedQueue> verifyBroadcast(ScoringRuleCommand command, BroadcastOperatorTestHarness harness, - ConcurrentLinkedQueue> scoringRuleCommandResults, List expectedReplies) throws Exception { + private ConcurrentLinkedQueue> verifyBroadcast(ScoringRuleCommand command, + BroadcastOperatorTestHarness harness, + ConcurrentLinkedQueue> scoringRuleCommandResults, + List expectedReplies) + throws Exception { harness.processBroadcastElement(new StreamRecord<>(command)); if (scoringRuleCommandResults == null) { scoringRuleCommandResults = harness.getSideOutput(ScoringJob.Descriptors.rulesResultSink); } - for(ScoringRuleCommandResult expectedReply : expectedReplies) { + for (ScoringRuleCommandResult expectedReply : expectedReplies) { ScoringRuleCommandResult reply = scoringRuleCommandResults.remove().getValue(); assertEquals(expectedReply, reply); } @@ -313,13 +402,13 @@ private ConcurrentLinkedQueue> verifyBroa } private ScoringRuleCommand buildScoringRuleCommand(DynamicRuleCommandType type, ScoringRule rule, String ruleId) { - return ScoringRuleCommand.builder(). - id(UUID.randomUUID().toString()). - ts(Instant.now().toEpochMilli()).id(UUID.randomUUID().toString()). - type(type). - rule(rule). - ruleId(ruleId). - headers(Collections.emptyMap()). - build(); + return ScoringRuleCommand.builder() + .id(UUID.randomUUID().toString()) + .ts(Instant.now().toEpochMilli()).id(UUID.randomUUID().toString()) + .type(type) + .rule(rule) + .ruleId(ruleId) + .headers(Collections.emptyMap()) + .build(); } } diff --git a/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TypeTests.java b/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TypeTests.java index c137659d..3d22157c 100644 --- a/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TypeTests.java +++ b/flink-cyber/flink-alert-scoring/src/test/java/com/cloudera/cyber/scoring/TypeTests.java @@ -12,12 +12,12 @@ package com.cloudera.cyber.scoring; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.junit.Test; - import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertThat; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.junit.Test; + public class TypeTests { @Test public void testScoringRule() { diff --git a/flink-cyber/flink-commands/json-commands/src/main/java/com/cloudera/cyber/json/ReplaceJsonProperties.java b/flink-cyber/flink-commands/json-commands/src/main/java/com/cloudera/cyber/json/ReplaceJsonProperties.java index 74f74442..25624665 100644 --- a/flink-cyber/flink-commands/json-commands/src/main/java/com/cloudera/cyber/json/ReplaceJsonProperties.java +++ b/flink-cyber/flink-commands/json-commands/src/main/java/com/cloudera/cyber/json/ReplaceJsonProperties.java @@ -15,13 +15,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.extern.slf4j.Slf4j; - -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Iterator; import java.util.Properties; +import lombok.extern.slf4j.Slf4j; @Slf4j public class ReplaceJsonProperties { @@ -43,7 +45,8 @@ public static void main(String[] args) throws IOException { } } - private static void injectEnvironmentProperties(String templateRestConfigPath, String generatedRestConfigPath, Properties restProperties) throws IOException { + private static void injectEnvironmentProperties(String templateRestConfigPath, String generatedRestConfigPath, + Properties restProperties) throws IOException { byte[] templateBytes = Files.readAllBytes(Paths.get(templateRestConfigPath)); @@ -53,7 +56,7 @@ private static void injectEnvironmentProperties(String templateRestConfigPath, S JsonNode nextConfig = rootIter.next(); JsonNode propertiesNode = nextConfig.get("properties"); if (propertiesNode != null && propertiesNode.isObject()) { - ObjectNode propertiesObject = (ObjectNode)propertiesNode; + ObjectNode propertiesObject = (ObjectNode) propertiesNode; for (String propertyName : restProperties.stringPropertyNames()) { if (propertiesObject.has(propertyName)) { propertiesObject.put(propertyName, restProperties.getProperty(propertyName)); diff --git a/flink-cyber/flink-commands/json-commands/src/test/java/com/cloudera/cyber/json/ReplaceJsonPropertiesTest.java b/flink-cyber/flink-commands/json-commands/src/test/java/com/cloudera/cyber/json/ReplaceJsonPropertiesTest.java index 9ecdfdf6..e2d216e7 100644 --- a/flink-cyber/flink-commands/json-commands/src/test/java/com/cloudera/cyber/json/ReplaceJsonPropertiesTest.java +++ b/flink-cyber/flink-commands/json-commands/src/test/java/com/cloudera/cyber/json/ReplaceJsonPropertiesTest.java @@ -13,24 +13,27 @@ package com.cloudera.cyber.json; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Paths; -import java.util.*; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; public class ReplaceJsonPropertiesTest { @@ -44,20 +47,20 @@ public void testJsonReplace() throws IOException { ReplaceJsonProperties.main(args); List> actualPropertyMaps = getProperties(outputFile.getAbsolutePath()); List> expectedPropertyMaps = new ArrayList<>(); - expectedPropertyMaps.add(ImmutableMap.builder(). - put("server", "myhost:1080"). - put("protocol", "http"). - put("dga_access_key", "dga_model_key"). - put("bearer_token", "btoken_value"). - build()); + expectedPropertyMaps.add(ImmutableMap.builder() + .put("server", "myhost:1080") + .put("protocol", "http") + .put("dga_access_key", "dga_model_key") + .put("bearer_token", "btoken_value") + .build()); - expectedPropertyMaps.add(ImmutableMap.builder(). - put("server", "myhost:1080"). - put("protocol", "http"). - put("other_access_key", "other_model_key"). - put("bearer_token", "btoken_value"). - put("empty_prop", ""). - build()); + expectedPropertyMaps.add(ImmutableMap.builder() + .put("server", "myhost:1080") + .put("protocol", "http") + .put("other_access_key", "other_model_key") + .put("bearer_token", "btoken_value") + .put("empty_prop", "") + .build()); Assert.assertEquals(expectedPropertyMaps, actualPropertyMaps); } diff --git a/flink-cyber/flink-commands/kafka-commands/src/main/java/com/cloudera/cyber/kafka/Config.java b/flink-cyber/flink-commands/kafka-commands/src/main/java/com/cloudera/cyber/kafka/Config.java index 09fa4537..15c107ad 100644 --- a/flink-cyber/flink-commands/kafka-commands/src/main/java/com/cloudera/cyber/kafka/Config.java +++ b/flink-cyber/flink-commands/kafka-commands/src/main/java/com/cloudera/cyber/kafka/Config.java @@ -12,18 +12,16 @@ package com.cloudera.cyber.kafka; +import static com.hortonworks.registries.schemaregistry.client.SchemaRegistryClient.Configuration.SASL_JAAS_CONFIG; + import com.hortonworks.registries.schemaregistry.client.SchemaRegistryClient; import com.hortonworks.registries.schemaregistry.serdes.avro.kafka.KafkaAvroDeserializer; - import java.io.FileInputStream; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.UUID; - -import static com.hortonworks.registries.schemaregistry.client.SchemaRegistryClient.Configuration.SASL_JAAS_CONFIG; - import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.StringDeserializer; @@ -52,9 +50,11 @@ public Properties getKafkaConsumerProperties() { Properties kafkaConsumerProperties = getPropertiesWithKafkaPrefix(); kafkaConsumerProperties.putAll(readSchemaRegistryProperties(properties)); kafkaConsumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - kafkaConsumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class.getName()); + kafkaConsumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + KafkaAvroDeserializer.class.getName()); - overridePrintDefault(kafkaConsumerProperties, ConsumerConfig.GROUP_ID_CONFIG, "flink_cyber_command_line".concat(UUID.randomUUID().toString())); + overridePrintDefault(kafkaConsumerProperties, ConsumerConfig.GROUP_ID_CONFIG, + "flink_cyber_command_line".concat(UUID.randomUUID().toString())); overridePrintDefault(kafkaConsumerProperties, ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); kafkaConsumerProperties.put("specific.avro.reader", false); @@ -62,14 +62,16 @@ public Properties getKafkaConsumerProperties() { } private void overridePrintDefault(Properties kafkaConsumerProperties, String kafkaProperty, String defaultValue) { - kafkaConsumerProperties.put(kafkaProperty, properties.getOrDefault(PRINT_KAFKA_PROPERTY_PREFIX.concat(kafkaProperty), defaultValue)); + kafkaConsumerProperties.put(kafkaProperty, + properties.getOrDefault(PRINT_KAFKA_PROPERTY_PREFIX.concat(kafkaProperty), defaultValue)); } private Properties getPropertiesWithKafkaPrefix() { Properties filteredProperties = new Properties(); int prefixLength = Config.KAFKA_PREFIX.length(); - properties.stringPropertyNames().stream().filter(k -> k.startsWith(Config.KAFKA_PREFIX)).forEach(k -> filteredProperties.setProperty(k.substring(prefixLength), properties.getProperty(k))); + properties.stringPropertyNames().stream().filter(k -> k.startsWith(Config.KAFKA_PREFIX)) + .forEach(k -> filteredProperties.setProperty(k.substring(prefixLength), properties.getProperty(k))); return filteredProperties; } diff --git a/flink-cyber/flink-commands/kafka-commands/src/main/java/com/cloudera/cyber/kafka/TopicAvroToJson.java b/flink-cyber/flink-commands/kafka-commands/src/main/java/com/cloudera/cyber/kafka/TopicAvroToJson.java index 44fdeab9..6d6bc738 100644 --- a/flink-cyber/flink-commands/kafka-commands/src/main/java/com/cloudera/cyber/kafka/TopicAvroToJson.java +++ b/flink-cyber/flink-commands/kafka-commands/src/main/java/com/cloudera/cyber/kafka/TopicAvroToJson.java @@ -12,6 +12,10 @@ package com.cloudera.cyber.kafka; +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumWriter; @@ -23,11 +27,6 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; - @Slf4j public class TopicAvroToJson { @@ -63,7 +62,8 @@ public static void main(String[] args) { } } - private static void consume(Properties consumerProperties, String topic, boolean pretty, int maxRetries, int maxRecords) { + private static void consume(Properties consumerProperties, String topic, boolean pretty, int maxRetries, + int maxRecords) { try (KafkaConsumer consumer = new KafkaConsumer<>(consumerProperties)) { boolean gotResponse = false; diff --git a/flink-cyber/flink-commands/scoring-commands/src/main/java/com/cloudera/cyber/scoring/UpsertScoringRule.java b/flink-cyber/flink-commands/scoring-commands/src/main/java/com/cloudera/cyber/scoring/UpsertScoringRule.java index ce894964..3344543b 100644 --- a/flink-cyber/flink-commands/scoring-commands/src/main/java/com/cloudera/cyber/scoring/UpsertScoringRule.java +++ b/flink-cyber/flink-commands/scoring-commands/src/main/java/com/cloudera/cyber/scoring/UpsertScoringRule.java @@ -20,6 +20,17 @@ import com.google.common.base.Preconditions; import com.hortonworks.registries.schemaregistry.serdes.avro.kafka.KafkaAvroDeserializer; import com.hortonworks.registries.schemaregistry.serdes.avro.kafka.KafkaAvroSerializer; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.Optional; +import java.util.Properties; +import java.util.UUID; +import javax.script.ScriptException; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -34,18 +45,6 @@ import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; -import javax.script.ScriptException; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.Optional; -import java.util.Properties; -import java.util.UUID; - @Slf4j public class UpsertScoringRule { @@ -79,19 +78,20 @@ public static void main(String[] args) { String ruleOutputTopic = applicationProperties.getRequired("query.output.topic"); String commandId = UUID.randomUUID().toString(); - final DynamicRuleCommandType dynamicRuleCommandType = Optional.ofNullable(applicationProperties.get("rule.command.type")) - .map(Object::toString) - .map(DynamicRuleCommandType::valueOf) - .orElse(DynamicRuleCommandType.UPSERT); - - ScoringRuleCommand command = ScoringRuleCommand.builder(). - id(commandId). - type(dynamicRuleCommandType). - ts(Instant.now().toEpochMilli()). - ruleId(ruleToUpsert.getId()). - rule(ruleToUpsert). - headers(Collections.emptyMap()). - build(); + final DynamicRuleCommandType dynamicRuleCommandType = + Optional.ofNullable(applicationProperties.get("rule.command.type")) + .map(Object::toString) + .map(DynamicRuleCommandType::valueOf) + .orElse(DynamicRuleCommandType.UPSERT); + + ScoringRuleCommand command = ScoringRuleCommand.builder() + .id(commandId) + .type(dynamicRuleCommandType) + .ts(Instant.now().toEpochMilli()) + .ruleId(ruleToUpsert.getId()) + .rule(ruleToUpsert) + .headers(Collections.emptyMap()) + .build(); produce(command, applicationProperties, ruleInputTopic); consume(commandId, applicationProperties, ruleOutputTopic); @@ -121,7 +121,8 @@ private static void consume(String commandId, ParameterTool applicationPropertie Properties consumerProperties = Utils.readKafkaProperties(applicationProperties, "rule-config-console", true); consumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); consumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class.getName()); - consumerProperties.put(ConsumerConfig.GROUP_ID_CONFIG, "flink_cyber_command_line".concat(UUID.randomUUID().toString())); + consumerProperties.put(ConsumerConfig.GROUP_ID_CONFIG, + "flink_cyber_command_line".concat(UUID.randomUUID().toString())); consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); consumerProperties.put("specific.avro.reader", true); consumerProperties.putAll(Utils.readSchemaRegistryProperties(applicationProperties)); @@ -135,7 +136,8 @@ private static void consume(String commandId, ParameterTool applicationPropertie boolean gotResponse = false; while (!gotResponse && retries <= maxRetries) { consumer.subscribe(Collections.singletonList(topic)); - ConsumerRecords records = consumer.poll(Duration.ofSeconds(retryDuration)); + ConsumerRecords records = + consumer.poll(Duration.ofSeconds(retryDuration)); for (ConsumerRecord rec : records) { boolean responseMatches = commandId.equals(rec.value().getCmdId()); gotResponse = gotResponse || responseMatches; @@ -143,7 +145,7 @@ private static void consume(String commandId, ParameterTool applicationPropertie } retries++; } - if (!gotResponse){ + if (!gotResponse) { fail(String.format("Kafka read timed out after %s attempts", retries), new TimeoutException()); } } catch (Exception e) { @@ -174,7 +176,8 @@ private static void validateScoringRule(ScoringRule ruleToUpsert) throws ScriptE fail("Rule script cannot be null." + ruleToUpsert); } - ruleToUpsert.getType().engine(ruleToUpsert.getRuleScript()).eval("function test(message) { " + ruleToUpsert.getRuleScript() + " } "); + ruleToUpsert.getType().engine(ruleToUpsert.getRuleScript()) + .eval("function test(message) { " + ruleToUpsert.getRuleScript() + " } "); } } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/MessageUtils.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/MessageUtils.java index d2351a4f..068ea789 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/MessageUtils.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/MessageUtils.java @@ -12,8 +12,9 @@ package com.cloudera.cyber; -import com.google.common.base.Joiner; +import static java.util.stream.Collectors.toMap; +import com.google.common.base.Joiner; import java.time.Instant; import java.util.HashMap; import java.util.List; @@ -22,26 +23,26 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static java.util.stream.Collectors.toMap; - public class MessageUtils { public static final String MESSAGE_FIELD_DELIMITER = "."; + /** * Returns a new message combining the fields from the first message with the fields passed in. * * @param message Original message. * @param field Fields to add to the output message. - * @return Message with fields from the original message and the fields passed in. If fields is empty, return original unmodified message. + * @return Message with fields from the original message and the fields passed in. + * If fields is empty, return original unmodified message. */ public static Message addFields(Message message, Map field) { if (field != null && !field.isEmpty()) { return message.toBuilder() - .extensions(Stream.concat( - streamExtensions(message), - field.entrySet().stream() - ).collect(toMap(Map.Entry::getKey, Map.Entry::getValue, MessageUtils::merge))) - .build(); + .extensions(Stream.concat( + streamExtensions(message), + field.entrySet().stream() + ).collect(toMap(Map.Entry::getKey, Map.Entry::getValue, MessageUtils::merge))) + .build(); } else { return message; } @@ -49,7 +50,8 @@ public static Message addFields(Message message, Map field) { public static Message replaceFields(Message message, Map fields) { if (fields != null && !fields.isEmpty()) { - Map messageExtensions = message.getExtensions() != null ? message.getExtensions() : new HashMap<>(); + Map messageExtensions = + message.getExtensions() != null ? message.getExtensions() : new HashMap<>(); Map newMessageExtensions = new HashMap<>(messageExtensions); newMessageExtensions.putAll(fields); @@ -59,44 +61,50 @@ public static Message replaceFields(Message message, Map fields) } } - private static Stream> prefixMap(Map field, String prefix) { - return field.entrySet().stream().collect(toMap(k -> Joiner.on(MESSAGE_FIELD_DELIMITER).skipNulls().join(prefix, k.getKey()), Map.Entry::getValue, MessageUtils::merge)).entrySet().stream(); + private static Stream> prefixMap(Map field, String prefix) { + return field.entrySet().stream().collect( + toMap(k -> Joiner.on(MESSAGE_FIELD_DELIMITER).skipNulls().join(prefix, k.getKey()), Map.Entry::getValue, + MessageUtils::merge)).entrySet().stream(); } - public static Message enrich(Message message, Map enrichmentExtensions, List dataQualityMessages) { + public static Message enrich(Message message, Map enrichmentExtensions, + List dataQualityMessages) { return enrich(message, enrichmentExtensions, null, dataQualityMessages); } - public static Message enrich(Message message, Map enrichmentExtensions, String prefix, List dataQualityMessages) { + public static Message enrich(Message message, Map enrichmentExtensions, String prefix, + List dataQualityMessages) { if (!enrichmentExtensions.isEmpty() || !dataQualityMessages.isEmpty()) { return message.toBuilder() - .extensions(Stream.concat( - streamExtensions(message), - prefixMap(enrichmentExtensions, prefix) - ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, MessageUtils::merge))) - .dataQualityMessages(Stream.concat( - streamDataQualityMessages(message), - dataQualityMessages.stream() - ).distinct().collect(Collectors.toList())) - .build(); + .extensions(Stream.concat( + streamExtensions(message), + prefixMap(enrichmentExtensions, prefix) + ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, MessageUtils::merge))) + .dataQualityMessages(Stream.concat( + streamDataQualityMessages(message), + dataQualityMessages.stream() + ).distinct().collect(Collectors.toList())) + .build(); } else { // no changes to the message needed return message; } } + public static String merge(String str1, String str2) { return str2; } - public static void addQualityMessage(List messages, DataQualityMessageLevel level, String errorMessage, String fieldName, String feature) { - Optional duplicate = messages.stream(). - filter(m -> m.getLevel().equals(level.name()) && m.getMessage().equals(errorMessage)).findFirst(); + public static void addQualityMessage(List messages, DataQualityMessageLevel level, + String errorMessage, String fieldName, String feature) { + Optional duplicate = messages.stream() + .filter(m -> m.getLevel().equals(level.name()) && m.getMessage().equals(errorMessage)).findFirst(); if (!duplicate.isPresent()) { messages.add(DataQualityMessage.builder() - .level(level.name()) - .feature(feature) - .field(fieldName) - .message(errorMessage).build()); + .level(level.name()) + .feature(feature) + .field(fieldName) + .message(errorMessage).build()); } } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/ValidateUtils.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/ValidateUtils.java index 53fbac0f..d16a35dc 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/ValidateUtils.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/ValidateUtils.java @@ -12,14 +12,13 @@ package com.cloudera.cyber; -import org.springframework.util.CollectionUtils; - import java.util.Collection; import java.util.Collections; import java.util.Map.Entry; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.springframework.util.CollectionUtils; public final class ValidateUtils { @@ -31,7 +30,9 @@ private ValidateUtils() { public static void validatePhoenixName(String value, String parameter) { if (!PHOENIX_NAME_REGEXP.matcher(value).matches() || UNDERSCORE_PATTERN.matcher(value).matches()) { - throw new IllegalArgumentException(String.format("Invalid value %s for parameter '%s'. It can only contain alphanumerics or underscore(a-z, A-Z, 0-9, _)", value, parameter)); + throw new IllegalArgumentException(String.format( + "Invalid value %s for parameter '%s'. It can only contain alphanumerics or underscore(a-z, A-Z, 0-9, _)", + value, parameter)); } } @@ -40,11 +41,11 @@ public static Collection getDuplicates(Collection list, Function entry.getValue() > 1) - .map(Entry::getKey) - .collect(Collectors.toList()); + .collect(Collectors.groupingBy(classifier, Collectors.counting())) + .entrySet() + .stream().filter(entry -> entry.getValue() > 1) + .map(Entry::getKey) + .collect(Collectors.toList()); } public static Collection getDuplicates(Collection collections) { diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/enrichment/Enrichment.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/enrichment/Enrichment.java index cbb931ea..45b2e7c6 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/enrichment/Enrichment.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/enrichment/Enrichment.java @@ -15,7 +15,6 @@ import com.cloudera.cyber.DataQualityMessage; import com.cloudera.cyber.DataQualityMessageLevel; - import java.util.List; import java.util.Map; import java.util.Optional; @@ -39,15 +38,17 @@ protected String getName(String enrichmentName) { return String.join(DELIMITER, prefix, enrichmentName); } - public List addQualityMessage(List messages, DataQualityMessageLevel level, String message) { - Optional duplicate = messages.stream(). - filter(m -> m.getLevel().equals(level.name()) && m.getField().equals(fieldName) && m.getFeature().equals(feature) && m.getMessage().equals(message)).findFirst(); + public List addQualityMessage(List messages, DataQualityMessageLevel level, + String message) { + Optional duplicate = messages.stream() + .filter(m -> m.getLevel().equals(level.name()) && m.getField().equals(fieldName) + && m.getFeature().equals(feature) && m.getMessage().equals(message)).findFirst(); if (!duplicate.isPresent()) { messages.add(DataQualityMessage.builder() - .level(level.name()) - .feature(feature) - .field(fieldName) - .message(message).build()); + .level(level.name()) + .feature(feature) + .field(fieldName) + .message(message).build()); } return messages; } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/enrichment/MetronGeoEnrichment.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/enrichment/MetronGeoEnrichment.java index b1c67664..0cca118a 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/enrichment/MetronGeoEnrichment.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/enrichment/MetronGeoEnrichment.java @@ -13,7 +13,6 @@ package com.cloudera.cyber.enrichment; import com.google.common.base.Joiner; - import java.util.Map; public class MetronGeoEnrichment extends Enrichment { @@ -29,7 +28,7 @@ protected String getName(String enrichmentName) { @Override public void enrich(Map extensions, String enrichmentName, Object enrichmentValue) { if (enrichmentValue != null) { - String extensionName =getName(enrichmentName); + String extensionName = getName(enrichmentName); extensions.put(extensionName, enrichmentValue.toString()); } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/CacheMetrics.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/CacheMetrics.java index 5ce746c8..0c9c395f 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/CacheMetrics.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/CacheMetrics.java @@ -18,10 +18,10 @@ import org.apache.flink.metrics.MetricGroup; public class CacheMetrics implements StatsCounter { - private transient final MetricGroup hbaseCache; - private transient final Counter hits; - private transient final Counter miss; - private transient final Counter evict; + private final transient MetricGroup hbaseCache; + private final transient Counter hits; + private final transient Counter miss; + private final transient Counter evict; public CacheMetrics(MetricGroup hbaseCache) { this.hbaseCache = hbaseCache; diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/ConfigConstants.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/ConfigConstants.java index 221e35af..46ee89f4 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/ConfigConstants.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/ConfigConstants.java @@ -12,8 +12,7 @@ package com.cloudera.cyber.flink; -public final class ConfigConstants -{ +public final class ConfigConstants { public static final String PARAMS_ALLOWED_LATENESS = "allowed.lateness"; public static final String PARAMS_TOPIC_INPUT = "topic.input"; public static final String PARAMS_TOPIC_OUTPUT = "topic.output"; diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/EventTimeAndCountTrigger.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/EventTimeAndCountTrigger.java index 042ee794..73cd9daf 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/EventTimeAndCountTrigger.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/EventTimeAndCountTrigger.java @@ -28,15 +28,19 @@ public class EventTimeAndCountTrigger extends Trigger { private final ReducingStateDescriptor stateDesc; private EventTimeAndCountTrigger(long maxCount) { - this.stateDesc = new ReducingStateDescriptor("count", new EventTimeAndCountTrigger.Sum(), LongSerializer.INSTANCE); + this.stateDesc = + new ReducingStateDescriptor("count", new EventTimeAndCountTrigger.Sum(), LongSerializer.INSTANCE); this.maxCount = maxCount; } - public TriggerResult onElement(Object element, long timestamp, TimeWindow window, TriggerContext ctx) throws Exception { - ReducingState count = (ReducingState)ctx.getPartitionedState(this.stateDesc); + public TriggerResult onElement(Object element, long timestamp, TimeWindow window, TriggerContext ctx) + throws Exception { + ReducingState count = (ReducingState) ctx.getPartitionedState(this.stateDesc); count.add(1L); - log.finest(String.format("onElement: %s count: %d, timestamp %s, maxTime: %s, watermark: %d", element, count.get(), timestamp, window.maxTimestamp(), ctx.getCurrentWatermark())); - if ((Long)count.get() >= this.maxCount || window.maxTimestamp() <= ctx.getCurrentWatermark()) { + log.finest( + String.format("onElement: %s count: %d, timestamp %s, maxTime: %s, watermark: %d", element, count.get(), + timestamp, window.maxTimestamp(), ctx.getCurrentWatermark())); + if ((Long) count.get() >= this.maxCount || window.maxTimestamp() <= ctx.getCurrentWatermark()) { count.clear(); //ctx.registerEventTimeTimer(window.maxTimestamp()); return TriggerResult.FIRE_AND_PURGE; @@ -46,14 +50,17 @@ public TriggerResult onElement(Object element, long timestamp, TimeWindow window } } + @SuppressWarnings("checkstyle:EmptyCatchBlock") public TriggerResult onEventTime(long timestamp, TimeWindow window, TriggerContext ctx) { try { - log.finest(String.format("onEventTime: count: %d, timestamp %d, maxTime: %d, watermark: %d", ctx.getPartitionedState(this.stateDesc).get(), timestamp, window.maxTimestamp(), ctx.getCurrentWatermark())); - } catch (Exception e) { + log.finest(String.format("onEventTime: count: %d, timestamp %d, maxTime: %d, watermark: %d", + ctx.getPartitionedState(this.stateDesc).get(), timestamp, window.maxTimestamp(), + ctx.getCurrentWatermark())); + } catch (Exception ignored) { } if (timestamp == window.maxTimestamp()) { - ReducingState count = (ReducingState)ctx.getPartitionedState(this.stateDesc); + ReducingState count = (ReducingState) ctx.getPartitionedState(this.stateDesc); count.clear(); return TriggerResult.FIRE_AND_PURGE; } else { @@ -67,7 +74,7 @@ public TriggerResult onProcessingTime(long time, TimeWindow window, TriggerConte public void clear(TimeWindow window, TriggerContext ctx) throws Exception { ctx.deleteEventTimeTimer(window.maxTimestamp()); - ((ReducingState)ctx.getPartitionedState(this.stateDesc)).clear(); + ((ReducingState) ctx.getPartitionedState(this.stateDesc)).clear(); } public boolean canMerge() { diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/FlinkUtils.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/FlinkUtils.java index 89a465dd..2d263958 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/FlinkUtils.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/FlinkUtils.java @@ -12,9 +12,14 @@ package com.cloudera.cyber.flink; +import static com.cloudera.cyber.flink.Utils.readKafkaProperties; +import static com.cloudera.cyber.flink.Utils.readSchemaRegistryProperties; + import com.cloudera.cyber.Message; import com.cloudera.cyber.parser.MessageToParse; import com.cloudera.cyber.parser.MessageToParseDeserializer; +import java.util.Properties; +import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.serialization.SimpleStringSchema; @@ -34,12 +39,6 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; -import java.util.Properties; -import java.util.regex.Pattern; - -import static com.cloudera.cyber.flink.Utils.readKafkaProperties; -import static com.cloudera.cyber.flink.Utils.readSchemaRegistryProperties; - @Slf4j public class FlinkUtils { @@ -55,17 +54,22 @@ public FlinkUtils(Class type) { } public static void setupEnv(StreamExecutionEnvironment env, ParameterTool params) { - env.enableCheckpointing(params.getInt(PARAMS_CHECKPOINT_INTERVAL, DEFAULT_CHECKPOINT_INTERVAL), CheckpointingMode.EXACTLY_ONCE); + env.enableCheckpointing(params.getInt(PARAMS_CHECKPOINT_INTERVAL, DEFAULT_CHECKPOINT_INTERVAL), + CheckpointingMode.EXACTLY_ONCE); env.setParallelism(params.getInt(PARAMS_PARALLELISM, DEFAULT_PARALLELISM)); env.getConfig().setGlobalJobParameters(params); } - public static void executeEnv(StreamExecutionEnvironment env, String defaultJobName, ParameterTool params) throws Exception { - env.execute(params.get("flink.job.name",defaultJobName)); + public static void executeEnv(StreamExecutionEnvironment env, String defaultJobName, ParameterTool params) + throws Exception { + env.execute(params.get("flink.job.name", defaultJobName)); } public static KafkaSource createKafkaStringSource(String topic, Properties kafkaProperties) { - return KafkaSource.builder().setBootstrapServers(kafkaProperties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)).setTopics(topic).setValueOnlyDeserializer(new SimpleStringSchema()).setProperties(kafkaProperties).build(); + return KafkaSource.builder() + .setBootstrapServers(kafkaProperties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) + .setTopics(topic).setValueOnlyDeserializer(new SimpleStringSchema()).setProperties(kafkaProperties) + .build(); } public KafkaSink createKafkaSink(final String topic, String groupId, final ParameterTool params) { @@ -75,15 +79,15 @@ public KafkaSink createKafkaSink(final String topic, String groupId, final Pa log.info("Creating Kafka Sink for {}, using {}", topic, kafkaProperties); KafkaRecordSerializationSchema schema = ClouderaRegistryAvroKafkaRecordSerializationSchema - .builder(topic) - .setConfig(readSchemaRegistryProperties(params)) - .build(); + .builder(topic) + .setConfig(readSchemaRegistryProperties(params)) + .build(); return KafkaSink.builder() - .setBootstrapServers(kafkaProperties.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) - .setRecordSerializer(schema) - .setKafkaProducerConfig(kafkaProperties) - .build(); + .setBootstrapServers(kafkaProperties.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) + .setRecordSerializer(schema) + .setKafkaProducerConfig(kafkaProperties) + .build(); } public KafkaSource createKafkaGenericSource(String topic, ParameterTool params, String groupId) { @@ -93,15 +97,15 @@ public KafkaSource createKafkaGenericSource(String topic, ParameterTool param Properties kafkaProperties = readKafkaProperties(params, groupId, true); log.info(String.format("Creating Kafka Source for %s, using %s", topic, kafkaProperties)); KafkaDeserializationSchema schema = ClouderaRegistryAvroKafkaDeserializationSchema - .builder(type) - .setConfig(readSchemaRegistryProperties(params)) - .build(); - - return KafkaSource.builder().setTopics(topic). - setBootstrapServers(kafkaProperties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)). - setProperties(kafkaProperties). - setDeserializer(KafkaRecordDeserializationSchema.of(schema)). - build(); + .builder(type) + .setConfig(readSchemaRegistryProperties(params)) + .build(); + + return KafkaSource.builder().setTopics(topic) + .setBootstrapServers(kafkaProperties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) + .setProperties(kafkaProperties) + .setDeserializer(KafkaRecordDeserializationSchema.of(schema)) + .build(); } private KafkaSourceBuilder createKafkaSourceBuilder(ParameterTool params, String groupId) { @@ -109,41 +113,47 @@ private KafkaSourceBuilder createKafkaSourceBuilder(ParameterTool params, Str Properties kafkaProperties = readKafkaProperties(params, groupId, true); KafkaDeserializationSchema schema = ClouderaRegistryAvroKafkaDeserializationSchema - .builder(type) - .setConfig(readSchemaRegistryProperties(params)) - .build(); + .builder(type) + .setConfig(readSchemaRegistryProperties(params)) + .build(); - return KafkaSource.builder(). - setBootstrapServers(kafkaProperties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)). - setProperties(kafkaProperties). - setDeserializer(KafkaRecordDeserializationSchema.of(schema)); + return KafkaSource.builder() + .setBootstrapServers(kafkaProperties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) + .setProperties(kafkaProperties) + .setDeserializer(KafkaRecordDeserializationSchema.of(schema)); } + public static KafkaSource createKafkaSource(String topic, ParameterTool params, String groupId) { Preconditions.checkNotNull(topic, "Must specific input topic"); - return new FlinkUtils<>(Message.class).createKafkaSourceBuilder(params, groupId).setTopics(topic).build(); + return new FlinkUtils<>(Message.class).createKafkaSourceBuilder(params, groupId).setTopics(topic).build(); } public static KafkaSource createKafkaSource(Pattern topic, ParameterTool params, String groupId) { Preconditions.checkNotNull(topic, "Must specific input topic pattern"); - return new FlinkUtils<>(Message.class).createKafkaSourceBuilder(params, groupId).setTopicPattern(topic).build(); + return new FlinkUtils<>(Message.class).createKafkaSourceBuilder(params, groupId).setTopicPattern(topic).build(); } - public static DataStream createRawKafkaSource(StreamExecutionEnvironment env, ParameterTool params, String groupId, KafkaRecordDeserializationSchema deserializationSchema) { - String inputTopic = params.get(ConfigConstants.PARAMS_TOPIC_INPUT,""); + public static DataStream createRawKafkaSource(StreamExecutionEnvironment env, ParameterTool params, + String groupId, + KafkaRecordDeserializationSchema deserializationSchema) { + String inputTopic = params.get(ConfigConstants.PARAMS_TOPIC_INPUT, ""); String pattern = params.get(ConfigConstants.PARAMS_TOPIC_PATTERN, ""); - log.info(String.format("createRawKafkaSource topic: '%s', pattern: '%s', good: %b", inputTopic, pattern, !(inputTopic.isEmpty() && pattern.isEmpty()))); + log.info(String.format("createRawKafkaSource topic: '%s', pattern: '%s', good: %b", inputTopic, pattern, + !(inputTopic.isEmpty() && pattern.isEmpty()))); Preconditions.checkArgument(!(inputTopic.isEmpty() && pattern.isEmpty()), - String.format("Must specify at least one of %s or %s", ConfigConstants.PARAMS_TOPIC_INPUT, ConfigConstants.PARAMS_TOPIC_PATTERN)); + String.format("Must specify at least one of %s or %s", ConfigConstants.PARAMS_TOPIC_INPUT, + ConfigConstants.PARAMS_TOPIC_PATTERN)); Properties kafkaProperties = readKafkaProperties(params, groupId, true); kafkaProperties.putIfAbsent(ConsumerConfig.GROUP_ID_CONFIG, groupId); - KafkaSourceBuilder kafkaSourceBuilder = KafkaSource.builder().setDeserializer(deserializationSchema).setProperties(kafkaProperties); + KafkaSourceBuilder kafkaSourceBuilder = + KafkaSource.builder().setDeserializer(deserializationSchema).setProperties(kafkaProperties); if (pattern != null) { kafkaSourceBuilder.setTopicPattern(Pattern.compile(pattern)); @@ -154,7 +164,8 @@ public static DataStream createRawKafkaSource(StreamExecutionEnvironment return env.fromSource(kafkaSourceBuilder.build(), WatermarkStrategy.noWatermarks(), "Kafka Source"); } - public static DataStream createRawKafkaSource(StreamExecutionEnvironment env, ParameterTool params, String groupId) { - return createRawKafkaSource(env, params,groupId, new MessageToParseDeserializer()); + public static DataStream createRawKafkaSource(StreamExecutionEnvironment env, ParameterTool params, + String groupId) { + return createRawKafkaSource(env, params, groupId, new MessageToParseDeserializer()); } } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/SourcesWithHeaders.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/SourcesWithHeaders.java index 3e008005..10ec4cde 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/SourcesWithHeaders.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/SourcesWithHeaders.java @@ -12,6 +12,14 @@ package com.cloudera.cyber.flink; +import static com.cloudera.cyber.flink.Utils.readKafkaProperties; +import static com.cloudera.cyber.flink.Utils.readSchemaRegistryProperties; +import static java.util.stream.Collectors.toList; + +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.utils.ParameterTool; @@ -30,19 +38,8 @@ import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; -import java.util.List; -import java.util.Properties; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static com.cloudera.cyber.flink.Utils.readKafkaProperties; -import static com.cloudera.cyber.flink.Utils.readSchemaRegistryProperties; -import static java.util.stream.Collectors.toList; - /** * TODO - all the request and response stuff should be in wrapper objects instead of using the HasHeader interface. - * - * @param */ @Slf4j public class SourcesWithHeaders { @@ -60,14 +57,14 @@ public KafkaSource createSourceWithHeaders(String topic, ParameterTool params Properties kafkaProperties = readKafkaProperties(params, groupId, true); KafkaDeserializationSchema delegate = ClouderaRegistryAvroKafkaDeserializationSchema - .builder(type) - .setConfig(readSchemaRegistryProperties(params)) - .build(); + .builder(type) + .setConfig(readSchemaRegistryProperties(params)) + .build(); KafkaRecordDeserializationSchema schema = new HeaderDeserializer(delegate); - return KafkaSource.builder().setTopics(topic).setDeserializer(schema). - setProperties(kafkaProperties).build(); + return KafkaSource.builder().setTopics(topic).setDeserializer(schema) + .setProperties(kafkaProperties).build(); } public KafkaSink createKafkaSink(final String topic, String groupId, final ParameterTool params) { @@ -77,16 +74,16 @@ public KafkaSink createKafkaSink(final String topic, String groupId, final Pa log.info("Creating Kafka Sink for {}, using {}", topic, kafkaProperties); KafkaRecordSerializationSchema delegate = ClouderaRegistryAvroKafkaRecordSerializationSchema - .builder(topic) - .setConfig(readSchemaRegistryProperties(params)) - .build(); + .builder(topic) + .setConfig(readSchemaRegistryProperties(params)) + .build(); HeaderSerializer schema = new HeaderSerializer(delegate); - return KafkaSink.builder(). - setKafkaProducerConfig(kafkaProperties). - setRecordSerializer(schema). - setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE). - build(); + return KafkaSink.builder() + .setKafkaProducerConfig(kafkaProperties) + .setRecordSerializer(schema) + .setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE) + .build(); } private class HeaderDeserializer implements KafkaRecordDeserializationSchema { @@ -105,7 +102,8 @@ public TypeInformation getProducedType() { public void deserialize(ConsumerRecord consumerRecord, Collector collector) { try { T deserialize = delegate.deserialize(consumerRecord); - deserialize.setHeaders(StreamSupport.stream(consumerRecord.headers().spliterator(), false).collect(Collectors.toMap(Header::key, v -> new String(v.value())))); + deserialize.setHeaders(StreamSupport.stream(consumerRecord.headers().spliterator(), false) + .collect(Collectors.toMap(Header::key, v -> new String(v.value())))); collector.collect(deserialize); } catch (Exception e) { e.printStackTrace(); @@ -123,8 +121,11 @@ public HeaderSerializer(KafkaRecordSerializationSchema delegate) { @Override public ProducerRecord serialize(T t, KafkaSinkContext kafkaSinkContext, Long ts) { ProducerRecord serialize = delegate.serialize(t, kafkaSinkContext, ts); - List

headers = t.getHeaders().entrySet().stream().map(h -> new RecordHeader(h.getKey(), h.getValue().getBytes())).collect(toList()); - return new ProducerRecord<>(serialize.topic(), serialize.partition(), serialize.key(), serialize.value(), headers); + List
headers = + t.getHeaders().entrySet().stream().map(h -> new RecordHeader(h.getKey(), h.getValue().getBytes())) + .collect(toList()); + return new ProducerRecord<>(serialize.topic(), serialize.partition(), serialize.key(), serialize.value(), + headers); } } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/TimedBoundedOutOfOrdernessTimestampExtractor.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/TimedBoundedOutOfOrdernessTimestampExtractor.java index 3f70d680..6579cb89 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/TimedBoundedOutOfOrdernessTimestampExtractor.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/TimedBoundedOutOfOrdernessTimestampExtractor.java @@ -16,7 +16,8 @@ import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor; import org.apache.flink.streaming.api.windowing.time.Time; -public class TimedBoundedOutOfOrdernessTimestampExtractor extends BoundedOutOfOrdernessTimestampExtractor { +public class TimedBoundedOutOfOrdernessTimestampExtractor + extends BoundedOutOfOrdernessTimestampExtractor { public TimedBoundedOutOfOrdernessTimestampExtractor(Time lateness) { super(lateness); diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/Utils.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/Utils.java index 61720151..7c5b3574 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/Utils.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/Utils.java @@ -12,28 +12,14 @@ package com.cloudera.cyber.flink; +import static com.hortonworks.registries.schemaregistry.client.SchemaRegistryClient.Configuration.SASL_JAAS_CONFIG; +import static org.apache.flink.configuration.GlobalConfiguration.loadConfiguration; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.Resources; import com.hortonworks.registries.schemaregistry.client.SchemaRegistryClient; - -import java.util.concurrent.TimeUnit; -import java.util.function.Function; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.client.cli.CliFrontend; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.core.fs.FSDataInputStream; -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.Path; -import org.apache.flink.util.Preconditions; -import org.apache.flink.util.encrypttool.EncryptTool; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; - import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -47,11 +33,22 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.function.ToLongFunction; - -import static com.hortonworks.registries.schemaregistry.client.SchemaRegistryClient.Configuration.SASL_JAAS_CONFIG; -import static org.apache.flink.configuration.GlobalConfiguration.loadConfiguration; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.client.cli.CliFrontend; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.fs.FSDataInputStream; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.Path; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.encrypttool.EncryptTool; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; @Slf4j public class Utils { @@ -96,18 +93,20 @@ private static Properties kafkaDefaultSettings(Properties kafkaProperties, Strin // interceptor currently unable to work with flink transactional kafka // https://docs.google.com/document/d/19jIN_POJvZPV466V5DolBKJxlqWOxYz-2gJV4e5cYtE/edit -// kafkaProperties.put(consumer ? ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG : ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, -// consumer ? -// "com.hortonworks.smm.kafka.monitoring.interceptors.MonitoringConsumerInterceptor" : -// "com.hortonworks.smm.kafka.monitoring.interceptors.MonitoringProducerInterceptor"); + //kafkaProperties.put(consumer ? ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG : ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, + // consumer ? + // "com.hortonworks.smm.kafka.monitoring.interceptors.MonitoringConsumerInterceptor" : + // "com.hortonworks.smm.kafka.monitoring.interceptors.MonitoringProducerInterceptor"); groupId = (String) kafkaProperties.getOrDefault(ConsumerConfig.GROUP_ID_CONFIG, groupId); if (!consumer) { kafkaProperties.putIfAbsent(ProducerConfig.ACKS_CONFIG, "all"); - kafkaProperties.put(ProducerConfig.CLIENT_ID_CONFIG, String.format("%s-producer-%d", groupId, nextKafkaClientId.incrementAndGet())); + kafkaProperties.put(ProducerConfig.CLIENT_ID_CONFIG, + String.format("%s-producer-%d", groupId, nextKafkaClientId.incrementAndGet())); } else { kafkaProperties.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); - kafkaProperties.put(ConsumerConfig.CLIENT_ID_CONFIG, String.format("%s-consumer-%d", groupId, nextKafkaClientId.incrementAndGet())); + kafkaProperties.put(ConsumerConfig.CLIENT_ID_CONFIG, + String.format("%s-consumer-%d", groupId, nextKafkaClientId.incrementAndGet())); } return kafkaProperties; @@ -132,9 +131,11 @@ public static Map readSchemaRegistryProperties(Map sslClientConfig = new HashMap<>(); String sslKey = K_SCHEMA_REG_SSL_CLIENT_KEY + "." + K_TRUSTSTORE_PATH; - sslClientConfig.put(K_TRUSTSTORE_PATH, isSensitive(sslKey) ? decrypt(params.get(sslKey)) : params.get(sslKey)); + sslClientConfig.put(K_TRUSTSTORE_PATH, + isSensitive(sslKey) ? decrypt(params.get(sslKey)) : params.get(sslKey)); sslKey = K_SCHEMA_REG_SSL_CLIENT_KEY + "." + K_TRUSTSTORE_PASSWORD; - sslClientConfig.put(K_TRUSTSTORE_PASSWORD, isSensitive(sslKey) ? decrypt(params.get(sslKey)) : params.get(sslKey)); + sslClientConfig.put(K_TRUSTSTORE_PASSWORD, + isSensitive(sslKey) ? decrypt(params.get(sslKey)) : params.get(sslKey)); sslClientConfig.put(K_KEYSTORE_PASSWORD, ""); //ugly hack needed for SchemaRegistryClient //schemaRegistryConf.put(K_SCHEMA_REG_SSL_CLIENT_KEY, sslClientConfig); @@ -151,9 +152,11 @@ public static Map readSchemaRegistryProperties(ParameterTool par if (schemaRegistryUrl.startsWith("https")) { Map sslClientConfig = new HashMap<>(); String sslKey = K_SCHEMA_REG_SSL_CLIENT_KEY + "." + K_TRUSTSTORE_PATH; - sslClientConfig.put(K_TRUSTSTORE_PATH, isSensitive(sslKey, params) ? decrypt(params.getRequired(sslKey)) : params.getRequired(sslKey)); + sslClientConfig.put(K_TRUSTSTORE_PATH, + isSensitive(sslKey, params) ? decrypt(params.getRequired(sslKey)) : params.getRequired(sslKey)); sslKey = K_SCHEMA_REG_SSL_CLIENT_KEY + "." + K_TRUSTSTORE_PASSWORD; - sslClientConfig.put(K_TRUSTSTORE_PASSWORD, isSensitive(sslKey, params) ? decrypt(params.getRequired(sslKey)) : params.getRequired(sslKey)); + sslClientConfig.put(K_TRUSTSTORE_PASSWORD, + isSensitive(sslKey, params) ? decrypt(params.getRequired(sslKey)) : params.getRequired(sslKey)); sslClientConfig.put(K_KEYSTORE_PASSWORD, ""); //ugly hack needed for SchemaRegistryClient schemaRegistryConf.put(K_SCHEMA_REG_SSL_CLIENT_KEY, sslClientConfig); @@ -169,7 +172,8 @@ private static T jsonToObject(String json, TypeReference typeReference) t return MAPPER.readValue(json, typeReference); } - public static T readResourceFile(String resourceLocation, Class cls, TypeReference typeReference) throws IOException { + public static T readResourceFile(String resourceLocation, Class cls, TypeReference typeReference) + throws IOException { return jsonToObject(readResourceFile(resourceLocation, cls), typeReference); } @@ -245,8 +249,10 @@ public static boolean isTimeEqual(Long unit1, String unitType1, Long unit2, Stri return TimeUnit.valueOf(unitType1).toMillis(unit1) == TimeUnit.valueOf(unitType2).toMillis(unit2); } - public static boolean isTimeEqual(T object1, T object2, ToLongFunction timeUnitSelector, Function timeUnitTypeSelector) { - return isTimeEqual(timeUnitSelector.applyAsLong(object1), timeUnitTypeSelector.apply(object1), timeUnitSelector.applyAsLong(object2), timeUnitTypeSelector.apply(object2)); + public static boolean isTimeEqual(T object1, T object2, ToLongFunction timeUnitSelector, + Function timeUnitTypeSelector) { + return isTimeEqual(timeUnitSelector.applyAsLong(object1), timeUnitTypeSelector.apply(object1), + timeUnitSelector.applyAsLong(object2), timeUnitTypeSelector.apply(object2)); } private static class ConfigHolder { @@ -264,20 +270,22 @@ public static String getResourceAsString(String file) { } public static ParameterTool getParamToolsFromProperties(String[] pathToPropertyFiles) { - return Arrays.stream(pathToPropertyFiles).filter(pathToPropertyFile -> pathToPropertyFile.endsWith(".properties")).map(Path::new).reduce(ParameterTool.fromMap(new HashMap<>()), (parameterTool, path) -> { - try { - FileSystem fileSystem = path.getFileSystem(); - if (fileSystem.exists(path)) { - try (FSDataInputStream fsDataInputStream = fileSystem.open(path)) { - ParameterTool nextParamTool = ParameterTool.fromPropertiesFile(fsDataInputStream); - return parameterTool.mergeWith(nextParamTool); - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return parameterTool; - }, ParameterTool::mergeWith); + return Arrays.stream(pathToPropertyFiles) + .filter(pathToPropertyFile -> pathToPropertyFile.endsWith(".properties")).map(Path::new) + .reduce(ParameterTool.fromMap(new HashMap<>()), (parameterTool, path) -> { + try { + FileSystem fileSystem = path.getFileSystem(); + if (fileSystem.exists(path)) { + try (FSDataInputStream fsDataInputStream = fileSystem.open(path)) { + ParameterTool nextParamTool = ParameterTool.fromPropertiesFile(fsDataInputStream); + return parameterTool.mergeWith(nextParamTool); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return parameterTool; + }, ParameterTool::mergeWith); } @@ -296,14 +304,16 @@ public static String readFile(String path) throws IOException { } - public static byte[] sign(String s, PrivateKey key) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException { + public static byte[] sign(String s, PrivateKey key) + throws NoSuchAlgorithmException, InvalidKeyException, SignatureException { Signature sig = Signature.getInstance("SHA1WithRSA"); sig.initSign(key); sig.update(s.getBytes(StandardCharsets.UTF_8)); return sig.sign(); } - public static boolean verify(String s, byte[] signature, PublicKey key) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException { + public static boolean verify(String s, byte[] signature, PublicKey key) + throws NoSuchAlgorithmException, InvalidKeyException, SignatureException { Signature sig = Signature.getInstance("SHA1WithRSA"); sig.initVerify(key); sig.update(s.getBytes(StandardCharsets.UTF_8)); diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/operators/MessageConcatenate.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/operators/MessageConcatenate.java index bc77fd48..d4af199c 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/operators/MessageConcatenate.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/flink/operators/MessageConcatenate.java @@ -14,12 +14,11 @@ import com.cloudera.cyber.GroupedMessage; import com.cloudera.cyber.Message; -import org.apache.flink.api.common.functions.AggregateFunction; - import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.flink.api.common.functions.AggregateFunction; public class MessageConcatenate implements AggregateFunction, GroupedMessage> { diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FlinkFileTemplateLoader.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FlinkFileTemplateLoader.java index 9e3ef458..c95c8974 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FlinkFileTemplateLoader.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FlinkFileTemplateLoader.java @@ -1,13 +1,12 @@ package com.cloudera.cyber.generator; import freemarker.cache.TemplateLoader; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.Path; - import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.Path; @Slf4j public class FlinkFileTemplateLoader implements TemplateLoader { @@ -34,10 +33,11 @@ public Object findTemplateSource(String templateFile) throws IOException { @Override public long getLastModified(Object templatePathObject) { if (!(templatePathObject instanceof Path)) { - throw new IllegalArgumentException("templatePathObject wasn't a flink Path, but a: " + templatePathObject.getClass().getName()); + throw new IllegalArgumentException( + "templatePathObject wasn't a flink Path, but a: " + templatePathObject.getClass().getName()); } - Path templatePath = (Path)templatePathObject; + Path templatePath = (Path) templatePathObject; try { return templatePath.getFileSystem().getFileStatus(templatePath).getModificationTime(); @@ -64,7 +64,8 @@ private Path safePathConversion(Object pathObject) { if (pathObject instanceof Path) { return (Path) pathObject; } else { - throw new IllegalArgumentException(String.format("templatePathObject expected %s, but got %s ", Path.class, pathObject.getClass().getName())); + throw new IllegalArgumentException(String.format("templatePathObject expected %s, but got %s ", Path.class, + pathObject.getClass().getName())); } } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FreemarkerImmediateGenerator.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FreemarkerImmediateGenerator.java index 97228b4e..d454e547 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FreemarkerImmediateGenerator.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FreemarkerImmediateGenerator.java @@ -15,7 +15,6 @@ import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; - import java.io.IOException; import java.io.StringWriter; import java.io.Writer; @@ -28,17 +27,17 @@ public class FreemarkerImmediateGenerator { Configuration cfg = new Configuration(Configuration.VERSION_2_3_30); private static final RandomGenerators utils = new RandomGenerators(); - public FreemarkerImmediateGenerator() { + public FreemarkerImmediateGenerator() { cfg.setClassLoaderForTemplateLoading(Thread.currentThread().getContextClassLoader(), ""); - cfg.setCacheStorage(new freemarker.cache.MruCacheStorage(50,150)); - cfg.setTemplateUpdateDelayMilliseconds(3600*24*1000); + cfg.setCacheStorage(new freemarker.cache.MruCacheStorage(50, 150)); + cfg.setTemplateUpdateDelayMilliseconds(3600 * 24 * 1000); } public String generateEntry(String template) throws IOException, TemplateException { SyntheticEntry entry = SyntheticEntry.builder().ts( - LocalDateTime.now().toInstant(ZoneOffset.UTC).toEpochMilli()) - .utils(utils) - .build(); + LocalDateTime.now().toInstant(ZoneOffset.UTC).toEpochMilli()) + .utils(utils) + .build(); Template temp = cfg.getTemplate(template); Writer out = new StringWriter(); @@ -46,14 +45,15 @@ public String generateEntry(String template) throws IOException, TemplateExcepti return out.toString(); } - public String replaceByFile(String pathToFile, Map params) throws IOException, TemplateException { + public String replaceByFile(String pathToFile, Map params) throws IOException, TemplateException { Template template = cfg.getTemplate(pathToFile); Writer out = new StringWriter(); template.process(params, out); return out.toString(); } - public String replaceByTemplate(String templateString, Map params) throws IOException, TemplateException { + public String replaceByTemplate(String templateString, Map params) + throws IOException, TemplateException { Template template = new Template("templateName", templateString, cfg); Writer out = new StringWriter(); template.process(params, out); diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FreemarkerTemplateSource.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FreemarkerTemplateSource.java index af47da16..5bf5f819 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FreemarkerTemplateSource.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/FreemarkerTemplateSource.java @@ -17,14 +17,6 @@ import freemarker.cache.TemplateLoader; import freemarker.template.Configuration; import freemarker.template.Template; -import lombok.extern.java.Log; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Parser; -import org.apache.commons.math3.distribution.EnumeratedDistribution; -import org.apache.commons.math3.util.Pair; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction; - import java.io.StringWriter; import java.io.Writer; import java.nio.charset.Charset; @@ -35,9 +27,16 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import lombok.extern.java.Log; +import org.apache.avro.Schema; +import org.apache.avro.Schema.Parser; +import org.apache.commons.math3.distribution.EnumeratedDistribution; +import org.apache.commons.math3.util.Pair; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction; @Log -public class FreemarkerTemplateSource implements ParallelSourceFunction> { +public class FreemarkerTemplateSource implements ParallelSourceFunction> { private final EnumeratedDistribution files; private volatile boolean isRunning = true; @@ -57,7 +56,7 @@ private static class JsonStringToAvro implements Function { JsonStringToAvro(String schemaString) { Parser avroSchemaParser = new Parser(); - this.avroSchema = avroSchemaParser.parse(schemaString); + this.avroSchema = avroSchemaParser.parse(schemaString); } @Override @@ -75,14 +74,13 @@ public byte[] apply(String s) { } - public FreemarkerTemplateSource(GeneratorConfig generatorConfig, long maxRecords, int eps) { // normalise weights List files = generatorConfig.getGenerationSources(); Double total = files.stream().mapToDouble(GenerationSource::getWeight).sum(); List> weights = files.stream() - .map(e -> Pair.create(e, e.getWeight() / total)) - .collect(Collectors.toList()); + .map(e -> Pair.create(e, e.getWeight() / total)) + .collect(Collectors.toList()); // create a reverse sorted version of the weights this.files = new EnumeratedDistribution<>(weights); @@ -97,24 +95,25 @@ public FreemarkerTemplateSource(GeneratorConfig generatorConfig, long maxRecords } @Override - public void run(SourceContext> sourceContext) throws Exception { + public void run(SourceContext> sourceContext) throws Exception { Configuration cfg = new Configuration(Configuration.VERSION_2_3_30); TemplateLoader templateLoader = new ClassTemplateLoader(Thread.currentThread().getContextClassLoader(), ""); if (templateBaseDir != null) { FlinkFileTemplateLoader flinkFileLoader = new FlinkFileTemplateLoader(templateBaseDir); - templateLoader = new MultiTemplateLoader(new TemplateLoader[]{flinkFileLoader, templateLoader}); + templateLoader = new MultiTemplateLoader(new TemplateLoader[] {flinkFileLoader, templateLoader}); } cfg.setTemplateLoader(templateLoader); - cfg.setCacheStorage(new freemarker.cache.MruCacheStorage(50,50)); - cfg.setTemplateUpdateDelayMilliseconds(3600*24*1000); + cfg.setCacheStorage(new freemarker.cache.MruCacheStorage(50, 50)); + cfg.setTemplateUpdateDelayMilliseconds(3600 * 24 * 1000); Map> topicOutputConverter = new HashMap<>(); - for(GenerationSource generationSource : generationSources) { + for (GenerationSource generationSource : generationSources) { if (generationSource.getOutputAvroSchema() != null) { - topicOutputConverter.put(generationSource.getTopic(), new JsonStringToAvro(generationSource.getOutputAvroSchema())); + topicOutputConverter.put(generationSource.getTopic(), + new JsonStringToAvro(generationSource.getOutputAvroSchema())); } else { topicOutputConverter.put(generationSource.getTopic(), new TextToBytes()); } @@ -144,20 +143,22 @@ public void run(SourceContext> sourceContext) throws Excep Writer out = new StringWriter(); SyntheticEntry entry = SyntheticEntry.builder().ts( - Instant.now().toEpochMilli()) - .utils(utils) - .params(file.getRandomParameters()) - .build(); + Instant.now().toEpochMilli()) + .utils(utils) + .params(file.getRandomParameters()) + .build(); temp.process(entry, out); if (count % 100 == 0) { Instant endTime = Instant.now(); - log.info(String.format("Produced %d records from template %s to topic %s from: %s to: %s", count, file.getFile(), file.getTopic(), startTime, endTime)); + log.info(String.format("Produced %d records from template %s to topic %s from: %s to: %s", count, + file.getFile(), file.getTopic(), startTime, endTime)); startTime = endTime; } - sourceContext.collect(Tuple2.of(file.getTopic(), topicOutputConverter.get(file.getTopic()).apply(out.toString()))); + sourceContext.collect( + Tuple2.of(file.getTopic(), topicOutputConverter.get(file.getTopic()).apply(out.toString()))); if (eps > 0) { Thread.sleep(ms, ns); } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/GenerationSource.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/GenerationSource.java index 247f4f65..c5bd1d28 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/GenerationSource.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/GenerationSource.java @@ -13,11 +13,6 @@ package com.cloudera.cyber.generator; import com.cloudera.cyber.generator.scenario.GeneratorScenario; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.io.IOUtils; - import java.io.IOException; import java.io.InputStream; import java.io.Serializable; @@ -25,6 +20,10 @@ import java.util.Collections; import java.util.Map; import java.util.Objects; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.io.IOUtils; @Data @AllArgsConstructor @@ -52,7 +51,7 @@ public void readAvroSchema(String baseDir) throws IOException { try (InputStream resourceStream = getClass().getClassLoader().getResourceAsStream(outputAvroSchemaFile)) { if (resourceStream != null) { outputAvroSchema = IOUtils.toString( - Objects.requireNonNull(resourceStream), Charset.defaultCharset()); + Objects.requireNonNull(resourceStream), Charset.defaultCharset()); } } @@ -60,7 +59,7 @@ public void readAvroSchema(String baseDir) throws IOException { if (outputAvroSchema == null) { try (InputStream schemaStream = Utils.openFileStream(baseDir, outputAvroSchemaFile)) { outputAvroSchema = IOUtils.toString( - Objects.requireNonNull(schemaStream), Charset.defaultCharset()); + Objects.requireNonNull(schemaStream), Charset.defaultCharset()); } } } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/GeneratorConfig.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/GeneratorConfig.java index 25507ce3..d93f15e8 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/GeneratorConfig.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/GeneratorConfig.java @@ -1,9 +1,8 @@ package com.cloudera.cyber.generator; -import lombok.Data; - import java.io.IOException; import java.util.List; +import lombok.Data; @Data public class GeneratorConfig { @@ -11,7 +10,7 @@ public class GeneratorConfig { List generationSources; public void open() throws IOException { - for(GenerationSource source : getGenerationSources()) { + for (GenerationSource source : getGenerationSources()) { source.readAvroSchema(baseDirectory); } } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/RandomGenerators.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/RandomGenerators.java index f855f52f..3c2be381 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/RandomGenerators.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/RandomGenerators.java @@ -13,10 +13,9 @@ package com.cloudera.cyber.generator; import com.github.javafaker.Faker; -import org.apache.commons.net.util.SubnetUtils; - import java.io.Serializable; import java.util.concurrent.ThreadLocalRandom; +import org.apache.commons.net.util.SubnetUtils; public class RandomGenerators implements Serializable { private static final Faker faker = new Faker(); @@ -36,7 +35,7 @@ public static int randomInt(int from, int to) { return ThreadLocalRandom.current().nextInt(from, to + 1); } - public static String randomChoice(String ...options) { + public static String randomChoice(String... options) { return options[ThreadLocalRandom.current().nextInt(options.length)]; } @@ -55,6 +54,7 @@ public static String canonicalize(String host) { public static String randomEmail() { return faker.internet().emailAddress(); } + public static String randomUser() { return faker.name().username(); } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/SyntheticEntry.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/SyntheticEntry.java index 2aefccdd..15498b41 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/SyntheticEntry.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/SyntheticEntry.java @@ -12,15 +12,14 @@ package com.cloudera.cyber.generator; +import java.util.Map; import lombok.Builder; import lombok.Data; -import java.util.Map; - @Data @Builder public class SyntheticEntry { private long ts; private RandomGenerators utils; - private Map params; + private Map params; } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/SyntheticThreatEntry.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/SyntheticThreatEntry.java index 4866ef33..cdd0751e 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/SyntheticThreatEntry.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/SyntheticThreatEntry.java @@ -12,7 +12,6 @@ package com.cloudera.cyber.generator; -import com.cloudera.cyber.generator.RandomGenerators; import lombok.Builder; import lombok.Data; diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGenerator.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGenerator.java index 4c6fec33..b2d73609 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGenerator.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGenerator.java @@ -16,8 +16,6 @@ import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; -import org.apache.commons.io.IOUtils; - import java.io.IOException; import java.io.StringWriter; import java.io.Writer; @@ -26,6 +24,7 @@ import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.IntStream; +import org.apache.commons.io.IOUtils; public class ThreatGenerator { private List lines; @@ -51,21 +50,19 @@ public ThreatGenerator() { private static final RandomGenerators utils = new RandomGenerators(); /** - * Choose a random threat template, insert the IP address and return the string + * Choose a random threat template, insert the IP address and return the string. + * *

* random line from the threats resource, treat as freemarker and expose - * - * @param ip - * @return */ public String generateThreat(String ip) throws IOException, TemplateException { String template = String.valueOf(ThreadLocalRandom.current().nextInt(lines.size() - 1)); SyntheticThreatEntry entry = SyntheticThreatEntry.builder().ts( - LocalDateTime.now().toInstant(ZoneOffset.UTC).toEpochMilli()) - .utils(utils) - .ip(ip) - .build(); + LocalDateTime.now().toInstant(ZoneOffset.UTC).toEpochMilli()) + .utils(utils) + .ip(ip) + .build(); // figure out which template we're using, i.e. weighted by the files map Template temp = cfg.getTemplate(template); diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGeneratorMap.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGeneratorMap.java index a5e9ea10..2b2ffb66 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGeneratorMap.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGeneratorMap.java @@ -12,12 +12,11 @@ package com.cloudera.cyber.generator; +import java.nio.charset.Charset; import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; -import java.nio.charset.Charset; - public class ThreatGeneratorMap extends RichMapFunction> { private transient ThreatGenerator threatGenerator; private final String topicName; diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/Utils.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/Utils.java index 57996eeb..bb251df2 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/Utils.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/Utils.java @@ -12,42 +12,46 @@ package com.cloudera.cyber.generator; +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.*; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.io.JsonDecoder; import org.apache.flink.core.fs.Path; -import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; - public class Utils { - public static GenericRecord jsonDecodeToAvroGenericRecord(String json, Schema schema) { - try { - JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(schema, json); - DatumReader datumReader = new GenericDatumReader<>(schema); - return datumReader.read(null, jsonDecoder); - } catch (IOException e) { - return null; - } - } + public static GenericRecord jsonDecodeToAvroGenericRecord(String json, Schema schema) { + try { + JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(schema, json); + DatumReader datumReader = new GenericDatumReader<>(schema); + return datumReader.read(null, jsonDecoder); + } catch (IOException e) { + return null; + } + } - public static byte[] jsonDecodeToAvroByteArray(String json, Schema schema) { - GenericRecord record = jsonDecodeToAvroGenericRecord(json, schema); - DatumWriter datumWriter = new GenericDatumWriter<>(schema); - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(out, null); - datumWriter.write(record, binaryEncoder); - return out.toByteArray(); - } catch (IOException exception) { - return null; - } - } + public static byte[] jsonDecodeToAvroByteArray(String json, Schema schema) { + GenericRecord record = jsonDecodeToAvroGenericRecord(json, schema); + DatumWriter datumWriter = new GenericDatumWriter<>(schema); + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(out, null); + datumWriter.write(record, binaryEncoder); + return out.toByteArray(); + } catch (IOException exception) { + return null; + } + } public static InputStream openFileStream(String baseDir, String filePath) throws IOException { Path possiblePath = new Path(filePath); diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/scenario/GeneratorScenario.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/scenario/GeneratorScenario.java index 32a24152..fb7a270a 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/scenario/GeneratorScenario.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/scenario/GeneratorScenario.java @@ -3,6 +3,11 @@ import com.cloudera.cyber.generator.RandomGenerators; import com.cloudera.cyber.generator.Utils; import com.google.common.base.Preconditions; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.MappingIterator; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectReader; @@ -10,15 +15,10 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvParser; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.List; -import java.util.Map; - public class GeneratorScenario { - protected final static String NOT_ENOUGH_LINES_ERROR = "Scenario CSV file must contain a header and at least one value line"; + protected static final String NOT_ENOUGH_LINES_ERROR = + "Scenario CSV file must contain a header and at least one value line"; private final List lines; private final ObjectReader csvReader; @@ -29,7 +29,7 @@ public static GeneratorScenario load(String baseDir, String csvFile) throws IOEx List lines = IOUtils.readLines(csvStream, Charset.defaultCharset()); Preconditions.checkState(lines.size() >= 2, NOT_ENOUGH_LINES_ERROR); scenario = new GeneratorScenario(lines); - } + } return scenario; } @@ -41,7 +41,7 @@ private GeneratorScenario(List lines) throws IOException { } public Map randomParameters() throws IOException { - int randomIndex = RandomGenerators.randomInt(1,lines.size() - 1); + int randomIndex = RandomGenerators.randomInt(1, lines.size() - 1); return csvReader.readValue(lines.get(randomIndex)); } @@ -50,16 +50,16 @@ private ObjectReader createCsvReader(List header) { header.forEach(builder::addColumn); final CsvMapper mapper = new CsvMapper(); return mapper - .readerForMapOf(String.class) - .with(builder.build()); + .readerForMapOf(String.class) + .with(builder.build()); } private List readCsvHeader(String header) throws IOException { final CsvMapper mapper = new CsvMapper(); MappingIterator> it = mapper - .readerForListOf(String.class) - .with(CsvParser.Feature.WRAP_AS_ARRAY) - .readValues(header); + .readerForListOf(String.class) + .with(CsvParser.Feature.WRAP_AS_ARRAY) + .readValues(header); return it.nextValue(); } diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/libs/CyberFunctionDefinition.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/libs/CyberFunctionDefinition.java index 2a8053c3..cfc2aa5f 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/libs/CyberFunctionDefinition.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/libs/CyberFunctionDefinition.java @@ -13,11 +13,6 @@ package com.cloudera.cyber.libs; import com.cloudera.cyber.CyberFunction; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.atteo.classindex.ClassIndex; - import java.lang.reflect.Method; import java.lang.reflect.Parameter; import java.util.Collections; @@ -26,6 +21,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.atteo.classindex.ClassIndex; @Getter @Slf4j @@ -36,9 +35,11 @@ public class CyberFunctionDefinition { private final List parameters; private final Class implementationClass; - private static final List CYBER_FUNCTION_DEFINITIONS = StreamSupport.stream(ClassIndex.getAnnotated(CyberFunction.class).spliterator(), false) - .map(funcClass -> new CyberFunctionDefinition(funcClass.getAnnotation(CyberFunction.class).value(), funcClass)) - .filter(f -> f.isValid).collect(Collectors.toList()); + private static final List CYBER_FUNCTION_DEFINITIONS = + StreamSupport.stream(ClassIndex.getAnnotated(CyberFunction.class).spliterator(), false) + .map(funcClass -> new CyberFunctionDefinition(funcClass.getAnnotation(CyberFunction.class).value(), + funcClass)) + .filter(f -> f.isValid).collect(Collectors.toList()); public static Stream findAll() { return CYBER_FUNCTION_DEFINITIONS.stream(); @@ -50,15 +51,20 @@ public CyberFunctionDefinition(String functionName, Class implementationClass List parameters = Collections.emptyList(); boolean isValid = false; if (StringUtils.isNotBlank(functionName)) { - Optional evalMethod = Stream.of(implementationClass.getDeclaredMethods()).filter(m -> m.getName().equals("eval")).findFirst(); + Optional evalMethod = + Stream.of(implementationClass.getDeclaredMethods()).filter(m -> m.getName().equals("eval")) + .findFirst(); if (evalMethod.isPresent()) { - parameters = Stream.of(evalMethod.get().getParameters()).collect(Collectors.toList()); - isValid = true; + parameters = Stream.of(evalMethod.get().getParameters()).collect(Collectors.toList()); + isValid = true; } else { - log.error("Cyber Function defined in class {} does not have an eval method. Function {} will not be defined in javascript.", implementationClass.getCanonicalName(), functionName); + log.error( + "Cyber Function defined in class {} does not have an eval method. Function {} will not be defined in javascript.", + implementationClass.getCanonicalName(), functionName); } } else { - log.error("Cyber Function defined in class {} has a blank function name.", implementationClass.getCanonicalName()); + log.error("Cyber Function defined in class {} has a blank function name.", + implementationClass.getCanonicalName()); } this.isValid = isValid; this.parameters = parameters; diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/parser/MessageToParseDeserializer.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/parser/MessageToParseDeserializer.java index 685ddc0e..b2a51260 100644 --- a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/parser/MessageToParseDeserializer.java +++ b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/parser/MessageToParseDeserializer.java @@ -12,24 +12,24 @@ package com.cloudera.cyber.parser; +import java.io.IOException; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.connector.kafka.source.reader.deserializer.KafkaRecordDeserializationSchema; import org.apache.flink.util.Collector; import org.apache.kafka.clients.consumer.ConsumerRecord; -import java.io.IOException; - public class MessageToParseDeserializer implements KafkaRecordDeserializationSchema { @Override - public void deserialize(ConsumerRecord consumerRecord, Collector collector) throws IOException { + public void deserialize(ConsumerRecord consumerRecord, Collector collector) + throws IOException { collector.collect(MessageToParse.builder() - .originalBytes(consumerRecord.value()) - .topic(consumerRecord.topic()) - .offset(consumerRecord.offset()) - .partition(consumerRecord.partition()) - .key(consumerRecord.key()) - .build()); + .originalBytes(consumerRecord.value()) + .topic(consumerRecord.topic()) + .offset(consumerRecord.offset()) + .partition(consumerRecord.partition()) + .key(consumerRecord.key()) + .build()); } @Override diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/MessageUtilsTest.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/MessageUtilsTest.java index 19fc1662..1f167200 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/MessageUtilsTest.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/MessageUtilsTest.java @@ -12,17 +12,20 @@ package com.cloudera.cyber; +import static java.util.stream.Collectors.toList; + import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.time.Instant; -import java.util.*; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; - public class MessageUtilsTest { private static final String TEST_FEATURE = "feature"; @@ -38,10 +41,14 @@ public class MessageUtilsTest { private final HashMap SECOND_FIELD_MAP = new HashMap<>(); private final HashMap ALL_FIELDS_MAP = new HashMap<>(); - private final List DATA_QUALITY_MESSAGES_1 = Collections.singletonList(new DataQualityMessage(DataQualityMessageLevel.INFO.name(), TEST_FEATURE, TEST_FIELD_1, TEST_MESSAGE_1)); - private final List DATA_QUALITY_MESSAGES_1_DEEP_COPY = DATA_QUALITY_MESSAGES_1.stream().map(m -> m.toBuilder().build()).collect(toList()); - private final List DATA_QUALITY_MESSAGES_2 = Collections.singletonList(new DataQualityMessage(DataQualityMessageLevel.ERROR.name(), TEST_FEATURE, TEST_FIELD_2, TEST_MESSAGE_2)); - private final List ALL_DATA_QUALITY_MESSAGES = Stream.concat(DATA_QUALITY_MESSAGES_1.stream(), DATA_QUALITY_MESSAGES_2.stream()).collect(toList()); + private final List DATA_QUALITY_MESSAGES_1 = Collections.singletonList( + new DataQualityMessage(DataQualityMessageLevel.INFO.name(), TEST_FEATURE, TEST_FIELD_1, TEST_MESSAGE_1)); + private final List DATA_QUALITY_MESSAGES_1_DEEP_COPY = + DATA_QUALITY_MESSAGES_1.stream().map(m -> m.toBuilder().build()).collect(toList()); + private final List DATA_QUALITY_MESSAGES_2 = Collections.singletonList( + new DataQualityMessage(DataQualityMessageLevel.ERROR.name(), TEST_FEATURE, TEST_FIELD_2, TEST_MESSAGE_2)); + private final List ALL_DATA_QUALITY_MESSAGES = + Stream.concat(DATA_QUALITY_MESSAGES_1.stream(), DATA_QUALITY_MESSAGES_2.stream()).collect(toList()); @Before public void initTestExtensionMaps() { @@ -72,13 +79,13 @@ public void testAddEmptyFields() { } - @Test + @Test public void testEnrichExtensions() { Message input = TestUtils.createMessage(); List expectedDataQualityMessages = new ArrayList<>(); Message output1 = MessageUtils.enrich(input, FIRST_FIELD_MAP, expectedDataQualityMessages); - Assert.assertNotSame(input, output1); + Assert.assertNotSame(input, output1); Assert.assertEquals(FIRST_FIELD_MAP, output1.getExtensions()); Assert.assertEquals(Collections.emptyList(), output1.getDataQualityMessages()); @@ -124,10 +131,11 @@ public void testReplaceExtensions() { String extensionsToChangeOriginalValue = "old_value"; String extensionsToChangeNewValue = "new_value"; Message input = TestUtils.createMessage(ImmutableMap.of(extensionStaysSame, extensionStaysSameValue, - extensionToChange, extensionsToChangeOriginalValue)); - Message output = MessageUtils.replaceFields(input, ImmutableMap.of(extensionToChange, extensionsToChangeNewValue)); + extensionToChange, extensionsToChangeOriginalValue)); + Message output = + MessageUtils.replaceFields(input, ImmutableMap.of(extensionToChange, extensionsToChangeNewValue)); Assert.assertEquals(ImmutableMap.of(extensionStaysSame, extensionStaysSameValue, - extensionToChange, extensionsToChangeNewValue), output.getExtensions()); + extensionToChange, extensionsToChangeNewValue), output.getExtensions()); } @Test @@ -144,7 +152,8 @@ private void testReplaceNoOriginalEventExtensions(Map originalEv String extensionToChange = "change_me"; String extensionsToChangeNewValue = "new_value"; Message input = TestUtils.createMessage(originalEventExtensions); - Message output = MessageUtils.replaceFields(input, ImmutableMap.of(extensionToChange, extensionsToChangeNewValue)); + Message output = + MessageUtils.replaceFields(input, ImmutableMap.of(extensionToChange, extensionsToChangeNewValue)); Assert.assertEquals(ImmutableMap.of(extensionToChange, extensionsToChangeNewValue), output.getExtensions()); } @@ -160,7 +169,7 @@ public void testReplaceExtensionsEmptyReplaceExtensions() { void testNoReplaceValues(Map noReplaceValues) { Map originalExtensions = ImmutableMap.of("field_1", "value_1", - "field_2", "value_2"); + "field_2", "value_2"); Message input = TestUtils.createMessage(originalExtensions); Message output = MessageUtils.replaceFields(input, noReplaceValues); Assert.assertEquals(originalExtensions, output.getExtensions()); diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestMessageSerializer.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestMessageSerializer.java index b9d72d78..e0b09909 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestMessageSerializer.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestMessageSerializer.java @@ -12,6 +12,14 @@ package com.cloudera.cyber; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.hortonworks.registries.schemaregistry.SchemaCompatibility; import com.hortonworks.registries.schemaregistry.SchemaIdVersion; import com.hortonworks.registries.schemaregistry.SchemaMetadata; @@ -21,57 +29,48 @@ import com.hortonworks.registries.schemaregistry.errors.SchemaNotFoundException; import com.hortonworks.registries.schemaregistry.serdes.avro.AvroSnapshotSerializer; import com.hortonworks.registries.schemaregistry.serdes.avro.AvroUtils; +import java.time.Instant; +import java.util.Collections; import org.apache.avro.Schema; import org.junit.Before; import org.junit.Test; -import java.time.Instant; -import java.util.Collections; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class TestMessageSerializer { private AvroSnapshotSerializer avroSnapshotSerializer; private Message testMessage() { return Message.builder() - .ts(Instant.now().toEpochMilli()) - .originalSource(SignedSourceKey.builder() - .topic("test") - .partition(0) - .offset(0) - .signature(new byte[128]) - .build()) - .extensions(Collections.singletonMap("test", "value")) - .message("") - .source("test") - .build(); + .ts(Instant.now().toEpochMilli()) + .originalSource(SignedSourceKey.builder() + .topic("test") + .partition(0) + .offset(0) + .signature(new byte[128]) + .build()) + .extensions(Collections.singletonMap("test", "value")) + .message("") + .source("test") + .build(); } private ThreatIntelligence testTi() { return ThreatIntelligence.builder() - .ts(Instant.now().toEpochMilli()) - .fields(Collections.singletonMap("test", "value")) - .observableType("testType") - .observable("testObservable") - .build(); + .ts(Instant.now().toEpochMilli()) + .fields(Collections.singletonMap("test", "value")) + .observableType("testType") + .observable("testObservable") + .build(); } @Before public void init() throws SchemaNotFoundException, InvalidSchemaException, IncompatibleSchemaException { ISchemaRegistryClient testClient = mock(ISchemaRegistryClient.class); - when(testClient.uploadSchemaVersion(any(),any(),any(),any())) - .thenReturn(new SchemaIdVersion(1L, 1)); + when(testClient.uploadSchemaVersion(any(), any(), any(), any())) + .thenReturn(new SchemaIdVersion(1L, 1)); when(testClient.addSchemaVersion(any(SchemaMetadata.class), any())) - .thenReturn(new SchemaIdVersion(1L, 1, 1L)); + .thenReturn(new SchemaIdVersion(1L, 1, 1L)); avroSnapshotSerializer = new AvroSnapshotSerializer(testClient); avroSnapshotSerializer.init(Collections.emptyMap()); @@ -83,10 +82,10 @@ public void testSimpleAvroBased() { Message test = testMessage(); SchemaMetadata schemaMetadata = new SchemaMetadata.Builder("test") - .schemaGroup("kafka") - .description("test") - .type("avro") - .compatibility(SchemaCompatibility.FORWARD).build(); + .schemaGroup("kafka") + .description("test") + .type("avro") + .compatibility(SchemaCompatibility.FORWARD).build(); byte[] serialize = avroSnapshotSerializer.serialize(test, schemaMetadata); assertThat("Bytes are made", serialize.length, greaterThan(100)); @@ -99,7 +98,7 @@ public void testMessageSchema() { System.out.println(schema.toString()); assertThat("Schema Computed for Message", schema, notNullValue()); assertThat("Schema has fields", schema.getField("extensions"), notNullValue()); - assertThat("Schema has timestamp field", schema.getField("ts").name(),equalTo("ts")); + assertThat("Schema has timestamp field", schema.getField("ts").name(), equalTo("ts")); } @Test @@ -115,10 +114,10 @@ public void testTIMessageSerializer() { ThreatIntelligence test = testTi(); SchemaMetadata schemaMetadata = new SchemaMetadata.Builder("test") - .schemaGroup("kafka") - .description("test") - .type("avro") - .compatibility(SchemaCompatibility.FORWARD).build(); + .schemaGroup("kafka") + .description("test") + .type("avro") + .compatibility(SchemaCompatibility.FORWARD).build(); byte[] serialize = avroSnapshotSerializer.serialize(test, schemaMetadata); diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestMessageTypeInformation.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestMessageTypeInformation.java index 90728b5b..b5c392e4 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestMessageTypeInformation.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestMessageTypeInformation.java @@ -12,12 +12,12 @@ package com.cloudera.cyber; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.junit.Test; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.notNullValue; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.junit.Test; + public class TestMessageTypeInformation { @Test diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestSigning.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestSigning.java index 64fb956f..dd7f4155 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestSigning.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestSigning.java @@ -12,20 +12,27 @@ package com.cloudera.cyber; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + import com.cloudera.cyber.flink.Utils; +import java.security.InvalidKeyException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.SignatureException; import org.apache.commons.lang3.ArrayUtils; import org.hamcrest.collection.IsArrayWithSize; import org.junit.Test; -import java.security.*; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - public class TestSigning { - + @Test - public void testSign() throws NoSuchProviderException, NoSuchAlgorithmException, SignatureException, InvalidKeyException { + public void testSign() + throws NoSuchProviderException, NoSuchAlgorithmException, SignatureException, InvalidKeyException { KeyPairGenerator gen = KeyPairGenerator.getInstance("RSA", "SunRsaSign"); gen.initialize(1024, new SecureRandom()); @@ -42,5 +49,5 @@ public void testSign() throws NoSuchProviderException, NoSuchAlgorithmException, assertThat("verifies messed about data fails", Utils.verify("fsdfsaf", out, pair.getPublic()), is(false)); assertThat("verifies wrong key fails", Utils.verify(data, out, pair2.getPublic()), is(false)); } - + } diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestUtils.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestUtils.java index f2a06d7f..a0819874 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestUtils.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/TestUtils.java @@ -12,8 +12,9 @@ package com.cloudera.cyber; -import com.cloudera.cyber.parser.MessageToParse; +import static java.lang.String.format; +import com.cloudera.cyber.parser.MessageToParse; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; @@ -27,8 +28,6 @@ import java.util.Map; import java.util.Stack; -import static java.lang.String.format; - public class TestUtils { public static String findDir(String name) { @@ -56,11 +55,11 @@ public static String findDir(File startDir, String name) { public static SignedSourceKey source(String topic, int partition, long offset) { return SignedSourceKey.builder() - .topic(topic) - .partition(partition) - .offset(offset) - .signature(new byte[128]) - .build(); + .topic(topic) + .partition(partition) + .offset(offset) + .signature(new byte[128]) + .build(); } public static SignedSourceKey source() { @@ -73,13 +72,13 @@ public static Message createMessage(Map extensions) { public static Message createMessage(long timestamp, String source, Map extensions) { return Message.builder() - .ts(timestamp) - .source(source) - .extensions(extensions) - .message("") - .originalSource( - source("topic", 0, 0)). - build(); + .ts(timestamp) + .source(source) + .extensions(extensions) + .message("") + .originalSource( + source("topic", 0, 0)) + .build(); } public static Message createMessage() { @@ -92,15 +91,16 @@ public static MessageToParse.MessageToParseBuilder createMessageToParse(String s public static MessageToParse.MessageToParseBuilder createMessageToParse(String source, String topic) { return MessageToParse.builder() - .originalBytes(source.getBytes(StandardCharsets.UTF_8)) - .topic(topic) - .offset(0) - .partition(0); + .originalBytes(source.getBytes(StandardCharsets.UTF_8)) + .topic(topic) + .offset(0) + .partition(0); } public static SignedSourceKey createOriginal(String topic) { return SignedSourceKey.builder().topic(topic).offset(0).partition(0).signature(new byte[128]).build(); } + public static SignedSourceKey createOriginal() { return createOriginal("test"); } @@ -120,7 +120,7 @@ public static File createTempDir(File dir) throws IOException { * Create directory that is automatically cleaned up after the * JVM shuts down through use of a Runtime shutdown hook. * - * @param dir Directory to create, including missing parent directories + * @param dir Directory to create, including missing parent directories * @param cleanup true/false * @return File handle to the created temp directory */ @@ -150,7 +150,7 @@ public static File createTempDir(String prefix) throws IOException { * Create directory that is optionally cleaned up after the * JVM shuts down through use of a Runtime shutdown hook. * - * @param prefix Prefix to apply to temp directory name + * @param prefix Prefix to apply to temp directory name * @param cleanup true/false * @return File handle to the created temp directory * @throws IOException Unable to create temp directory @@ -216,7 +216,7 @@ public static String read(File in) throws IOException { /** * Reads file contents into a String * - * @param in Input file + * @param in Input file * @param charset charset to use for reading * @return contents of input file * @throws IOException diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/ValidateUtilsTest.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/ValidateUtilsTest.java index 75ac55e0..175bb2d0 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/ValidateUtilsTest.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/ValidateUtilsTest.java @@ -12,16 +12,17 @@ package com.cloudera.cyber; -import com.cloudera.cyber.flink.Utils; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.entry; +import com.cloudera.cyber.flink.Utils; import java.util.concurrent.TimeUnit; - import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.windowing.time.Time; import org.junit.Test; -import static org.assertj.core.api.Assertions.*; - public class ValidateUtilsTest { @@ -58,26 +59,34 @@ public void validTimeCompare() { @Test public void invalidPhoenixNameTest() { - assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("1name", testParam)).isInstanceOf(IllegalArgumentException.class).hasMessage("Invalid value 1name for parameter '" + testParam + "'. It can only contain alphanumerics or underscore(a-z, A-Z, 0-9, _)"); - assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("$name1", testParam)).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("_n^ame_", testParam)).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("!name1", testParam)).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("n###ame1", testParam)).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("__________", testParam)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("1name", testParam)).isInstanceOf( + IllegalArgumentException.class).hasMessage("Invalid value 1name for parameter '" + testParam + + "'. It can only contain alphanumerics or underscore(a-z, A-Z, 0-9, _)"); + assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("$name1", testParam)).isInstanceOf( + IllegalArgumentException.class); + assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("_n^ame_", testParam)).isInstanceOf( + IllegalArgumentException.class); + assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("!name1", testParam)).isInstanceOf( + IllegalArgumentException.class); + assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("n###ame1", testParam)).isInstanceOf( + IllegalArgumentException.class); + assertThatThrownBy(() -> ValidateUtils.validatePhoenixName("__________", testParam)).isInstanceOf( + IllegalArgumentException.class); } @Test public void testPropToolsMerge() { ParameterTool parameterTool = Utils.getParamToolsFromProperties( - new String[]{"src/test/resources/test.properties", "src/test/resources/test1.properties", "src/test/resources/test2.properties"}); + new String[] {"src/test/resources/test.properties", "src/test/resources/test1.properties", + "src/test/resources/test2.properties"}); assertThat(parameterTool.toMap()).containsOnly( - entry("test.property", "test-property-rewrited"), - entry("test.property1", "test-property1"), - entry("test1.property", "test1-property"), - entry("test1.property1", "test1-property1-rewrited"), - entry("test2.property", "test2-property"), - entry("test2.property1", "test2-property1") + entry("test.property", "test-property-rewrited"), + entry("test.property1", "test-property1"), + entry("test1.property", "test1-property"), + entry("test1.property1", "test1-property1-rewrited"), + entry("test2.property", "test2-property"), + entry("test2.property1", "test2-property1") ); } } \ No newline at end of file diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/GenerationSourceTest.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/GenerationSourceTest.java index ab4875d1..1e41d5d9 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/GenerationSourceTest.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/GenerationSourceTest.java @@ -1,7 +1,6 @@ package com.cloudera.cyber.generator; -import org.junit.Assert; -import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.FileNotFoundException; import java.io.IOException; @@ -9,8 +8,8 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Assert; +import org.junit.Test; public class GenerationSourceTest { @@ -24,7 +23,7 @@ public void testReadSchema() throws IOException { @Test public void testSchemaDoesNotExist() { assertThatThrownBy(() -> verifySchemaRead("doesnt_exist", false)).isInstanceOf(FileNotFoundException.class) - .hasMessage("Basedir: '' File: 'doesnt_exist'"); + .hasMessage("Basedir: '' File: 'doesnt_exist'"); } private void verifySchemaRead(String schemaPath, boolean isNull) throws IOException { @@ -49,12 +48,13 @@ public void testScenarioFileParameters() throws IOException { @Test public void testScenarioFileDoesntExist() { - assertThatThrownBy(() -> testScenarioFile("doesnt_exist.csv", Collections.emptySet())).isInstanceOf(FileNotFoundException.class) - .hasMessage("Basedir: '' File: 'doesnt_exist.csv'"); + assertThatThrownBy(() -> testScenarioFile("doesnt_exist.csv", Collections.emptySet())).isInstanceOf( + FileNotFoundException.class) + .hasMessage("Basedir: '' File: 'doesnt_exist.csv'"); } private void testScenarioFile(String scenarioFilePath, Set expectedKeys) throws IOException { - GenerationSource gs = new GenerationSource("file", "topic", null, 1.0, scenarioFilePath, null, null ); + GenerationSource gs = new GenerationSource("file", "topic", null, 1.0, scenarioFilePath, null, null); gs.readScenarioFile(""); Assert.assertEquals(scenarioFilePath == null, gs.getScenario() == null); diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/GeneratorUtilsTest.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/GeneratorUtilsTest.java index 2419eb58..c9b92787 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/GeneratorUtilsTest.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/GeneratorUtilsTest.java @@ -1,15 +1,14 @@ package com.cloudera.cyber.generator; -import com.nimbusds.jose.util.IOUtils; -import org.junit.Assert; -import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.nimbusds.jose.util.IOUtils; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Assert; +import org.junit.Test; public class GeneratorUtilsTest { @@ -21,7 +20,8 @@ public void testStreamFile() throws IOException { String file = "test_file.txt"; verifyStreamFile(dir, file, EXPECTED_FILE_CONTENTS); verifyStreamFile("", dir.concat("/").concat(file), EXPECTED_FILE_CONTENTS); - assertThatThrownBy(() -> verifyStreamFile("/doesnt_exist", file, null)).isInstanceOf(FileNotFoundException.class).hasMessage("Basedir: '/doesnt_exist' File: 'test_file.txt'"); + assertThatThrownBy(() -> verifyStreamFile("/doesnt_exist", file, null)).isInstanceOf( + FileNotFoundException.class).hasMessage("Basedir: '/doesnt_exist' File: 'test_file.txt'"); } private void verifyStreamFile(String baseDir, String file, String expectedResult) throws IOException { diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/scenario/GeneratorScenarioTest.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/scenario/GeneratorScenarioTest.java index dabe7c80..d4f05f62 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/scenario/GeneratorScenarioTest.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/generator/scenario/GeneratorScenarioTest.java @@ -1,15 +1,18 @@ package com.cloudera.cyber.generator.scenario; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.junit.Assert; import org.junit.Test; -import java.io.IOException; -import java.util.*; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; - public class GeneratorScenarioTest { private static final String IP_DEST_ADDR = "ip_dst_addr"; private static final String DOMAIN_NAME = "domain_name"; @@ -19,17 +22,20 @@ public class GeneratorScenarioTest { public void testCsvScenario() throws IOException { GeneratorScenario scenario = GeneratorScenario.load("src/test/resources", "scenario.csv"); - Map expectedValue1 = ImmutableMap.of(IP_DEST_ADDR, "1.1.1.1", DOMAIN_NAME, "www.google.com", ACTION, "CONNECT"); - Map expectedValue2 = ImmutableMap.of(IP_DEST_ADDR, "2.2.2.2", DOMAIN_NAME, "www.amazon.com", ACTION, "MISS"); + Map expectedValue1 = + ImmutableMap.of(IP_DEST_ADDR, "1.1.1.1", DOMAIN_NAME, "www.google.com", ACTION, "CONNECT"); + Map expectedValue2 = + ImmutableMap.of(IP_DEST_ADDR, "2.2.2.2", DOMAIN_NAME, "www.amazon.com", ACTION, "MISS"); List> expectedValues = Lists.newArrayList(expectedValue1, expectedValue2); - List> actualRandomParameters = new ArrayList<>(); + List> actualRandomParameters = new ArrayList<>(); Set actualIpDstAddrValues = new HashSet<>(); int count = 100; - for(int i = 0; i < count; i++) { + for (int i = 0; i < count; i++) { Map randomParameters = scenario.randomParameters(); - Assert.assertTrue(String.format("randomParameters = %s does not match expected values", randomParameters), expectedValues.contains(randomParameters)); + Assert.assertTrue(String.format("randomParameters = %s does not match expected values", randomParameters), + expectedValues.contains(randomParameters)); actualIpDstAddrValues.add(randomParameters.get(IP_DEST_ADDR)); actualRandomParameters.add(randomParameters); } @@ -40,8 +46,8 @@ public void testCsvScenario() throws IOException { @Test public void testCsvFileNotFound() { String notFoundFileName = "doesnt_exist.csv"; - assertThatThrownBy(() ->GeneratorScenario.load("", notFoundFileName)).isInstanceOf(IOException.class). - hasMessage(String.format("Basedir: '' File: '%s'", notFoundFileName)); + assertThatThrownBy(() -> GeneratorScenario.load("", notFoundFileName)).isInstanceOf(IOException.class) + .hasMessage(String.format("Basedir: '' File: '%s'", notFoundFileName)); } @Test @@ -55,9 +61,9 @@ public void testHeaderOnlyFile() { } private void testNotEnoughLines(String scenarioFile) { - assertThatThrownBy(() ->GeneratorScenario.load("", scenarioFile)). - isInstanceOf(IllegalStateException.class). - hasMessage(GeneratorScenario.NOT_ENOUGH_LINES_ERROR); + assertThatThrownBy(() -> GeneratorScenario.load("", scenarioFile)) + .isInstanceOf(IllegalStateException.class) + .hasMessage(GeneratorScenario.NOT_ENOUGH_LINES_ERROR); } } diff --git a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/libs/TestCyberFunctionUtils.java b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/libs/TestCyberFunctionUtils.java index cc68932a..e224b75c 100644 --- a/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/libs/TestCyberFunctionUtils.java +++ b/flink-cyber/flink-common/src/test/java/com/cloudera/cyber/libs/TestCyberFunctionUtils.java @@ -12,15 +12,14 @@ package com.cloudera.cyber.libs; -import org.junit.Test; - -import java.util.List; -import java.util.stream.Collectors; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.Test; + public class TestCyberFunctionUtils { @Test diff --git a/flink-cyber/flink-cyber-api/enrichment_config.md b/flink-cyber/flink-cyber-api/enrichment_config.md index dbcd5a3c..9ec9dc86 100644 --- a/flink-cyber/flink-cyber-api/enrichment_config.md +++ b/flink-cyber/flink-cyber-api/enrichment_config.md @@ -1,24 +1,26 @@ # Enrichment Config File -The enrichment config file defines what enrichments to apply to each message source. It is a text file in json format. + +The enrichment config file defines what enrichments to apply to each message source. It is a text file in json format. ## EnrichmentsConfig Json -| Json Field | Type | Description | Required/Default | -| -----------------| -------- | -------------------------- | ----------------| -| source | string | Source of message to enrich. Apply these enrichments to the specified source. | required | -| kind | enum [LOCAL,HBASE] | Defines where the enrichment data is stored. | required | -| fields | list of EnrichmentFields | Maps message fields to enrichments | required | +| Json Field | Type | Description | Required/Default | +|------------|--------------------------|--------------------------------------------------------------------------------|------------------| +| source | string | Source of message to enrich. Apply these enrichments to the specified source. | required | +| kind | enum [LOCAL,HBASE] | Defines where the enrichment data is stored. | required | +| fields | list of EnrichmentFields | Maps message fields to enrichments | required | ### EnrichmentFields -| Json Field | Type | Description | Required/Default | -| -----------------| -------- | -------------------------- | ----------------| -| name | string | Use this field value as the key to the enrichment. | required | -| enrichmentType | string | Name of the enrichment type to apply. | required | +| Json Field | Type | Description | Required/Default | +|----------------|--------|----------------------------------------------------|------------------| +| name | string | Use this field value as the key to the enrichment. | required | +| enrichmentType | string | Name of the enrichment type to apply. | required | ## Example In the example below, the triaging job will apply the following enrichments to messages with squid source: + 1. Lookup domain in the malicious_domain map stored in Flink state. 2. Lookup domain in the domain_category map stored in Flink state. 3. Lookup domain in the majestic_million mapping stored in HBase. diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/AvroTypes.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/AvroTypes.java index 5afdfda5..4be6243c 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/AvroTypes.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/AvroTypes.java @@ -12,6 +12,12 @@ package com.cloudera.cyber; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificRecordBase; @@ -20,32 +26,28 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; - public class AvroTypes { - public static final Schema timestampMilliType = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)); + public static final Schema timestampMilliType = + LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)); public static final Schema uuidType = LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)); - public static TypeInformation overrideTypes(AvroTypeInfo avro, Class clazz, Map> fields) { + public static TypeInformation overrideTypes(AvroTypeInfo avro, Class clazz, + Map> fields) { return Types.POJO(clazz, Arrays.stream(avro.getFieldNames()) - .collect(toMap( - k -> k, - v -> fields.containsKey(v) ? fields.get(v) : - avro.getTypeAt(v) - ))); + .collect(toMap( + k -> k, + v -> fields.containsKey(v) ? fields.get(v) : avro.getTypeAt(v) + ))); } public static Map utf8toStringMap(Object value$) { - if (value$ == null) return null; + if (value$ == null) { + return null; + } try { return ((Map) value$).entrySet().stream().collect(toMap( - k -> k.getKey().toString(), - k -> k.getValue().toString() + k -> k.getKey().toString(), + k -> k.getValue().toString() )); } catch (ClassCastException e) { return (Map) value$; @@ -54,9 +56,11 @@ public static Map utf8toStringMap(Object value$) { public static List toListOf(Class cls, Object value$) { // TODO - ensure the serialization of the contained object is correct - if (value$ == null) return null; + if (value$ == null) { + return null; + } return ((List) value$).stream() - .map(o -> (T) o) - .collect(toList()); + .map(o -> (T) o) + .collect(toList()); } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/CyberFunction.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/CyberFunction.java index 0f462aba..a11ea27e 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/CyberFunction.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/CyberFunction.java @@ -12,20 +12,18 @@ package com.cloudera.cyber; -import org.atteo.classindex.IndexAnnotated; - import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import org.atteo.classindex.IndexAnnotated; @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) @IndexAnnotated public @interface CyberFunction { /** - * The name given to the function in its UDF forms - * @return + * The name given to the function in its UDF forms. */ String value(); } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessage.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessage.java index 36e97f59..a8862da2 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessage.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessage.java @@ -33,25 +33,23 @@ @TypeInfo(DataQualityMessageTypeInfo.class) public class DataQualityMessage extends SpecificRecordBase implements SpecificRecord { + public static final Schema SCHEMA$ = SchemaBuilder + .record(DataQualityMessage.class.getName()) + .namespace(DataQualityMessage.class.getPackage().getName()) + .fields() + .requiredString("level") + .requiredString("feature") + .requiredString("field") + .requiredString("message") + .endRecord(); + public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( + new String[] {"level", "feature", "field", "message"}, + Types.STRING, Types.STRING, Types.STRING, Types.STRING); private String level; private String feature; private String field; private String message; - public static final Schema SCHEMA$ = SchemaBuilder - .record(DataQualityMessage.class.getName()) - .namespace(DataQualityMessage.class.getPackage().getName()) - .fields() - .requiredString("level") - .requiredString("feature") - .requiredString("field") - .requiredString("message") - .endRecord(); - - public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( - new String[]{"level", "feature", "field", "message"}, - Types.STRING, Types.STRING, Types.STRING, Types.STRING); - public Row toRow() { return Row.of(level, feature, field, message); } @@ -63,23 +61,37 @@ public Schema getSchema() { public Object get(int field$) { switch (field$) { - case 0: return level; - case 1: return feature; - case 2: return field; - case 3: return message; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + return level; + case 1: + return feature; + case 2: + return field; + case 3: + return message; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") + @SuppressWarnings(value = "unchecked") public void put(int field$, Object value$) { switch (field$) { - case 0: level = value$.toString(); break; - case 1: feature = value$.toString(); break; - case 2: field = value$.toString(); break; - case 3: message = value$.toString(); break; - default: throw new AvroRuntimeException("Bad index"); + case 0: + level = value$.toString(); + break; + case 1: + feature = value$.toString(); + break; + case 2: + field = value$.toString(); + break; + case 3: + message = value$.toString(); + break; + default: + throw new AvroRuntimeException("Bad index"); } } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessageLevel.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessageLevel.java index a5b19d62..3b96f965 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessageLevel.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessageLevel.java @@ -12,19 +12,18 @@ package com.cloudera.cyber; -import org.apache.avro.Schema; +import static java.util.stream.Collectors.toList; import java.util.Arrays; - -import static java.util.stream.Collectors.toList; +import org.apache.avro.Schema; public enum DataQualityMessageLevel { ERROR, WARNING, INFO; private static Schema schema = Schema.createEnum(DataQualityMessageLevel.class.getName(), - "", - DataQualityMessageLevel.class.getPackage().getName(), - Arrays.stream(values()).map(e -> e.toString()).collect(toList())); + "", + DataQualityMessageLevel.class.getPackage().getName(), + Arrays.stream(values()).map(e -> e.toString()).collect(toList())); public static Schema getSchema() { return schema; diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessageTypeInfo.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessageTypeInfo.java index 7f41e9e0..4515d957 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessageTypeInfo.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DataQualityMessageTypeInfo.java @@ -12,13 +12,12 @@ package com.cloudera.cyber; +import java.lang.reflect.Type; +import java.util.Map; import org.apache.flink.api.common.typeinfo.TypeInfoFactory; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; -import java.lang.reflect.Type; -import java.util.Map; - public class DataQualityMessageTypeInfo extends TypeInfoFactory { @Override public TypeInformation createTypeInfo(Type type, Map> map) { diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DedupeMessage.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DedupeMessage.java index 605bb719..bdaa46b0 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DedupeMessage.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/DedupeMessage.java @@ -12,6 +12,10 @@ package com.cloudera.cyber; +import static com.cloudera.cyber.AvroTypes.utf8toStringMap; + +import java.util.Map; +import java.util.UUID; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -21,33 +25,28 @@ import org.apache.avro.specific.SpecificRecord; import org.apache.avro.specific.SpecificRecordBase; -import java.util.Map; -import java.util.UUID; - -import static com.cloudera.cyber.AvroTypes.utf8toStringMap; - @Data -@Builder(toBuilder=true) +@Builder(toBuilder = true) @NoArgsConstructor @AllArgsConstructor public class DedupeMessage extends SpecificRecordBase implements SpecificRecord, IdentifiedMessage, Timestamped { + static Schema schema = + SchemaBuilder.record(DedupeMessage.class.getName()).namespace(DedupeMessage.class.getPackage().getName()) + .fields().requiredString("id") + .requiredLong("ts") + .requiredLong("startTs") + .requiredLong("count") + .requiredBoolean("late") + .name("fields").type(Schema.createMap(Schema.create(Schema.Type.STRING))).noDefault() + .endRecord(); @Builder.Default private String id = UUID.randomUUID().toString(); private long ts; private long startTs; private long count; private boolean late; - private Map fields; - - static Schema schema = SchemaBuilder.record(DedupeMessage.class.getName()).namespace(DedupeMessage.class.getPackage().getName()) - .fields().requiredString("id") - .requiredLong("ts") - .requiredLong("startTs") - .requiredLong("count") - .requiredBoolean("late") - .name("fields").type(Schema.createMap(Schema.create(Schema.Type.STRING))).noDefault() - .endRecord(); + private Map fields; @Override public Schema getSchema() { @@ -56,27 +55,47 @@ public Schema getSchema() { public java.lang.Object get(int field$) { switch (field$) { - case 0: return id; - case 1: return ts; - case 2: return startTs; - case 3: return count; - case 4: return late; - case 5: return fields; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + return id; + case 1: + return ts; + case 2: + return startTs; + case 3: + return count; + case 4: + return late; + case 5: + return fields; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") + @SuppressWarnings(value = "unchecked") public void put(int field$, java.lang.Object value$) { switch (field$) { - case 0: id = value$.toString(); break; - case 1: ts = (java.lang.Long)value$; break; - case 2: startTs = (java.lang.Long)value$; break; - case 3: count = (java.lang.Long)value$; break; - case 4: late = (java.lang.Boolean)value$; break; - case 5: fields = utf8toStringMap(value$); break; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + id = value$.toString(); + break; + case 1: + ts = (java.lang.Long) value$; + break; + case 2: + startTs = (java.lang.Long) value$; + break; + case 3: + count = (java.lang.Long) value$; + break; + case 4: + late = (java.lang.Boolean) value$; + break; + case 5: + fields = utf8toStringMap(value$); + break; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/EnrichmentEntry.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/EnrichmentEntry.java index d5a5846c..a7b572c4 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/EnrichmentEntry.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/EnrichmentEntry.java @@ -12,6 +12,9 @@ package com.cloudera.cyber; +import static com.cloudera.cyber.AvroTypes.utf8toStringMap; + +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -22,28 +25,24 @@ import org.apache.avro.specific.SpecificRecord; import org.apache.avro.specific.SpecificRecordBase; -import java.util.Map; - -import static com.cloudera.cyber.AvroTypes.utf8toStringMap; - @Data @Builder @NoArgsConstructor @AllArgsConstructor public class EnrichmentEntry extends SpecificRecordBase implements SpecificRecord, Timestamped { + public static final Schema SCHEMA$ = + SchemaBuilder.record(EnrichmentEntry.class.getName()).namespace(EnrichmentEntry.class.getPackage().getName()) + .fields() + .requiredString("key") + .requiredString("type") + .requiredLong("ts") + .name("entries").type(Schema.createMap(SchemaBuilder.builder().stringType())).noDefault() + .endRecord(); private String key; private String type; private long ts; - private Map entries; - - public static final Schema SCHEMA$ = SchemaBuilder.record(EnrichmentEntry.class.getName()).namespace(EnrichmentEntry.class.getPackage().getName()) - .fields() - .requiredString("key") - .requiredString("type") - .requiredLong("ts") - .name("entries").type(Schema.createMap(SchemaBuilder.builder().stringType())).noDefault() - .endRecord(); + private Map entries; @Override public Schema getSchema() { @@ -53,23 +52,37 @@ public Schema getSchema() { // Used by DatumWriter. Applications should not call. public Object get(int field$) { switch (field$) { - case 0: return key; - case 1: return type; - case 2: return ts; - case 3: return entries; - default: throw new AvroRuntimeException("Bad index"); + case 0: + return key; + case 1: + return type; + case 2: + return ts; + case 3: + return entries; + default: + throw new AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") + @SuppressWarnings(value = "unchecked") public void put(int field$, Object value$) { switch (field$) { - case 0: key = value$.toString(); break; - case 1: type = value$.toString(); break; - case 2: ts = (Long)value$; break; - case 3: entries = utf8toStringMap(value$); break; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + key = value$.toString(); + break; + case 1: + type = value$.toString(); + break; + case 2: + ts = (Long) value$; + break; + case 3: + entries = utf8toStringMap(value$); + break; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/EnrichmentEntryTypeFactory.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/EnrichmentEntryTypeFactory.java index 403b23fa..9219cc65 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/EnrichmentEntryTypeFactory.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/EnrichmentEntryTypeFactory.java @@ -12,6 +12,12 @@ package com.cloudera.cyber; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import org.apache.flink.api.common.typeinfo.TypeInfoFactory; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeinfo.Types; @@ -19,24 +25,18 @@ import org.apache.flink.api.java.typeutils.PojoTypeInfo; import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - public class EnrichmentEntryTypeFactory extends TypeInfoFactory { @Override public TypeInformation createTypeInfo(Type type, Map> map) { AvroTypeInfo avroOut = new AvroTypeInfo<>(EnrichmentEntry.class); List fields = Arrays.stream(avroOut.getFieldNames()) - .map(e -> - new PojoField(avroOut.getPojoFieldAt(avroOut.getFieldIndex(e)).getField(), - e == "entries" ? - Types.MAP(Types.STRING, Types.STRING) : - avroOut.getPojoFieldAt(avroOut.getFieldIndex(e)).getTypeInformation()) - ).collect(Collectors.toList()); + .map(e -> + new PojoField(avroOut.getPojoFieldAt(avroOut.getFieldIndex(e)).getField(), + Objects.equals(e, "entries") + ? Types.MAP(Types.STRING, Types.STRING) + : avroOut.getPojoFieldAt(avroOut.getFieldIndex(e)).getTypeInformation()) + ).collect(Collectors.toList()); return new PojoTypeInfo(EnrichmentEntry.class, fields); } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/FieldRegistry.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/FieldRegistry.java index 3aea92d1..6f37780e 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/FieldRegistry.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/FieldRegistry.java @@ -12,20 +12,18 @@ package com.cloudera.cyber; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; - import java.util.HashMap; import java.util.Map; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; public class FieldRegistry implements IFieldRegistry { - private Map fields = new HashMap<>(); - private static FieldRegistry instance; + private Map fields = new HashMap<>(); public static FieldRegistry instance() { - if(instance == null) { + if (instance == null) { instance = new FieldRegistry(); } return instance; diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/GroupedMessage.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/GroupedMessage.java index 34a58dfc..a6f917c1 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/GroupedMessage.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/GroupedMessage.java @@ -12,6 +12,10 @@ package com.cloudera.cyber; +import static com.cloudera.cyber.AvroTypes.toListOf; + +import java.util.List; +import java.util.UUID; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -22,29 +26,24 @@ import org.apache.avro.specific.SpecificRecord; import org.apache.avro.specific.SpecificRecordBase; -import java.util.List; -import java.util.UUID; - -import static com.cloudera.cyber.AvroTypes.toListOf; - @Data @Builder @NoArgsConstructor @AllArgsConstructor public class GroupedMessage extends SpecificRecordBase implements SpecificRecord, IdentifiedMessage, Timestamped { + public static final Schema SCHEMA$ = + SchemaBuilder.record(GroupedMessage.class.getName()).namespace(GroupedMessage.class.getPackage().getName()) + .fields() + .requiredString("id") + .name("messages").type( + SchemaBuilder.array().items(Message.SCHEMA$) + ).noDefault() + .endRecord(); @Builder.Default private String id = UUID.randomUUID().toString(); private List messages; - public static final Schema SCHEMA$ = SchemaBuilder.record(GroupedMessage.class.getName()).namespace(GroupedMessage.class.getPackage().getName()) - .fields() - .requiredString("id") - .name("messages").type( - SchemaBuilder.array().items(Message.SCHEMA$) - ).noDefault() - .endRecord(); - @Override public Schema getSchema() { return SCHEMA$; @@ -52,19 +51,27 @@ public Schema getSchema() { public java.lang.Object get(int field$) { switch (field$) { - case 0: return id; - case 1: return messages; - default: throw new AvroRuntimeException("Bad index"); + case 0: + return id; + case 1: + return messages; + default: + throw new AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") + @SuppressWarnings(value = "unchecked") public void put(int field$, Object value$) { switch (field$) { - case 0: id = value$.toString(); break; - case 1: messages = toListOf(Message.class, value$); break; - default: throw new AvroRuntimeException("Bad index"); + case 0: + id = value$.toString(); + break; + case 1: + messages = toListOf(Message.class, value$); + break; + default: + throw new AvroRuntimeException("Bad index"); } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/IdentifiedMessage.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/IdentifiedMessage.java index c4045ba9..e455b0aa 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/IdentifiedMessage.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/IdentifiedMessage.java @@ -12,7 +12,8 @@ package com.cloudera.cyber; -public interface IdentifiedMessage extends Timestamped{ +public interface IdentifiedMessage extends Timestamped { String getId(); + long getTs(); } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java index ea7e61e1..63965964 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java @@ -12,7 +12,16 @@ package com.cloudera.cyber; +import static com.cloudera.cyber.AvroTypes.toListOf; +import static com.cloudera.cyber.AvroTypes.utf8toStringMap; +import static java.util.stream.Collectors.toMap; + import com.cloudera.cyber.avro.AvroSchemas; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -28,73 +37,69 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.types.Row; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static com.cloudera.cyber.AvroTypes.toListOf; -import static com.cloudera.cyber.AvroTypes.utf8toStringMap; -import static java.util.stream.Collectors.toMap; - @Data @Builder(toBuilder = true) @NoArgsConstructor @AllArgsConstructor public class Message extends SpecificRecordBase implements SpecificRecord, IdentifiedMessage, Timestamped { + public static final Schema SCHEMA$ = + AvroSchemas.createRecordBuilder(Message.class.getPackage().getName(), Message.class.getName()) + .fields() + .requiredString("id") + .name("ts").type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))) + .noDefault() + .name("originalSource").type(SignedSourceKey.SCHEMA$).noDefault() + .requiredString("message") + .name("threats").type().optional() + .type(SchemaBuilder.map().values(SchemaBuilder.array().items(ThreatIntelligence.SCHEMA$))) + .name("extensions").type(Schema.createMap(Schema.create(Schema.Type.STRING))).noDefault() + .requiredString("source") + .name("dataQualityMessages").type().optional().type(Schema.createArray(DataQualityMessage.SCHEMA$)) + .endRecord(); + public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( + new String[] {"id", "ts", "originalSource", "message", "threats", "extensions", "source", + "dataQualityMessages"}, + Types.STRING, Types.LONG, SignedSourceKey.FLINK_TYPE_INFO, Types.STRING, + Types.MAP(Types.STRING, Types.OBJECT_ARRAY(ThreatIntelligence.FLINK_TYPE_INFO)), + Types.MAP(Types.STRING, Types.STRING), Types.STRING, Types.OBJECT_ARRAY(DataQualityMessage.FLINK_TYPE_INFO)); @Builder.Default - @NonNull private String id = UUID.randomUUID().toString(); + @NonNull + private String id = UUID.randomUUID().toString(); private long ts; - @NonNull private SignedSourceKey originalSource; + @NonNull + private SignedSourceKey originalSource; @Builder.Default - @NonNull private String message = ""; + @NonNull + private String message = ""; private Map> threats; private Map extensions; - @NonNull private String source; + @NonNull + private String source; private List dataQualityMessages; - public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(Message.class.getPackage().getName(), Message.class.getName()) - .fields() - .requiredString("id") - .name("ts").type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))).noDefault() - .name("originalSource").type(SignedSourceKey.SCHEMA$).noDefault() - .requiredString("message") - .name("threats").type().optional().type(SchemaBuilder.map().values(SchemaBuilder.array().items(ThreatIntelligence.SCHEMA$))) - .name("extensions").type(Schema.createMap(Schema.create(Schema.Type.STRING))).noDefault() - .requiredString("source") - .name("dataQualityMessages").type().optional().type(Schema.createArray(DataQualityMessage.SCHEMA$)) - .endRecord(); - - public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( - new String[]{"id", "ts", "originalSource", "message", "threats", "extensions", "source", "dataQualityMessages"}, - Types.STRING, Types.LONG, SignedSourceKey.FLINK_TYPE_INFO,Types.STRING, - Types.MAP(Types.STRING, Types.OBJECT_ARRAY(ThreatIntelligence.FLINK_TYPE_INFO)), - Types.MAP(Types.STRING, Types.STRING), Types.STRING, Types.OBJECT_ARRAY(DataQualityMessage.FLINK_TYPE_INFO)); + public static Schema getClassSchema() { + return SCHEMA$; + } public Row toRow() { return Row.of(id, - ts, - originalSource.toRow(), - message, - threats == null ? null : threats.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - e -> e.getValue().stream() - .map(ThreatIntelligence::toRow) - .toArray(Row[]::new))), - extensions, - source, - dataQualityMessages == null ? null : dataQualityMessages.stream() - .map(DataQualityMessage::toRow) - .toArray(Row[]::new) + ts, + originalSource.toRow(), + message, + threats == null ? null : threats.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().stream() + .map(ThreatIntelligence::toRow) + .toArray(Row[]::new))), + extensions, + source, + dataQualityMessages == null ? null : dataQualityMessages.stream() + .map(DataQualityMessage::toRow) + .toArray(Row[]::new) ); } - public static Schema getClassSchema() { - return SCHEMA$; - } - @Override public Schema getSchema() { return SCHEMA$; @@ -102,42 +107,69 @@ public Schema getSchema() { public java.lang.Object get(int field$) { switch (field$) { - case 0: return id; - case 1: return ts; - case 2: return originalSource; - case 3: return message; - case 4: return threats; - case 5: return extensions; - case 6: return source; - case 7: return dataQualityMessages == null ? null : dataQualityMessages instanceof List ? dataQualityMessages : Arrays.asList(dataQualityMessages); - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + return id; + case 1: + return ts; + case 2: + return originalSource; + case 3: + return message; + case 4: + return threats; + case 5: + return extensions; + case 6: + return source; + case 7: + return dataQualityMessages == null ? null : + dataQualityMessages instanceof List ? dataQualityMessages : Arrays.asList(dataQualityMessages); + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") + @SuppressWarnings(value = "unchecked") public void put(int field$, java.lang.Object value$) { switch (field$) { - case 0: id = value$.toString(); break; - case 1: ts = (java.lang.Long)value$; break; - case 2: originalSource = (com.cloudera.cyber.SignedSourceKey)value$; break; + case 0: + id = value$.toString(); + break; + case 1: + ts = (java.lang.Long) value$; + break; + case 2: + originalSource = (com.cloudera.cyber.SignedSourceKey) value$; + break; case 3: message = value$.toString(); break; - case 4: threats = toTiMap(value$); break; - case 5: extensions = utf8toStringMap(value$); break; - case 6: source = value$.toString(); break; - case 7: dataQualityMessages = toListOf(DataQualityMessage.class, value$); break; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 4: + threats = toTiMap(value$); + break; + case 5: + extensions = utf8toStringMap(value$); + break; + case 6: + source = value$.toString(); + break; + case 7: + dataQualityMessages = toListOf(DataQualityMessage.class, value$); + break; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } private Map> toTiMap(Object value$) { - if(value$ == null) return null; + if (value$ == null) { + return null; + } Map> o = (Map>) value$; return o.entrySet().stream().collect(toMap( - k -> k.getKey().toString(), - v -> toListOf(ThreatIntelligence.class, v.getValue()) + k -> k.getKey().toString(), + v -> toListOf(ThreatIntelligence.class, v.getValue()) )); } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/MessageTypeFactory.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/MessageTypeFactory.java index 65340d58..d82b23de 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/MessageTypeFactory.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/MessageTypeFactory.java @@ -12,34 +12,33 @@ package com.cloudera.cyber; -import org.apache.flink.api.common.typeinfo.TypeInfoFactory; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; +import static com.cloudera.cyber.AvroTypes.overrideTypes; import java.lang.reflect.Type; import java.util.Collections; import java.util.HashMap; import java.util.Map; - -import static com.cloudera.cyber.AvroTypes.overrideTypes; +import org.apache.flink.api.common.typeinfo.TypeInfoFactory; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; public class MessageTypeFactory extends TypeInfoFactory { @Override public TypeInformation createTypeInfo(Type type, Map> map) { return overrideTypes(new AvroTypeInfo<>(Message.class), - Message.class, - new HashMap>() { - { - put("dataQualityMessages", Types.OBJECT_ARRAY(TypeInformation.of(DataQualityMessage.class))); - put("extensions", Types.MAP(Types.STRING, Types.STRING)); - put("originalSource", overrideTypes(new AvroTypeInfo<>(SignedSourceKey.class), - SignedSourceKey.class, - Collections.singletonMap("signature", Types.PRIMITIVE_ARRAY(Types.BYTE)))); - put("threats", Types.OBJECT_ARRAY(overrideTypes(new AvroTypeInfo<>(ThreatIntelligence.class), - ThreatIntelligence.class, - Collections.singletonMap("fields", Types.MAP(Types.STRING, Types.STRING))))); - } - }); + Message.class, + new HashMap>() { + { + put("dataQualityMessages", Types.OBJECT_ARRAY(TypeInformation.of(DataQualityMessage.class))); + put("extensions", Types.MAP(Types.STRING, Types.STRING)); + put("originalSource", overrideTypes(new AvroTypeInfo<>(SignedSourceKey.class), + SignedSourceKey.class, + Collections.singletonMap("signature", Types.PRIMITIVE_ARRAY(Types.BYTE)))); + put("threats", Types.OBJECT_ARRAY(overrideTypes(new AvroTypeInfo<>(ThreatIntelligence.class), + ThreatIntelligence.class, + Collections.singletonMap("fields", Types.MAP(Types.STRING, Types.STRING))))); + } + }); } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/SignedSourceKey.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/SignedSourceKey.java index 06b0c124..fbe67c1c 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/SignedSourceKey.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/SignedSourceKey.java @@ -12,6 +12,7 @@ package com.cloudera.cyber; +import java.nio.ByteBuffer; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -25,44 +26,39 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.types.Row; -import java.nio.ByteBuffer; - @Data @Builder //@TypeInfo(SignedSourceKeyTypeFactory.class) @NoArgsConstructor @AllArgsConstructor public class SignedSourceKey extends SpecificRecordBase implements SpecificRecord { + public static final Schema SCHEMA$ = SchemaBuilder + .record(SignedSourceKey.class.getName()) + .namespace(SignedSourceKey.class.getPackage().getName()) + .fields() + .requiredString("topic") + .requiredInt("partition") + .requiredLong("offset") + .name("signature").type().bytesBuilder().endBytes().noDefault() + .endRecord(); + public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( + new String[] {"topic", "partition", "offset", "signature"}, + Types.STRING, Types.INT, Types.LONG, Types.PRIMITIVE_ARRAY(Types.BYTE)); private String topic; private int partition; private long offset; private byte[] signature; - public static final Schema SCHEMA$ = SchemaBuilder - .record(SignedSourceKey.class.getName()) - .namespace(SignedSourceKey.class.getPackage().getName()) - .fields() - .requiredString("topic") - .requiredInt("partition") - .requiredLong("offset") - .name("signature").type().bytesBuilder().endBytes().noDefault() - .endRecord(); - - public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( - new String[]{"topic", "partition", "offset", "signature"}, - Types.STRING, Types.INT, Types.LONG, Types.PRIMITIVE_ARRAY(Types.BYTE)); - - public Row toRow() { return Row.of(topic, partition, offset, signature); } -// static public class SignedSourceKeyBuilder { -// public SignedSourceKeyBuilder signature(byte[] bytes) { -// this.signature = ByteBuffer.wrap(bytes); -// return this; -// } -// } + // static public class SignedSourceKeyBuilder { + // public SignedSourceKeyBuilder signature(byte[] bytes) { + // this.signature = ByteBuffer.wrap(bytes); + // return this; + // } + // } @Override public Schema getSchema() { @@ -71,23 +67,37 @@ public Schema getSchema() { @Override public Object get(int field) { - switch(field) { - case 0: return topic; - case 1: return partition; - case 2: return offset; - case 3: return ByteBuffer.wrap(signature); - default: throw new AvroRuntimeException("Bad index"); + switch (field) { + case 0: + return topic; + case 1: + return partition; + case 2: + return offset; + case 3: + return ByteBuffer.wrap(signature); + default: + throw new AvroRuntimeException("Bad index"); } } @Override public void put(int field, Object value) { - switch(field) { - case 0: this.topic = value.toString(); break; - case 1: this.partition = (int) value; break; - case 2: this.offset = (long) value; break; - case 3: this.signature = (value instanceof byte[]) ? (byte[])value: ((ByteBuffer) value).array(); break; - default: throw new AvroRuntimeException("Bad index"); + switch (field) { + case 0: + this.topic = value.toString(); + break; + case 1: + this.partition = (int) value; + break; + case 2: + this.offset = (long) value; + break; + case 3: + this.signature = (value instanceof byte[]) ? (byte[]) value : ((ByteBuffer) value).array(); + break; + default: + throw new AvroRuntimeException("Bad index"); } } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/SignedSourceKeyTypeFactory.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/SignedSourceKeyTypeFactory.java index 9f96e5cd..2878d727 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/SignedSourceKeyTypeFactory.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/SignedSourceKeyTypeFactory.java @@ -12,22 +12,23 @@ package com.cloudera.cyber; -import org.apache.flink.api.common.typeinfo.TypeInfoFactory; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; - import java.lang.reflect.Type; import java.util.HashMap; import java.util.Map; +import org.apache.flink.api.common.typeinfo.TypeInfoFactory; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; public class SignedSourceKeyTypeFactory extends TypeInfoFactory { @Override public TypeInformation createTypeInfo(Type type, Map> map) { - return Types.POJO(SignedSourceKey.class, new HashMap>() {{ - put("topic", Types.STRING); - put("partition", Types.INT); - put("offset", Types.LONG); - put("signature", Types.PRIMITIVE_ARRAY(Types.BYTE)); - }}); + return Types.POJO(SignedSourceKey.class, new HashMap>() { + { + put("topic", Types.STRING); + put("partition", Types.INT); + put("offset", Types.LONG); + put("signature", Types.PRIMITIVE_ARRAY(Types.BYTE)); + } + }); } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/ThreatIntelligence.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/ThreatIntelligence.java index 1341277b..00280140 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/ThreatIntelligence.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/ThreatIntelligence.java @@ -12,6 +12,10 @@ package com.cloudera.cyber; +import static com.cloudera.cyber.AvroTypes.utf8toStringMap; + +import java.util.Map; +import java.util.UUID; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -26,17 +30,24 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.types.Row; -import java.util.Map; -import java.util.UUID; - -import static com.cloudera.cyber.AvroTypes.utf8toStringMap; - @Data @Builder @NoArgsConstructor @AllArgsConstructor @TypeInfo(ThreatIntelligenceTypeFactory.class) public class ThreatIntelligence extends SpecificRecordBase implements SpecificRecord, IdentifiedMessage, Timestamped { + public static final Schema SCHEMA$ = SchemaBuilder.record(ThreatIntelligence.class.getName()) + .namespace(ThreatIntelligence.class.getPackage().getName()) + .fields() + .requiredString("id") + .requiredLong("ts") + .requiredString("observable") + .requiredString("observableType") + .name("fields").type(Schema.createMap(SchemaBuilder.builder().stringType())).noDefault() + .endRecord(); + public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( + new String[] {"id", "ts", "observable", "observableType", "fields"}, + Types.STRING, Types.LONG, Types.STRING, Types.STRING, Types.MAP(Types.STRING, Types.STRING)); @Builder.Default private String id = UUID.randomUUID().toString(); private long ts; @@ -44,19 +55,6 @@ public class ThreatIntelligence extends SpecificRecordBase implements SpecificRe private String observableType; private Map fields; - public static final Schema SCHEMA$ = SchemaBuilder.record(ThreatIntelligence.class.getName()).namespace(ThreatIntelligence.class.getPackage().getName()) - .fields() - .requiredString("id") - .requiredLong("ts") - .requiredString("observable") - .requiredString("observableType") - .name("fields").type(Schema.createMap(SchemaBuilder.builder().stringType())).noDefault() - .endRecord(); - - public static final TypeInformation FLINK_TYPE_INFO = Types.ROW_NAMED( - new String[]{"id", "ts", "observable", "observableType", "fields"}, - Types.STRING, Types.LONG, Types.STRING, Types.STRING, Types.MAP(Types.STRING, Types.STRING)); - public Row toRow() { return Row.of(id, ts, observable, observableType, fields); } @@ -69,26 +67,43 @@ public Schema getSchema() { @Override public Object get(int field$) { switch (field$) { - case 0: return id; - case 1: return ts; - case 2: return observable; - case 3: return observableType; - case 4: return fields; - default: throw new AvroRuntimeException("Bad index"); + case 0: + return id; + case 1: + return ts; + case 2: + return observable; + case 3: + return observableType; + case 4: + return fields; + default: + throw new AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") + @SuppressWarnings(value = "unchecked") @Override public void put(int field$, Object value$) { switch (field$) { - case 0: id = value$.toString(); break; - case 1: ts = (Long)value$; break; - case 2: observable = value$.toString(); break; - case 3: observableType = value$.toString(); break; - case 4: fields = utf8toStringMap(value$); break; - default: throw new AvroRuntimeException("Bad index"); + case 0: + id = value$.toString(); + break; + case 1: + ts = (Long) value$; + break; + case 2: + observable = value$.toString(); + break; + case 3: + observableType = value$.toString(); + break; + case 4: + fields = utf8toStringMap(value$); + break; + default: + throw new AvroRuntimeException("Bad index"); } } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/ThreatIntelligenceTypeFactory.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/ThreatIntelligenceTypeFactory.java index 2e702103..5d2ddaf8 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/ThreatIntelligenceTypeFactory.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/ThreatIntelligenceTypeFactory.java @@ -12,26 +12,25 @@ package com.cloudera.cyber; -import org.apache.flink.api.common.typeinfo.TypeInfoFactory; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; +import static com.cloudera.cyber.AvroTypes.overrideTypes; import java.lang.reflect.Type; import java.util.HashMap; import java.util.Map; - -import static com.cloudera.cyber.AvroTypes.overrideTypes; +import org.apache.flink.api.common.typeinfo.TypeInfoFactory; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; public class ThreatIntelligenceTypeFactory extends TypeInfoFactory { @Override public TypeInformation createTypeInfo(Type type, Map> map) { return overrideTypes(new AvroTypeInfo<>(ThreatIntelligence.class), - ThreatIntelligence.class, - new HashMap>() { - { - put("fields", Types.MAP(Types.STRING, Types.STRING)); - } - }); + ThreatIntelligence.class, + new HashMap>() { + { + put("fields", Types.MAP(Types.STRING, Types.STRING)); + } + }); } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java index 8309f15a..4283fc3f 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java @@ -4,16 +4,20 @@ public class AvroSchemas { - public static SchemaBuilder.RecordBuilder createRecordBuilder(String namespace, String recordName) { + public static SchemaBuilder.RecordBuilder createRecordBuilder(String namespace, + String recordName) { return createRecordBuilder(namespace, recordName, "ts"); } - public static SchemaBuilder.RecordBuilder createRecordBuilder(String namespace, String recordName, String tsFieldName) { - SchemaBuilder.RecordBuilder recordBuilder = SchemaBuilder.record(recordName).namespace(namespace); + public static SchemaBuilder.RecordBuilder createRecordBuilder(String namespace, + String recordName, + String tsFieldName) { + SchemaBuilder.RecordBuilder recordBuilder = + SchemaBuilder.record(recordName).namespace(namespace); if (tsFieldName != null) { recordBuilder - .prop("ssb.rowtimeAttribute", tsFieldName) - .prop("ssb.watermarkExpression", String.format("`%s` - INTERVAL '30' SECOND", tsFieldName)); + .prop("ssb.rowtimeAttribute", tsFieldName) + .prop("ssb.watermarkExpression", String.format("`%s` - INTERVAL '30' SECOND", tsFieldName)); } return recordBuilder; } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/Command.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/Command.java index 04d82f63..6654e6db 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/Command.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/Command.java @@ -12,15 +12,14 @@ package com.cloudera.cyber.commands; +import static com.cloudera.cyber.AvroTypes.utf8toStringMap; + +import java.util.Map; import lombok.Data; import lombok.NoArgsConstructor; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificRecordBase; -import java.util.Map; - -import static com.cloudera.cyber.AvroTypes.utf8toStringMap; - @Data @NoArgsConstructor public abstract class Command extends SpecificRecordBase { @@ -39,25 +38,36 @@ protected Command(CommandBuilder b) { public java.lang.Object get(int field$) { switch (field$) { - case 0: return type; - case 1: return payload; - case 2: return headers; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + return type; + case 1: + return payload; + case 2: + return headers; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") + @SuppressWarnings(value = "unchecked") public void put(int field$, java.lang.Object value$) { switch (field$) { - case 0: type = CommandType.valueOf(value$.toString()); break; - case 1: payload = (T)value$; break; - case 2: headers = utf8toStringMap(value$); break; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + type = CommandType.valueOf(value$.toString()); + break; + case 1: + payload = (T) value$; + break; + case 2: + headers = utf8toStringMap(value$); + break; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } - public static abstract class CommandBuilder, B extends CommandBuilder> { + public abstract static class CommandBuilder, B extends CommandBuilder> { private CommandType type; private T payload; private Map headers; @@ -82,7 +92,8 @@ public B headers(Map headers) { public abstract C build(); public String toString() { - return "Command.CommandBuilder(super=" + super.toString() + ", type=" + this.type + ", payload=" + this.payload + ", headers=" + this.headers + ")"; + return "Command.CommandBuilder(super=" + super.toString() + ", type=" + this.type + ", payload=" + + this.payload + ", headers=" + this.headers + ")"; } } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/CommandResponse.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/CommandResponse.java index 4be3e0cd..55d479ea 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/CommandResponse.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/CommandResponse.java @@ -12,19 +12,18 @@ package com.cloudera.cyber.commands; +import static com.cloudera.cyber.AvroTypes.utf8toStringMap; + import com.cloudera.cyber.flink.HasHeaders; +import java.util.Collections; +import java.util.List; +import java.util.Map; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificRecordBase; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static com.cloudera.cyber.AvroTypes.utf8toStringMap; - @EqualsAndHashCode(callSuper = false) @Data @NoArgsConstructor @@ -32,7 +31,7 @@ public abstract class CommandResponse extends SpecificRecordBase implements H private boolean success; private String message; private List content; - private Map headers; + private Map headers; protected CommandResponse(CommandResponseBuilder b) { this.success = b.success; @@ -46,28 +45,42 @@ protected CommandResponse(CommandResponseBuilder b) { public java.lang.Object get(int field$) { switch (field$) { - case 0: return success; - case 1: return message; - case 2: return content; - case 3: return headers; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + return success; + case 1: + return message; + case 2: + return content; + case 3: + return headers; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. public void put(int field$, java.lang.Object value$) { switch (field$) { - case 0: success = (boolean) value$; break; - case 1: message = value$ == null ? "" : value$.toString(); break; - case 2: content = putContent(value$); break; - case 3: headers = utf8toStringMap(value$); break; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + success = (boolean) value$; + break; + case 1: + message = value$ == null ? "" : value$.toString(); + break; + case 2: + content = putContent(value$); + break; + case 3: + headers = utf8toStringMap(value$); + break; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } abstract List putContent(Object value$); - public static abstract class CommandResponseBuilder, B extends CommandResponseBuilder> { + public abstract static class CommandResponseBuilder, B extends CommandResponseBuilder> { private boolean success; private String message; private List content; @@ -100,7 +113,8 @@ public B headers(Map headers) { public abstract C build(); public String toString() { - return "CommandResponse.CommandResponseBuilder(super=" + super.toString() + ", success=" + this.success + ", message=" + this.message + ", content=" + this.content + ", headers=" + this.headers + ")"; + return "CommandResponse.CommandResponseBuilder(super=" + super.toString() + ", success=" + this.success + + ", message=" + this.message + ", content=" + this.content + ", headers=" + this.headers + ")"; } } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/EnrichmentCommand.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/EnrichmentCommand.java index 825f86ea..18bc1858 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/EnrichmentCommand.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/EnrichmentCommand.java @@ -14,31 +14,30 @@ import com.cloudera.cyber.EnrichmentEntry; import com.cloudera.cyber.flink.HasHeaders; +import java.util.Arrays; +import java.util.stream.Collectors; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; -import java.util.Arrays; -import java.util.stream.Collectors; - @Data @NoArgsConstructor @ToString(callSuper = true) public class EnrichmentCommand extends Command implements HasHeaders { public static final Schema SCHEMA$ = SchemaBuilder - .record(EnrichmentCommand.class.getName()) - .namespace(EnrichmentCommand.class.getPackage().getName()) - .fields() - .name("type").type(Schema.createEnum(CommandType.class.getName(), - "", - CommandType.class.getPackage().getName(), - Arrays.stream(CommandType.values()).map(v -> v.name()).collect(Collectors.toList()))).noDefault() - .name("payload").type().optional().type(EnrichmentEntry.SCHEMA$) - .name("headers").type(Schema.createMap(Schema.create(Schema.Type.STRING))).noDefault() - .endRecord(); + .record(EnrichmentCommand.class.getName()) + .namespace(EnrichmentCommand.class.getPackage().getName()) + .fields() + .name("type").type(Schema.createEnum(CommandType.class.getName(), + "", + CommandType.class.getPackage().getName(), + Arrays.stream(CommandType.values()).map(v -> v.name()).collect(Collectors.toList()))).noDefault() + .name("payload").type().optional().type(EnrichmentEntry.SCHEMA$) + .name("headers").type(Schema.createMap(Schema.create(Schema.Type.STRING))).noDefault() + .endRecord(); protected EnrichmentCommand(EnrichmentCommandBuilder b) { super(b); @@ -48,16 +47,17 @@ protected EnrichmentCommand(EnrichmentCommandBuilder b) { return new EnrichmentCommandBuilderImpl(); } - @Override - public Schema getSchema() { + public static Schema getClassSchema() { return SCHEMA$; - }; + } - public static Schema getClassSchema() { + @Override + public Schema getSchema() { return SCHEMA$; } - public static abstract class EnrichmentCommandBuilder> extends CommandBuilder { + public abstract static class EnrichmentCommandBuilder> + extends CommandBuilder { protected abstract B self(); public abstract C build(); @@ -67,7 +67,8 @@ public String toString() { } } - private static final class EnrichmentCommandBuilderImpl extends EnrichmentCommandBuilder { + private static final class EnrichmentCommandBuilderImpl + extends EnrichmentCommandBuilder { private EnrichmentCommandBuilderImpl() { } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/EnrichmentCommandResponse.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/EnrichmentCommandResponse.java index f7d7d09a..ef1f7aa5 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/EnrichmentCommandResponse.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/commands/EnrichmentCommandResponse.java @@ -12,7 +12,11 @@ package com.cloudera.cyber.commands; +import static com.cloudera.cyber.AvroTypes.toListOf; + import com.cloudera.cyber.EnrichmentEntry; +import java.util.Collections; +import java.util.List; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -20,25 +24,21 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; -import java.util.Collections; -import java.util.List; - -import static com.cloudera.cyber.AvroTypes.toListOf; - @EqualsAndHashCode(callSuper = true) @SuperBuilder @Data @NoArgsConstructor public class EnrichmentCommandResponse extends CommandResponse { - + private static final Schema SCHEMA$ = SchemaBuilder - .record(EnrichmentCommandResponse.class.getName()) - .namespace(EnrichmentCommandResponse.class.getPackage().getName()) - .fields() - .requiredBoolean("success").requiredString("message") - .name("content").type(Schema.createArray(EnrichmentEntry.SCHEMA$)).noDefault() - .name("headers").type(Schema.createMap(Schema.create(Schema.Type.STRING))).withDefault(Collections.emptyMap()) - .endRecord(); + .record(EnrichmentCommandResponse.class.getName()) + .namespace(EnrichmentCommandResponse.class.getPackage().getName()) + .fields() + .requiredBoolean("success").requiredString("message") + .name("content").type(Schema.createArray(EnrichmentEntry.SCHEMA$)).noDefault() + .name("headers").type(Schema.createMap(Schema.create(Schema.Type.STRING))).withDefault(Collections.emptyMap()) + .endRecord(); + @Override public Schema getSchema() { return SCHEMA$; diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/enrichment/lookup/config/EnrichmentConfig.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/enrichment/lookup/config/EnrichmentConfig.java index 9bf5b600..0af93155 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/enrichment/lookup/config/EnrichmentConfig.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/enrichment/lookup/config/EnrichmentConfig.java @@ -12,14 +12,13 @@ package com.cloudera.cyber.enrichment.lookup.config; +import java.io.Serializable; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.io.Serializable; -import java.util.List; - @Builder @Data @NoArgsConstructor diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/enrichment/lookup/config/EnrichmentField.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/enrichment/lookup/config/EnrichmentField.java index a8ae7666..65fb3a63 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/enrichment/lookup/config/EnrichmentField.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/enrichment/lookup/config/EnrichmentField.java @@ -12,15 +12,12 @@ package com.cloudera.cyber.enrichment.lookup.config; +import java.io.Serializable; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - @Builder @Data @NoArgsConstructor diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/flink/HasHeaders.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/flink/HasHeaders.java index d35fa4ac..809c4fbb 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/flink/HasHeaders.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/flink/HasHeaders.java @@ -15,7 +15,7 @@ import java.util.Map; public interface HasHeaders { - void setHeaders(Map headers); - Map getHeaders(); + + void setHeaders(Map headers); } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingColumnDto.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingColumnDto.java index 33d75307..0144f8b1 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingColumnDto.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingColumnDto.java @@ -38,7 +38,7 @@ public String getKafkaName() { } } - public String getRawKafkaName(){ + public String getRawKafkaName() { return kafkaName; } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingDto.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingDto.java index bcde54dd..a2ada4a5 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingDto.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingDto.java @@ -1,10 +1,6 @@ package com.cloudera.cyber.indexing; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - import java.util.Collections; import java.util.List; import java.util.Objects; @@ -12,6 +8,9 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @NoArgsConstructor @@ -29,11 +28,11 @@ public class MappingDto { public Set getIgnoreFields() { return ignoreFields == null ? Collections.singleton("") : - Stream.concat(ignoreFields.stream(), columnMapping.stream() - .filter(dto -> "extensions".equals(dto.getPath()) && dto.getIsMap()) - .map(dto -> Optional.ofNullable(dto.getRawKafkaName()).orElse(dto.getName()))) - .filter(Objects::nonNull) - .map(String::toLowerCase) - .collect(Collectors.toSet()); + Stream.concat(ignoreFields.stream(), columnMapping.stream() + .filter(dto -> "extensions".equals(dto.getPath()) && dto.getIsMap()) + .map(dto -> Optional.ofNullable(dto.getRawKafkaName()).orElse(dto.getName()))) + .filter(Objects::nonNull) + .map(String::toLowerCase) + .collect(Collectors.toSet()); } } diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/parser/MessageToParse.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/parser/MessageToParse.java index 2aeb93a5..26f98131 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/parser/MessageToParse.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/parser/MessageToParse.java @@ -12,6 +12,7 @@ package com.cloudera.cyber.parser; +import java.nio.ByteBuffer; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -21,55 +22,75 @@ import org.apache.avro.specific.SpecificRecord; import org.apache.avro.specific.SpecificRecordBase; -import java.nio.ByteBuffer; - @Data @Builder @NoArgsConstructor @AllArgsConstructor public class MessageToParse extends SpecificRecordBase implements SpecificRecord { + public static final Schema SCHEMA$ = + SchemaBuilder.record(MessageToParse.class.getName()).namespace(MessageToParse.class.getPackage().getName()) + .fields() + .requiredBytes("originalBytes") + .requiredString("topic") + .requiredInt("partition") + .requiredLong("offset") + .optionalBytes("key") + .endRecord(); private byte[] originalBytes; private String topic; private int partition; private long offset; private byte[] key; - public static final Schema SCHEMA$ = SchemaBuilder.record(MessageToParse.class.getName()).namespace(MessageToParse.class.getPackage().getName()) - .fields() - .requiredBytes("originalBytes") - .requiredString("topic") - .requiredInt("partition") - .requiredLong("offset") - .optionalBytes("key") - .endRecord(); - - public static Schema getClassSchema() { return SCHEMA$; } + public static Schema getClassSchema() { + return SCHEMA$; + } @Override - public Schema getSchema() { return SCHEMA$; } + public Schema getSchema() { + return SCHEMA$; + } @Override public java.lang.Object get(int field$) { switch (field$) { - case 0: return ByteBuffer.wrap(originalBytes); - case 1: return topic; - case 2: return partition; - case 3: return offset; - case 4: return (key != null) ? ByteBuffer.wrap(key) : null; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + return ByteBuffer.wrap(originalBytes); + case 1: + return topic; + case 2: + return partition; + case 3: + return offset; + case 4: + return (key != null) ? ByteBuffer.wrap(key) : null; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } @Override public void put(int field$, java.lang.Object value$) { switch (field$) { - case 0: originalBytes = (value$ instanceof byte[]) ? (byte[])value$: ((ByteBuffer) value$).array(); break; - case 1: topic = value$.toString(); break; - case 2: partition = (int)value$; break; - case 3: offset = (long)value$; break; - case 4: key = (value$ == null) ? null : ((value$ instanceof byte[]) ? (byte[])value$: ((ByteBuffer) value$).array()); break; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + case 0: + originalBytes = (value$ instanceof byte[]) ? (byte[]) value$ : ((ByteBuffer) value$).array(); + break; + case 1: + topic = value$.toString(); + break; + case 2: + partition = (int) value$; + break; + case 3: + offset = (long) value$; + break; + case 4: + key = (value$ == null) ? null : + ((value$ instanceof byte[]) ? (byte[]) value$ : ((ByteBuffer) value$).array()); + break; + default: + throw new org.apache.avro.AvroRuntimeException("Bad index"); } } diff --git a/flink-cyber/flink-cyber-api/src/test/java/com/cloudera/cyber/SerializationTests.java b/flink-cyber/flink-cyber-api/src/test/java/com/cloudera/cyber/SerializationTests.java index 95cbfbd3..c6d179ba 100644 --- a/flink-cyber/flink-cyber-api/src/test/java/com/cloudera/cyber/SerializationTests.java +++ b/flink-cyber/flink-cyber-api/src/test/java/com/cloudera/cyber/SerializationTests.java @@ -12,10 +12,22 @@ package com.cloudera.cyber; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + import com.cloudera.cyber.commands.CommandType; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.commands.EnrichmentCommandResponse; import com.cloudera.cyber.parser.MessageToParse; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.avro.specific.SpecificRecordBase; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -23,20 +35,28 @@ import org.apache.flink.util.InstantiationUtil; import org.junit.Test; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +public class SerializationTests { -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertThat; + { + Map map = new HashMap() {{ + put("a", "a"); + put("b", "b"); + }}; + ThreatIntelligence ti = ThreatIntelligence.builder().fields(map) + .observable("ob").observableType("ip").ts(0).build(); + } -public class SerializationTests { + public static T test(T obj) throws IOException { + Class cls = obj.getClass(); + AvroTypeInfo ti = new AvroTypeInfo(cls); + TypeSerializer serializer = ti.createSerializer(new ExecutionConfig()); + + byte[] bytes = InstantiationUtil.serializeToByteArray(serializer, obj); + T out = InstantiationUtil.deserializeFromByteArray(serializer, bytes); + + assertThat(out, notNullValue()); + return out; + } @Test public void testThreatIntelligence() throws IOException { @@ -45,7 +65,7 @@ public void testThreatIntelligence() throws IOException { put("b", "b"); }}; ThreatIntelligence ti = ThreatIntelligence.builder().fields(map) - .observable("ob").observableType("ip").ts(0).build(); + .observable("ob").observableType("ip").ts(0).build(); ThreatIntelligence test = test(ti); assertThat(test.getFields(), equalTo(map)); @@ -58,42 +78,33 @@ public void testEnrichmentEntry() throws IOException { put("b", "b"); }}; EnrichmentEntry entry = EnrichmentEntry.builder() - .type("ip") - .key("test") - .entries(map) - .ts(0) - .build(); + .type("ip") + .key("test") + .entries(map) + .ts(0) + .build(); EnrichmentEntry test = test(entry); assertThat(test.getEntries(), equalTo(map)); } - { - Map map = new HashMap() {{ - put("a", "a"); - put("b", "b"); - }}; - ThreatIntelligence ti = ThreatIntelligence.builder().fields(map) - .observable("ob").observableType("ip").ts(0).build(); - } - - @Test public void testMessageWithDataQuality() throws IOException { Map map = Collections.singletonMap("test", "value"); Message m = Message.builder() - .source("test") - .message("") - .ts(0) - .originalSource(SignedSourceKey.builder().topic("test").offset(0).partition(0).signature(new byte[128]).build()) - .extensions(map) - .dataQualityMessages(Arrays.asList( - DataQualityMessage.builder() - .field("test") - .level("INFO") - .feature("feature") - .message("test message") - .build() - )).build(); + .source("test") + .message("") + .ts(0) + .originalSource( + SignedSourceKey.builder().topic("test").offset(0).partition(0).signature(new byte[128]).build()) + .extensions(map) + .dataQualityMessages(Arrays.asList( + DataQualityMessage.builder() + .field("test") + .level("INFO") + .feature("feature") + .message("test message") + .build() + )).build(); Message test = test(m); assertThat(test.getExtensions(), equalTo(map)); assertThat(test.getDataQualityMessages().get(0).getMessage(), equalTo("test message")); @@ -101,28 +112,28 @@ public void testMessageWithDataQuality() throws IOException { @Test public void testMessageToParse() throws IOException { - MessageToParse messageToParse = MessageToParse.builder(). - offset(3).partition(1). - originalBytes("this is a test".getBytes(UTF_8)). - topic("test_topic"). - build(); + MessageToParse messageToParse = MessageToParse.builder() + .offset(3).partition(1) + .originalBytes("this is a test".getBytes(UTF_8)) + .topic("test_topic") + .build(); MessageToParse output = test(messageToParse); assertThat(output, equalTo(messageToParse)); } - + @Test public void testEnrichmentCommand() throws IOException { Map entryMap = Collections.singletonMap("test", "value"); EnrichmentCommand command = EnrichmentCommand.builder() - .type(CommandType.ADD) - .headers(Collections.singletonMap("test", "header")) - .payload(EnrichmentEntry.builder() - .type("test") - .ts(0) - .entries(entryMap) - .key("test") - .build()) - .build(); + .type(CommandType.ADD) + .headers(Collections.singletonMap("test", "header")) + .payload(EnrichmentEntry.builder() + .type("test") + .ts(0) + .entries(entryMap) + .key("test") + .build()) + .build(); EnrichmentCommand output = test(command); assertThat(output.getPayload().getEntries(), equalTo(entryMap)); } @@ -132,16 +143,16 @@ public void testEnrichmentCommandResponse() throws IOException { Map entryMap = Collections.singletonMap("test", "value"); Map headerMap = Collections.singletonMap("test", "header"); EnrichmentCommandResponse cr = EnrichmentCommandResponse.builder() - .success(true) - .message("") - .content(Arrays.asList(EnrichmentEntry.builder() - .type("test") - .ts(0) - .entries(entryMap) - .key("test") - .build())) - .headers(headerMap) - .build(); + .success(true) + .message("") + .content(Arrays.asList(EnrichmentEntry.builder() + .type("test") + .ts(0) + .entries(entryMap) + .key("test") + .build())) + .headers(headerMap) + .build(); EnrichmentCommandResponse output = test(cr); assertThat(output.getHeaders(), hasEntry("test", "header")); assertThat(output.getContent().get(0).getEntries(), hasEntry("test", "value")); @@ -149,22 +160,10 @@ public void testEnrichmentCommandResponse() throws IOException { private List createTi(Map tiFields) { return Arrays.asList(ThreatIntelligence.builder() - .fields(tiFields) - .observable("ob") - .observableType("ip") - .ts(0) - .build()); - } - - public static T test(T obj) throws IOException { - Class cls = obj.getClass(); - AvroTypeInfo ti = new AvroTypeInfo(cls); - TypeSerializer serializer = ti.createSerializer(new ExecutionConfig()); - - byte[] bytes = InstantiationUtil.serializeToByteArray(serializer, obj); - T out = InstantiationUtil.deserializeFromByteArray(serializer, bytes); - - assertThat(out, notNullValue()); - return out; + .fields(tiFields) + .observable("ob") + .observableType("ip") + .ts(0) + .build()); } } diff --git a/flink-cyber/flink-cyber-api/src/test/java/com/cloudera/cyber/TypeTests.java b/flink-cyber/flink-cyber-api/src/test/java/com/cloudera/cyber/TypeTests.java index ae367424..f1c32117 100644 --- a/flink-cyber/flink-cyber-api/src/test/java/com/cloudera/cyber/TypeTests.java +++ b/flink-cyber/flink-cyber-api/src/test/java/com/cloudera/cyber/TypeTests.java @@ -12,18 +12,18 @@ package com.cloudera.cyber; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + import com.cloudera.cyber.commands.EnrichmentCommand; +import java.util.Map; import org.apache.flink.api.common.typeinfo.TypeHint; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.avro.typeutils.AvroTypeInfo; import org.junit.Test; -import java.util.Map; - -import static org.hamcrest.Matchers.*; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThat; - public class TypeTests { @Test diff --git a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/Dedupe.java b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/Dedupe.java index b3cf552d..a5257ea9 100644 --- a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/Dedupe.java +++ b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/Dedupe.java @@ -17,28 +17,30 @@ import com.cloudera.cyber.dedupe.impl.CreateKeyFromMessage; import com.cloudera.cyber.dedupe.impl.SumAndMaxTs; import com.cloudera.cyber.flink.EventTimeAndCountTrigger; +import java.util.List; +import java.util.Map; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.util.OutputTag; -import java.util.List; -import java.util.Map; - public class Dedupe { - public static SingleOutputStreamOperator dedupe(DataStream source, List sessionKey, Long maxTime, Long maxCount, OutputTag lateData, Time allowedLateness) { + public static SingleOutputStreamOperator dedupe(DataStream source, List sessionKey, + Long maxTime, Long maxCount, + OutputTag lateData, + Time allowedLateness) { return source - .map(new CreateKeyFromMessage(sessionKey)) - .keyBy(new KeySelector>() { - @Override - public Map getKey(DedupeMessage dedupeMessage) throws Exception { - return dedupeMessage.getFields(); - } - }) - .timeWindow(Time.milliseconds(maxTime)) - .sideOutputLateData(lateData) - .trigger(EventTimeAndCountTrigger.of(maxCount)) - .aggregate(new SumAndMaxTs()); + .map(new CreateKeyFromMessage(sessionKey)) + .keyBy(new KeySelector>() { + @Override + public Map getKey(DedupeMessage dedupeMessage) throws Exception { + return dedupeMessage.getFields(); + } + }) + .timeWindow(Time.milliseconds(maxTime)) + .sideOutputLateData(lateData) + .trigger(EventTimeAndCountTrigger.of(maxCount)) + .aggregate(new SumAndMaxTs()); } } diff --git a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJob.java b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJob.java index 6dd35eec..3ce3f1fb 100644 --- a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJob.java +++ b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJob.java @@ -12,8 +12,12 @@ package com.cloudera.cyber.dedupe; +import static com.cloudera.cyber.dedupe.Dedupe.dedupe; + import com.cloudera.cyber.DedupeMessage; import com.cloudera.cyber.Message; +import java.util.Arrays; +import java.util.List; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; @@ -22,20 +26,18 @@ import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.util.OutputTag; -import java.util.Arrays; -import java.util.List; - -import static com.cloudera.cyber.dedupe.Dedupe.dedupe; - /** * Deduplication Job * + *

* Provide a set of fields to group on, a limit for the time window and the count of duplicates * before emitting, and receive a message with the values of the deduped fields, a count and the * maximum and minimum timestamp for the message rolled into the duplicate. - * + * + *

* This will need to be output to a separate kafka topic for each de-dupe * + *

* TODO - maybe. Add a salt to the key for keys that are likely to be high duplicate * this will avoid all the messages from the same key being shunted to one task * Note that to do this really smartly, we could auto watch the count metrics and @@ -58,9 +60,11 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws Long maxCount = params.getLong(PARAM_DEDUPE_MAX_COUNT, 0); DataStream source = createSource(env, params, key, maxTime); - final OutputTag lateData = new OutputTag("late-data"){}; - Time allowedLateness = Time.milliseconds(params.getLong(PARAM_DEDUPE_LATENESS, 0L)); - SingleOutputStreamOperator results = dedupe(source, key, maxTime, maxCount, lateData, allowedLateness); + final OutputTag lateData = new OutputTag("late-data") { + }; + Time allowedLateness = Time.milliseconds(params.getLong(PARAM_DEDUPE_LATENESS, 0L)); + SingleOutputStreamOperator results = + dedupe(source, key, maxTime, maxCount, lateData, allowedLateness); writeResults(params, results); //printResults(results); @@ -75,5 +79,7 @@ private void printResults(SingleOutputStreamOperator results) { } protected abstract void writeResults(ParameterTool params, DataStream results); - protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, List sessionKey, Long sessionTimeout); + + protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, + List sessionKey, Long sessionTimeout); } diff --git a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java index 74ebf72d..8d3346fd 100644 --- a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java +++ b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java @@ -12,10 +12,15 @@ package com.cloudera.cyber.dedupe; +import static com.cloudera.cyber.flink.FlinkUtils.createKafkaSource; + import com.cloudera.cyber.DedupeMessage; import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.kafka.sink.KafkaSink; @@ -23,23 +28,17 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.time.Duration; -import java.util.Arrays; -import java.util.List; - -import static com.cloudera.cyber.flink.FlinkUtils.createKafkaSource; - public class DedupeJobKafka extends DedupeJob { public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); ParameterTool params = Utils.getParamToolsFromProperties(args); new DedupeJobKafka() - .createPipeline(params) - .execute("Flink Deduplicator"); + .createPipeline(params) + .execute("Flink Deduplicator"); } /** - * Returns a consumer group id for the deduper ensuring that each topic is only processed once with the same keys + * Returns a consumer group id for the deduper ensuring that each topic is only processed once with the same keys. * * @param inputTopic topic to read from * @param sessionKey the keys being used to sessionise @@ -47,31 +46,34 @@ public static void main(String[] args) throws Exception { */ private String createGroupId(String inputTopic, List sessionKey, long sessionTimeout) { List parts = Arrays.asList("dedupe", - inputTopic, - String.valueOf(sessionKey.hashCode()), - String.valueOf(sessionTimeout)); + inputTopic, + String.valueOf(sessionKey.hashCode()), + String.valueOf(sessionTimeout)); return String.join(".", parts); } @Override protected void writeResults(ParameterTool params, DataStream results) { KafkaSink sink = new FlinkUtils<>(DedupeMessage.class).createKafkaSink( - params.getRequired("topic.enrichment"), - "dedup", - params); + params.getRequired("topic.enrichment"), + "dedup", + params); results.sinkTo(sink).name("Kafka Results").uid("kafka.results"); } @Override - protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, List sessionKey, Long sessionTimeout) { + protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, + List sessionKey, Long sessionTimeout) { String inputTopic = params.getRequired("topic.input"); String groupId = createGroupId(inputTopic, sessionKey, sessionTimeout); - WatermarkStrategy watermarkStrategy = WatermarkStrategy.forBoundedOutOfOrderness(Duration.ofMillis(params.getLong(PARAM_DEDUPE_LATENESS, 0L))). - withTimestampAssigner((event, timestamp) -> event.getTs()); + WatermarkStrategy watermarkStrategy = WatermarkStrategy.forBoundedOutOfOrderness( + Duration.ofMillis(params.getLong(PARAM_DEDUPE_LATENESS, 0L))) + .thTimestampAssigner( + (event, timestamp) -> event.getTs()); return env.fromSource(createKafkaSource(inputTopic, params, groupId), watermarkStrategy, "Kafka Source") - .uid("kafka.input"); + .uid("kafka.input"); } } diff --git a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/CreateKeyFromMessage.java b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/CreateKeyFromMessage.java index ced69ce3..5e20b748 100644 --- a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/CreateKeyFromMessage.java +++ b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/CreateKeyFromMessage.java @@ -14,10 +14,9 @@ import com.cloudera.cyber.DedupeMessage; import com.cloudera.cyber.Message; -import org.apache.flink.api.common.functions.RichMapFunction; - import java.util.List; import java.util.stream.Collectors; +import org.apache.flink.api.common.functions.RichMapFunction; public class CreateKeyFromMessage extends RichMapFunction { List key; @@ -29,13 +28,13 @@ public CreateKeyFromMessage(List key) { @Override public DedupeMessage map(Message message) { return DedupeMessage.builder() - .count(1L) - .fields(key.stream() - .collect(Collectors.toMap( - f -> f, - f->message.get(f).toString())) - ) - .ts(message.getTs()) - .build(); + .count(1L) + .fields(key.stream() + .collect(Collectors.toMap( + f -> f, + f -> message.get(f).toString())) + ) + .ts(message.getTs()) + .build(); } } diff --git a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/SumAndMax.java b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/SumAndMax.java index 91dc5f8d..16522c4e 100644 --- a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/SumAndMax.java +++ b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/SumAndMax.java @@ -12,11 +12,10 @@ package com.cloudera.cyber.dedupe.impl; +import java.util.Map; import lombok.Builder; import lombok.Data; -import java.util.Map; - @Data @Builder public class SumAndMax { @@ -30,10 +29,10 @@ public SumAndMax merge(SumAndMax other) { throw new IllegalStateException("Unmatched SumAndMax"); } return SumAndMax.builder() - .fields(this.getFields()) - .maxTs(Math.max(this.getMaxTs(), other.getMaxTs())) - .minTs(Math.min(this.getMinTs(), other.getMinTs())) - .sum(this.getSum() + other.getSum()) - .build(); + .fields(this.getFields()) + .maxTs(Math.max(this.getMaxTs(), other.getMaxTs())) + .minTs(Math.min(this.getMinTs(), other.getMinTs())) + .sum(this.getSum() + other.getSum()) + .build(); } } diff --git a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/SumAndMaxTs.java b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/SumAndMaxTs.java index f843dffb..efa44367 100644 --- a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/SumAndMaxTs.java +++ b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/impl/SumAndMaxTs.java @@ -22,10 +22,10 @@ public class SumAndMaxTs implements AggregateFunction cidr(SingleOutputStreamOperator source, List ipFields, String confPath) throws IOException { - RegionCidrEnrichmentConfiguration regionCidrEnrichmentConfiguration = JSONUtils.INSTANCE.load(readConfigFile(confPath), RegionCidrEnrichmentConfiguration.class); + public static SingleOutputStreamOperator cidr(SingleOutputStreamOperator source, + List ipFields, String confPath) throws IOException { + RegionCidrEnrichmentConfiguration regionCidrEnrichmentConfiguration = + JSONUtils.INSTANCE.load(readConfigFile(confPath), RegionCidrEnrichmentConfiguration.class); validate(regionCidrEnrichmentConfiguration); return source - .map(new IpRegionMap(regionCidrEnrichmentConfiguration, ipFields)) - .name("IP CIDR enrichment").uid("region-cidr"); + .map(new IpRegionMap(regionCidrEnrichmentConfiguration, ipFields)) + .name("IP CIDR enrichment").uid("region-cidr"); } private static String readConfigFile(String stringPath) throws IOException { @@ -65,23 +67,30 @@ private static String readConfigFile(String stringPath) throws IOException { private static void validate(RegionCidrEnrichmentConfiguration regionCidrEnrichmentMap) { List regionNames = regionCidrEnrichmentMap.values().stream() - .flatMap(map -> map.keySet().stream()) - .collect(Collectors.toList()); - List cidrs = regionCidrEnrichmentMap.values().stream(). - flatMap(map -> map.values().stream() - .flatMap(Collection::stream)) - .collect(Collectors.toList()); + .flatMap(map -> map.keySet().stream()) + .collect(Collectors.toList()); + List cidrs = regionCidrEnrichmentMap.values().stream() + .flatMap(map -> map.values().stream() + .flatMap(Collection::stream)) + .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(ValidateUtils.getDuplicates(regionNames))) { - throw new ValidationException(String.format("Configuration contains duplicates in region names: '%s'", String.join(",", regionNames))); + throw new ValidationException(String.format("Configuration contains duplicates in region names: '%s'", + String.join(",", regionNames))); } if (CollectionUtils.isNotEmpty(ValidateUtils.getDuplicates(cidrs))) { - throw new ValidationException(String.format("Configuration contains duplicates in cidrs: '%s'", String.join(",", cidrs))); + throw new ValidationException( + String.format("Configuration contains duplicates in cidrs: '%s'", String.join(",", cidrs))); } for (String cidr : cidrs) { try { IPAddress ipAddress = new IPAddressString(cidr).toAddress(); - if (ipAddress.getPrefixLength() == null || ipAddress.getCount().equals(BigInteger.ONE) && ipAddress.getPrefixLength() != 32){ - throw new ValidationException(String.format("Wrong format for cidr: '%s'. Only the subnet is allowed to be used as Cidr values. 1.2.0.0/16 - is the subnet; 1.2.3.4/16 is a single address, since the host is 3.4; 1.2.3.4 is a single address, since have no prefix", cidr)); + if (ipAddress.getPrefixLength() == null + || ipAddress.getCount().equals(BigInteger.ONE) && ipAddress.getPrefixLength() != 32) { + throw new ValidationException(String.format( + "Wrong format for cidr: '%s'. Only the subnet is allowed to be used as Cidr values. " + + "1.2.0.0/16 - is the subnet; 1.2.3.4/16 is a single address, since the host is 3.4; " + + "1.2.3.4 is a single address, since have no prefix", + cidr)); } } catch (AddressStringException e) { throw new ValidationException(e.getMessage()); diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionCidrJob.java b/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionCidrJob.java index e09dd240..79b5fcb7 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionCidrJob.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionCidrJob.java @@ -15,14 +15,13 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import java.util.Arrays; -import java.util.List; - public abstract class IpRegionCidrJob { public static final String PARAM_CIDR_IP_FIELDS = "cidr.ip_fields"; public static final String PARAM_CIDR_CONFIG_PATH = "cidr.config_file_path"; @@ -43,6 +42,7 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws protected abstract void writeResults(ParameterTool params, DataStream results); - protected abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params, List ipFields); + protected abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, + ParameterTool params, List ipFields); } \ No newline at end of file diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionCidrJobKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionCidrJobKafka.java index 738a3219..c5a70664 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionCidrJobKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionCidrJobKafka.java @@ -15,6 +15,8 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; +import java.util.Arrays; +import java.util.List; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.kafka.sink.KafkaSink; @@ -23,48 +25,46 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.Arrays; -import java.util.List; - public class IpRegionCidrJobKafka extends IpRegionCidrJob { public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); ParameterTool params = Utils.getParamToolsFromProperties(args); new IpRegionCidrJobKafka() - .createPipeline(params) - .execute("Flink IP CIDR enrichment"); + .createPipeline(params) + .execute("Flink IP CIDR enrichment"); } @Override protected void writeResults(ParameterTool params, DataStream results) { KafkaSink sink = new FlinkUtils<>(Message.class).createKafkaSink( - params.getRequired("topic.output"), - "enrichment-region-cidr", - params); + params.getRequired("topic.output"), + "enrichment-region-cidr", + params); results.sinkTo(sink).name("Kafka Results").uid("kafka.results"); } /** - * Returns a consumer group id for the geocoder ensuring that each topic is only processed once with the same fields + * Returns a consumer group id for the geocoder ensuring that each topic is only processed once with the same fields. * * @param inputTopic topic to read from - * @param ipFields the ip fields to be geocoded + * @param ipFields the ip fields to be geocoded * @return Kafka group id for geocoder */ private String createGroupId(String inputTopic, List ipFields) { List parts = Arrays.asList("ipRegionCidr", - inputTopic, - String.valueOf(ipFields.hashCode())); + inputTopic, + String.valueOf(ipFields.hashCode())); return String.join(".", parts); } @Override - protected SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params, List ipFields) { + protected SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params, + List ipFields) { String inputTopic = params.getRequired("topic.input"); - return env.fromSource(FlinkUtils.createKafkaSource(inputTopic, - params, createGroupId(inputTopic, ipFields)), WatermarkStrategy.noWatermarks(), "Kafka Source") - .uid("kafka.input"); + return env.fromSource(FlinkUtils.createKafkaSource(inputTopic, + params, createGroupId(inputTopic, ipFields)), WatermarkStrategy.noWatermarks(), "Kafka Source") + .uid("kafka.input"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionMap.java b/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionMap.java index ef5b849b..f3bbb383 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionMap.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/IpRegionMap.java @@ -3,8 +3,8 @@ import com.cloudera.cyber.DataQualityMessage; import com.cloudera.cyber.Message; import com.cloudera.cyber.MessageUtils; -import com.cloudera.cyber.enrichment.cidr.impl.types.RegionCidrEnrichmentConfiguration; import com.cloudera.cyber.enrichment.cidr.impl.IpRegionCidrEnrichment; +import com.cloudera.cyber.enrichment.cidr.impl.types.RegionCidrEnrichmentConfiguration; import inet.ipaddr.IPAddressString; import java.util.ArrayList; import java.util.HashMap; @@ -39,7 +39,8 @@ public Message map(Message message) throws Exception { Map extensions = new HashMap<>(); for (String ipFieldName : ipFieldNames) { Optional.ofNullable(messageFields.get(ipFieldName)) - .ifPresent(ipFieldValue -> regionCidrEnrichment.lookup(ipFieldName, ipFieldValue, extensions, qualityMessages )); + .ifPresent(ipFieldValue -> regionCidrEnrichment.lookup(ipFieldName, ipFieldValue, extensions, + qualityMessages)); } newMessage = MessageUtils.enrich(message, extensions, qualityMessages); } @@ -49,7 +50,7 @@ public Message map(Message message) throws Exception { @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - Map regionCidrMap = new HashMap<>(); + Map regionCidrMap = new HashMap<>(); for (Map> regionCidrs : cidrEnrichmentMap.values()) { for (Entry> regionCidrEntry : regionCidrs.entrySet()) { for (String cidr : regionCidrEntry.getValue()) { diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/impl/IpRegionCidrEnrichment.java b/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/impl/IpRegionCidrEnrichment.java index 1388637e..99a23495 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/impl/IpRegionCidrEnrichment.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-cidr/src/main/java/com/cloudera/cyber/enrichment/cidr/impl/IpRegionCidrEnrichment.java @@ -5,7 +5,6 @@ import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.enrichment.Enrichment; import com.cloudera.cyber.enrichment.MetronGeoEnrichment; -import inet.ipaddr.IPAddress; import inet.ipaddr.IPAddressString; import java.util.Comparator; import java.util.List; @@ -19,22 +18,27 @@ @Slf4j public class IpRegionCidrEnrichment { private Map regionCidrMap; - public final static String FEATURE_NAME = "region"; + public static final String FEATURE_NAME = "region"; - public void lookup(String ipFieldName, String ipFieldValue, Map extensions, List qualityMessages) { + public void lookup(String ipFieldName, String ipFieldValue, Map extensions, + List qualityMessages) { IPAddressString ipAddressString = new IPAddressString(ipFieldValue); Enrichment enrichment = new MetronGeoEnrichment(ipFieldName, null); if (ipAddressString.getAddress() == null) { - MessageUtils.addQualityMessage(qualityMessages, DataQualityMessageLevel.ERROR, "Unable to parse message with ip " + ipFieldValue, ipFieldName, FEATURE_NAME); + MessageUtils.addQualityMessage(qualityMessages, DataQualityMessageLevel.ERROR, + "Unable to parse message with ip " + ipFieldValue, ipFieldName, FEATURE_NAME); log.error("Unable to parse ip in a message {}", extensions); return; } Optional regionNameOptional = regionCidrMap.entrySet().stream() - .filter(entry -> entry.getKey().getAddress().toIPv6().contains(ipAddressString.getAddress().toIPv6())) - .min(Comparator.comparing(entry -> entry.getKey().getAddress() - .getCount())).map(Entry::getValue); + .filter(entry -> entry.getKey().getAddress().toIPv6() + .contains(ipAddressString.getAddress() + .toIPv6())) + .min(Comparator.comparing( + entry -> entry.getKey().getAddress() + .getCount())).map(Entry::getValue); if (regionNameOptional.isPresent()) { enrichment.enrich(extensions, FEATURE_NAME, regionNameOptional.get()); diff --git a/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/EnrichmentJob.java b/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/EnrichmentJob.java index 267af0ca..3e5ade13 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/EnrichmentJob.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/EnrichmentJob.java @@ -12,6 +12,14 @@ package com.cloudera.cyber.enrichment; +import static com.cloudera.cyber.enrichment.cidr.IpRegionCidrJob.PARAM_CIDR_CONFIG_PATH; +import static com.cloudera.cyber.enrichment.cidr.IpRegionCidrJob.PARAM_CIDR_IP_FIELDS; +import static com.cloudera.cyber.enrichment.geocode.IpGeoJob.PARAM_ASN_DATABASE_PATH; +import static com.cloudera.cyber.enrichment.geocode.IpGeoJob.PARAM_ASN_FIELDS; +import static com.cloudera.cyber.enrichment.geocode.IpGeoJob.PARAM_GEO_DATABASE_PATH; +import static com.cloudera.cyber.enrichment.geocode.IpGeoJob.PARAM_GEO_FIELDS; +import static com.cloudera.cyber.enrichment.hbase.HbaseJob.PARAMS_ENRICHMENT_CONFIG; + import com.cloudera.cyber.Message; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.commands.EnrichmentCommandResponse; @@ -32,6 +40,11 @@ import com.cloudera.cyber.scoring.ScoringJob; import com.cloudera.cyber.scoring.ScoringRuleCommand; import com.cloudera.cyber.scoring.ScoringRuleCommandResult; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.utils.ParameterTool; @@ -39,17 +52,6 @@ import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; - -import static com.cloudera.cyber.enrichment.cidr.IpRegionCidrJob.PARAM_CIDR_CONFIG_PATH; -import static com.cloudera.cyber.enrichment.cidr.IpRegionCidrJob.PARAM_CIDR_IP_FIELDS; -import static com.cloudera.cyber.enrichment.geocode.IpGeoJob.*; -import static com.cloudera.cyber.enrichment.hbase.HbaseJob.PARAMS_ENRICHMENT_CONFIG; - @Slf4j public abstract class EnrichmentJob { private static final String PARAMS_LOOKUPS_CONFIG_FILE = "lookups.config.file"; @@ -65,6 +67,7 @@ public abstract class EnrichmentJob { private static final String PARAMS_ENABLE_RULES = "rules.enabled"; private static final String PARAMS_ENABLE_STELLAR = "stellar.enabled"; + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws IOException { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FlinkUtils.setupEnv(env, params); @@ -72,25 +75,29 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws SingleOutputStreamOperator messages = createSource(env, params); DataStream enrichments = createEnrichmentSource(env, params); - List enrichmentConfigs = ConfigUtils.allConfigs(Files.readAllBytes(Paths.get(params.getRequired(PARAMS_LOOKUPS_CONFIG_FILE)))); - SingleOutputStreamOperator localEnrichments = enrichments.filter(new FilterEnrichmentType(enrichmentConfigs, EnrichmentKind.LOCAL)); - SingleOutputStreamOperator hbaseEnrichments = enrichments.filter(new FilterEnrichmentType(enrichmentConfigs, EnrichmentKind.HBASE)); - - SingleOutputStreamOperator geoEnriched = params.getBoolean(PARAMS_ENABLE_GEO, true) ? - IpGeo.geo(messages, - Arrays.asList(params.getRequired(PARAM_GEO_FIELDS).split(",")), - params.getRequired(PARAM_GEO_DATABASE_PATH)) : messages; - SingleOutputStreamOperator asnEnriched = params.getBoolean(PARAMS_ENABLE_ASN, true) ? - IpGeo.asn(geoEnriched, - Arrays.asList(params.getRequired(PARAM_ASN_FIELDS).split(",")), - params.getRequired(PARAM_ASN_DATABASE_PATH)) : geoEnriched; - - SingleOutputStreamOperator cidrEnriched = params.getBoolean(PARAMS_ENABLE_CIDR, false) ? - IpRegionCidr.cidr(asnEnriched, - Arrays.asList(params.getRequired(PARAM_CIDR_IP_FIELDS).split(",")), - params.getRequired(PARAM_CIDR_CONFIG_PATH)) : asnEnriched; - - Tuple2, DataStream> enriched = LookupJob.enrich(localEnrichments, cidrEnriched, enrichmentConfigs); + List enrichmentConfigs = + ConfigUtils.allConfigs(Files.readAllBytes(Paths.get(params.getRequired(PARAMS_LOOKUPS_CONFIG_FILE)))); + SingleOutputStreamOperator localEnrichments = + enrichments.filter(new FilterEnrichmentType(enrichmentConfigs, EnrichmentKind.LOCAL)); + SingleOutputStreamOperator hbaseEnrichments = + enrichments.filter(new FilterEnrichmentType(enrichmentConfigs, EnrichmentKind.HBASE)); + + SingleOutputStreamOperator geoEnriched = params.getBoolean(PARAMS_ENABLE_GEO, true) + ? IpGeo.geo(messages, + Arrays.asList(params.getRequired(PARAM_GEO_FIELDS).split(",")), + params.getRequired(PARAM_GEO_DATABASE_PATH)) : messages; + SingleOutputStreamOperator asnEnriched = params.getBoolean(PARAMS_ENABLE_ASN, true) + ? IpGeo.asn(geoEnriched, + Arrays.asList(params.getRequired(PARAM_ASN_FIELDS).split(",")), + params.getRequired(PARAM_ASN_DATABASE_PATH)) : geoEnriched; + + SingleOutputStreamOperator cidrEnriched = params.getBoolean(PARAMS_ENABLE_CIDR, false) + ? IpRegionCidr.cidr(asnEnriched, + Arrays.asList(params.getRequired(PARAM_CIDR_IP_FIELDS).split(",")), + params.getRequired(PARAM_CIDR_CONFIG_PATH)) : asnEnriched; + + Tuple2, DataStream> enriched = + LookupJob.enrich(localEnrichments, cidrEnriched, enrichmentConfigs); DataStream enrichmentCommandResponses = enriched.f1; @@ -103,18 +110,19 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws } - DataStream hbased = hbaseEnabled ? - HbaseJob.enrich(enriched.f0, enrichmentConfigs, enrichmentsStorageConfig) : enriched.f0; + DataStream hbased = hbaseEnabled + ? HbaseJob.enrich(enriched.f0, enrichmentConfigs, enrichmentsStorageConfig) : enriched.f0; // rest based enrichments - DataStream rested = params.getBoolean(PARAMS_ENABLE_REST, true) ? - RestLookupJob.enrich(hbased, params.getRequired(PARAMS_REST_CONFIG_FILE)) : hbased; + DataStream rested = params.getBoolean(PARAMS_ENABLE_REST, true) + ? RestLookupJob.enrich(hbased, params.getRequired(PARAMS_REST_CONFIG_FILE)) : hbased; // Run threatQ integrations DataStream tqed; DataStream threatqEnrichments = null; if (threatqEnabled) { - List threatQconfigs = ThreatQJob.parseConfigs(Files.readAllBytes(Paths.get(params.getRequired(PARAMS_THREATQ_CONFIG_FILE)))); + List threatQconfigs = ThreatQJob.parseConfigs( + Files.readAllBytes(Paths.get(params.getRequired(PARAMS_THREATQ_CONFIG_FILE)))); log.info("ThreatQ Configs {}", threatQconfigs); tqed = ThreatQJob.enrich(rested, threatQconfigs, enrichmentsStorageConfig); threatqEnrichments = createThreatQSource(env, params); @@ -124,7 +132,9 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws DataStream hbaseTqEnrichResults = null; if (hbaseEnabled || threatqEnabled) { - hbaseTqEnrichResults = writeHbaseThreatQEnrichmentsToHbaseAndRespond(params, env, hbaseEnrichments, threatqEnrichments, enrichmentsStorageConfig); + hbaseTqEnrichResults = + writeHbaseThreatQEnrichmentsToHbaseAndRespond(params, env, hbaseEnrichments, threatqEnrichments, + enrichmentsStorageConfig); } @@ -136,14 +146,16 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws String configDir = params.getRequired(StellarEnrichmentJob.PARAMS_CONFIG_DIR); String geoDatabasePath = params.getRequired(PARAM_GEO_DATABASE_PATH); String asnDatabasePath = params.getRequired(PARAM_ASN_DATABASE_PATH); - stellarStream = StellarEnrichmentJob.enrich(tqed, StellarEnrichmentJob.loadFiles(configDir), geoDatabasePath, asnDatabasePath); + stellarStream = + StellarEnrichmentJob.enrich(tqed, StellarEnrichmentJob.loadFiles(configDir), geoDatabasePath, + asnDatabasePath); } else { stellarStream = tqed; } // disabled by default - NOT IMPLEMENTED - DataStream ruled = params.getBoolean(PARAMS_ENABLE_RULES, false) ? - doRules(stellarStream) : stellarStream; + DataStream ruled = params.getBoolean(PARAMS_ENABLE_RULES, false) + ? doRules(stellarStream) : stellarStream; DataStream scoring = doScoring(ruled, env, params); @@ -151,15 +163,19 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws return env; } - private DataStream writeHbaseThreatQEnrichmentsToHbaseAndRespond(ParameterTool params, StreamExecutionEnvironment env, - DataStream hbaseEnrichments, DataStream threatqEnrichments, - EnrichmentsConfig enrichmentsStorageConfig) { + private DataStream writeHbaseThreatQEnrichmentsToHbaseAndRespond( + ParameterTool params, + StreamExecutionEnvironment env, + DataStream hbaseEnrichments, + DataStream threatqEnrichments, + EnrichmentsConfig enrichmentsStorageConfig) { // write the threatq and hbase enrichments to hbase - return new HbaseJobRawKafka().writeEnrichments(env, params, unionStreams(hbaseEnrichments, threatqEnrichments), enrichmentsStorageConfig); + return new HbaseJobRawKafka().writeEnrichments(env, params, unionStreams(hbaseEnrichments, threatqEnrichments), + enrichmentsStorageConfig); } - private static DataStream unionStreams(DataStream stream1, DataStream stream2) { + private static DataStream unionStreams(DataStream stream1, DataStream stream2) { if (stream1 != null && stream2 != null) { return stream1.union(stream2); } else if (stream1 != null) { @@ -170,32 +186,42 @@ private static DataStream unionStreams(DataStream stream1, DataStream doRules(DataStream in) { return in; } - private DataStream doScoring(DataStream in, StreamExecutionEnvironment env, ParameterTool params) { + private DataStream doScoring(DataStream in, StreamExecutionEnvironment env, + ParameterTool params) { DataStream rulesSource = createRulesSource(env, params); SingleOutputStreamOperator results = ScoringJob.enrich(in, rulesSource, params); writeScoredRuleCommandResult(params, results.getSideOutput(ScoringJob.COMMAND_RESULT_OUTPUT_TAG)); return results; } - protected abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, + ParameterTool params); - protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream results); + protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream results); - protected abstract DataStream createEnrichmentSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract DataStream createEnrichmentSource(StreamExecutionEnvironment env, + ParameterTool params); - protected abstract void writeEnrichmentQueryResults(StreamExecutionEnvironment env, ParameterTool params, DataStream sideOutput); + protected abstract void writeEnrichmentQueryResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream sideOutput); - protected abstract DataStream createThreatQSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract DataStream createThreatQSource(StreamExecutionEnvironment env, + ParameterTool params); - protected abstract DataStream createRulesSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract DataStream createRulesSource(StreamExecutionEnvironment env, + ParameterTool params); - protected abstract void writeScoredRuleCommandResult(ParameterTool params, DataStream results); + protected abstract void writeScoredRuleCommandResult(ParameterTool params, + DataStream results); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/EnrichmentJobKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/EnrichmentJobKafka.java index 59a66648..54a75d7d 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/EnrichmentJobKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/EnrichmentJobKafka.java @@ -12,6 +12,11 @@ package com.cloudera.cyber.enrichment; +import static com.cloudera.cyber.enrichment.lookup.LookupJobKafka.PARAMS_QUERY_OUTPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; +import static com.cloudera.cyber.flink.Utils.readKafkaProperties; + import com.cloudera.cyber.Message; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.commands.EnrichmentCommandResponse; @@ -21,6 +26,7 @@ import com.cloudera.cyber.scoring.ScoredMessage; import com.cloudera.cyber.scoring.ScoringRuleCommand; import com.cloudera.cyber.scoring.ScoringRuleCommandResult; +import java.util.Properties; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.kafka.sink.KafkaSink; @@ -30,13 +36,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.Properties; - -import static com.cloudera.cyber.enrichment.lookup.LookupJobKafka.PARAMS_QUERY_OUTPUT; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; -import static com.cloudera.cyber.flink.Utils.readKafkaProperties; - public class EnrichmentJobKafka extends EnrichmentJob { private static final String PARAMS_TOPIC_ENRICHMENT_INPUT = "enrichment.topic.input"; @@ -49,33 +48,41 @@ public class EnrichmentJobKafka extends EnrichmentJob { public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); ParameterTool params = Utils.getParamToolsFromProperties(args); - FlinkUtils.executeEnv(new EnrichmentJobKafka().createPipeline(params),"Triaging Job - default",params); + FlinkUtils.executeEnv(new EnrichmentJobKafka().createPipeline(params), "Triaging Job - default", params); } @Override - protected void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream reduction) { - reduction.sinkTo(new FlinkUtils<>(ScoredMessage.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), "enrichments-combined", params)) - .name("Kafka Triaging Scored Sink").uid("kafka-sink"); + protected void writeResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream reduction) { + reduction.sinkTo(new FlinkUtils<>(ScoredMessage.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), + "enrichments-combined", params)) + .name("Kafka Triaging Scored Sink").uid("kafka-sink"); } @Override public SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params) { return env.fromSource( - FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), - params, params.get(PARAMS_GROUP_ID, DEFAULT_GROUP_ID)), - WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); + FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), + params, params.get(PARAMS_GROUP_ID, DEFAULT_GROUP_ID)), + WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); } @Override - protected DataStream createEnrichmentSource(StreamExecutionEnvironment env, ParameterTool params) { - KafkaSource source = new FlinkUtils<>(EnrichmentCommand.class).createKafkaGenericSource(params.getRequired(PARAMS_TOPIC_ENRICHMENT_INPUT), params, params.get(PARAMS_GROUP_ID, DEFAULT_GROUP_ID)); - return env.fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Enrichments").uid("kafka-enrichment-source"); + protected DataStream createEnrichmentSource(StreamExecutionEnvironment env, + ParameterTool params) { + KafkaSource source = new FlinkUtils<>(EnrichmentCommand.class).createKafkaGenericSource( + params.getRequired(PARAMS_TOPIC_ENRICHMENT_INPUT), params, params.get(PARAMS_GROUP_ID, DEFAULT_GROUP_ID)); + return env.fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Enrichments") + .uid("kafka-enrichment-source"); } @Override - protected void writeEnrichmentQueryResults(StreamExecutionEnvironment env, ParameterTool params, DataStream sideOutput) { - sideOutput.sinkTo(new FlinkUtils<>(EnrichmentCommandResponse.class).createKafkaSink(params.getRequired(PARAMS_QUERY_OUTPUT), "enrichment-combined-command", params)) - .name("Triaging Query Sink").uid("kafka-enrichment-query-sink"); + protected void writeEnrichmentQueryResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream sideOutput) { + sideOutput.sinkTo( + new FlinkUtils<>(EnrichmentCommandResponse.class).createKafkaSink(params.getRequired(PARAMS_QUERY_OUTPUT), + "enrichment-combined-command", params)) + .name("Triaging Query Sink").uid("kafka-enrichment-query-sink"); } @Override @@ -85,21 +92,26 @@ protected DataStream createThreatQSource(StreamExecutionEnvir Properties kafkaProperties = readKafkaProperties(params, groupId, true); - return env.fromSource(FlinkUtils.createKafkaStringSource(topic, kafkaProperties), WatermarkStrategy.noWatermarks(),"ThreatQ Source").uid("threatq-kafka") - .flatMap(new ThreatQParserFlatMap()).name("ThreatQ Parser").uid("threatq-parser"); + return env.fromSource(FlinkUtils.createKafkaStringSource(topic, kafkaProperties), + WatermarkStrategy.noWatermarks(), "ThreatQ Source").uid("threatq-kafka") + .flatMap(new ThreatQParserFlatMap()).name("ThreatQ Parser").uid("threatq-parser"); } @Override protected DataStream createRulesSource(StreamExecutionEnvironment env, ParameterTool params) { String topic = params.getRequired("query.input.topic"); - KafkaSource source = new FlinkUtils<>(ScoringRuleCommand.class).createKafkaGenericSource(topic, params, SCORING_RULES_GROUP_ID); - return env.fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Score Rule Source").uid("kafka.input.rule.command"); + KafkaSource source = + new FlinkUtils<>(ScoringRuleCommand.class).createKafkaGenericSource(topic, params, + SCORING_RULES_GROUP_ID); + return env.fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Score Rule Source") + .uid("kafka.input.rule.command"); } @Override protected void writeScoredRuleCommandResult(ParameterTool params, DataStream results) { String topic = params.getRequired("query.output.topic"); - KafkaSink sink = new FlinkUtils<>(ScoringRuleCommandResult.class).createKafkaSink(topic, SCORING_RULES_GROUP_ID, params); + KafkaSink sink = + new FlinkUtils<>(ScoringRuleCommandResult.class).createKafkaSink(topic, SCORING_RULES_GROUP_ID, params); results.sinkTo(sink).name("Kafka Score Rule Command Results").uid("kafka.output.rule.command.results"); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/FilterEnrichmentType.java b/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/FilterEnrichmentType.java index d1844245..d7504416 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/FilterEnrichmentType.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-combined/src/main/java/com/cloudera/cyber/enrichment/FilterEnrichmentType.java @@ -16,18 +16,17 @@ import com.cloudera.cyber.enrichment.lookup.config.EnrichmentConfig; import com.cloudera.cyber.enrichment.lookup.config.EnrichmentField; import com.cloudera.cyber.enrichment.lookup.config.EnrichmentKind; -import org.apache.flink.api.common.functions.FilterFunction; - import java.util.ArrayList; import java.util.List; +import org.apache.flink.api.common.functions.FilterFunction; public class FilterEnrichmentType implements FilterFunction { private final List enrichmentTypes = new ArrayList<>(); public FilterEnrichmentType(List enrichmentConfigStream, EnrichmentKind kind) { - for(EnrichmentConfig enrichmentConfig : enrichmentConfigStream) { + for (EnrichmentConfig enrichmentConfig : enrichmentConfigStream) { if (enrichmentConfig.getKind().equals(kind)) { - for(EnrichmentField field : enrichmentConfig.getFields()) { + for (EnrichmentField field : enrichmentConfig.getFields()) { enrichmentTypes.add(field.getEnrichmentType()); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpAsnMap.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpAsnMap.java index 75bab6ab..dbb662b3 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpAsnMap.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpAsnMap.java @@ -16,6 +16,10 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.enrichment.geocode.impl.IpAsnEnrichment; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; @@ -23,11 +27,6 @@ import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.configuration.Configuration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - @EqualsAndHashCode(callSuper = true) @Data @AllArgsConstructor diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeo.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeo.java index 0d9c9091..20c9926d 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeo.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeo.java @@ -13,22 +13,22 @@ package com.cloudera.cyber.enrichment.geocode; import com.cloudera.cyber.Message; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; - import java.util.List; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; public class IpGeo { - public static SingleOutputStreamOperator geo(SingleOutputStreamOperator source, List ipFields, String geoDatabasePath) { + public static SingleOutputStreamOperator geo(SingleOutputStreamOperator source, + List ipFields, String geoDatabasePath) { return source - .map(new IpGeoMap(geoDatabasePath, ipFields, null)) - .name("IP Geo").uid("maxmind-geo"); + .map(new IpGeoMap(geoDatabasePath, ipFields, null)) + .name("IP Geo").uid("maxmind-geo"); } - public static SingleOutputStreamOperator asn(SingleOutputStreamOperator source, List ipFields, String asnDatabasePath) { + public static SingleOutputStreamOperator asn(SingleOutputStreamOperator source, + List ipFields, String asnDatabasePath) { return source - .map(new IpAsnMap(asnDatabasePath, ipFields, null)) - .name("IP ASN").uid("maxmind-asn"); + .map(new IpAsnMap(asnDatabasePath, ipFields, null)) + .name("IP ASN").uid("maxmind-asn"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoJob.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoJob.java index 2b3f243e..2e05b56e 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoJob.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoJob.java @@ -14,20 +14,19 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; +import java.util.Arrays; +import java.util.List; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import java.util.Arrays; -import java.util.List; - public abstract class IpGeoJob { public static final String PARAM_GEO_FIELDS = "geo.ip_fields"; public static final String PARAM_GEO_DATABASE_PATH = "geo.database_path"; - public static final String PARAM_ASN_FIELDS = "asn.ip_fields";; + public static final String PARAM_ASN_FIELDS = "asn.ip_fields"; public static final String PARAM_ASN_DATABASE_PATH = "asn.database_path"; protected StreamExecutionEnvironment createPipeline(ParameterTool params) { @@ -47,6 +46,7 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) { protected abstract void writeResults(ParameterTool params, DataStream results); - protected abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params, List ipFields); + protected abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, + ParameterTool params, List ipFields); } \ No newline at end of file diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoJobKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoJobKafka.java index 069bb281..a6b76077 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoJobKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoJobKafka.java @@ -15,6 +15,8 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; +import java.util.Arrays; +import java.util.List; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.kafka.sink.KafkaSink; @@ -23,47 +25,45 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.Arrays; -import java.util.List; - public class IpGeoJobKafka extends IpGeoJob { public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); ParameterTool params = Utils.getParamToolsFromProperties(args); new IpGeoJobKafka() - .createPipeline(params) - .execute("Flink IP Geocode"); + .createPipeline(params) + .execute("Flink IP Geocode"); } @Override protected void writeResults(ParameterTool params, DataStream results) { KafkaSink sink = new FlinkUtils<>(Message.class).createKafkaSink( - params.getRequired("topic.output"), - "enrichment-geocode", - params); + params.getRequired("topic.output"), + "enrichment-geocode", + params); results.sinkTo(sink).name("Kafka Results").uid("kafka.results"); } /** - * Returns a consumer group id for the geocoder ensuring that each topic is only processed once with the same fields + * Returns a consumer group id for the geocoder ensuring that each topic is only processed once with the same fields. * * @param inputTopic topic to read from - * @param ipFields the ip fields to be geocoded + * @param ipFields the ip fields to be geocoded * @return Kafka group id for geocoder */ private String createGroupId(String inputTopic, List ipFields) { List parts = Arrays.asList("ipgeo", - inputTopic, - String.valueOf(ipFields.hashCode())); + inputTopic, + String.valueOf(ipFields.hashCode())); return String.join(".", parts); } @Override - protected SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params, List ipFields) { + protected SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params, + List ipFields) { String inputTopic = params.getRequired("topic.input"); - return env.fromSource(FlinkUtils.createKafkaSource(inputTopic, - params,createGroupId(inputTopic, ipFields)), WatermarkStrategy.noWatermarks(), "Kafka Source") - .uid("kafka.input"); + return env.fromSource(FlinkUtils.createKafkaSource(inputTopic, + params, createGroupId(inputTopic, ipFields)), WatermarkStrategy.noWatermarks(), "Kafka Source") + .uid("kafka.input"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoMap.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoMap.java index 6a988474..5065a7b9 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoMap.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/IpGeoMap.java @@ -17,6 +17,11 @@ import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.enrichment.geocode.impl.IpGeoEnrichment; import com.cloudera.cyber.enrichment.geocode.impl.types.GeoEnrichmentFields; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; @@ -24,12 +29,6 @@ import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.configuration.Configuration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - @EqualsAndHashCode(callSuper = true) @Data @AllArgsConstructor @@ -37,7 +36,7 @@ public class IpGeoMap extends RichMapFunction { public static final String GEOCODE_FEATURE = "geo"; public static final GeoEnrichmentFields[] AGGREGATE_GEO_FIELDS = - new GeoEnrichmentFields[]{GeoEnrichmentFields.CITY, GeoEnrichmentFields.COUNTRY}; + new GeoEnrichmentFields[] {GeoEnrichmentFields.CITY, GeoEnrichmentFields.COUNTRY}; public static final GeoEnrichmentFields[] ALL_GEO_FIELDS = GeoEnrichmentFields.values(); private final String geocodeDatabasePath; private final List ipFieldNames; @@ -52,7 +51,8 @@ public Message map(Message message) { Map geoExtensions = new HashMap<>(); for (String ipFieldName : ipFieldNames) { Object ipFieldValue = messageFields.get(ipFieldName); - geoEnrichment.lookup(ipFieldName, ipFieldValue, getGeoFieldSet(ipFieldValue), geoExtensions, qualityMessages); + geoEnrichment.lookup(ipFieldName, ipFieldValue, getGeoFieldSet(ipFieldValue), geoExtensions, + qualityMessages); } newMessage = MessageUtils.enrich(message, geoExtensions, qualityMessages); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/IpAsnEnrichment.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/IpAsnEnrichment.java index b7737188..581c3a34 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/IpAsnEnrichment.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/IpAsnEnrichment.java @@ -16,10 +16,8 @@ import com.cloudera.cyber.DataQualityMessageLevel; import com.cloudera.cyber.enrichment.Enrichment; import com.cloudera.cyber.enrichment.SingleValueEnrichment; -import com.cloudera.cyber.enrichment.geocode.impl.types.GeoFields; import com.maxmind.geoip2.DatabaseProvider; import com.maxmind.geoip2.model.AsnResponse; - import java.net.InetAddress; import java.util.Collection; import java.util.List; @@ -42,7 +40,8 @@ public IpAsnEnrichment(String path) { super(path); } - public void lookup(Enrichment enrichment, Object ipFieldValue, Map extensions, List qualityMessages) { + public void lookup(Enrichment enrichment, Object ipFieldValue, Map extensions, + List qualityMessages) { InetAddress ipAddress = convertToIpAddress(enrichment, ipFieldValue, qualityMessages); if (ipAddress != null) { try { @@ -53,17 +52,20 @@ public void lookup(Enrichment enrichment, Object ipFieldValue, Map extensions, List qualityMessages) { + public void lookup(String fieldName, Object ipFieldValue, Map extensions, + List qualityMessages) { lookup(SingleValueEnrichment::new, fieldName, ipFieldValue, extensions, qualityMessages); } - public void lookup(BiFunction enrichmentBiFunction, String fieldName, Object ipFieldValue, Map extensions, List qualityMessages) { + public void lookup(BiFunction enrichmentBiFunction, String fieldName, + Object ipFieldValue, Map extensions, List qualityMessages) { if (ipFieldValue instanceof Collection) { Enrichment enrichment = enrichmentBiFunction.apply(fieldName, ASN_FEATURE); //noinspection unchecked diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/IpGeoEnrichment.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/IpGeoEnrichment.java index d9cfe0f9..c1225e84 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/IpGeoEnrichment.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/IpGeoEnrichment.java @@ -19,7 +19,6 @@ import com.cloudera.cyber.enrichment.geocode.impl.types.GeoFields; import com.maxmind.geoip2.DatabaseProvider; import com.maxmind.geoip2.model.CityResponse; - import java.net.InetAddress; import java.util.AbstractMap; import java.util.Collection; @@ -53,33 +52,44 @@ public IpGeoEnrichment(String path) { * * @param ipFieldValue A valid IPv4 or IPv6 address represented in a string. * @param geoFieldSet Only Geocoding fields specified are returned. - * For example if the IP has a city in the database but city is not included in this set, the city will not be returned. + * For example if the IP has a city in the database but city is not included in this set, + * the city will not be returned. */ - private void lookup(Enrichment enrichment, Function nameFunction, Object ipFieldValue, GeoFields[] geoFieldSet, Map geoEnrichments, List qualityMessages) { + private void lookup(Enrichment enrichment, Function nameFunction, Object ipFieldValue, + GeoFields[] geoFieldSet, Map geoEnrichments, + List qualityMessages) { InetAddress ipAddress = convertToIpAddress(enrichment, ipFieldValue, qualityMessages); if (ipAddress != null) { try { Optional response = database.tryCity(ipAddress); - response.ifPresent(cityResponse -> Stream.of(geoFieldSet).map(field -> new AbstractMap.SimpleEntry<>(nameFunction.apply(field), field.getFunction().apply(cityResponse))). - filter(entry -> Objects.nonNull(entry.getValue())). - forEach(entry -> enrichment.enrich(geoEnrichments, entry.getKey(), entry.getValue()))); + response.ifPresent(cityResponse -> Stream.of(geoFieldSet).map(field -> new AbstractMap.SimpleEntry<>( + nameFunction.apply(field), field.getFunction().apply(cityResponse))) + .filter(entry -> Objects.nonNull(entry.getValue())) + .forEach(entry -> enrichment.enrich(geoEnrichments, + entry.getKey(), entry.getValue()))); } catch (Exception e) { - enrichment.addQualityMessage(qualityMessages, DataQualityMessageLevel.ERROR, String.format(GEOCODE_FAILED_MESSAGE, e.getMessage())); + enrichment.addQualityMessage(qualityMessages, DataQualityMessageLevel.ERROR, + String.format(GEOCODE_FAILED_MESSAGE, e.getMessage())); } } } - public void lookup(String fieldName, Object ipFieldValue, GeoFields[] geoFieldSet, Map geoEnrichments, List qualityMessages) { + public void lookup(String fieldName, Object ipFieldValue, GeoFields[] geoFieldSet, + Map geoEnrichments, List qualityMessages) { lookup(SingleValueEnrichment::new, fieldName, ipFieldValue, geoFieldSet, geoEnrichments, qualityMessages); } - public void lookup(BiFunction enrichmentBiFunction, String fieldName, Object ipFieldValue, GeoFields[] geoFieldSet, Map geoEnrichments, List qualityMessages) { + public void lookup(BiFunction enrichmentBiFunction, String fieldName, + Object ipFieldValue, GeoFields[] geoFieldSet, Map geoEnrichments, + List qualityMessages) { if (ipFieldValue instanceof Collection) { Enrichment enrichment = enrichmentBiFunction.apply(fieldName, GEOCODE_FEATURE); //noinspection unchecked - ((Collection) ipFieldValue).forEach(ip -> lookup(enrichment, GeoFields::getPluralName, ip, geoFieldSet, geoEnrichments, qualityMessages)); + ((Collection) ipFieldValue).forEach( + ip -> lookup(enrichment, GeoFields::getPluralName, ip, geoFieldSet, geoEnrichments, qualityMessages)); } else if (ipFieldValue != null) { - lookup(enrichmentBiFunction.apply(fieldName, GEOCODE_FEATURE), GeoFields::getSingularName, ipFieldValue, geoFieldSet, geoEnrichments, qualityMessages); + lookup(enrichmentBiFunction.apply(fieldName, GEOCODE_FEATURE), GeoFields::getSingularName, ipFieldValue, + geoFieldSet, geoEnrichments, qualityMessages); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/MaxMindBase.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/MaxMindBase.java index 24deb5d5..a3868c2e 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/MaxMindBase.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/MaxMindBase.java @@ -12,14 +12,19 @@ package com.cloudera.cyber.enrichment.geocode.impl; -import com.cloudera.cyber.cache.Memoizer; -import com.google.common.base.Preconditions; import com.cloudera.cyber.DataQualityMessage; import com.cloudera.cyber.DataQualityMessageLevel; +import com.cloudera.cyber.cache.Memoizer; import com.cloudera.cyber.enrichment.Enrichment; +import com.google.common.base.Preconditions; import com.maxmind.db.CHMCache; import com.maxmind.geoip2.DatabaseProvider; import com.maxmind.geoip2.DatabaseReader; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.List; +import java.util.function.Function; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -28,15 +33,10 @@ import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.List; -import java.util.function.Function; - @Slf4j public abstract class MaxMindBase { - private static final Function MEMOIZER = Memoizer.memoize(MaxMindBase::loadDatabaseProvider); + private static final Function MEMOIZER = + Memoizer.memoize(MaxMindBase::loadDatabaseProvider); public static final String FIELD_VALUE_IS_NOT_A_STRING = "'%s' is not a String."; public static final String FIELD_VALUE_IS_NOT_A_VALID_IP_ADDRESS = "'%s' is not a valid IP address."; public static final String MAXMIND_FAILED_MESSAGE = "Maxmind lookup failed '%s'"; @@ -53,7 +53,8 @@ protected MaxMindBase(DatabaseProvider database) { } protected MaxMindBase(String geocodeDatabasePath) { - Preconditions.checkArgument(StringUtils.isNotBlank(geocodeDatabasePath), "The path to Maxmind database is blank '%s'", geocodeDatabasePath); + Preconditions.checkArgument(StringUtils.isNotBlank(geocodeDatabasePath), + "The path to Maxmind database is blank '%s'", geocodeDatabasePath); DatabaseProvider databaseProvider = getDatabaseProvider(geocodeDatabasePath); Preconditions.checkNotNull(databaseProvider); this.database = databaseProvider; @@ -76,7 +77,8 @@ private static DatabaseProvider loadDatabaseProvider(String geocodeDatabasePath) return reader; } - private static DatabaseReader createDatabaseReader(String geocodeDatabasePath, FileSystem fileSystem) throws IOException { + private static DatabaseReader createDatabaseReader(String geocodeDatabasePath, FileSystem fileSystem) + throws IOException { DatabaseReader reader; try (FSDataInputStream dbStream = fileSystem.open(new Path(geocodeDatabasePath))) { reader = new DatabaseReader.Builder(dbStream).withCache(new CHMCache()).build(); @@ -89,30 +91,34 @@ private static DatabaseReader createDatabaseReader(String geocodeDatabasePath, F } - protected InetAddress convertToIpAddress(Enrichment enrichment, Object ipValueObject, List qualityMessages) { + protected InetAddress convertToIpAddress(Enrichment enrichment, Object ipValueObject, + List qualityMessages) { InetAddress inetAddress = null; if (ipValueObject instanceof String) { String ipValue = (String) ipValueObject; if (InetAddressValidator.getInstance().isValid(ipValue)) { try { inetAddress = InetAddress.getByName(ipValue); - if (inetAddress.isSiteLocalAddress() || - inetAddress.isAnyLocalAddress() || - inetAddress.isLinkLocalAddress() || - inetAddress.isLoopbackAddress() || - inetAddress.isMulticastAddress()) { + if (inetAddress.isSiteLocalAddress() + || inetAddress.isAnyLocalAddress() + || inetAddress.isLinkLocalAddress() + || inetAddress.isLoopbackAddress() + || inetAddress.isMulticastAddress()) { // internal network addresses won't have geo info so stop here inetAddress = null; } } catch (UnknownHostException e) { // this should not happen - checks for valid IP prior to call - enrichment.addQualityMessage(qualityMessages, DataQualityMessageLevel.INFO, String.format(MAXMIND_FAILED_MESSAGE, e.getMessage())); + enrichment.addQualityMessage(qualityMessages, DataQualityMessageLevel.INFO, + String.format(MAXMIND_FAILED_MESSAGE, e.getMessage())); } } else { - enrichment.addQualityMessage(qualityMessages, DataQualityMessageLevel.INFO, String.format(FIELD_VALUE_IS_NOT_A_VALID_IP_ADDRESS, ipValue)); + enrichment.addQualityMessage(qualityMessages, DataQualityMessageLevel.INFO, + String.format(FIELD_VALUE_IS_NOT_A_VALID_IP_ADDRESS, ipValue)); } } else { - enrichment.addQualityMessage(qualityMessages, DataQualityMessageLevel.INFO, String.format(FIELD_VALUE_IS_NOT_A_STRING, ipValueObject.toString())); + enrichment.addQualityMessage(qualityMessages, DataQualityMessageLevel.INFO, + String.format(FIELD_VALUE_IS_NOT_A_STRING, ipValueObject.toString())); } return inetAddress; diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/GeoEnrichmentFields.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/GeoEnrichmentFields.java index 5f296296..c860b414 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/GeoEnrichmentFields.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/GeoEnrichmentFields.java @@ -13,11 +13,10 @@ package com.cloudera.cyber.enrichment.geocode.impl.types; import com.maxmind.geoip2.model.CityResponse; +import java.util.function.Function; import lombok.AllArgsConstructor; import lombok.Getter; -import java.util.function.Function; - /** * All geocode enrichments that could be returned for an IP. */ diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/GeoFields.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/GeoFields.java index 8611f29c..dfd0e5ae 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/GeoFields.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/GeoFields.java @@ -16,9 +16,8 @@ import com.maxmind.geoip2.record.Country; import com.maxmind.geoip2.record.Location; import com.maxmind.geoip2.record.Subdivision; -import org.apache.commons.lang3.StringUtils; - import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; public interface GeoFields { @@ -59,7 +58,7 @@ static Object getCountry(CityResponse cityResponse) { static Object getState(CityResponse cityResponse) { Subdivision subdivision = cityResponse.getMostSpecificSubdivision(); String stateName = null; - if (subdivision != null ) { + if (subdivision != null) { stateName = convertEmptyToNull(subdivision.getName()); } @@ -85,7 +84,8 @@ static Object getLongitude(CityResponse cityResponse) { } /** - * Converts null to empty string + * Converts null to empty string. + * * @param raw The raw object * @return Empty string if null, or the String value if not */ diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/MetronGeoEnrichmentFields.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/MetronGeoEnrichmentFields.java index 49268abc..2c0508ad 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/MetronGeoEnrichmentFields.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/main/java/com/cloudera/cyber/enrichment/geocode/impl/types/MetronGeoEnrichmentFields.java @@ -13,16 +13,13 @@ package com.cloudera.cyber.enrichment.geocode.impl.types; import com.maxmind.geoip2.model.CityResponse; -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.awt.*; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.Getter; @AllArgsConstructor @Getter @@ -30,7 +27,8 @@ public enum MetronGeoEnrichmentFields implements GeoFields { LOC_ID(cityResponse -> cityResponse.getCity().getGeoNameId(), "locID", "locID"), CITY(GeoFields::getCity, "city", "city"), COUNTRY(GeoFields::getCountry, "country", "country"), - POSTAL_CODE(cityResponse -> GeoFields.convertEmptyToNull(cityResponse.getPostal().getCode()), "postalCode", "postalCode"), + POSTAL_CODE(cityResponse -> GeoFields.convertEmptyToNull(cityResponse.getPostal().getCode()), "postalCode", + "postalCode"), DMA_CODE(cityResponse -> cityResponse.getLocation().getMetroCode(), "dmaCode", "dmaCode"), LATITUDE(GeoFields::getLatitude, "latitude", "latitudes"), LONGITUDE(GeoFields::getLongitude, "longitude", "longitudes"), @@ -57,7 +55,8 @@ private static Object getLocationPoint(CityResponse cityResponse) { if (latitudeRaw == null || longitudeRaw == null) { return null; } else { - return GeoFields.convertNullToEmptyString(latitudeRaw) + "," + GeoFields.convertNullToEmptyString(longitudeRaw); + return GeoFields.convertNullToEmptyString(latitudeRaw) + "," + + GeoFields.convertNullToEmptyString(longitudeRaw); } } @@ -65,9 +64,9 @@ private static Object getLocationPoint(CityResponse cityResponse) { static { singularNameMap = Arrays.stream(MetronGeoEnrichmentFields.values()).collect( - Collectors.collectingAndThen( - Collectors.toMap(MetronGeoEnrichmentFields::getSingularName, Function.identity()), - Collections::unmodifiableMap)); + Collectors.collectingAndThen( + Collectors.toMap(MetronGeoEnrichmentFields::getSingularName, Function.identity()), + Collections::unmodifiableMap)); } public static MetronGeoEnrichmentFields fromSingularName(String singularName) { diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/test/java/com/cloudera/cyber/enrichment/geocode/IpAsnMapTest.java b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/test/java/com/cloudera/cyber/enrichment/geocode/IpAsnMapTest.java index f3bad131..4a552ae9 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/test/java/com/cloudera/cyber/enrichment/geocode/IpAsnMapTest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/src/test/java/com/cloudera/cyber/enrichment/geocode/IpAsnMapTest.java @@ -12,18 +12,21 @@ package com.cloudera.cyber.enrichment.geocode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + import com.cloudera.cyber.DataQualityMessage; import com.cloudera.cyber.Message; import com.cloudera.cyber.TestUtils; +import java.io.File; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.flink.configuration.Configuration; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.util.*; -import static org.assertj.core.api.Assertions.*; - public class IpAsnMapTest { private static final String IP_FIELD_NAME = "ip_dst_addr"; private static final List ENRICH_FIELD_NAMES = Collections.singletonList(IP_FIELD_NAME); @@ -85,9 +88,9 @@ public void testThrowsAsnDatabaseDoesNotExist() { File databaseFile = new File(doesntExistPath); Assert.assertFalse(databaseFile.exists()); IpAsnMap map = new IpAsnMap(doesntExistPath, ENRICH_FIELD_NAMES, null); - assertThatThrownBy(() -> map.open(new Configuration())). - isInstanceOfAny(IllegalStateException.class). - hasMessage("Could not read asn database %s", doesntExistPath); + assertThatThrownBy(() -> map.open(new Configuration())) + .isInstanceOfAny(IllegalStateException.class) + .hasMessage("Could not read asn database %s", doesntExistPath); } @Test @@ -97,16 +100,16 @@ public void testThrowsAsnDatabaseEmptyFile() { Assert.assertTrue(databaseFile.exists()); Assert.assertTrue(databaseFile.length() > 0); IpAsnMap map = new IpAsnMap(emptyFilePath, ENRICH_FIELD_NAMES, null); - assertThatThrownBy(() ->map.open(new Configuration())).isInstanceOfAny(IllegalStateException.class). - hasMessage("Could not read asn database %s", emptyFilePath); + assertThatThrownBy(() ->map.open(new Configuration())).isInstanceOfAny(IllegalStateException.class) + .hasMessage("Could not read asn database %s", emptyFilePath); } @Test public void testThrowsBadFilesystem() { String badFilesystemPath = "bad:/src/test/resources/geolite/invalid_maxmind_db"; IpAsnMap map = new IpAsnMap(badFilesystemPath, ENRICH_FIELD_NAMES, null); - assertThatThrownBy(() ->map.open(new Configuration())).isInstanceOfAny(IllegalStateException.class). - hasMessage("Could not read asn database %s", badFilesystemPath); + assertThatThrownBy(() ->map.open(new Configuration())).isInstanceOfAny(IllegalStateException.class) + .hasMessage("Could not read asn database %s", badFilesystemPath); } private Message testAsnMap(Map inputFields) { diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoader.java b/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoader.java index 82439663..e43ed168 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoader.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoader.java @@ -28,7 +28,9 @@ protected StreamExecutionEnvironment runPipeline(ParameterTool params) throws Ex return env; } - protected abstract void writeResults(ParameterTool params, EnrichmentsConfig enrichmentsConfig,String enrichmentType, DataStream enrichmentSource, StreamExecutionEnvironment env); + protected abstract void writeResults(ParameterTool params, EnrichmentsConfig enrichmentsConfig, + String enrichmentType, DataStream enrichmentSource, + StreamExecutionEnvironment env); protected abstract void load(StreamExecutionEnvironment env, ParameterTool params) throws Exception; diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoaderCSV.java b/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoaderCSV.java index d93d6066..11901642 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoaderCSV.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoaderCSV.java @@ -12,12 +12,18 @@ package com.cloudera.cyber.enrichment.load; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_SIMPLE; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME; + import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentConfig; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfig; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig; import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; @@ -28,13 +34,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_SIMPLE; -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME; - @Slf4j public abstract class BatchEnrichmentLoaderCSV extends BatchEnrichmentLoader { @@ -48,25 +47,28 @@ public abstract class BatchEnrichmentLoaderCSV extends BatchEnrichmentLoader { @Override protected void load(StreamExecutionEnvironment env, ParameterTool params) { - String path = params.getRequired(ENRICHMENT_SOURCE_FILE); + String path = params.getRequired(ENRICHMENT_SOURCE_FILE); boolean ignoreFirstLine = params.getBoolean(ENRICHMENT_SKIP_FIRST_LINE, false); List columnNames = Arrays.asList(params.getRequired(ENRICHMENT_COLUMNS).split(",", -1)); String enrichmentType = params.getRequired(ENRICHMENT_TYPE); EnrichmentsConfig enrichmentsConfig = getEnrichmentsConfig(params, enrichmentType, columnNames); EnrichmentConfig enrichmentConfig = enrichmentsConfig.getEnrichmentConfigs().get(enrichmentType); CsvReaderFormat csvFormat = - CsvToEnrichmentCommandDeserializer.createCsvReaderFormat(enrichmentConfig, columnNames, ignoreFirstLine, enrichmentType); + CsvToEnrichmentCommandDeserializer.createCsvReaderFormat(enrichmentConfig, columnNames, ignoreFirstLine, + enrichmentType); Path csvPath = new Path(path); FileSource source = FileSource.forRecordStreamFormat(csvFormat, csvPath).build(); - DataStream csvEnrichment = env.fromSource(source, WatermarkStrategy.noWatermarks(), "Enrichment CSV"); + DataStream csvEnrichment = + env.fromSource(source, WatermarkStrategy.noWatermarks(), "Enrichment CSV"); writeResults(params, enrichmentsConfig, enrichmentType, csvEnrichment, env); } - private EnrichmentsConfig getEnrichmentsConfig(ParameterTool params, String enrichmentType, List columnNames) { + private EnrichmentsConfig getEnrichmentsConfig(ParameterTool params, String enrichmentType, + List columnNames) { String enrichmentConfigFile = params.get(PARAMS_ENRICHMENT_FILE); String keyFieldName = params.get(ENRICHMENT_KEY_FIELD_NAME); EnrichmentsConfig enrichmentsConfig; @@ -75,33 +77,43 @@ private EnrichmentsConfig getEnrichmentsConfig(ParameterTool params, String enri if (enrichmentConfigFile != null) { enrichmentsConfig = EnrichmentsConfig.load(enrichmentConfigFile); enrichmentConfig = enrichmentsConfig.getEnrichmentConfigs().get(enrichmentType); - Preconditions.checkNotNull(enrichmentConfig, String.format("No configuration for enrichmentType '%s' found in enrichments config file '%s'", enrichmentType, enrichmentConfigFile)); + Preconditions.checkNotNull(enrichmentConfig, + String.format("No configuration for enrichmentType '%s' found in enrichments config file '%s'", + enrichmentType, enrichmentConfigFile)); Preconditions.checkState(keyFieldName == null, - String.format("Key field name should not be specified when using %s configuration.", PARAMS_ENRICHMENT_FILE)); + String.format("Key field name should not be specified when using %s configuration.", + PARAMS_ENRICHMENT_FILE)); List valueFields = enrichmentConfig.getFieldMapping().getValueFields(); if (valueFields != null) { - List valueFieldsInColumns = columnNames.stream().filter(valueFields::contains).collect(Collectors.toList()); + List valueFieldsInColumns = + columnNames.stream().filter(valueFields::contains).collect(Collectors.toList()); Preconditions.checkState(!valueFieldsInColumns.isEmpty(), - String.format("Columns specified in csv '%s' do not contain any value fields '%s'", String.join(",", columnNames), String.join(",", valueFields))); + String.format("Columns specified in csv '%s' do not contain any value fields '%s'", + String.join(",", columnNames), String.join(",", valueFields))); } log.info("Using enrichment config file {}", enrichmentConfig); } else { - Preconditions.checkNotNull(keyFieldName, String.format("Missing key field configuration %s", ENRICHMENT_KEY_FIELD_NAME)); + Preconditions.checkNotNull(keyFieldName, + String.format("Missing key field configuration %s", ENRICHMENT_KEY_FIELD_NAME)); String enrichmentsTable = params.getRequired(PARAMS_ENRICHMENTS_TABLE); enrichmentsConfig = new EnrichmentsConfig(); - enrichmentsConfig.getStorageConfigs().put(DEFAULT_ENRICHMENT_STORAGE_NAME, new EnrichmentStorageConfig(HBASE_SIMPLE, enrichmentsTable, null)); - EnrichmentFieldsConfig fieldsConfig = new EnrichmentFieldsConfig(Lists.newArrayList(keyFieldName), null, null, null); - enrichmentConfig = new EnrichmentConfig(DEFAULT_ENRICHMENT_STORAGE_NAME, fieldsConfig); + enrichmentsConfig.getStorageConfigs().put(DEFAULT_ENRICHMENT_STORAGE_NAME, + new EnrichmentStorageConfig(HBASE_SIMPLE, enrichmentsTable, null)); + EnrichmentFieldsConfig fieldsConfig = + new EnrichmentFieldsConfig(Lists.newArrayList(keyFieldName), null, null, null); + enrichmentConfig = new EnrichmentConfig(DEFAULT_ENRICHMENT_STORAGE_NAME, fieldsConfig); enrichmentsConfig.getEnrichmentConfigs().put(enrichmentType, enrichmentConfig); log.info("Using default config {}", enrichmentConfig); } List keyFields = enrichmentConfig.getFieldMapping().getKeyFields(); - List missingKeyFields = keyFields.stream().filter(kf -> !columnNames.contains(kf)).collect(Collectors.toList()); + List missingKeyFields = + keyFields.stream().filter(kf -> !columnNames.contains(kf)).collect(Collectors.toList()); Preconditions.checkState(missingKeyFields.isEmpty(), - String.format("Columns '%s' are missing key field(s) '%s'", String.join(",", columnNames), String.join(", ", missingKeyFields))); + String.format("Columns '%s' are missing key field(s) '%s'", String.join(",", columnNames), + String.join(", ", missingKeyFields))); return enrichmentsConfig; } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoaderCSVHbaseKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoaderCSVHbaseKafka.java index 1d5e3179..5c63095f 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoaderCSVHbaseKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/BatchEnrichmentLoaderCSVHbaseKafka.java @@ -32,18 +32,22 @@ public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); ParameterTool params = Utils.getParamToolsFromProperties(args); FlinkUtils.executeEnv(new BatchEnrichmentLoaderCSVHbaseKafka().runPipeline(params), - String.format("Enrichment %s - batch load", params.get(ENRICHMENT_TYPE, "unspecified")), - params); + String.format("Enrichment %s - batch load", params.get(ENRICHMENT_TYPE, "unspecified")), + params); } @Override - protected void writeResults(ParameterTool params, EnrichmentsConfig enrichmentsConfig, String enrichmentType, DataStream enrichmentSource, StreamExecutionEnvironment env) { + protected void writeResults(ParameterTool params, EnrichmentsConfig enrichmentsConfig, String enrichmentType, + DataStream enrichmentSource, StreamExecutionEnvironment env) { String topic = params.get(PARAMS_TOPIC_ENRICHMENT_INPUT); if (topic != null) { - enrichmentSource.sinkTo(new FlinkUtils<>(EnrichmentCommand.class).createKafkaSink(topic, "enrichment_loader", params)).name("Kafka Enrichment Command Sink"); + enrichmentSource.sinkTo( + new FlinkUtils<>(EnrichmentCommand.class).createKafkaSink(topic, "enrichment_loader", params)) + .name("Kafka Enrichment Command Sink"); } else { String hbaseTable = enrichmentsConfig.getStorageForEnrichmentType(enrichmentType).getHbaseTableName(); - HBaseSinkFunction hbaseSink = new HbaseEnrichmentCommandSink(hbaseTable, enrichmentsConfig, params); + HBaseSinkFunction hbaseSink = + new HbaseEnrichmentCommandSink(hbaseTable, enrichmentsConfig, params); enrichmentSource.addSink(hbaseSink); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/CsvToEnrichmentCommandDeserializer.java b/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/CsvToEnrichmentCommandDeserializer.java index b10fd93d..3d998c01 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/CsvToEnrichmentCommandDeserializer.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-load/src/main/java/com/cloudera/cyber/enrichment/load/CsvToEnrichmentCommandDeserializer.java @@ -5,6 +5,13 @@ import com.cloudera.cyber.commands.CommandType; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentConfig; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.csv.CsvReaderFormat; @@ -16,14 +23,6 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - @Slf4j public class CsvToEnrichmentCommandDeserializer extends StdDeserializer { private static final String NULL_STRING_VALUE = "null"; @@ -35,8 +34,8 @@ public class CsvToEnrichmentCommandDeserializer extends StdDeserializer keyFieldNames, String keyDelimiter, - List valueFieldNames) { + List keyFieldNames, String keyDelimiter, + List valueFieldNames) { super(EnrichmentCommand.class); this.enrichmentType = enrichmentType; this.commandType = commandType; @@ -47,7 +46,8 @@ protected CsvToEnrichmentCommandDeserializer(String enrichmentType, CommandType @Override - public EnrichmentCommand deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + public EnrichmentCommand deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { JsonNode node = jsonParser.getCodec().readTree(jsonParser); Map enrichmentValues = new HashMap<>(); valueFieldNames.forEach(valueFieldName -> { @@ -57,34 +57,44 @@ public EnrichmentCommand deserialize(JsonParser jsonParser, DeserializationConte } }); - String enrichmentKey = keyFieldNames.stream().map(keyFieldName -> node.get(keyFieldName).asText(NULL_STRING_VALUE)).collect(Collectors.joining(keyDelimiter)); + String enrichmentKey = + keyFieldNames.stream().map(keyFieldName -> node.get(keyFieldName).asText(NULL_STRING_VALUE)) + .collect(Collectors.joining(keyDelimiter)); - EnrichmentEntry enrichmentEntry = EnrichmentEntry.builder().ts(MessageUtils.getCurrentTimestamp()). - type(enrichmentType). - key(enrichmentKey).entries(enrichmentValues).build(); - return EnrichmentCommand.builder(). - headers(Collections.emptyMap()). - type(commandType).payload(enrichmentEntry).build(); + EnrichmentEntry enrichmentEntry = EnrichmentEntry.builder().ts(MessageUtils.getCurrentTimestamp()) + .type(enrichmentType) + .key(enrichmentKey).entries(enrichmentValues).build(); + return EnrichmentCommand.builder() + .headers(Collections.emptyMap()) + .type(commandType).payload(enrichmentEntry).build(); } - public static CsvReaderFormat createCsvReaderFormat(EnrichmentConfig enrichmentConfig, List columnNames, boolean ignoreFirstLine, String enrichmentType) { + public static CsvReaderFormat createCsvReaderFormat(EnrichmentConfig enrichmentConfig, + List columnNames, + boolean ignoreFirstLine, + String enrichmentType) { List keyFields = enrichmentConfig.getFieldMapping().getKeyFields(); List configuredValueFields = enrichmentConfig.getFieldMapping().getValueFields(); - List valueFieldNames = columnNames.stream().filter(f -> !f.isEmpty() && !keyFields.contains(f) && (configuredValueFields == null || configuredValueFields.contains(f))).collect(Collectors.toList()); + List valueFieldNames = columnNames.stream().filter(f -> !f.isEmpty() && !keyFields.contains(f) + && (configuredValueFields == null + || configuredValueFields.contains(f))) + .collect(Collectors.toList()); // create a CSV file source from the specified file CsvMapper mapper = new CsvMapper(); CsvSchema.Builder schemaBuilder = CsvSchema.builder(); // add a field for each column, any ignored column is named ignore_ - IntStream.range(0, columnNames.size()). - mapToObj(index -> !columnNames.get(index).isEmpty() ? columnNames.get(index) : String.format("ignore_%d", index)). - forEach(f -> schemaBuilder.addColumn(f, CsvSchema.ColumnType.STRING)); + IntStream.range(0, columnNames.size()) + .mapToObj(index -> !columnNames.get(index).isEmpty() ? columnNames.get(index) + : String.format("ignore_%d", index)) + .forEach(f -> schemaBuilder.addColumn(f, CsvSchema.ColumnType.STRING)); CsvSchema schema = schemaBuilder.build().withSkipFirstDataRow(ignoreFirstLine).withLineSeparator("\n"); SimpleModule module = new SimpleModule(); - module.addDeserializer(EnrichmentCommand.class, new CsvToEnrichmentCommandDeserializer(enrichmentType, CommandType.ADD, keyFields, ":", valueFieldNames)); + module.addDeserializer(EnrichmentCommand.class, + new CsvToEnrichmentCommandDeserializer(enrichmentType, CommandType.ADD, keyFields, ":", valueFieldNames)); mapper.registerModule(module); return CsvReaderFormat.forSchema(mapper, schema, TypeInformation.of(EnrichmentCommand.class)); diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/src/test/java/com/cloudera/cyber/enrichment/load/CsvToEnrichmentCommandDeserializerTest.java b/flink-cyber/flink-enrichment/flink-enrichment-load/src/test/java/com/cloudera/cyber/enrichment/load/CsvToEnrichmentCommandDeserializerTest.java index c507178a..2629bd03 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-load/src/test/java/com/cloudera/cyber/enrichment/load/CsvToEnrichmentCommandDeserializerTest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-load/src/test/java/com/cloudera/cyber/enrichment/load/CsvToEnrichmentCommandDeserializerTest.java @@ -7,6 +7,13 @@ import com.cloudera.cyber.enrichment.hbase.config.EnrichmentConfig; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfig; import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; import org.apache.flink.api.common.io.InputStreamFSInputWrapper; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.file.src.reader.StreamFormat; @@ -14,10 +21,6 @@ import org.junit.Assert; import org.junit.Test; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.*; - public class CsvToEnrichmentCommandDeserializerTest { private static final String TEST_ENRICHMENT_NAME = "TestEnrichment"; @@ -43,13 +46,13 @@ public void testSingleLineWithSingleKeyAndValue() throws IOException { String columns = "rank,TldRank,domain,TLD,RefSubNets,RefIPs,IDN_Domain,IDN_TLD,PrevGlobalRank,PrevTldRank,PrevRefSubNets,PrevRefIPs"; - List expectedCommands = Collections.singletonList(EnrichmentCommand.builder(). - type(CommandType.ADD). - payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME). - key("facebook.com"). - entries(ImmutableMap.of(MM_VALUE_FIELD, "1")).ts(MessageUtils.getCurrentTimestamp()).build()). - build()); + List expectedCommands = Collections.singletonList(EnrichmentCommand.builder() + .type(CommandType.ADD) + .payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME) + .key("facebook.com") + .entries(ImmutableMap.of(MM_VALUE_FIELD, "1")).ts(MessageUtils.getCurrentTimestamp()).build()) + .build()); testCSVFormat(false, columns, MAJESTIC_MILLION_ENRICHMENT_CONFIG, "1,1,facebook.com,com,494155,2890130,facebook.com,com,1,1,493314,2868293\n", expectedCommands ); } @@ -59,14 +62,14 @@ public void testMultiLineWithSingleKeyAndValue() throws IOException { String columns = "rank,TldRank,domain,TLD,RefSubNets,RefIPs,IDN_Domain,IDN_TLD,PrevGlobalRank,PrevTldRank,PrevRefSubNets,PrevRefIPs"; - String csv = "1,1,facebook.com,com,494155,2890130,facebook.com,com,1,1,493314,2868293\n" + - "5,5,instagram.com,com,354772,1748958,instagram.com,com,5,5,353741,1732258\n"; + String csv = "1,1,facebook.com,com,494155,2890130,facebook.com,com,1,1,493314,2868293\n" + + "5,5,instagram.com,com,354772,1748958,instagram.com,com,5,5,353741,1732258\n"; List expectedCommands = new ArrayList<>(); - expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME).key("facebook.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "1")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); - expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME).key("instagram.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "5")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); + expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME).key("facebook.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "1")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); + expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME).key("instagram.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "5")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); testCSVFormat(false, columns, MAJESTIC_MILLION_ENRICHMENT_CONFIG, csv, expectedCommands ); } @@ -75,14 +78,14 @@ public void testMultiLineWithMinimalColumnNames() throws IOException { String columns = "rank,,domain,,,,,,,,,"; - String csv = "1,1,facebook.com,com,494155,2890130,facebook.com,com,1,1,493314,2868293\n" + - "5,5,instagram.com,com,354772,1748958,instagram.com,com,5,5,353741,1732258\n"; + String csv = "1,1,facebook.com,com,494155,2890130,facebook.com,com,1,1,493314,2868293\n" + + "5,5,instagram.com,com,354772,1748958,instagram.com,com,5,5,353741,1732258\n"; List expectedCommands = new ArrayList<>(); - expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME).key("facebook.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "1")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); - expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME).key("instagram.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "5")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); + expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME).key("facebook.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "1")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); + expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME).key("instagram.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "5")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); testCSVFormat(false, columns, MAJESTIC_MILLION_ENRICHMENT_CONFIG, csv, expectedCommands ); } @@ -91,15 +94,15 @@ public void testMultiLineSkipHeader() throws IOException { String columns = "rank,,domain,,,,,,,,,"; - String csv = columns + "\n" + - "1,1,facebook.com,com,494155,2890130,facebook.com,com,1,1,493314,2868293\n" + - "5,5,instagram.com,com,354772,1748958,instagram.com,com,5,5,353741,1732258\n"; + String csv = columns + "\n" + + "1,1,facebook.com,com,494155,2890130,facebook.com,com,1,1,493314,2868293\n" + + "5,5,instagram.com,com,354772,1748958,instagram.com,com,5,5,353741,1732258\n"; List expectedCommands = new ArrayList<>(); - expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME).key("facebook.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "1")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); - expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME).key("instagram.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "5")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); + expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME).key("facebook.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "1")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); + expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME).key("instagram.com").entries(ImmutableMap.of(MM_VALUE_FIELD, "5")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); testCSVFormat(true, columns, MAJESTIC_MILLION_ENRICHMENT_CONFIG, csv, expectedCommands ); } @@ -108,15 +111,15 @@ public void testMultipleKeysAndValuesNoHeader() throws IOException { String columns = "source,indicator_type,indicator,score"; - String csv = columns + "\n" + - "abuse,IPADDRESS,1.1.1.1,45.0\n" + - "vendor,DOMAIN,this.is.a.bad.one.xxx,99.0\n"; + String csv = columns + "\n" + + "abuse,IPADDRESS,1.1.1.1,45.0\n" + + "vendor,DOMAIN,this.is.a.bad.one.xxx,99.0\n"; List expectedCommands = new ArrayList<>(); - expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME).key("1.1.1.1:IPADDRESS").entries(ImmutableMap.of(TI_SOURCE_FIELD, "abuse", TI_SCORE_FIELD, "45.0")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); - expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder(). - type(TEST_ENRICHMENT_NAME).key("this.is.a.bad.one.xxx:DOMAIN").entries(ImmutableMap.of(TI_SOURCE_FIELD, "vendor", TI_SCORE_FIELD, "99.0")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); + expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME).key("1.1.1.1:IPADDRESS").entries(ImmutableMap.of(TI_SOURCE_FIELD, "abuse", TI_SCORE_FIELD, "45.0")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); + expectedCommands.add(EnrichmentCommand.builder().type(CommandType.ADD).payload(EnrichmentEntry.builder() + .type(TEST_ENRICHMENT_NAME).key("this.is.a.bad.one.xxx:DOMAIN").entries(ImmutableMap.of(TI_SOURCE_FIELD, "vendor", TI_SCORE_FIELD, "99.0")).ts(MessageUtils.getCurrentTimestamp()).build()).build()); testCSVFormat(true, columns, TI_ENRICHMENT_CONFIG, csv, expectedCommands ); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/src/test/java/com/cloudera/cyber/enrichment/load/HbaseBatchEnrichmentCSVLoaderTest.java b/flink-cyber/flink-enrichment/flink-enrichment-load/src/test/java/com/cloudera/cyber/enrichment/load/HbaseBatchEnrichmentCSVLoaderTest.java index 23e0abb9..9e98fff8 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-load/src/test/java/com/cloudera/cyber/enrichment/load/HbaseBatchEnrichmentCSVLoaderTest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-load/src/test/java/com/cloudera/cyber/enrichment/load/HbaseBatchEnrichmentCSVLoaderTest.java @@ -17,14 +17,6 @@ import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig; import com.google.common.collect.ImmutableMap; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.test.util.CollectingSink; -import org.apache.flink.test.util.JobTester; -import org.junit.Assert; -import org.junit.Test; - import java.io.File; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -33,6 +25,13 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.test.util.CollectingSink; +import org.apache.flink.test.util.JobTester; +import org.junit.Assert; +import org.junit.Test; public class HbaseBatchEnrichmentCSVLoaderTest extends BatchEnrichmentLoaderCSV { @@ -40,33 +39,33 @@ public class HbaseBatchEnrichmentCSVLoaderTest extends BatchEnrichmentLoaderCSV private final CollectingSink sink = new CollectingSink<>(); private static final String MAJESTIC_MILLION_CF = "majestic_million"; - private static final Map firstExtensions = ImmutableMap.builder(). - put("GlobalRank", "1"). - put("TldRank", "1"). - put("TLD", "com"). - put("RefSubNets", "494155"). - put("RefIPs", "2890130"). - put("IDN_Domain", "facebook.com"). - put("IDN_TLD", "com"). - put("PrevGlobalRank", "1"). - put("PrevTldRank", "1"). - put("PrevRefSubNets", "493314"). - put("PrevRefIPs", "2868293"). - build(); - - private static final Map lastExtensions = ImmutableMap.builder(). - put("GlobalRank", "9"). - put("TldRank", "1"). - put("TLD", "org"). - put("RefSubNets", "285566"). - put("RefIPs", "1126132"). - put("IDN_Domain", "wikipedia.org"). - put("IDN_TLD", "org"). - put("PrevGlobalRank", "9"). - put("PrevTldRank", "1"). - put("PrevRefSubNets", "284262"). - put("PrevRefIPs", "1111606"). - build(); + private static final Map firstExtensions = ImmutableMap.builder() + .put("GlobalRank", "1") + .put("TldRank", "1") + .put("TLD", "com") + .put("RefSubNets", "494155") + .put("RefIPs", "2890130") + .put("IDN_Domain", "facebook.com") + .put("IDN_TLD", "com") + .put("PrevGlobalRank", "1") + .put("PrevTldRank", "1") + .put("PrevRefSubNets", "493314") + .put("PrevRefIPs", "2868293") + .build(); + + private static final Map lastExtensions = ImmutableMap.builder() + .put("GlobalRank", "9") + .put("TldRank", "1") + .put("TLD", "org") + .put("RefSubNets", "285566") + .put("RefIPs", "1126132") + .put("IDN_Domain", "wikipedia.org") + .put("IDN_TLD", "org") + .put("PrevGlobalRank", "9") + .put("PrevTldRank", "1") + .put("PrevRefSubNets", "284262") + .put("PrevRefIPs", "1111606") + .build(); private static final Map> extensionsWithAllData = ImmutableMap.of("facebook.com", firstExtensions, @@ -88,26 +87,26 @@ public void testCSVIgnoreColumns() throws Exception { } public void testLoadFromEnrichmentStorageConfig(String testFileName, String enrichmentFields, Map> enrichmentsToVerify) throws Exception { - ParameterTool params = ParameterTool.fromMap(ImmutableMap.builder(). - put(ENRICHMENT_SOURCE_FILE, "./src/test/resources/majestic_million_sample.csv"). - put(ENRICHMENT_SKIP_FIRST_LINE, "true"). - put(ENRICHMENT_COLUMNS, enrichmentFields). - put(PARAMS_ENRICHMENT_FILE, getJsonPath(testFileName)). - put(ENRICHMENT_TYPE, MAJESTIC_MILLION_CF). - put(PARAMS_ENRICHMENTS_TABLE, "enrichments"). - build()); + ParameterTool params = ParameterTool.fromMap(ImmutableMap.builder() + .put(ENRICHMENT_SOURCE_FILE, "./src/test/resources/majestic_million_sample.csv") + .put(ENRICHMENT_SKIP_FIRST_LINE, "true") + .put(ENRICHMENT_COLUMNS, enrichmentFields) + .put(PARAMS_ENRICHMENT_FILE, getJsonPath(testFileName)) + .put(ENRICHMENT_TYPE, MAJESTIC_MILLION_CF) + .put(PARAMS_ENRICHMENTS_TABLE, "enrichments") + .build()); testLoad(params, enrichmentsToVerify); } private void testLoadStream(String enrichmentFields, Map> enrichmentsToVerify) throws Exception { - ParameterTool params = ParameterTool.fromMap(ImmutableMap.builder(). - put(ENRICHMENT_SOURCE_FILE, "./src/test/resources/majestic_million_sample.csv"). - put(ENRICHMENT_SKIP_FIRST_LINE, "true"). - put(ENRICHMENT_COLUMNS, enrichmentFields). - put(ENRICHMENT_KEY_FIELD_NAME, "Domain"). - put(ENRICHMENT_TYPE, MAJESTIC_MILLION_CF). - put(PARAMS_ENRICHMENTS_TABLE, "enrichments"). - build()); + ParameterTool params = ParameterTool.fromMap(ImmutableMap.builder() + .put(ENRICHMENT_SOURCE_FILE, "./src/test/resources/majestic_million_sample.csv") + .put(ENRICHMENT_SKIP_FIRST_LINE, "true") + .put(ENRICHMENT_COLUMNS, enrichmentFields) + .put(ENRICHMENT_KEY_FIELD_NAME, "Domain") + .put(ENRICHMENT_TYPE, MAJESTIC_MILLION_CF) + .put(PARAMS_ENRICHMENTS_TABLE, "enrichments") + .build()); testLoad(params, enrichmentsToVerify); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/src/main/java/com/cloudera/cyber/enrichment/ConfigUtils.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/src/main/java/com/cloudera/cyber/enrichment/ConfigUtils.java index 766df056..d1198e89 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/src/main/java/com/cloudera/cyber/enrichment/ConfigUtils.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/src/main/java/com/cloudera/cyber/enrichment/ConfigUtils.java @@ -12,12 +12,14 @@ package com.cloudera.cyber.enrichment; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; + import com.cloudera.cyber.enrichment.lookup.config.EnrichmentConfig; import com.cloudera.cyber.enrichment.lookup.config.EnrichmentField; import com.cloudera.cyber.enrichment.lookup.config.EnrichmentKind; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; - import java.io.IOException; import java.util.Collection; import java.util.List; @@ -26,37 +28,35 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.mapping; - public class ConfigUtils { public static final String PARAMS_CONFIG_FILE = "config.file"; public static Map> typeToFields(List allConfigs, EnrichmentKind kind) { return allConfigs.stream() - .filter(c -> c.getKind() == kind) - .flatMap(c -> c.getFields().stream() - .collect(groupingBy(EnrichmentField::getEnrichmentType, - mapping(EnrichmentField::getName, Collectors.toList()))) - .entrySet().stream() - ) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> - Stream.of(a, b) - .flatMap(Collection::stream) - .collect(Collectors.toList()) - )); + .filter(c -> c.getKind() == kind) + .flatMap(c -> c.getFields().stream() + .collect(groupingBy(EnrichmentField::getEnrichmentType, + mapping(EnrichmentField::getName, Collectors.toList()))) + .entrySet().stream() + ) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> + Stream.of(a, b) + .flatMap(Collection::stream) + .collect(Collectors.toList()) + )); } public static Set enrichmentTypes(List allConfigs, EnrichmentKind kind) { return allConfigs.stream() - .filter(f -> f.getKind() == kind) - .flatMap(s -> s.getFields().stream().map(EnrichmentField::getEnrichmentType)).collect(Collectors.toSet()); + .filter(f -> f.getKind() == kind) + .flatMap(s -> s.getFields().stream().map(EnrichmentField::getEnrichmentType)) + .collect(Collectors.toSet()); } public static List allConfigs(byte[] configJson) throws IOException { return new ObjectMapper().readValue( - configJson, - new TypeReference>() { - }); + configJson, + new TypeReference>() { + }); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/EnrichmentLookupBuilder.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/EnrichmentLookupBuilder.java index f70e0e69..a7ff0a8f 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/EnrichmentLookupBuilder.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/EnrichmentLookupBuilder.java @@ -2,7 +2,6 @@ import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; import com.cloudera.cyber.hbase.LookupKey; - import java.io.Serializable; public interface EnrichmentLookupBuilder extends Serializable { diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseEnrichmentFunction.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseEnrichmentFunction.java index cd3030ef..e2f4a184 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseEnrichmentFunction.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseEnrichmentFunction.java @@ -12,19 +12,23 @@ package com.cloudera.cyber.enrichment.hbase; -import org.apache.flink.table.functions.ScalarFunction; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; +import static java.util.stream.Collectors.toMap; import java.io.IOException; import java.util.Collections; import java.util.Map; - -import static java.util.stream.Collectors.toMap; +import org.apache.flink.table.functions.ScalarFunction; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; /** - * Return a lookup for a given Hbase enrichment at a given time + * Return a lookup for a given Hbase enrichment at a given time. + * *

* This function also performs local positive result caching * @@ -44,16 +48,16 @@ public Map eval(long timestamp, String type, String key) { Table table = connection.getTable(hbaseTable); Result result = table.get(get); - if (!result.getExists()) + if (!result.getExists()) { return Collections.emptyMap(); + } return result.getFamilyMap(cf).entrySet().stream() - .collect(toMap( - k -> new String(k.getKey()), - v -> new String(v.getValue()) - )); - } catch ( - IOException e) { + .collect(toMap( + k -> new String(k.getKey()), + v -> new String(v.getValue()) + )); + } catch (IOException e) { return Collections.emptyMap(); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseEnrichmentMapFunction.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseEnrichmentMapFunction.java index 2c7a4723..eb8227ef 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseEnrichmentMapFunction.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseEnrichmentMapFunction.java @@ -12,6 +12,8 @@ package com.cloudera.cyber.enrichment.hbase; +import static java.util.stream.Collectors.toMap; + import com.cloudera.cyber.Message; import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; @@ -22,30 +24,28 @@ import com.cloudera.cyber.hbase.AbstractHbaseMapFunction; import com.cloudera.cyber.hbase.LookupKey; import com.google.common.base.Joiner; +import java.util.List; +import java.util.Map; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.flink.util.Collector; -import java.util.List; -import java.util.Map; - -import static java.util.stream.Collectors.toMap; - @Slf4j @Getter public class HbaseEnrichmentMapFunction extends AbstractHbaseMapFunction { private final Map> fieldToLookup; private final EnrichmentsConfig enrichmentStorageConfig; - public HbaseEnrichmentMapFunction(List configs, com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig enrichmentStorageConfig) { + public HbaseEnrichmentMapFunction(List configs, + com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig enrichmentStorageConfig) { super(); fieldToLookup = configs.stream() - .filter(c -> c.getKind().equals(EnrichmentKind.HBASE)) - .collect(toMap( - EnrichmentConfig::getSource, EnrichmentConfig::getFields) - ); + .filter(c -> c.getKind().equals(EnrichmentKind.HBASE)) + .collect(toMap( + EnrichmentConfig::getSource, EnrichmentConfig::getFields) + ); log.info("Applying HBase enrichments to the following sources: {}", String.join(" ,", fieldToLookup.keySet())); @@ -63,18 +63,22 @@ public void processElement(Message message, Context context, Collector if (CollectionUtils.isNotEmpty(enrichmentFields)) { messageCounter.inc(1); collector.collect( - MessageUtils.addFields(message, enrichmentFields.stream().flatMap( - field -> { - String enrichmentType = field.getEnrichmentType(); - String key = message.getExtensions().get(field.getName()); + MessageUtils.addFields(message, enrichmentFields.stream().flatMap( + field -> { + String enrichmentType = field.getEnrichmentType(); + String key = message.getExtensions().get(field.getName()); - if (key == null) return notFound().entrySet().stream(); - return hbaseLookup(message.getTs(), - buildLookup(enrichmentType, key), - Joiner.on(".").join(field.getName(), field.getEnrichmentType()) - ).entrySet().stream(); - }) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b)))); + if (key == null) { + return notFound().entrySet().stream(); + } + return hbaseLookup(message.getTs(), + buildLookup(enrichmentType, key), + Joiner.on(".") + .join(field.getName(), field.getEnrichmentType()) + ).entrySet().stream(); + }) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, + (a, b) -> b)))); } else { collector.collect(message); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseJob.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseJob.java index 6b73ad9f..d96c33c7 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseJob.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseJob.java @@ -12,31 +12,31 @@ package com.cloudera.cyber.enrichment.hbase; +import static com.cloudera.cyber.enrichment.ConfigUtils.PARAMS_CONFIG_FILE; +import static com.cloudera.cyber.enrichment.ConfigUtils.allConfigs; + import com.cloudera.cyber.Message; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.commands.EnrichmentCommandResponse; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig; import com.cloudera.cyber.enrichment.lookup.config.EnrichmentConfig; import com.cloudera.cyber.flink.FlinkUtils; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; - -import static com.cloudera.cyber.enrichment.ConfigUtils.PARAMS_CONFIG_FILE; -import static com.cloudera.cyber.enrichment.ConfigUtils.allConfigs; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; public abstract class HbaseJob { public static final String PARAMS_ENRICHMENT_CONFIG = "enrichments.config"; - public static DataStream enrich(DataStream source, List configs, EnrichmentsConfig enrichmentsConfig) { + public static DataStream enrich(DataStream source, List configs, + EnrichmentsConfig enrichmentsConfig) { return source.process(new HbaseEnrichmentMapFunction(configs, enrichmentsConfig)) - .name("HBase Enrichment Mapper").uid("hbase-map"); + .name("HBase Enrichment Mapper").uid("hbase-map"); } protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws IOException { @@ -52,22 +52,27 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws DataStream result = enrich(source, allConfigs(configJson), enrichmentsConfig); writeResults(env, params, result); - DataStream enrichmentCommandResponses = writeEnrichments(env, params, enrichmentSource, enrichmentsConfig); + DataStream enrichmentCommandResponses = + writeEnrichments(env, params, enrichmentSource, enrichmentsConfig); writeQueryResults(env, params, enrichmentCommandResponses); return env; } - public abstract DataStream writeEnrichments(StreamExecutionEnvironment env, ParameterTool params, + public abstract DataStream writeEnrichments(StreamExecutionEnvironment env, + ParameterTool params, DataStream enrichmentSource, EnrichmentsConfig enrichmentsConfig); - protected abstract void writeQueryResults(StreamExecutionEnvironment env, ParameterTool params, DataStream sideOutput); + protected abstract void writeQueryResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream sideOutput); - protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream result); + protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream result); protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); - protected abstract DataStream createEnrichmentSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract DataStream createEnrichmentSource(StreamExecutionEnvironment env, + ParameterTool params); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseJobRawKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseJobRawKafka.java index d22b8f87..2a23b551 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseJobRawKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/HbaseJobRawKafka.java @@ -12,8 +12,10 @@ package com.cloudera.cyber.enrichment.hbase; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; + import com.cloudera.cyber.Message; -import com.cloudera.cyber.MessageTypeFactory; import com.cloudera.cyber.commands.CommandType; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.commands.EnrichmentCommandResponse; @@ -21,6 +23,8 @@ import com.cloudera.cyber.enrichment.hbase.writer.HbaseEnrichmentCommandSink; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; +import java.util.Collections; +import java.util.List; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.hbase.sink.HBaseSinkFunction; @@ -28,71 +32,83 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.Collections; -import java.util.List; - -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; - public class HbaseJobRawKafka extends HbaseJob { private static final String PARAMS_TOPIC_ENRICHMENT_INPUT = "enrichment.topic.input"; public static final String PARAMS_QUERY_OUTPUT = "enrichment.topic.query.output"; private static final String DEFAULT_GROUP_ID = "enrichment-lookups-hbase"; - public static DataStream enrichmentCommandsToHbase(ParameterTool params, DataStream enrichmentSource, EnrichmentsConfig enrichmentsConfig, List tables) { + public static DataStream enrichmentCommandsToHbase(ParameterTool params, + DataStream enrichmentSource, + EnrichmentsConfig enrichmentsConfig, + List tables) { // filter out commands that don't require changes to Hbase - DataStream hbaseMods = enrichmentSource.filter(c -> (c.getType().equals(CommandType.ADD) || c.getType().equals(CommandType.DELETE))).name("ADD-DELETE CommandType Filter"); + DataStream hbaseMods = enrichmentSource + .filter(c -> (c.getType().equals(CommandType.ADD) || c.getType().equals(CommandType.DELETE))) + .name("ADD-DELETE CommandType Filter"); // add one sink for each table - hbase table sinks are table specific tables.forEach(table -> { - HBaseSinkFunction tableSink = new HbaseEnrichmentCommandSink(table, enrichmentsConfig, params); - hbaseMods.filter(command -> table.equals(enrichmentsConfig.getStorageForEnrichmentType(command.getPayload().getType()).getHbaseTableName())).name("Hbase Table ".concat(table).concat(" Filter")).addSink(tableSink).name("Enrichment ".concat(table).concat(" HBase Sink")); + HBaseSinkFunction tableSink = + new HbaseEnrichmentCommandSink(table, enrichmentsConfig, params); + hbaseMods.filter(command -> table.equals( + enrichmentsConfig.getStorageForEnrichmentType(command.getPayload().getType()).getHbaseTableName())) + .name("Hbase Table ".concat(table).concat(" Filter")).addSink(tableSink) + .name("Enrichment ".concat(table).concat(" HBase Sink")); }); return hbaseMods.map(c -> EnrichmentCommandResponse.builder() - .success(true) - .message(c.getType().name().concat(" HBase Enrichment")) - .content(Collections.singletonList(c.getPayload())) - .headers(c.getHeaders()) - .build()); + .success(true) + .message(c.getType().name().concat(" HBase Enrichment")) + .content(Collections.singletonList(c.getPayload())) + .headers(c.getHeaders()) + .build()); } public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); - new HbaseJobRawKafka().createPipeline(Utils.getParamToolsFromProperties(args)).execute("Enrichments - HBase Lookup"); + new HbaseJobRawKafka().createPipeline(Utils.getParamToolsFromProperties(args)) + .execute("Enrichments - HBase Lookup"); } @Override protected void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream reduction) { - reduction.sinkTo(new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), DEFAULT_GROUP_ID,params)); + reduction.sinkTo( + new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), DEFAULT_GROUP_ID, + params)); } @Override public DataStream createSource(StreamExecutionEnvironment env, ParameterTool params) { return env.fromSource( - FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, DEFAULT_GROUP_ID), - WatermarkStrategy.noWatermarks(), "Kafka Messages"); + FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, DEFAULT_GROUP_ID), + WatermarkStrategy.noWatermarks(), "Kafka Messages"); } @Override - protected DataStream createEnrichmentSource(StreamExecutionEnvironment env, ParameterTool params) { + protected DataStream createEnrichmentSource(StreamExecutionEnvironment env, + ParameterTool params) { return env.fromSource( - new FlinkUtils<>(EnrichmentCommand.class).createKafkaGenericSource(params.getRequired(PARAMS_TOPIC_ENRICHMENT_INPUT), params, DEFAULT_GROUP_ID), - WatermarkStrategy.noWatermarks(), "Kafka Enrichment Commands"); + new FlinkUtils<>(EnrichmentCommand.class).createKafkaGenericSource( + params.getRequired(PARAMS_TOPIC_ENRICHMENT_INPUT), params, DEFAULT_GROUP_ID), + WatermarkStrategy.noWatermarks(), "Kafka Enrichment Commands"); } @Override - public DataStream writeEnrichments(StreamExecutionEnvironment env, ParameterTool params, DataStream enrichmentSource, + public DataStream writeEnrichments(StreamExecutionEnvironment env, ParameterTool params, + DataStream enrichmentSource, EnrichmentsConfig enrichmentsConfig) { List tables = enrichmentsConfig.getReferencedTables(); return enrichmentCommandsToHbase(params, enrichmentSource, enrichmentsConfig, tables); } - protected void writeQueryResults(StreamExecutionEnvironment env, ParameterTool params, DataStream enrichmentResults) { - enrichmentResults.sinkTo(new FlinkUtils<>(EnrichmentCommandResponse.class).createKafkaSink(params.getRequired(PARAMS_QUERY_OUTPUT), "enrichment-combined-command", params)) - .name("HBase Enrichment Response Sink").uid("kafka-enrichment-query-sink"); + protected void writeQueryResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream enrichmentResults) { + enrichmentResults.sinkTo( + new FlinkUtils<>(EnrichmentCommandResponse.class).createKafkaSink(params.getRequired(PARAMS_QUERY_OUTPUT), + "enrichment-combined-command", params)) + .name("HBase Enrichment Response Sink").uid("kafka-enrichment-query-sink"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MapMergeFunction.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MapMergeFunction.java index 1e968595..b6eba0f5 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MapMergeFunction.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MapMergeFunction.java @@ -12,17 +12,17 @@ package com.cloudera.cyber.enrichment.hbase; -import org.apache.flink.table.functions.ScalarFunction; +import static java.util.stream.Collectors.toMap; import java.util.Map; import java.util.stream.Stream; - -import static java.util.stream.Collectors.toMap; +import org.apache.flink.table.functions.ScalarFunction; /** * Add the results of enrichments to a map * + *

* The will merge two or more maps together. It will not overwrite values from the later maps */ public class MapMergeFunction extends ScalarFunction { @@ -32,13 +32,13 @@ public MapMergeFunction() { public Map eval(Map... merge) { return Stream.of(merge) - .map(s -> s.entrySet().stream()) - .flatMap(s -> s) - .collect(toMap( - k -> k.getKey(), - v -> v.getValue(), - (o, n) -> o - )); + .map(s -> s.entrySet().stream()) + .flatMap(s -> s) + .collect(toMap( + k -> k.getKey(), + v -> v.getValue(), + (o, n) -> o + )); } @Override diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MetronLookupKey.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MetronLookupKey.java index 317f5964..35fbb47a 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MetronLookupKey.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MetronLookupKey.java @@ -1,7 +1,11 @@ package com.cloudera.cyber.enrichment.hbase; import com.cloudera.cyber.hbase.LookupKey; -import lombok.*; +import java.util.Collections; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.hbase.client.Get; @@ -11,16 +15,13 @@ import org.apache.metron.enrichment.converter.EnrichmentValue; import org.apache.metron.enrichment.lookup.LookupKV; -import java.util.Collections; -import java.util.Map; - @Getter @EqualsAndHashCode(callSuper = true) @SuperBuilder @Slf4j @ToString(callSuper = true) public class MetronLookupKey extends LookupKey { - private final EnrichmentConverter converter = new EnrichmentConverter(); + private final EnrichmentConverter converter = new EnrichmentConverter(); private final String enrichmentType; @Override @@ -33,7 +34,7 @@ public Map resultToMap(Result result) { try { LookupKV keyValue = converter.fromResult(result, getCf()); return keyValue.getValue().getMetadata(); - } catch(Exception e) { + } catch (Exception e) { log.error(String.format("Unable to convert result Map: '%s'", result), e); return Collections.emptyMap(); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MetronLookupKeyBuilder.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MetronLookupKeyBuilder.java index 2015b696..47bcc6b3 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MetronLookupKeyBuilder.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/MetronLookupKeyBuilder.java @@ -6,8 +6,8 @@ public class MetronLookupKeyBuilder implements EnrichmentLookupBuilder { @Override public LookupKey build(EnrichmentStorageConfig storageConfig, String enrichmentType, String fieldValue) { - return MetronLookupKey.builder().tableName(storageConfig.getHbaseTableName()). - cf(storageConfig.getColumnFamily()). - enrichmentType(enrichmentType).key(fieldValue).build(); + return MetronLookupKey.builder().tableName(storageConfig.getHbaseTableName()) + .cf(storageConfig.getColumnFamily()) + .enrichmentType(enrichmentType).key(fieldValue).build(); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/PrefixMapFunction.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/PrefixMapFunction.java index a610856b..8a6084a8 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/PrefixMapFunction.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/PrefixMapFunction.java @@ -12,18 +12,17 @@ package com.cloudera.cyber.enrichment.hbase; -import org.apache.flink.table.functions.ScalarFunction; +import static java.util.stream.Collectors.toMap; import java.util.Map; - -import static java.util.stream.Collectors.toMap; +import org.apache.flink.table.functions.ScalarFunction; public class PrefixMapFunction extends ScalarFunction { public Map eval(Map map, String prefix) { return map.entrySet().stream() - .collect(toMap( + .collect(toMap( k -> prefix + k.getKey(), v -> v.getValue()) - ); + ); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/SimpleLookupKey.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/SimpleLookupKey.java index b2dd89d3..a30fe960 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/SimpleLookupKey.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/SimpleLookupKey.java @@ -1,19 +1,20 @@ package com.cloudera.cyber.enrichment.hbase; import com.cloudera.cyber.hbase.LookupKey; -import lombok.*; +import java.util.HashMap; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; import lombok.experimental.SuperBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; -import java.util.HashMap; -import java.util.Map; - @Getter @SuperBuilder @EqualsAndHashCode(callSuper = true) -@ToString(callSuper=true) +@ToString(callSuper = true) public class SimpleLookupKey extends LookupKey { @Override @@ -27,7 +28,8 @@ public Get toGet() { @Override public Map resultToMap(Result result) { Map hbaseMap = new HashMap<>(); - result.getFamilyMap(Bytes.toBytes(getCf())).forEach((k,v) -> hbaseMap.put( Bytes.toString(k), Bytes.toString(v))); + result.getFamilyMap(Bytes.toBytes(getCf())) + .forEach((k, v) -> hbaseMap.put(Bytes.toString(k), Bytes.toString(v))); return hbaseMap; } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/SimpleLookupKeyBuilder.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/SimpleLookupKeyBuilder.java index 12978753..b3d2fee3 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/SimpleLookupKeyBuilder.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/SimpleLookupKeyBuilder.java @@ -8,8 +8,9 @@ public class SimpleLookupKeyBuilder implements EnrichmentLookupBuilder { @Override public LookupKey build(EnrichmentStorageConfig storageConfig, String enrichmentType, String fieldValue) { - String columnFamily = StringUtils.isEmpty(storageConfig.getColumnFamily()) ? enrichmentType : storageConfig.getColumnFamily(); - return SimpleLookupKey.builder().tableName(storageConfig.getHbaseTableName()). - cf(columnFamily).key(fieldValue).build(); + String columnFamily = + StringUtils.isEmpty(storageConfig.getColumnFamily()) ? enrichmentType : storageConfig.getColumnFamily(); + return SimpleLookupKey.builder().tableName(storageConfig.getHbaseTableName()) + .cf(columnFamily).key(fieldValue).build(); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentConfig.java index 05b41ba7..a277fa8f 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentConfig.java @@ -12,30 +12,31 @@ package com.cloudera.cyber.enrichment.hbase.config; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME; + import com.google.common.base.Preconditions; +import java.io.Serializable; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import java.io.Serializable; -import java.util.Map; - -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME; - @Data @Slf4j @AllArgsConstructor @NoArgsConstructor public class EnrichmentConfig implements Serializable { - public static final String ENRICHMENT_CONFIG_MISSING_STORAGE_ERROR = "EnrichmentConfig %s: references undefined storage %s."; + public static final String ENRICHMENT_CONFIG_MISSING_STORAGE_ERROR = + "EnrichmentConfig %s: references undefined storage %s."; public static final String ENRICHMENT_CONFIG_MISSING_FIELD = "EnrichmentConfig %s, field mapping not specified"; private String storage; private EnrichmentFieldsConfig fieldMapping; public void validate(String enrichmentType, Map storageConfigs) { - Preconditions.checkState(storage == null || storageConfigs.containsKey(storage), String.format(ENRICHMENT_CONFIG_MISSING_STORAGE_ERROR, enrichmentType, storage)); + Preconditions.checkState(storage == null || storageConfigs.containsKey(storage), + String.format(ENRICHMENT_CONFIG_MISSING_STORAGE_ERROR, enrichmentType, storage)); Preconditions.checkNotNull(fieldMapping, String.format(ENRICHMENT_CONFIG_MISSING_FIELD, enrichmentType)); fieldMapping.validate(enrichmentType); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentFieldsConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentFieldsConfig.java index 971c2493..8963771c 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentFieldsConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentFieldsConfig.java @@ -13,33 +13,40 @@ package com.cloudera.cyber.enrichment.hbase.config; import com.google.common.base.Preconditions; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.collections.CollectionUtils; - import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.collections.CollectionUtils; @Data @AllArgsConstructor @NoArgsConstructor public class EnrichmentFieldsConfig implements Serializable { - public static final String FIELD_CONFIG_INVALID_KEY_FIELD = "EnrichmentFieldsConfig %s: EnrichmentConfig.keyFields is null or empty. Must contain at least one field name"; - public static final String RESERVED_ENRICH_DEFINES_KEY_FIELD = "EnrichmentFieldsConfig %s: EnrichmentConfig.keyFields for reserved enrichment type should be empty or null"; - public static final String RESERVED_ENRICH_DEFINES_VALUE_FIELD = "EnrichmentFieldsConfig %s: EnrichmentConfig.valueFields for reserved enrichment type should be empty or null"; - public static final String RESERVED_ENRICH_DEFINES_DELIMITER = "EnrichmentFieldsConfig %s: EnrichmentConfig.delimiter for reserved enrichment type should be null"; - public static final String FIELD_CONFIG_DUPLICATE_KEY_FIELD = "EnrichmentFieldsConfig %s: EnrichmentConfig.keyFields has duplicate values. All key field names must be unique."; - public static final String FIELD_CONFIG_INVALID_VALUE_FIELD = "EnrichmentFieldsConfig %s:EnrichmentConfig.valueFields is empty. Must be null or list of fields"; - public static final String FIELD_CONFIG_DUPLICATE_VALUE_FIELD = "EnrichmentFieldsConfig %s: EnrichmentConfig.valueFields has duplicate values. All value field names must be unique."; + public static final String FIELD_CONFIG_INVALID_KEY_FIELD = + "EnrichmentFieldsConfig %s: EnrichmentConfig.keyFields is null or empty. Must contain at least one field name"; + public static final String RESERVED_ENRICH_DEFINES_KEY_FIELD = + "EnrichmentFieldsConfig %s: EnrichmentConfig.keyFields for reserved enrichment type should be empty or null"; + public static final String RESERVED_ENRICH_DEFINES_VALUE_FIELD = + "EnrichmentFieldsConfig %s: EnrichmentConfig.valueFields for reserved enrichment type should be empty or null"; + public static final String RESERVED_ENRICH_DEFINES_DELIMITER = + "EnrichmentFieldsConfig %s: EnrichmentConfig.delimiter for reserved enrichment type should be null"; + public static final String FIELD_CONFIG_DUPLICATE_KEY_FIELD = + "EnrichmentFieldsConfig %s: EnrichmentConfig.keyFields has duplicate values. All key field names must be unique."; + public static final String FIELD_CONFIG_INVALID_VALUE_FIELD = + "EnrichmentFieldsConfig %s:EnrichmentConfig.valueFields is empty. Must be null or list of fields"; + public static final String FIELD_CONFIG_DUPLICATE_VALUE_FIELD = + "EnrichmentFieldsConfig %s: EnrichmentConfig.valueFields has duplicate values. All value field names must be unique."; public static final String DEFAULT_KEY_DELIMITER = ":"; public static final String THREATQ_ENRICHMENT_NAME = "threatq"; public static final String FIRST_SEEN_ENRICHMENT_NAME = "first_seen"; - private static final Set RESERVED_ENRICHMENT_NAMES = Stream.of(THREATQ_ENRICHMENT_NAME, FIRST_SEEN_ENRICHMENT_NAME).collect(Collectors.toSet()); + private static final Set RESERVED_ENRICHMENT_NAMES = + Stream.of(THREATQ_ENRICHMENT_NAME, FIRST_SEEN_ENRICHMENT_NAME).collect(Collectors.toSet()); private static final ArrayList EMPTY_LIST = new ArrayList<>(); /** @@ -82,14 +89,21 @@ public ArrayList getStreamingSources() { public void validate(String enrichmentType) { if (RESERVED_ENRICHMENT_NAMES.contains(enrichmentType)) { - Preconditions.checkState(CollectionUtils.isEmpty(keyFields), String.format(RESERVED_ENRICH_DEFINES_KEY_FIELD, enrichmentType) ); - Preconditions.checkState(CollectionUtils.isEmpty(valueFields), String.format(RESERVED_ENRICH_DEFINES_VALUE_FIELD, enrichmentType)); - Preconditions.checkState(keyDelimiter == null, String.format(RESERVED_ENRICH_DEFINES_DELIMITER, enrichmentType)); + Preconditions.checkState(CollectionUtils.isEmpty(keyFields), + String.format(RESERVED_ENRICH_DEFINES_KEY_FIELD, enrichmentType)); + Preconditions.checkState(CollectionUtils.isEmpty(valueFields), + String.format(RESERVED_ENRICH_DEFINES_VALUE_FIELD, enrichmentType)); + Preconditions.checkState(keyDelimiter == null, + String.format(RESERVED_ENRICH_DEFINES_DELIMITER, enrichmentType)); } else { - Preconditions.checkState(CollectionUtils.isNotEmpty(keyFields), String.format(FIELD_CONFIG_INVALID_KEY_FIELD, enrichmentType)); - Preconditions.checkState(isUnique(keyFields), String.format(FIELD_CONFIG_DUPLICATE_KEY_FIELD, enrichmentType)); - Preconditions.checkState(valueFields == null || !valueFields.isEmpty(), String.format(FIELD_CONFIG_INVALID_VALUE_FIELD, enrichmentType)); - Preconditions.checkState(valueFields == null || isUnique(valueFields), String.format(FIELD_CONFIG_DUPLICATE_VALUE_FIELD, enrichmentType)); + Preconditions.checkState(CollectionUtils.isNotEmpty(keyFields), + String.format(FIELD_CONFIG_INVALID_KEY_FIELD, enrichmentType)); + Preconditions.checkState(isUnique(keyFields), + String.format(FIELD_CONFIG_DUPLICATE_KEY_FIELD, enrichmentType)); + Preconditions.checkState(valueFields == null || !valueFields.isEmpty(), + String.format(FIELD_CONFIG_INVALID_VALUE_FIELD, enrichmentType)); + Preconditions.checkState(valueFields == null || isUnique(valueFields), + String.format(FIELD_CONFIG_DUPLICATE_VALUE_FIELD, enrichmentType)); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentStorageConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentStorageConfig.java index eea4879a..ca37b92c 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentStorageConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentStorageConfig.java @@ -13,29 +13,35 @@ package com.cloudera.cyber.enrichment.hbase.config; import com.google.common.base.Preconditions; +import java.io.Serializable; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.apache.commons.lang.StringUtils; -import java.io.Serializable; - @Data @AllArgsConstructor @NoArgsConstructor public class EnrichmentStorageConfig implements Serializable { - public static final String STORAGE_CONFIG_TABLE_NOT_SET_ERROR = "EnrichmentStorageConfig %s: Hbase table name must be set and non-empty"; - public static final String STORAGE_CONFIG_COLUMN_FAMILY_NOT_SET_ERROR = "EnrichmentStorageConfig %s: Hbase metron enrichment format requires column family"; - public static final String STORAGE_CONFIG_COLUMN_FAMILY_SET_ERROR = "EnrichmentStorageConfig %s: Hbase simple enrichment format does not require a column family"; + public static final String STORAGE_CONFIG_TABLE_NOT_SET_ERROR = + "EnrichmentStorageConfig %s: Hbase table name must be set and non-empty"; + public static final String STORAGE_CONFIG_COLUMN_FAMILY_NOT_SET_ERROR = + "EnrichmentStorageConfig %s: Hbase metron enrichment format requires column family"; + public static final String STORAGE_CONFIG_COLUMN_FAMILY_SET_ERROR = + "EnrichmentStorageConfig %s: Hbase simple enrichment format does not require a column family"; private EnrichmentStorageFormat format; private String hbaseTableName; private String columnFamily; public void validate(String storageType) { - Preconditions.checkState(StringUtils.isNotEmpty(hbaseTableName), STORAGE_CONFIG_TABLE_NOT_SET_ERROR, storageType); - Preconditions.checkState(format.equals(EnrichmentStorageFormat.HBASE_SIMPLE) || StringUtils.isNotEmpty(columnFamily), String.format(STORAGE_CONFIG_COLUMN_FAMILY_NOT_SET_ERROR, storageType)); - Preconditions.checkState(format.equals(EnrichmentStorageFormat.HBASE_METRON) || StringUtils.isEmpty(columnFamily), String.format - (STORAGE_CONFIG_COLUMN_FAMILY_SET_ERROR, storageType)); + Preconditions.checkState(StringUtils.isNotEmpty(hbaseTableName), STORAGE_CONFIG_TABLE_NOT_SET_ERROR, + storageType); + Preconditions.checkState( + format.equals(EnrichmentStorageFormat.HBASE_SIMPLE) || StringUtils.isNotEmpty(columnFamily), + String.format(STORAGE_CONFIG_COLUMN_FAMILY_NOT_SET_ERROR, storageType)); + Preconditions.checkState( + format.equals(EnrichmentStorageFormat.HBASE_METRON) || StringUtils.isEmpty(columnFamily), + String.format(STORAGE_CONFIG_COLUMN_FAMILY_SET_ERROR, storageType)); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentStorageFormat.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentStorageFormat.java index a94f37f6..aab4e835 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentStorageFormat.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentStorageFormat.java @@ -26,7 +26,8 @@ public enum EnrichmentStorageFormat { private final EnrichmentCommandMutationConverter mutationConverter; private final EnrichmentLookupBuilder lookupBuilder; - EnrichmentStorageFormat(EnrichmentCommandMutationConverter mutationConverter, EnrichmentLookupBuilder lookupBuilder) { + EnrichmentStorageFormat(EnrichmentCommandMutationConverter mutationConverter, + EnrichmentLookupBuilder lookupBuilder) { this.mutationConverter = mutationConverter; this.lookupBuilder = lookupBuilder; } @@ -35,6 +36,8 @@ public EnrichmentCommandMutationConverter getMutationConverter() { return mutationConverter; } - public EnrichmentLookupBuilder getLookupBuilder() { return lookupBuilder; } + public EnrichmentLookupBuilder getLookupBuilder() { + return lookupBuilder; + } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentsConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentsConfig.java index 83f8b88f..a150b511 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentsConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentsConfig.java @@ -14,13 +14,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import lombok.Data; -import org.apache.commons.lang3.StringUtils; - import java.io.File; import java.io.Serializable; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; /** * Maps enrichment type to information about the key and value fields of the enrichment and @@ -28,23 +29,27 @@ */ @Data public class EnrichmentsConfig implements Serializable { - public static final String NO_STORAGE_TYPE_NAME_SPECIFIED_ERROR = "Null or empty string are not valid storageTypeNames"; - public static final String NO_ENRICHMENT_TYPE_NAME_SPECIFIED_ERROR = "Null or empty string are not valid enrichmentTypeNames"; - public static final String ENRICHMENT_CONFIG_FILE_DESERIALIZATION_ERROR = "Could not deserialize enrichments configuration file '%s'"; + public static final String NO_STORAGE_TYPE_NAME_SPECIFIED_ERROR = + "Null or empty string are not valid storageTypeNames"; + public static final String NO_ENRICHMENT_TYPE_NAME_SPECIFIED_ERROR = + "Null or empty string are not valid enrichmentTypeNames"; + public static final String ENRICHMENT_CONFIG_FILE_DESERIALIZATION_ERROR = + "Could not deserialize enrichments configuration file '%s'"; public static final String DEFAULT_ENRICHMENT_STORAGE_NAME = "default"; public static final String MISSING_STORAGE_ERROR = "Enrichment storage does not contain configuration for %s"; /** * maps a storage config name to a storage Config. */ - private HashMap storageConfigs ; + private HashMap storageConfigs; /** - * Maps enrichment type to its configuration + * Maps enrichment type to its configuration. */ private HashMap enrichmentConfigs; - public EnrichmentsConfig(Map storageConfigs, Map enrichmentConfigs) { + public EnrichmentsConfig(Map storageConfigs, + Map enrichmentConfigs) { if (storageConfigs != null) { this.storageConfigs = new HashMap<>(storageConfigs); } else { @@ -76,7 +81,8 @@ public static EnrichmentsConfig load(String filePath) { public void validate() { - Preconditions.checkState(storageConfigs.containsKey(DEFAULT_ENRICHMENT_STORAGE_NAME), String.format(MISSING_STORAGE_ERROR, DEFAULT_ENRICHMENT_STORAGE_NAME)); + Preconditions.checkState(storageConfigs.containsKey(DEFAULT_ENRICHMENT_STORAGE_NAME), + String.format(MISSING_STORAGE_ERROR, DEFAULT_ENRICHMENT_STORAGE_NAME)); storageConfigs.forEach((storageType, storageConfig) -> { Preconditions.checkState(StringUtils.isNotBlank(storageType), NO_STORAGE_TYPE_NAME_SPECIFIED_ERROR); storageConfig.validate(storageType); @@ -89,21 +95,28 @@ public void validate() { /** * Return a distinct list of tables specified in the storage configs that are referenced by enrichment types. + * * @return list of tables used by enrichment types */ public List getReferencedTables() { - List referencedStorage = enrichmentConfigs.values().stream().map(EnrichmentConfig::getStorage).distinct().collect(Collectors.toList()); + List referencedStorage = + enrichmentConfigs.values().stream().map(EnrichmentConfig::getStorage).distinct() + .collect(Collectors.toList()); return getTablesForStorage(referencedStorage); } public List getReferencedTablesForSource(String source) { - List storageForSources = enrichmentConfigs.values().stream().filter(c -> c.getFieldMapping().getStreamingSources().contains(source)).map(EnrichmentConfig::getStorage). - distinct().collect(Collectors.toList()); + List storageForSources = enrichmentConfigs.values().stream() + .filter(c -> c.getFieldMapping().getStreamingSources() + .contains(source)) + .map(EnrichmentConfig::getStorage) + .distinct().collect(Collectors.toList()); return getTablesForStorage(storageForSources); } private List getTablesForStorage(List storageNames) { - return storageNames.stream().map(storageConfigs::get).map(EnrichmentStorageConfig::getHbaseTableName).distinct().collect(Collectors.toList()); + return storageNames.stream().map(storageConfigs::get).map(EnrichmentStorageConfig::getHbaseTableName).distinct() + .collect(Collectors.toList()); } public EnrichmentStorageConfig getStorageForEnrichmentType(String enrichmentType) { @@ -117,8 +130,9 @@ public EnrichmentStorageConfig getStorageForEnrichmentType(String enrichmentType } public List getStreamingEnrichmentSources() { - return enrichmentConfigs.values().stream(). - flatMap(c -> c.getFieldMapping().getStreamingSources().stream()).distinct().collect(Collectors.toList()); + return enrichmentConfigs.values().stream() + .flatMap(c -> c.getFieldMapping().getStreamingSources().stream()).distinct() + .collect(Collectors.toList()); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/EnrichmentCommandMutationConverter.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/EnrichmentCommandMutationConverter.java index a784d78b..ef716dea 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/EnrichmentCommandMutationConverter.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/EnrichmentCommandMutationConverter.java @@ -14,9 +14,8 @@ import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; -import org.apache.hadoop.hbase.client.Mutation; - import java.io.Serializable; +import org.apache.hadoop.hbase.client.Mutation; public interface EnrichmentCommandMutationConverter extends Serializable { Mutation convertToMutation(EnrichmentStorageConfig storageConfig, EnrichmentCommand enrichmentCommand); diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/HbaseEnrichmentMutationConverter.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/HbaseEnrichmentMutationConverter.java index 03c115c1..342b9b83 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/HbaseEnrichmentMutationConverter.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/HbaseEnrichmentMutationConverter.java @@ -31,7 +31,8 @@ public void open() { @Override public Mutation convertToMutation(EnrichmentCommand enrichmentCommand) { - EnrichmentStorageConfig storageConfig = enrichmentsConfig.getStorageForEnrichmentType(enrichmentCommand.getPayload().getType()); + EnrichmentStorageConfig storageConfig = + enrichmentsConfig.getStorageForEnrichmentType(enrichmentCommand.getPayload().getType()); EnrichmentCommandMutationConverter mutationConverter = storageConfig.getFormat().getMutationConverter(); return mutationConverter.convertToMutation(storageConfig, enrichmentCommand); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/MetronHbaseEnrichmentMutationConverter.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/MetronHbaseEnrichmentMutationConverter.java index 7af23c7d..3500d3be 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/MetronHbaseEnrichmentMutationConverter.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/MetronHbaseEnrichmentMutationConverter.java @@ -14,20 +14,20 @@ import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; +import java.util.HashMap; import org.apache.hadoop.hbase.client.Mutation; import org.apache.metron.enrichment.converter.EnrichmentConverter; import org.apache.metron.enrichment.converter.EnrichmentKey; import org.apache.metron.enrichment.converter.EnrichmentValue; -import java.util.HashMap; - public class MetronHbaseEnrichmentMutationConverter implements EnrichmentCommandMutationConverter { - private final EnrichmentConverter converter = new EnrichmentConverter(); + private final EnrichmentConverter converter = new EnrichmentConverter(); @Override public Mutation convertToMutation(EnrichmentStorageConfig storageConfig, EnrichmentCommand enrichmentCommand) { - EnrichmentKey key = new EnrichmentKey(enrichmentCommand.getPayload().getType(), enrichmentCommand.getPayload().getKey()); + EnrichmentKey key = + new EnrichmentKey(enrichmentCommand.getPayload().getType(), enrichmentCommand.getPayload().getKey()); EnrichmentValue value = new EnrichmentValue(new HashMap<>(enrichmentCommand.getPayload().getEntries())); return converter.toPut(storageConfig.getColumnFamily(), key, value); diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/SimpleHbaseEnrichmentMutationConverter.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/SimpleHbaseEnrichmentMutationConverter.java index f23c06b1..61c1c7f3 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/SimpleHbaseEnrichmentMutationConverter.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/mutators/SimpleHbaseEnrichmentMutationConverter.java @@ -12,6 +12,10 @@ package com.cloudera.cyber.enrichment.hbase.mutators; +import static com.cloudera.cyber.enrichment.EnrichmentUtils.CF_ID; +import static com.cloudera.cyber.enrichment.EnrichmentUtils.Q_KEY; +import static com.cloudera.cyber.enrichment.EnrichmentUtils.enrichmentKey; + import com.cloudera.cyber.EnrichmentEntry; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; @@ -20,8 +24,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; -import static com.cloudera.cyber.enrichment.EnrichmentUtils.*; - public class SimpleHbaseEnrichmentMutationConverter implements EnrichmentCommandMutationConverter { @Override public Mutation convertToMutation(EnrichmentStorageConfig storageConfig, EnrichmentCommand enrichmentCommand) { @@ -32,10 +34,14 @@ public Mutation convertToMutation(EnrichmentStorageConfig storageConfig, Enrichm Put put = new Put(enrichmentKey(enrichmentEntry.getKey())); put.addColumn(CF_ID, Q_KEY, Bytes.toBytes(enrichmentEntry.getKey())); // add the map for the entries - enrichmentEntry.getEntries().forEach((k, v) -> put.addColumn(Bytes.toBytes(enrichmentEntry.getType()), Bytes.toBytes(k), Bytes.toBytes(v))); + enrichmentEntry.getEntries().forEach( + (k, v) -> put.addColumn(Bytes.toBytes(enrichmentEntry.getType()), Bytes.toBytes(k), + Bytes.toBytes(v))); return put; case DELETE: return new Delete(Bytes.toBytes(enrichmentEntry.getKey())); + default: + break; } // this should not happen - Enrichment commands filtered into ADD and DELETE before sink diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/writer/HbaseEnrichmentCommandSink.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/writer/HbaseEnrichmentCommandSink.java index 1535795f..9adb50ca 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/writer/HbaseEnrichmentCommandSink.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/main/java/com/cloudera/cyber/enrichment/hbase/writer/HbaseEnrichmentCommandSink.java @@ -13,14 +13,13 @@ package com.cloudera.cyber.enrichment.hbase.writer; import com.cloudera.cyber.commands.EnrichmentCommand; -import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig; import com.cloudera.cyber.enrichment.hbase.mutators.HbaseEnrichmentMutationConverter; import com.cloudera.cyber.hbase.AbstractHbaseSinkFunction; import org.apache.flink.api.java.utils.ParameterTool; -public class HbaseEnrichmentCommandSink extends AbstractHbaseSinkFunction { - public HbaseEnrichmentCommandSink(String hTableName, EnrichmentsConfig enrichmentsConfig, ParameterTool params) { - super(hTableName, new HbaseEnrichmentMutationConverter(enrichmentsConfig), params, "numEnrichmentsWritten"); +public class HbaseEnrichmentCommandSink extends AbstractHbaseSinkFunction { + public HbaseEnrichmentCommandSink(String hbaseTableName, EnrichmentsConfig enrichmentsConfig, ParameterTool params) { + super(hbaseTableName, new HbaseEnrichmentMutationConverter(enrichmentsConfig), params, "numEnrichmentsWritten"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/test/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentsConfigTest.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/test/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentsConfigTest.java index b389f6c0..a97a20f6 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/test/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentsConfigTest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/src/test/java/com/cloudera/cyber/enrichment/hbase/config/EnrichmentsConfigTest.java @@ -12,23 +12,22 @@ package com.cloudera.cyber.enrichment.hbase.config; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfigTest.KEY_FIELDS; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_METRON; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_SIMPLE; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.junit.Assert; -import org.junit.Test; - import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; - -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfigTest.KEY_FIELDS; -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_METRON; -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_SIMPLE; -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Assert; +import org.junit.Test; public class EnrichmentsConfigTest { @@ -108,8 +107,8 @@ public void testLoadInvalidSemanticsFile() { @Test public void testLoadFileDoesNotExist() { String doesNotExistFile = "file_does_not_exist.json"; - assertThatThrownBy(() -> testLoadJson(doesNotExistFile)).isInstanceOf(RuntimeException.class). - hasMessage(EnrichmentsConfig.ENRICHMENT_CONFIG_FILE_DESERIALIZATION_ERROR, doesNotExistFile) + assertThatThrownBy(() -> testLoadJson(doesNotExistFile)).isInstanceOf(RuntimeException.class) + .hasMessage(EnrichmentsConfig.ENRICHMENT_CONFIG_FILE_DESERIALIZATION_ERROR, doesNotExistFile) .hasCauseInstanceOf(IOException.class); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentBroadcastProcessFunction.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentBroadcastProcessFunction.java index 81259704..3b6da5b1 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentBroadcastProcessFunction.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentBroadcastProcessFunction.java @@ -18,6 +18,13 @@ import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.commands.EnrichmentCommandResponse; import com.google.common.base.Joiner; +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -30,11 +37,6 @@ import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction; import org.apache.flink.util.Collector; -import java.time.Instant; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - @NoArgsConstructor @RequiredArgsConstructor @Slf4j @@ -59,8 +61,10 @@ public void open(Configuration parameters) throws Exception { @Override - public void processElement(Message message, ReadOnlyContext readOnlyContext, Collector collector) throws Exception { - ReadOnlyBroadcastState> bc = readOnlyContext.getBroadcastState(broadcastDescriptors.get(type)); + public void processElement(Message message, ReadOnlyContext readOnlyContext, Collector collector) + throws Exception { + ReadOnlyBroadcastState> bc = + readOnlyContext.getBroadcastState(broadcastDescriptors.get(type)); log.debug("Process Message: {}", message); if (bc != null) { @@ -70,8 +74,8 @@ public void processElement(Message message, ReadOnlyContext readOnlyContext, Col if (value != null && bc.contains(value.toString())) { hits.inc(); hm.putAll(bc.get(value.toString()).entrySet().stream().collect(Collectors.toMap( - k -> fieldJoiner.join(field, type, k.getKey()), - Map.Entry::getValue + k -> fieldJoiner.join(field, type, k.getKey()), + Map.Entry::getValue ))); } } @@ -83,60 +87,81 @@ public void processElement(Message message, ReadOnlyContext readOnlyContext, Col } @Override - public void processBroadcastElement(EnrichmentCommand enrichmentCommand, Context context, Collector collector) throws Exception { + public void processBroadcastElement(EnrichmentCommand enrichmentCommand, Context context, + Collector collector) throws Exception { // add to the state EnrichmentEntry enrichmentEntry = enrichmentCommand.getPayload(); - BroadcastState> broadcastState = context.getBroadcastState(broadcastDescriptors.get(enrichmentEntry.getType())); + BroadcastState> broadcastState = + context.getBroadcastState(broadcastDescriptors.get(enrichmentEntry.getType())); log.info("Process Command: {}", enrichmentCommand); switch (enrichmentCommand.getType()) { case ADD: enrichments.inc(); broadcastState.put(enrichmentEntry.getKey(), enrichmentEntry.getEntries()); context.output(LookupJob.QUERY_RESULT, EnrichmentCommandResponse.builder() - .success(true) - .message("Added LOCAL enrichment") - .content(Collections.singletonList(enrichmentEntry)) - .headers(enrichmentCommand.getHeaders()) - .build()); + .success(true) + .message("Added LOCAL enrichment") + .content(Collections.singletonList( + enrichmentEntry)) + .headers(enrichmentCommand.getHeaders()) + .build()); break; case DELETE: broadcastState.remove(enrichmentEntry.getKey()); context.output(LookupJob.QUERY_RESULT, EnrichmentCommandResponse.builder() - .success(true) - .message("Deleted LOCAL enrichment") - .content(Collections.singletonList(enrichmentEntry)) - .headers(enrichmentCommand.getHeaders()) - .build()); + .success(true) + .message("Deleted LOCAL enrichment") + .content(Collections.singletonList( + enrichmentEntry)) + .headers(enrichmentCommand.getHeaders()) + .build()); break; case LIST: - context.output(LookupJob.QUERY_RESULT, EnrichmentCommandResponse.builder() - .success(true) - .message("Current enrichments of type "+ type) - .content(StreamSupport.stream(broadcastState.immutableEntries().spliterator(), true) - .map(e -> - EnrichmentEntry.builder() - .type(type) - .key(e.getKey()) - .entries(e.getValue()) - .ts(Instant.now().getEpochSecond()) - .build()).collect(Collectors.toList())) - .message("") - .headers(enrichmentCommand.getHeaders()) - .build()); + context.output(LookupJob.QUERY_RESULT, + EnrichmentCommandResponse.builder() + .success(true) + .message( + "Current enrichments of type " + + type) + .content(StreamSupport.stream( + broadcastState.immutableEntries() + .spliterator(), + true) + .map(e -> + EnrichmentEntry.builder() + .type(type) + .key(e.getKey()) + .entries( + e.getValue()) + .ts(Instant.now() + .getEpochSecond()) + .build()) + .collect( + Collectors.toList())) + .message("") + .headers(enrichmentCommand.getHeaders()) + .build()); break; case FIND: - context.output(LookupJob.QUERY_RESULT, EnrichmentCommandResponse.builder() - .success(true) - .content(Collections.singletonList(EnrichmentEntry.builder() - .type(type) - .key(enrichmentEntry.getKey()) - .entries(broadcastState.get(enrichmentCommand.getPayload().getKey())) - .ts(Instant.now().getEpochSecond()) - .build())) - .headers(enrichmentCommand.getHeaders()) - .message("Query enrichment") - .build()); + context.output(LookupJob.QUERY_RESULT, + EnrichmentCommandResponse.builder() + .success(true) + .content(Collections.singletonList( + EnrichmentEntry.builder() + .type(type) + .key(enrichmentEntry.getKey()) + .entries( + broadcastState.get( + enrichmentCommand.getPayload() + .getKey())) + .ts(Instant.now() + .getEpochSecond()) + .build())) + .headers(enrichmentCommand.getHeaders()) + .message("Query enrichment") + .build()); break; + default: log.warn("Unknown enrichment command type: {}", enrichmentCommand.getType()); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentKey.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentKey.java index aa4b5bdb..5ac59c26 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentKey.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentKey.java @@ -12,14 +12,13 @@ package com.cloudera.cyber.enrichment.lookup; +import java.io.Serializable; import lombok.Builder; import lombok.Data; -import java.io.Serializable; - @Builder @Data -public class EnrichmentKey implements Serializable { +public class EnrichmentKey implements Serializable { private String type; private String key; } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentLookupResult.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentLookupResult.java index fe5f637a..82d33ecd 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentLookupResult.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/EnrichmentLookupResult.java @@ -12,12 +12,11 @@ package com.cloudera.cyber.enrichment.lookup; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; -import java.util.Map; - @Data @Builder @AllArgsConstructor diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/LookupJob.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/LookupJob.java index c321521a..0b6a2f8c 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/LookupJob.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/LookupJob.java @@ -12,6 +12,11 @@ package com.cloudera.cyber.enrichment.lookup; +import static com.cloudera.cyber.enrichment.ConfigUtils.PARAMS_CONFIG_FILE; +import static com.cloudera.cyber.enrichment.ConfigUtils.allConfigs; +import static com.cloudera.cyber.enrichment.ConfigUtils.enrichmentTypes; +import static com.cloudera.cyber.enrichment.ConfigUtils.typeToFields; + import com.cloudera.cyber.Message; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.commands.EnrichmentCommandResponse; @@ -19,6 +24,12 @@ import com.cloudera.cyber.enrichment.lookup.config.EnrichmentKind; import com.cloudera.cyber.flink.CyberJob; import com.cloudera.cyber.flink.FlinkUtils; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.state.MapStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -31,44 +42,54 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.OutputTag; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.cloudera.cyber.enrichment.ConfigUtils.*; - @Slf4j public abstract class LookupJob implements CyberJob { - public static final OutputTag QUERY_RESULT = new OutputTag<>("query-result", TypeInformation.of(EnrichmentCommandResponse.class)); + public static final OutputTag QUERY_RESULT = + new OutputTag<>("query-result", TypeInformation.of(EnrichmentCommandResponse.class)); - public static Tuple2, DataStream> enrich(DataStream baseEnrichmentSource, - SingleOutputStreamOperator source, - List configs + public static Tuple2, DataStream> enrich( + DataStream baseEnrichmentSource, + SingleOutputStreamOperator source, + List configs ) { DataStream enrichmentSource = baseEnrichmentSource.keyBy(e -> EnrichmentKey.builder() - .type(e.getPayload().getType()) - .key(e.getPayload().getKey()) - .build() + .type(e.getPayload() + .getType()) + .key(e.getPayload() + .getKey()) + .build() ); Map> typeToFields = typeToFields(configs, EnrichmentKind.LOCAL); Set enrichmentTypes = enrichmentTypes(configs, EnrichmentKind.LOCAL); - Map>> broadcastDescriptors = enrichmentTypes.stream().collect(Collectors.toMap( - v -> v, - enrichmentType -> new MapStateDescriptor<>(enrichmentType, Types.STRING, Types.MAP(Types.STRING, Types.STRING))) - ); - - Map> enrichmentBroadcasts = enrichmentTypes.stream() - .map(enrichmentType -> - Tuple2.of(enrichmentType, enrichmentSource.filter(f -> f.getPayload().getType().equals(enrichmentType)).name("Filter: " + enrichmentType) - .broadcast(broadcastDescriptors.get(enrichmentType))) - ) - .collect(Collectors.toMap(v -> v.f0, k -> k.f1)); + Map>> broadcastDescriptors = + enrichmentTypes.stream().collect(Collectors.toMap( + v -> v, + enrichmentType -> new MapStateDescriptor<>(enrichmentType, Types.STRING, + Types.MAP(Types.STRING, Types.STRING))) + ); + + Map> enrichmentBroadcasts = + enrichmentTypes.stream() + .map(enrichmentType -> + Tuple2.of( + enrichmentType, + enrichmentSource.filter( + f -> f.getPayload() + .getType() + .equals( + enrichmentType)) + .name("Filter: " + + enrichmentType) + .broadcast( + broadcastDescriptors.get( + enrichmentType))) + ) + .collect(Collectors.toMap( + v -> v.f0, + k -> k.f1)); /* * Apply all the configs as a series of broadcast connections that the messages pass through @@ -82,14 +103,15 @@ public static Tuple2, DataStream> List fields = enrichmentBroadcast.getValue(); String type = enrichmentBroadcast.getKey(); pipeline = pipeline.connect(enrichmentBroadcasts.get(type)) - .process(new EnrichmentBroadcastProcessFunction(type, fields, broadcastDescriptors)).name("Process: " + type).uid("broadcast-process-" + type); + .process(new EnrichmentBroadcastProcessFunction(type, fields, broadcastDescriptors)) + .name("Process: " + type).uid("broadcast-process-" + type); if (enrichmentCommandResponses == null) { enrichmentCommandResponses = pipeline.getSideOutput(QUERY_RESULT); } else { enrichmentCommandResponses = enrichmentCommandResponses.union(pipeline.getSideOutput(QUERY_RESULT)); } } - + return Tuple2.of(pipeline, enrichmentCommandResponses); } @@ -104,17 +126,22 @@ public StreamExecutionEnvironment createPipeline(ParameterTool params) throws Ex byte[] configJson = Files.readAllBytes(Paths.get(params.getRequired(PARAMS_CONFIG_FILE))); - Tuple2, DataStream> pipeline = enrich(enrichmentSource, source, allConfigs(configJson)); + Tuple2, DataStream> pipeline = + enrich(enrichmentSource, source, allConfigs(configJson)); writeResults(env, params, pipeline.f0); writeQueryResults(env, params, pipeline.f1); return env; } - protected abstract void writeQueryResults(StreamExecutionEnvironment env, ParameterTool params, DataStream sideOutput); + protected abstract void writeQueryResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream sideOutput); - protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream reduction); + protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream reduction); - public abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params); + public abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, + ParameterTool params); - protected abstract DataStream createEnrichmentSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract DataStream createEnrichmentSource(StreamExecutionEnvironment env, + ParameterTool params); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/LookupJobKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/LookupJobKafka.java index 381f2c27..217b642d 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/LookupJobKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/src/main/java/com/cloudera/cyber/enrichment/lookup/LookupJobKafka.java @@ -12,6 +12,9 @@ package com.cloudera.cyber.enrichment.lookup; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; + import com.cloudera.cyber.Message; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.commands.EnrichmentCommandResponse; @@ -26,9 +29,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; - public class LookupJobKafka extends LookupJob { private static final String PARAMS_TOPIC_ENRICHMENT_INPUT = "enrichment.topic.input"; @@ -36,32 +36,40 @@ public class LookupJobKafka extends LookupJob { public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); - new LookupJobKafka().createPipeline(Utils.getParamToolsFromProperties(args)).execute("Enrichments - Local Lookup"); + new LookupJobKafka().createPipeline(Utils.getParamToolsFromProperties(args)) + .execute("Enrichments - Local Lookup"); } @Override - protected void writeQueryResults(StreamExecutionEnvironment env, ParameterTool params, DataStream sideOutput) { - sideOutput.sinkTo(new FlinkUtils<>(EnrichmentCommandResponse.class).createKafkaSink(params.getRequired(PARAMS_QUERY_OUTPUT), "enrichments-lookup-command", params)) - .name("Kafka Sink").uid("kafka-sink"); + protected void writeQueryResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream sideOutput) { + sideOutput.sinkTo( + new FlinkUtils<>(EnrichmentCommandResponse.class).createKafkaSink(params.getRequired(PARAMS_QUERY_OUTPUT), + "enrichments-lookup-command", params)) + .name("Kafka Sink").uid("kafka-sink"); } @Override protected void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream reduction) { - reduction.sinkTo(new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), "enrichents-lookup", params)) - .name("Kafka Sink").uid("kafka-sink"); + reduction.sinkTo(new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), + "enrichents-lookup", params)) + .name("Kafka Sink").uid("kafka-sink"); } @Override public SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params) { return env.fromSource( - FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, "enrichment-lookups-local"), - WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); + FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, "enrichment-lookups-local"), + WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); } @Override - protected DataStream createEnrichmentSource(StreamExecutionEnvironment env, ParameterTool params) { - KafkaSource enrichmentCommands = new SourcesWithHeaders<>(EnrichmentCommand.class). - createSourceWithHeaders(params.getRequired(PARAMS_TOPIC_ENRICHMENT_INPUT), params, "enrichment-lookups-local"); - return env.fromSource(enrichmentCommands, WatermarkStrategy.noWatermarks(),"Kafka Enrichments").uid("kafka-enrichment-source"); + protected DataStream createEnrichmentSource(StreamExecutionEnvironment env, + ParameterTool params) { + KafkaSource enrichmentCommands = new SourcesWithHeaders<>(EnrichmentCommand.class) + .createSourceWithHeaders(params.getRequired(PARAMS_TOPIC_ENRICHMENT_INPUT), params, + "enrichment-lookups-local"); + return env.fromSource(enrichmentCommands, WatermarkStrategy.noWatermarks(), "Kafka Enrichments") + .uid("kafka-enrichment-source"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequest.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequest.java index 03a0f19b..b527fe06 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequest.java @@ -12,24 +12,22 @@ package com.cloudera.cyber.enrichment.rest; +import static com.cloudera.cyber.DataQualityMessageLevel.ERROR; + import com.cloudera.cyber.DataQualityMessage; import com.cloudera.cyber.Message; import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.enrichment.SingleValueEnrichment; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; - import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.function.Predicate; import java.util.List; - -import static com.cloudera.cyber.DataQualityMessageLevel.ERROR; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; @Slf4j public class AsyncHttpRequest extends RichAsyncFunction { @@ -56,7 +54,7 @@ public AsyncHttpRequest(@NonNull RestEnrichmentConfig config) { @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - log.debug("Opening AsyncHttpRequest {}", config.toString()); + log.debug("Opening AsyncHttpRequest {}", config); // setup the client request = config.createRestEnrichmentRequest(); enrichment = new SingleValueEnrichment(config.getPrefix(), REST_ENRICHMENT_FEATURE); @@ -76,9 +74,10 @@ public void close() { private List createDataQualityMessages(List errorMessages) { if (!errorMessages.isEmpty()) { List dataQualityMessages = new ArrayList<>(); - errorMessages.forEach((errorMessage) -> enrichment.addQualityMessage(dataQualityMessages, ERROR, errorMessage)); + errorMessages.forEach( + (errorMessage) -> enrichment.addQualityMessage(dataQualityMessages, ERROR, errorMessage)); return dataQualityMessages; - } else { + } else { return Collections.emptyList(); } } @@ -88,22 +87,26 @@ public void asyncInvoke(Message message, ResultFuture resultFuture) { List configSource = config.getSources(); String messageSource = message.getSource(); if (matchAnySource || configSource.contains(messageSource)) { - request.getResult(!matchAnySource, message.getExtensions()).handleAsync((RestRequestResult restRequestResult, Throwable e) -> { - if (restRequestResult == null) { - restRequestResult = new RestRequestResult(); - } - if (e != null) { - restRequestResult.getErrors().add(e.getMessage()); - } - Collection result = Collections.singleton(MessageUtils.enrich(message, restRequestResult.getExtensions(), config.getPrefix(), createDataQualityMessages(restRequestResult.getErrors()))); - log.debug("Returned model result {}", result); - resultFuture.complete(result); - return restRequestResult; - }); + request.getResult(!matchAnySource, message.getExtensions()) + .handleAsync((RestRequestResult restRequestResult, Throwable e) -> { + if (restRequestResult == null) { + restRequestResult = new RestRequestResult(); + } + if (e != null) { + restRequestResult.getErrors().add(e.getMessage()); + } + Collection result = Collections.singleton( + MessageUtils.enrich(message, restRequestResult.getExtensions(), config.getPrefix(), + createDataQualityMessages(restRequestResult.getErrors()))); + log.debug("Returned model result {}", result); + resultFuture.complete(result); + return restRequestResult; + }); } else { // enrichment not relevant for this source - pass message through - log.debug("predicate returned false or event source {} does not match rest source {}", messageSource, configSource); + log.debug("predicate returned false or event source {} does not match rest source {}", messageSource, + configSource); resultFuture.complete(Collections.singleton(message)); } - } + } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/BasicAuthorizationConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/BasicAuthorizationConfig.java index f1fc1589..69a82103 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/BasicAuthorizationConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/BasicAuthorizationConfig.java @@ -12,14 +12,18 @@ package com.cloudera.cyber.enrichment.rest; -import lombok.*; +import java.nio.charset.StandardCharsets; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.apache.commons.codec.binary.Base64; import org.apache.commons.text.StringSubstitutor; import org.apache.flink.util.Preconditions; import org.apache.http.client.config.AuthSchemes; -import java.nio.charset.StandardCharsets; - /** * Credentials required for HTTP basic authorization. */ @@ -29,10 +33,14 @@ @Builder @Data public class BasicAuthorizationConfig extends EndpointAuthorizationConfig { - /** template resulting in the user name of the rest service basic auth credential. */ + /** + * template resulting in the user name of the rest service basic auth credential. + */ private String userNameTemplate; - /** template resulting in the password of the rest service basic auth credential. */ + /** + * template resulting in the password of the rest service basic auth credential. + */ private String passwordTemplate; @@ -41,7 +49,7 @@ public String generateAuthString(StringSubstitutor stringSubstitutor) { Preconditions.checkNotNull(userNameTemplate); Preconditions.checkNotNull(passwordTemplate); byte[] credentials = Base64.encodeBase64(String.join(":", stringSubstitutor.replace(userNameTemplate), - stringSubstitutor.replace(passwordTemplate)).getBytes(StandardCharsets.ISO_8859_1)); + stringSubstitutor.replace(passwordTemplate)).getBytes(StandardCharsets.ISO_8859_1)); return String.join(" ", AuthSchemes.BASIC, new String(credentials)); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/BearerTokenAuthorizationConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/BearerTokenAuthorizationConfig.java index 0233e8b1..57d7591b 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/BearerTokenAuthorizationConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/BearerTokenAuthorizationConfig.java @@ -12,7 +12,12 @@ package com.cloudera.cyber.enrichment.rest; -import lombok.*; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.apache.commons.text.StringSubstitutor; import org.apache.flink.util.Preconditions; diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/EndpointAuthorizationConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/EndpointAuthorizationConfig.java index 9af25f6f..59d7aabd 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/EndpointAuthorizationConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/EndpointAuthorizationConfig.java @@ -12,20 +12,19 @@ package com.cloudera.cyber.enrichment.rest; -import lombok.*; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.apache.commons.text.StringSubstitutor; import com.fasterxml.jackson.annotation.JsonSubTypes; - +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; +import lombok.Data; +import org.apache.commons.text.StringSubstitutor; @Data @JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "type") + use = JsonTypeInfo.Id.NAME, + property = "type") @JsonSubTypes({ - @JsonSubTypes.Type(value = BasicAuthorizationConfig.class, name = "basic"), - @JsonSubTypes.Type(value = BearerTokenAuthorizationConfig.class, name = "token") + @JsonSubTypes.Type(value = BasicAuthorizationConfig.class, name = "basic"), + @JsonSubTypes.Type(value = BearerTokenAuthorizationConfig.class, name = "token") }) public abstract class EndpointAuthorizationConfig implements Serializable { diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/GetRestRequest.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/GetRestRequest.java index 776ea616..e1a543fd 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/GetRestRequest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/GetRestRequest.java @@ -12,12 +12,11 @@ package com.cloudera.cyber.enrichment.rest; -import org.apache.http.client.methods.HttpGet; - -import javax.annotation.Nonnull; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; +import javax.annotation.Nonnull; +import org.apache.http.client.methods.HttpGet; public class GetRestRequest extends RestRequest { diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/PostRestRequest.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/PostRestRequest.java index 9da3eb00..d0a22693 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/PostRestRequest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/PostRestRequest.java @@ -12,15 +12,14 @@ package com.cloudera.cyber.enrichment.rest; -import org.apache.flink.util.Preconditions; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; - -import javax.annotation.Nonnull; import java.io.UnsupportedEncodingException; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; +import javax.annotation.Nonnull; +import org.apache.flink.util.Preconditions; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; public class PostRestRequest extends RestRequest { private final String entityTemplate; @@ -51,7 +50,8 @@ public RestRequestKey getKey(Map variables) { return new RestRequestKey(variables, urlTemplate, entityTemplate); } - protected void addEntityToRequest(@Nonnull HttpPost postRequest, @Nonnull String entityTemplate) throws UnsupportedEncodingException { + protected void addEntityToRequest(@Nonnull HttpPost postRequest, @Nonnull String entityTemplate) + throws UnsupportedEncodingException { postRequest.setEntity(new StringEntity(entityTemplate)); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestEnrichmentConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestEnrichmentConfig.java index c69f2baa..86ad4d8d 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestEnrichmentConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestEnrichmentConfig.java @@ -12,15 +12,18 @@ package com.cloudera.cyber.enrichment.rest; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; +import static com.cloudera.cyber.enrichment.rest.RestEnrichmentMethod.GET; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.concurrent.TimeUnit; - -import static com.cloudera.cyber.enrichment.rest.RestEnrichmentMethod.GET; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @@ -29,18 +32,26 @@ @NoArgsConstructor(force = true, access = AccessLevel.PUBLIC) public class RestEnrichmentConfig implements Serializable { - /** A StringSubstitutor template that puts the message extensions and rest properties into a URL to call */ - @JsonProperty(required=true) + /** + * A StringSubstitutor template that puts the message extensions and rest properties into a URL to call. + */ + @JsonProperty(required = true) private String endpointTemplate; - /** A StringSubstitutor template for the text entity of a POST. Omit for GET. **/ + /** + * A StringSubstitutor template for the text entity of a POST. Omit for GET. + **/ private String entityTemplate; - /** HTTP method required by the enrichment rest request. */ + /** + * HTTP method required by the enrichment rest request. + */ @Builder.Default private RestEnrichmentMethod method = GET; - /** TLS connection configuration keystores and truststores required by server. Null if client certificates are not required. */ + /** + * TLS connection configuration keystores and truststores required by server. Null if client certificates are not required. + */ @Builder.Default private TlsConfig tls = null; @@ -48,7 +59,7 @@ public class RestEnrichmentConfig implements Serializable { * Apply rest enrichment to the sources in this list. If source is ANY, enrichment will be applied to any event that defines the * variables used in the url and entity. */ - @JsonProperty(required=true) + @JsonProperty(required = true) private ArrayList sources; /** @@ -62,53 +73,73 @@ public class RestEnrichmentConfig implements Serializable { @Builder.Default private HashMap headers = new HashMap<>(); - /** Set of properties to be substituted in templates that do not vary by message. For example, API keys or auth tokens */ + /** + * Set of properties to be substituted in templates that do not vary by message. For example, API keys or auth tokens + */ private HashMap properties; - /** Flink async operator timeout in milliseconds */ + /** + * Flink async operator timeout in milliseconds. + */ @Builder.Default private int timeoutMillis = 1000; - /** Flink async currently open capacity */ + /** + * Flink async currently open capacity. + */ @Builder.Default private int capacity = 1000; - /** Max size of the local results cache */ + /** + * Max size of the local results cache. + */ @Builder.Default private int cacheSize = 10000; - /** Seconds before a successful cached rest result will be refreshed. */ + /** + * Seconds before a successful cached rest result will be refreshed. + */ @Builder.Default private long successCacheExpirationSeconds = TimeUnit.SECONDS.convert(30, TimeUnit.MINUTES); - /** Seconds before an unsuccessful cached rest result will be refreshed. */ + /** + * Seconds before an unsuccessful cached rest result will be refreshed. + */ @Builder.Default private long failureCacheExpirationSeconds = TimeUnit.SECONDS.convert(5, TimeUnit.MINUTES); - /** Prefix for extensions added to the enriched message, e.g. "modelName"*/ + /** + * Prefix for extensions added to the enriched message, e.g. "modelName" + */ private String prefix; /** - * An expression to determine whether the enrichment should run at all + * An expression to determine whether the enrichment should run at all. */ private String filterExpression; - /** Json path to boolean in json result indicating request status. If null, assume the HTTP status message indicates success or failure. */ + /** + * Json path to boolean in json result indicating request status. If null, assume the HTTP status message indicates success or failure. + */ private String successJsonPath; - /** Json path to map of results. By default, return the entire json document returned by the endpoint. */ + /** + * Json path to map of results. By default, return the entire json document returned by the endpoint. + */ @Builder.Default private String resultsJsonPath = "$"; public RestRequest createRestEnrichmentRequest() throws Exception { RestRequest request = null; switch (this.method) { - case POST : + case POST: request = new PostRestRequest(this); break; - case GET : + case GET: request = new GetRestRequest(this); break; + default: + break; } return request; } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJob.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJob.java index 1097142f..7b36fa04 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJob.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJob.java @@ -12,20 +12,15 @@ package com.cloudera.cyber.enrichment.rest; +import static com.cloudera.cyber.enrichment.ConfigUtils.PARAMS_CONFIG_FILE; +import static org.apache.commons.codec.digest.DigestUtils.md5; + import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.TimeCharacteristic; -import org.apache.flink.streaming.api.datastream.AsyncDataStream; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.util.Preconditions; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; @@ -33,40 +28,48 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; - -import static com.cloudera.cyber.enrichment.ConfigUtils.PARAMS_CONFIG_FILE; -import static org.apache.commons.codec.digest.DigestUtils.md5; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.datastream.AsyncDataStream; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.util.Preconditions; @Slf4j public abstract class RestLookupJob { protected static ObjectMapper getConfigObjectMapper() { return new ObjectMapper() - .activateDefaultTyping(BasicPolymorphicTypeValidator.builder(). - allowIfSubType(Map.class). - allowIfSubType(List.class). - allowIfBaseType(EndpointAuthorizationConfig.class). - build()) - .enable(SerializationFeature.INDENT_OUTPUT); + .activateDefaultTyping(BasicPolymorphicTypeValidator.builder() + .allowIfSubType(Map.class) + .allowIfSubType(List.class) + .allowIfBaseType(EndpointAuthorizationConfig.class) + .build()) + .enable(SerializationFeature.INDENT_OUTPUT); } - public static DataStream enrich(DataStream source, String enrichmentConfigPath) throws IOException { - List restConfig = RestLookupJob.parseConfigs(Files.readAllBytes(Paths.get(enrichmentConfigPath))); + public static DataStream enrich(DataStream source, String enrichmentConfigPath) + throws IOException { + List restConfig = + RestLookupJob.parseConfigs(Files.readAllBytes(Paths.get(enrichmentConfigPath))); return enrich(source, restConfig); } private static DataStream enrich(DataStream source, List configs) { return configs.stream().reduce(source, (in, config) -> { - Preconditions.checkNotNull(config.getSources(),"specify a list of source names or ANY to match any source"); - Preconditions.checkArgument(!config.getSources().isEmpty(), "specify a list of source names or ANY to match any source"); + Preconditions.checkNotNull(config.getSources(), + "specify a list of source names or ANY to match any source"); + Preconditions.checkArgument(!config.getSources().isEmpty(), + "specify a list of source names or ANY to match any source"); AsyncHttpRequest asyncHttpRequest = new AsyncHttpRequest(config); String processId = "rest-" + md5(source + config.getEndpointTemplate()); return AsyncDataStream.unorderedWait( - in, - asyncHttpRequest, config.getTimeoutMillis(), TimeUnit.MILLISECONDS, config.getCapacity()) - .name("REST - " + config.getEndpointTemplate() + " " + config.getSources()) - .uid("rest-" + processId); + in, + asyncHttpRequest, config.getTimeoutMillis(), TimeUnit.MILLISECONDS, config.getCapacity()) + .name("REST - " + config.getEndpointTemplate() + " " + config.getSources()) + .uid("rest-" + processId); }, (a, b) -> a); // TODO - does the combiner really make sense?); } @@ -87,13 +90,14 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws public static List parseConfigs(byte[] configJson) throws IOException { return getConfigObjectMapper().readValue( - configJson, - new TypeReference>() { - }); + configJson, + new TypeReference>() { + }); } protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); - protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream results); + protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream results); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJobKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJobKafka.java index 5208faf2..54152891 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJobKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJobKafka.java @@ -12,6 +12,9 @@ package com.cloudera.cyber.enrichment.rest; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; + import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; @@ -21,12 +24,10 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; - public class RestLookupJobKafka extends RestLookupJob { private static final String DEFAULT_GROUP_ID = "enrichment-rest"; + public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files"); new RestLookupJobKafka().createPipeline(Utils.getParamToolsFromProperties(args)).execute("Enrichment - REST"); @@ -34,13 +35,16 @@ public static void main(String[] args) throws Exception { @Override protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params) { - return env.fromSource(FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, DEFAULT_GROUP_ID), - WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); + return env.fromSource( + FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, DEFAULT_GROUP_ID), + WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); } @Override protected void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream results) { - results.sinkTo(new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), DEFAULT_GROUP_ID, params)) - .name("Kafka Sink").uid("kafka-sink"); + results.sinkTo( + new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), DEFAULT_GROUP_ID, + params)) + .name("Kafka Sink").uid("kafka-sink"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequest.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequest.java index 440ee0ea..646c4e4b 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequest.java @@ -13,27 +13,13 @@ package com.cloudera.cyber.enrichment.rest; +import static org.apache.http.conn.ssl.SSLConnectionSocketFactory.STRICT_HOSTNAME_VERIFIER; + import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.text.StringSubstitutor; -import org.apache.flink.util.Preconditions; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.PrivateKeyStrategy; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLContextBuilder; -import org.apache.http.conn.ssl.SSLContexts; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; - -import javax.annotation.Nonnull; -import javax.net.ssl.SSLContext; import java.io.Closeable; import java.io.FileInputStream; import java.io.IOException; @@ -48,13 +34,27 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - -import static org.apache.http.conn.ssl.SSLConnectionSocketFactory.STRICT_HOSTNAME_VERIFIER; +import javax.annotation.Nonnull; +import javax.net.ssl.SSLContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.text.StringSubstitutor; +import org.apache.flink.util.Preconditions; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.conn.ssl.PrivateKeyStrategy; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLContextBuilder; +import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; @Slf4j public abstract class RestRequest implements AsyncCacheLoader, Closeable { - public static final String REST_REQUEST_RETURNS_FAILED_STATUS = "Rest request returned a success code but the content indicated failure."; + public static final String REST_REQUEST_RETURNS_FAILED_STATUS = + "Rest request returned a success code but the content indicated failure."; public static final String REST_REQUEST_HTTP_FAILURE = "Rest request failed due to '%s'."; public static final String REST_REQUEST_ERROR_MESSAGE_WITH_KEY = "Rest request url='%s' entity='%s' failed '%s'"; private final String resultJsonPath; @@ -68,8 +68,10 @@ public abstract class RestRequest implements AsyncCacheLoader requestProperties = config.getProperties(); if (requestProperties != null && !requestProperties.isEmpty()) { this.propertySubstitutor = new StringSubstitutor(config.getProperties()); @@ -83,9 +85,10 @@ public RestRequest(RestEnrichmentConfig config) throws Exception { HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); setupConnectionSocketFactory(config, httpClientBuilder); this.client = httpClientBuilder.build(); - this.cache = Caffeine.newBuilder().maximumSize(config.getCacheSize()). - expireAfter(new RestRequestCacheExpiry(config.getSuccessCacheExpirationSeconds(), TimeUnit.SECONDS, config.getFailureCacheExpirationSeconds(), TimeUnit.SECONDS)) - .buildAsync(this); + this.cache = Caffeine.newBuilder().maximumSize(config.getCacheSize()) + .expireAfter(new RestRequestCacheExpiry(config.getSuccessCacheExpirationSeconds(), + TimeUnit.SECONDS, config.getFailureCacheExpirationSeconds(), TimeUnit.SECONDS)) + .buildAsync(this); } private static PrivateKeyStrategy getPrivateKeyStrategy(final String aliasName) { @@ -96,7 +99,8 @@ private static PrivateKeyStrategy getPrivateKeyStrategy(final String aliasName) } } - private static KeyStore loadKeyStore(String path, String password) throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException { + private static KeyStore loadKeyStore(String path, String password) + throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException { KeyStore store = KeyStore.getInstance("JKS"); store.load(new FileInputStream(path), password.toCharArray()); @@ -127,7 +131,8 @@ private void setupConnectionSocketFactory(RestEnrichmentConfig config, HttpClien // load the key store KeyStore keyStore = loadKeyStore(tlsConfig.getKeyStorePath(), tlsConfig.getKeyStorePassword()); - sslContextBuilder = sslContextBuilder.loadKeyMaterial(keyStore, tlsConfig.getKeyPassword().toCharArray(), getPrivateKeyStrategy(tlsConfig.getKeyAlias())); + sslContextBuilder = sslContextBuilder.loadKeyMaterial(keyStore, tlsConfig.getKeyPassword().toCharArray(), + getPrivateKeyStrategy(tlsConfig.getKeyAlias())); //Building the SSLContext using the build() method SSLContext sslcontext = sslContextBuilder.build(); @@ -140,7 +145,8 @@ private void setupConnectionSocketFactory(RestEnrichmentConfig config, HttpClien private void createHeaders(RestEnrichmentConfig config) { Map configuredHeaders = config.getHeaders(); if (configuredHeaders != null) { - this.headers = configuredHeaders.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> substituteProperties(e.getValue()))); + this.headers = configuredHeaders.entrySet().stream().collect( + Collectors.toMap(Map.Entry::getKey, e -> substituteProperties(e.getValue()))); } EndpointAuthorizationConfig authConfig = config.getAuthorization(); @@ -151,10 +157,13 @@ private void createHeaders(RestEnrichmentConfig config) { } protected void addErrorToResult(RestRequestKey key, RestRequestResult result, String errorMessage) { - result.getErrors().add(String.format(REST_REQUEST_ERROR_MESSAGE_WITH_KEY, key.getRestUri().toASCIIString(), key.getEntity(), errorMessage)); + result.getErrors() + .add(String.format(REST_REQUEST_ERROR_MESSAGE_WITH_KEY, key.getRestUri().toASCIIString(), key.getEntity(), + errorMessage)); } - protected CompletableFuture executeRequest(Executor executor, RestRequestKey key, HttpUriRequest request) { + protected CompletableFuture executeRequest(Executor executor, RestRequestKey key, + HttpUriRequest request) { return CompletableFuture.supplyAsync(() -> { headers.forEach(request::addHeader); @@ -178,7 +187,8 @@ protected CompletableFuture executeRequest(Executor executor, addErrorToResult(key, result, REST_REQUEST_RETURNS_FAILED_STATUS); } } else { - addErrorToResult(key, result, String.format(REST_REQUEST_HTTP_FAILURE, response.getStatusLine().toString())); + addErrorToResult(key, result, + String.format(REST_REQUEST_HTTP_FAILURE, response.getStatusLine().toString())); } } catch (Exception e) { addErrorToResult(key, result, e.getMessage()); @@ -199,7 +209,8 @@ public CompletableFuture getResult(boolean reportKeyErrors, M @Nonnull @Override - public abstract CompletableFuture asyncLoad(@Nonnull RestRequestKey key, @Nonnull Executor executor); + public abstract CompletableFuture asyncLoad(@Nonnull RestRequestKey key, + @Nonnull Executor executor); protected abstract RestRequestKey getKey(Map variables); diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestCacheExpiry.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestCacheExpiry.java index 0f0a3d28..51de022e 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestCacheExpiry.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestCacheExpiry.java @@ -13,22 +13,23 @@ package com.cloudera.cyber.enrichment.rest; import com.github.benmanes.caffeine.cache.Expiry; - -import javax.annotation.Nonnull; import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; public class RestRequestCacheExpiry implements Expiry { private final long successCacheDurationNanos; private final long failureCacheDurationNanos; - public RestRequestCacheExpiry(long successCacheDuration, TimeUnit successCacheDurationUnits, long failureCacheDuration, TimeUnit failureCacheDurationUnits) { + public RestRequestCacheExpiry(long successCacheDuration, TimeUnit successCacheDurationUnits, + long failureCacheDuration, TimeUnit failureCacheDurationUnits) { this.successCacheDurationNanos = TimeUnit.NANOSECONDS.convert(successCacheDuration, successCacheDurationUnits); this.failureCacheDurationNanos = TimeUnit.NANOSECONDS.convert(failureCacheDuration, failureCacheDurationUnits); } @Override - public long expireAfterCreate(@Nonnull RestRequestKey cacheKey, @Nonnull RestRequestResult cacheValue, long cacheDuration) { + public long expireAfterCreate(@Nonnull RestRequestKey cacheKey, @Nonnull RestRequestResult cacheValue, + long cacheDuration) { if (cacheValue.getErrors().isEmpty()) { return successCacheDurationNanos; } else { @@ -37,12 +38,14 @@ public long expireAfterCreate(@Nonnull RestRequestKey cacheKey, @Nonnull RestReq } @Override - public long expireAfterUpdate(@Nonnull RestRequestKey cacheKey, @Nonnull RestRequestResult cacheValue, long currentTime, long cacheDuration) { + public long expireAfterUpdate(@Nonnull RestRequestKey cacheKey, @Nonnull RestRequestResult cacheValue, + long currentTime, long cacheDuration) { return cacheDuration; } @Override - public long expireAfterRead(@Nonnull RestRequestKey cacheKey, @Nonnull RestRequestResult cacheValue, long currentTime, long cacheDuration) { + public long expireAfterRead(@Nonnull RestRequestKey cacheKey, @Nonnull RestRequestResult cacheValue, + long currentTime, long cacheDuration) { return cacheDuration; } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestKey.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestKey.java index 998a0379..46f82380 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestKey.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestKey.java @@ -13,6 +13,13 @@ package com.cloudera.cyber.enrichment.rest; import com.google.common.base.Joiner; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import lombok.Data; import lombok.EqualsAndHashCode; import org.apache.commons.lang3.StringUtils; @@ -20,10 +27,6 @@ import org.apache.commons.text.lookup.StringLookup; import org.apache.commons.text.lookup.StringLookupFactory; -import java.net.URI; -import java.util.*; -import java.util.stream.Collectors; - @Data @EqualsAndHashCode public class RestRequestKey { @@ -59,9 +62,9 @@ public String replace(String template) { undefinedVariables.clear(); String replacedString = substitutor.replace(template); // filter out any variables where the template provided a default value - undefinedVariables = undefinedVariables.stream(). - filter((var_name) -> replacedString.contains("${".concat(var_name))). - collect(Collectors.toSet()); + undefinedVariables = undefinedVariables.stream() + .filter((varName) -> replacedString.contains("${".concat(varName))) + .collect(Collectors.toSet()); return replacedString; } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestResult.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestResult.java index c8c5c19d..755d23af 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestResult.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestRequestResult.java @@ -12,15 +12,14 @@ package com.cloudera.cyber.enrichment.rest; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; @Data @EqualsAndHashCode diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/TlsConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/TlsConfig.java index d7322a6a..6d5a11f2 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/TlsConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/TlsConfig.java @@ -12,9 +12,12 @@ package com.cloudera.cyber.enrichment.rest; -import lombok.*; - import java.io.Serializable; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @Builder @@ -22,7 +25,8 @@ @NoArgsConstructor(force = true, access = AccessLevel.PUBLIC) public class TlsConfig implements Serializable { - /** When useTLS=true, the path to the .jks file containing the trusted certificate authorities. + /** + * When useTLS=true, the path to the .jks file containing the trusted certificate authorities. */ private String trustStorePath; diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequestTest.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequestTest.java index 30aedfb0..0811bc40 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequestTest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequestTest.java @@ -18,6 +18,16 @@ import com.cloudera.cyber.TestUtils; import com.cloudera.cyber.enrichment.rest.impl.MockRestServer; import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import lombok.Data; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; @@ -26,10 +36,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - public class AsyncHttpRequestTest extends RestRequestTest { private static MockRestServer mockRestServer; @@ -135,12 +141,12 @@ public void testGetError() throws Exception { String mockHostAndPort = mockRestServer.getMockHostAndPort(); String url = String.format("http://%s/asset?id=56", mockHostAndPort); String exceptionMessage = String.format(RestRequest.REST_REQUEST_HTTP_FAILURE, "HTTP/1.1 503 Service Unavailable"); - List expectedQualityMessages = Collections.singletonList(DataQualityMessage.builder(). - feature(AsyncHttpRequest.REST_ENRICHMENT_FEATURE). - field(MockRestServer.ASSET_PREFIX). - level(DataQualityMessageLevel.ERROR.name()). - message(String.format(AsyncHttpRequest.REST_REQUEST_FAILED_QUALITY_MESSAGE, url, "null", exceptionMessage)). - build()); + List expectedQualityMessages = Collections.singletonList(DataQualityMessage.builder() + .feature(AsyncHttpRequest.REST_ENRICHMENT_FEATURE) + .field(MockRestServer.ASSET_PREFIX) + .level(DataQualityMessageLevel.ERROR.name()) + .message(String.format(AsyncHttpRequest.REST_REQUEST_FAILED_QUALITY_MESSAGE, url, "null", exceptionMessage)) + .build()); testGetAsset(MockRestServer.ASSET_ID_FIELD_NAME, MockRestServer.SERVER_ERROR_ASSET_ID, expectedExtensions, expectedQualityMessages); } @@ -153,12 +159,12 @@ public void testExactSourceWithUndefinedVariablesInUrlAndEntity() throws Excepti }}; String messageText = String.format(RestRequestKey.UNDEFINED_VARIABLE_MESSAGE, MockRestServer.ASSET_ID_FIELD_NAME, RestRequestKey.URL_SOURCE); - List expectedQualityMessages = Collections.singletonList(DataQualityMessage.builder(). - feature(AsyncHttpRequest.REST_ENRICHMENT_FEATURE). - field(MockRestServer.ASSET_PREFIX). - level(DataQualityMessageLevel.ERROR.name()). - message(messageText). - build()); + List expectedQualityMessages = Collections.singletonList(DataQualityMessage.builder() + .feature(AsyncHttpRequest.REST_ENRICHMENT_FEATURE) + .field(MockRestServer.ASSET_PREFIX) + .level(DataQualityMessageLevel.ERROR.name()) + .message(messageText) + .build()); testGetAsset(badFieldName, MockRestServer.ASSET_ID, expectedExtensions, expectedQualityMessages); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/GetRestRequestTest.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/GetRestRequestTest.java index 7d0a11cc..85fba053 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/GetRestRequestTest.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/GetRestRequestTest.java @@ -14,14 +14,13 @@ import com.cloudera.cyber.enrichment.rest.impl.MockRestServer; import com.google.common.collect.Lists; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; -import java.util.HashMap; -import java.util.Map; - @Slf4j // tests will run either with or without tls when junit runs derived classes @Ignore @@ -44,8 +43,8 @@ public void testSimpleGet() throws Exception { @Test public void testPropertiesNull() throws Exception { - RestEnrichmentConfig config = mockRestServer.getBuilder(null).sources(Lists.newArrayList(MockRestServer.USER_SOURCE)). - endpointTemplate(String.format("%s://%s/user?name=${name}", mockRestServer.getMockProtocol(), mockRestServer.getMockHostAndPort())).build(); + RestEnrichmentConfig config = mockRestServer.getBuilder(null).sources(Lists.newArrayList(MockRestServer.USER_SOURCE)) + .endpointTemplate(String.format("%s://%s/user?name=${name}", mockRestServer.getMockProtocol(), mockRestServer.getMockHostAndPort())).build(); Map variables = new HashMap() {{ put("name", "Chris"); diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/impl/MockRestServer.java b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/impl/MockRestServer.java index f3f4b534..4a0e51df 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/impl/MockRestServer.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/impl/MockRestServer.java @@ -12,9 +12,24 @@ package com.cloudera.cyber.enrichment.rest.impl; +import static org.mockserver.integration.ClientAndServer.startClientAndServer; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; +import static org.mockserver.model.JsonBody.json; + +import com.cloudera.cyber.enrichment.rest.BasicAuthorizationConfig; +import com.cloudera.cyber.enrichment.rest.BearerTokenAuthorizationConfig; +import com.cloudera.cyber.enrichment.rest.RestEnrichmentConfig; +import com.cloudera.cyber.enrichment.rest.RestEnrichmentMethod; +import com.cloudera.cyber.enrichment.rest.TlsConfig; import com.google.common.base.Joiner; -import com.cloudera.cyber.enrichment.rest.*; import com.google.common.collect.Lists; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.net.ssl.HttpsURLConnection; import lombok.AllArgsConstructor; import lombok.Data; import org.apache.commons.codec.binary.Base64; @@ -28,15 +43,6 @@ import org.mockserver.socket.PortFactory; import org.mockserver.socket.tls.KeyStoreFactory; -import javax.net.ssl.HttpsURLConnection; -import java.nio.charset.StandardCharsets; -import java.util.*; - -import static org.mockserver.integration.ClientAndServer.startClientAndServer; -import static org.mockserver.model.HttpRequest.request; -import static org.mockserver.model.HttpResponse.response; -import static org.mockserver.model.JsonBody.json; - @Data public class MockRestServer { @@ -102,8 +108,8 @@ public MockRestServer(boolean enableTlsMutualAuth) { } public RestEnrichmentConfig.RestEnrichmentConfigBuilder getBuilder(HashMap properties) { - RestEnrichmentConfig.RestEnrichmentConfigBuilder builder = RestEnrichmentConfig.builder(). - capacity(3); + RestEnrichmentConfig.RestEnrichmentConfigBuilder builder = RestEnrichmentConfig.builder() + .capacity(3); boolean mutualTlsAuth = mockServer.isSecure(); if (mutualTlsAuth) { configureTLS(builder, null); @@ -118,13 +124,13 @@ public RestEnrichmentConfig.RestEnrichmentConfigBuilder getBuilder(HashMap() {{ + return getBuilder(authTokenProperties) + .sources(Lists.newArrayList(DNS_SOURCE)) + .prefix(DGA_MODEL_PREFIX) + .endpointTemplate("${protocol}://${server}/model") + .method(RestEnrichmentMethod.POST) + .headers(new HashMap() {{ put(HttpHeaders.CONTENT_TYPE, "application/json"); - }}). - authorization(BearerTokenAuthorizationConfig.builder().bearerTokenTemplate("${bearer_token}").build()). - entityTemplate("{\"accessKey\":\"${access_key}\",\"request\":{\"domain\":\"${domain}\"}}"). - successJsonPath("$['success']"). - resultsJsonPath("$['response']"); + }}) + .authorization(BearerTokenAuthorizationConfig.builder().bearerTokenTemplate("${bearer_token}").build()) + .entityTemplate("{\"accessKey\":\"${access_key}\",\"request\":{\"domain\":\"${domain}\"}}") + .successJsonPath("$['success']") + .resultsJsonPath("$['response']"); } public RestEnrichmentConfig.RestEnrichmentConfigBuilder configureGetUserRequest() { - return getBuilder(new HashMap<>()). - sources(Lists.newArrayList(USER_SOURCE)). - prefix(USER_PREFIX). - endpointTemplate("${protocol}://${server}/user?name=${name}"); + return getBuilder(new HashMap<>()) + .sources(Lists.newArrayList(USER_SOURCE)) + .prefix(USER_PREFIX) + .endpointTemplate("${protocol}://${server}/user?name=${name}"); } public RestEnrichmentConfig.RestEnrichmentConfigBuilder configureGetAssetRequest() { @@ -160,22 +166,22 @@ public RestEnrichmentConfig.RestEnrichmentConfigBuilder configureGetAssetRequest put("password", BASIC_PASSWORD); }}; - return getBuilder(properties). - sources(Lists.newArrayList(ASSET_SOURCE)). - prefix(ASSET_PREFIX). - endpointTemplate("${protocol}://${server}/asset?id=${id}"). - authorization(BasicAuthorizationConfig.builder(). - userNameTemplate("${user}"). - passwordTemplate("${password}") + return getBuilder(properties) + .sources(Lists.newArrayList(ASSET_SOURCE)) + .prefix(ASSET_PREFIX) + .endpointTemplate("${protocol}://${server}/asset?id=${id}") + .authorization(BasicAuthorizationConfig.builder() + .userNameTemplate("${user}") + .passwordTemplate("${password}") .build()); } private void initializeGetExpectations() { mockServer.when( - request(). - withMethod("GET"). - withPath("/user"). - withQueryStringParameter(USER_NAME_PROPERTY, USER_NAME) + request() + .withMethod("GET") + .withPath("/user") + .withQueryStringParameter(USER_NAME_PROPERTY, USER_NAME) ) .respond( response() @@ -184,10 +190,10 @@ private void initializeGetExpectations() { ); mockServer.when( - request(). - withMethod("GET"). - withPath("/asset"). - withHeader(HttpHeaders.AUTHORIZATION, createBasicAuth()) + request() + .withMethod("GET") + .withPath("/asset") + .withHeader(HttpHeaders.AUTHORIZATION, createBasicAuth()) .withQueryStringParameter("id", ASSET_ID) ) .respond( @@ -196,10 +202,10 @@ private void initializeGetExpectations() { .withBody(String.format("{%s: '%s'}", ASSET_LOCATION_PROPERTY, ASSET_LOCATION)) ); mockServer.when( - request(). - withMethod("GET"). - withPath("/asset"). - withHeader(HttpHeaders.AUTHORIZATION, createBasicAuth()) + request() + .withMethod("GET") + .withPath("/asset") + .withHeader(HttpHeaders.AUTHORIZATION, createBasicAuth()) .withQueryStringParameter("id", SERVER_ERROR_ASSET_ID) ) .respond( @@ -223,12 +229,12 @@ private void initializePostExpectations() { private void addModelResultExpectation(boolean success, String domain, boolean legit) { mockServer.when( - request(). - withMethod("POST"). - withPath("/model"). - withContentType(MediaType.APPLICATION_JSON). - withHeader(HttpHeaders.AUTHORIZATION, "Bearer ".concat(BEARER_TOKEN)). - withBody(json(String.format("{\"accessKey\":\"%s\",\"request\":{\"domain\":\"%s\"}}", ACCESS_KEY, domain), MatchType.STRICT)) + request() + .withMethod("POST") + .withPath("/model") + .withContentType(MediaType.APPLICATION_JSON) + .withHeader(HttpHeaders.AUTHORIZATION, "Bearer ".concat(BEARER_TOKEN)) + .withBody(json(String.format("{\"accessKey\":\"%s\",\"request\":{\"domain\":\"%s\"}}", ACCESS_KEY, domain), MatchType.STRICT)) ) .respond( response() @@ -239,12 +245,12 @@ private void addModelResultExpectation(boolean success, String domain, boolean l private void addModelResultClientErrorExpectation(String domain, int statusCode) { mockServer.when( - request(). - withMethod("POST"). - withPath("/model"). - withContentType(MediaType.APPLICATION_JSON). - withHeader(HttpHeaders.AUTHORIZATION, "Bearer ".concat(BEARER_TOKEN)). - withBody(json(String.format("{\"accessKey\":\"%s\",\"request\":{\"domain\":\"%s\"}}", ACCESS_KEY, domain), MatchType.STRICT)) + request() + .withMethod("POST") + .withPath("/model") + .withContentType(MediaType.APPLICATION_JSON) + .withHeader(HttpHeaders.AUTHORIZATION, "Bearer ".concat(BEARER_TOKEN)) + .withBody(json(String.format("{\"accessKey\":\"%s\",\"request\":{\"domain\":\"%s\"}}", ACCESS_KEY, domain), MatchType.STRICT)) ) .respond( response() diff --git a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/StellarEnrichmentJob.java b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/StellarEnrichmentJob.java index 68bc1ba6..edaf7890 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/StellarEnrichmentJob.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/StellarEnrichmentJob.java @@ -15,6 +15,11 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.enrichemnt.stellar.functions.flink.StellarEnrichMapFunction; import com.cloudera.cyber.enrichment.geocode.IpGeoJob; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; @@ -27,12 +32,6 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - @Slf4j public abstract class StellarEnrichmentJob { protected static final String STELLAR_ENRICHMENT_GROUP_ID = "enrichment-stellar"; @@ -40,8 +39,10 @@ public abstract class StellarEnrichmentJob { public static final String PARAMS_CONFIG_DIR = "stellar.config.dir"; - public static DataStream enrich(DataStream source, Map sensorConfigs, String geoDatabasePath, String asnDatabasePath) { - return source.map(new StellarEnrichMapFunction(sensorConfigs, geoDatabasePath, asnDatabasePath)).name("Stellar Enrichment Mapper").uid("stellar-flat-map"); + public static DataStream enrich(DataStream source, Map sensorConfigs, + String geoDatabasePath, String asnDatabasePath) { + return source.map(new StellarEnrichMapFunction(sensorConfigs, geoDatabasePath, asnDatabasePath)) + .name("Stellar Enrichment Mapper").uid("stellar-flat-map"); } protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws IOException { @@ -49,7 +50,9 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws String configFileDir = params.getRequired(PARAMS_CONFIG_DIR); DataStream source = createSource(env, params); - DataStream result = enrich(source, loadFiles(configFileDir), params.getRequired(IpGeoJob.PARAM_GEO_DATABASE_PATH), params.getRequired(IpGeoJob.PARAM_ASN_DATABASE_PATH)); + DataStream result = + enrich(source, loadFiles(configFileDir), params.getRequired(IpGeoJob.PARAM_GEO_DATABASE_PATH), + params.getRequired(IpGeoJob.PARAM_ASN_DATABASE_PATH)); writeResults(env, params, result); return env; } @@ -58,10 +61,13 @@ public static Map loadFiles(String pathToFiles) throws IOExcepti Map result = new HashMap<>(); FileSystem fileSystem = new Path(pathToFiles).getFileSystem(); final FileStatus[] fileStatusList = fileSystem.listStatus(new Path(pathToFiles)); - if (ArrayUtils.isEmpty(fileStatusList)){ - throw new RuntimeException(String.format("Provided config directory doesn't exist or empty [%s]!", pathToFiles)); + if (ArrayUtils.isEmpty(fileStatusList)) { + throw new RuntimeException( + String.format("Provided config directory doesn't exist or empty [%s]!", pathToFiles)); } - Path[] configPaths = Arrays.stream(fileStatusList).map(FileStatus::getPath).filter(path -> FilenameUtils.isExtension(path.getName(), "json")).toArray(Path[]::new); + Path[] configPaths = Arrays.stream(fileStatusList).map(FileStatus::getPath) + .filter(path -> FilenameUtils.isExtension(path.getName(), "json")) + .toArray(Path[]::new); for (Path path : configPaths) { result.put(FilenameUtils.removeExtension(path.getName()), readConfigFile(path)); } @@ -78,7 +84,8 @@ private static String readConfigFile(Path path) throws IOException { } } - protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream result); + protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream result); protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); diff --git a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/StellarEnrichmentJobKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/StellarEnrichmentJobKafka.java index 0f55b0a4..3e5ede34 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/StellarEnrichmentJobKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/StellarEnrichmentJobKafka.java @@ -12,6 +12,9 @@ package com.cloudera.cyber.enrichemnt.stellar; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; + import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; @@ -21,9 +24,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; - public class StellarEnrichmentJobKafka extends StellarEnrichmentJob { @@ -35,15 +35,16 @@ public static void main(String[] args) throws Exception { @Override protected void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream reduction) { - reduction.sinkTo(new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), STELLAR_ENRICHMENT_GROUP_ID, params)) - .name("Kafka Sink").uid("kafka-sink"); + reduction.sinkTo(new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), + STELLAR_ENRICHMENT_GROUP_ID, params)) + .name("Kafka Sink").uid("kafka-sink"); } @Override public DataStream createSource(StreamExecutionEnvironment env, ParameterTool params) { return env.fromSource( - FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, STELLAR_ENRICHMENT_GROUP_ID), - WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); + FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, STELLAR_ENRICHMENT_GROUP_ID), + WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/adapter/MetronGeoEnrichmentAdapter.java b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/adapter/MetronGeoEnrichmentAdapter.java index 35fc0988..b840cda4 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/adapter/MetronGeoEnrichmentAdapter.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/adapter/MetronGeoEnrichmentAdapter.java @@ -17,17 +17,15 @@ import com.cloudera.cyber.enrichment.geocode.IpGeoJob; import com.cloudera.cyber.enrichment.geocode.impl.IpGeoEnrichment; import com.cloudera.cyber.enrichment.geocode.impl.types.MetronGeoEnrichmentFields; -import lombok.extern.slf4j.Slf4j; -import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase; -import org.apache.metron.enrichment.cache.CacheKey; -import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; -import org.json.simple.JSONObject; - import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.metron.enrichment.cache.CacheKey; +import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; +import org.json.simple.JSONObject; @Slf4j public class MetronGeoEnrichmentAdapter implements EnrichmentAdapter, Serializable { @@ -39,7 +37,8 @@ public JSONObject enrich(CacheKey value) { JSONObject enriched; HashMap result = new HashMap<>(); List qualityMessages = new ArrayList<>(); - ipGeoEnrichment.lookup(MetronGeoEnrichment::new, value.getField(), value.getValue(), MetronGeoEnrichmentFields.values(), result, qualityMessages); + ipGeoEnrichment.lookup(MetronGeoEnrichment::new, value.getField(), value.getValue(), + MetronGeoEnrichmentFields.values(), result, qualityMessages); if (result.isEmpty()) { return new JSONObject(); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/functions/GeoEnrichmentFunctions.java b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/functions/GeoEnrichmentFunctions.java index 5c8c042a..a98bd68b 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/functions/GeoEnrichmentFunctions.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/functions/GeoEnrichmentFunctions.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.cloudera.cyber.enrichemnt.stellar.functions; import com.cloudera.cyber.enrichment.MetronGeoEnrichment; @@ -24,6 +25,11 @@ import com.cloudera.cyber.enrichment.geocode.impl.types.GeoFields; import com.cloudera.cyber.enrichment.geocode.impl.types.MetronGeoEnrichmentFields; import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -33,27 +39,25 @@ import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - @Slf4j @NoArgsConstructor(access = AccessLevel.PRIVATE) public class GeoEnrichmentFunctions { - private static final Map ASN_RESULT_TO_METRON_KEYS = ImmutableMap.of("org", "autonomous_system_organization", - "number", "autonomous_system_number", - "mask","network"); - @Stellar(name = "GET" - , namespace = "GEO" - , description = "Look up an IPV4 address and returns geographic information about it" - , params = { - "ip - The IPV4 address to lookup", - "fields - Optional list of GeoIP fields to grab. Options are locID, country, city, postalCode, dmaCode, latitude, longitude, location_point" - } - , returns = "If a Single field is requested a string of the field, If multiple fields a map of string of the fields, and null otherwise" + private static final Map ASN_RESULT_TO_METRON_KEYS = + ImmutableMap.of("org", "autonomous_system_organization", + "number", "autonomous_system_number", + "mask", "network"); + + @Stellar(name = "GET", + namespace = "GEO", + description = "Look up an IPV4 address and returns geographic information about it", + params = { + "ip - The IPV4 address to lookup", + "fields - Optional list of GeoIP fields to grab. " + + "Options are locID, country, city, postalCode, dmaCode, latitude, longitude, location_point" + }, + returns = "If a Single field is requested a string of the field, " + + "If multiple fields a map of string of the fields, and null otherwise" ) public static class GeoGet implements StellarFunction { boolean initialized = false; @@ -74,15 +78,19 @@ public Object apply(List args, Context context) throws ParseException { if (ip == null || ip.trim().isEmpty()) { return null; } - ipGeoEnrichment.lookup(MetronGeoEnrichment::new, null, ip, MetronGeoEnrichmentFields.values(), result, Collections.emptyList()); + ipGeoEnrichment.lookup(MetronGeoEnrichment::new, null, ip, MetronGeoEnrichmentFields.values(), result, + Collections.emptyList()); return result; } else if (args.size() == 2 && args.get(1) instanceof List) { // If fields are provided, return just those fields. String ip = (String) args.get(0); @SuppressWarnings("unchecked") - GeoFields[] geoFieldSet = ((List) args.get(1)).stream().map(MetronGeoEnrichmentFields::fromSingularName).toArray(GeoFields[]::new); - ipGeoEnrichment.lookup(MetronGeoEnrichment::new, null, ip, geoFieldSet, result, Collections.emptyList()); + GeoFields[] geoFieldSet = + ((List) args.get(1)).stream().map(MetronGeoEnrichmentFields::fromSingularName) + .toArray(GeoFields[]::new); + ipGeoEnrichment.lookup(MetronGeoEnrichment::new, null, ip, geoFieldSet, result, + Collections.emptyList()); if (geoFieldSet.length == 1) { if (MapUtils.isNotEmpty(result)) { return result.values().iterator().next(); @@ -107,7 +115,7 @@ public void initialize(Context context) { @SuppressWarnings("unchecked") private static Map getConfig(Context context) { return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false) - .orElse(new HashMap<>()); + .orElse(new HashMap<>()); } @Override @@ -117,14 +125,16 @@ public boolean isInitialized() { } - @Stellar(name = "GET" - , namespace = "ASN" - , description = "Look up an IPV4 address and returns Autonomous System Number information about it" - , params = { - "ip - The IPV4 address to lookup", - "fields - Optional list of ASN fields to grab. Options are autonomous_system_organization, autonomous_system_number, network" - } - , returns = "If a single field is requested a string of the field, If multiple fields a map of string of the fields, and null otherwise" + @Stellar(name = "GET", + namespace = "ASN", + description = "Look up an IPV4 address and returns Autonomous System Number information about it", + params = { + "ip - The IPV4 address to lookup", + "fields - Optional list of ASN fields to grab. " + + "Options are autonomous_system_organization, autonomous_system_number, network" + }, + returns = "If a single field is requested a string of the field, " + + "If multiple fields a map of string of the fields, and null otherwise" ) public static class AsnGet implements StellarFunction { @@ -138,7 +148,7 @@ public Object apply(List args, Context context) throws ParseException { } if (args.size() > 2) { throw new IllegalArgumentException( - "ASN_GET received more arguments than expected: " + args.size()); + "ASN_GET received more arguments than expected: " + args.size()); } HashMap result = new HashMap<>(); @@ -166,7 +176,8 @@ public Object apply(List args, Context context) throws ParseException { return result.get(fields.get(0)); } else if (MapUtils.isNotEmpty(result)) { // If multiple fields are requested, return all of them - return result.entrySet().stream().filter(entry -> fields.contains(entry.getKey())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return result.entrySet().stream().filter(entry -> fields.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } } @@ -174,7 +185,9 @@ public Object apply(List args, Context context) throws ParseException { } private static HashMap convertToMetronKeys(Map result) { - return result.entrySet().stream().collect(Collectors.toMap(e -> ASN_RESULT_TO_METRON_KEYS.getOrDefault(e.getKey(), e.getKey()), Map.Entry::getValue, (prev, next) -> next, HashMap::new)); + return result.entrySet().stream().collect( + Collectors.toMap(e -> ASN_RESULT_TO_METRON_KEYS.getOrDefault(e.getKey(), e.getKey()), + Map.Entry::getValue, (prev, next) -> next, HashMap::new)); } @Override @@ -189,7 +202,7 @@ public void initialize(Context context) { @SuppressWarnings("unchecked") private static Map getConfig(Context context) { return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false) - .orElse(new HashMap<>()); + .orElse(new HashMap<>()); } @Override diff --git a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/functions/flink/StellarEnrichMapFunction.java b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/functions/flink/StellarEnrichMapFunction.java index 6deea933..12e6211a 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/functions/flink/StellarEnrichMapFunction.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-stellar/src/main/java/com/cloudera/cyber/enrichemnt/stellar/functions/flink/StellarEnrichMapFunction.java @@ -12,6 +12,8 @@ package com.cloudera.cyber.enrichemnt.stellar.functions.flink; +import static org.apache.metron.stellar.common.Constants.STELLAR_CONTEXT_CONF; + import com.cloudera.cyber.DataQualityMessage; import com.cloudera.cyber.DataQualityMessageLevel; import com.cloudera.cyber.Message; @@ -20,6 +22,13 @@ import com.cloudera.cyber.enrichment.geocode.IpGeoJob; import com.cloudera.cyber.hbase.HbaseConfiguration; import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.ObjectUtils; import org.apache.flink.api.common.functions.RichMapFunction; @@ -39,22 +48,13 @@ import org.apache.metron.stellar.dsl.StellarFunctions; import org.json.simple.JSONObject; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.apache.metron.stellar.common.Constants.STELLAR_CONTEXT_CONF; - @Slf4j public class StellarEnrichMapFunction extends RichMapFunction { private static final String GEO_ADAPTER_NAME = "geo"; private static final String STELLAR_ADAPTER_NAME = "stellar"; - private static final Map ADAPTER_NAME_FEATURE = ImmutableMap.of(GEO_ADAPTER_NAME, "stellar_geo_feature", STELLAR_ADAPTER_NAME, "stellar_feature"); + private static final Map ADAPTER_NAME_FEATURE = + ImmutableMap.of(GEO_ADAPTER_NAME, "stellar_geo_feature", STELLAR_ADAPTER_NAME, "stellar_feature"); private final Map stringEnrichmentConfigs; private final String geoDatabasePath; private final String asnDatabasePath; @@ -91,10 +91,14 @@ private Map initSensorConfigs() throws java.io.I Map result = new HashMap<>(); for (Map.Entry stringConfigEntry : stringEnrichmentConfigs.entrySet()) { EnrichmentConfigurations enrichmentConfigurations = new EnrichmentConfigurations(); - enrichmentConfigurations.updateGlobalConfig(ImmutableMap.of(IpGeoJob.PARAM_GEO_DATABASE_PATH, geoDatabasePath, IpGeoJob.PARAM_ASN_DATABASE_PATH, asnDatabasePath)); + enrichmentConfigurations.updateGlobalConfig( + ImmutableMap.of(IpGeoJob.PARAM_GEO_DATABASE_PATH, geoDatabasePath, IpGeoJob.PARAM_ASN_DATABASE_PATH, + asnDatabasePath)); SensorEnrichmentConfig sensorEnrichmentConfig; - sensorEnrichmentConfig = JSONUtils.INSTANCE.load(stringConfigEntry.getValue(), SensorEnrichmentConfig.class); - sensorEnrichmentConfig.getConfiguration().putIfAbsent(STELLAR_CONTEXT_CONF, initializeStellarContext(enrichmentConfigurations)); + sensorEnrichmentConfig = + JSONUtils.INSTANCE.load(stringConfigEntry.getValue(), SensorEnrichmentConfig.class); + sensorEnrichmentConfig.getConfiguration().putIfAbsent(STELLAR_CONTEXT_CONF, + initializeStellarContext(enrichmentConfigurations)); enrichmentConfigurations.updateSensorEnrichmentConfig(ENRICHMENT, sensorEnrichmentConfig); result.put(stringConfigEntry.getKey(), sensorEnrichmentConfig); @@ -103,14 +107,18 @@ private Map initSensorConfigs() throws java.io.I } private Context initializeStellarContext(EnrichmentConfigurations enrichmentConfigurations) { - org.apache.hadoop.conf.Configuration hbaseConfig = HBaseConfigurationUtil.deserializeConfiguration(serializedHbaseConfig, HBaseConfigurationUtil.createHBaseConf()); + org.apache.hadoop.conf.Configuration hbaseConfig = + HBaseConfigurationUtil.deserializeConfiguration(serializedHbaseConfig, + HBaseConfigurationUtil.createHBaseConf()); Context stellarContext = new Context.Builder() - .with(Context.Capabilities.GLOBAL_CONFIG, () -> ImmutableMap.builder() - .putAll(enrichmentConfigurations.getGlobalConfig()) - .put(HbaseConfiguration.HBASE_CONFIG_NAME, hbaseConfig) - .build()) - .with(Context.Capabilities.STELLAR_CONFIG, enrichmentConfigurations::getGlobalConfig) - .build(); + .with(Context.Capabilities.GLOBAL_CONFIG, () -> ImmutableMap.builder() + .putAll( + enrichmentConfigurations.getGlobalConfig()) + .put(HbaseConfiguration.HBASE_CONFIG_NAME, + hbaseConfig) + .build()) + .with(Context.Capabilities.STELLAR_CONFIG, enrichmentConfigurations::getGlobalConfig) + .build(); StellarFunctions.initialize(stellarContext); return stellarContext; @@ -120,8 +128,10 @@ private Context initializeStellarContext(EnrichmentConfigurations enrichmentConf private Map> generateTasks(JSONObject message, SensorEnrichmentConfig config ) { Map> streamMessageMap = new HashMap<>(); - Map enrichmentFieldMap = EnrichmentStrategies.ENRICHMENT.getUnderlyingConfig(config).getFieldMap(); - Map fieldToHandler = EnrichmentStrategies.ENRICHMENT.getUnderlyingConfig(config).getEnrichmentConfigs(); + Map enrichmentFieldMap = + EnrichmentStrategies.ENRICHMENT.getUnderlyingConfig(config).getFieldMap(); + Map fieldToHandler = + EnrichmentStrategies.ENRICHMENT.getUnderlyingConfig(config).getEnrichmentConfigs(); Set enrichmentTypes = new HashSet<>(enrichmentFieldMap.keySet()); @@ -136,11 +146,13 @@ private Map> generateTasks(JSONObject message, SensorEn //How this is split depends on the ConfigHandler List enrichmentObject = retriever.getType() - .splitByFields(message - , fields - , field -> ((EnrichmentStrategy) EnrichmentStrategies.ENRICHMENT).fieldToEnrichmentKey(enrichmentType, field) - , retriever - ); + .splitByFields(message, + fields, + field -> ((EnrichmentStrategy) + EnrichmentStrategies.ENRICHMENT).fieldToEnrichmentKey( + enrichmentType, field), + retriever + ); streamMessageMap.put(enrichmentType, enrichmentObject); } return streamMessageMap; @@ -171,7 +183,9 @@ public Message map(Message message) { } } - private void collectData(SensorEnrichmentConfig sensorEnrichmentConfig, Map.Entry> task, EnrichmentAdapter adapter, JSONObject m, Map tmpMap, List dataQualityMessages) { + private void collectData(SensorEnrichmentConfig sensorEnrichmentConfig, Map.Entry> task, + EnrichmentAdapter adapter, JSONObject m, Map tmpMap, + List dataQualityMessages) { for (Object fieldValue : m.entrySet()) { Map.Entry fieldValueEntry = (Map.Entry) fieldValue; String field = fieldValueEntry.getKey(); @@ -184,7 +198,9 @@ private void collectData(SensorEnrichmentConfig sensorEnrichmentConfig, Map.Entr }); } catch (Exception e) { log.error("Error with " + task.getKey() + " failed: " + e.getMessage(), e); - MessageUtils.addQualityMessage(dataQualityMessages, DataQualityMessageLevel.ERROR, "Error with " + task.getKey() + " failed: " + e.getMessage(), field, ADAPTER_NAME_FEATURE.get(task.getKey())); + MessageUtils.addQualityMessage(dataQualityMessages, DataQualityMessageLevel.ERROR, + "Error with " + task.getKey() + " failed: " + e.getMessage(), field, + ADAPTER_NAME_FEATURE.get(task.getKey())); } } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQConfig.java b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQConfig.java index 2eb7ba25..34eef124 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQConfig.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQConfig.java @@ -12,13 +12,12 @@ package com.cloudera.cyber.enrichment.threatq; +import java.io.Serializable; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.io.Serializable; - @Data @Builder @AllArgsConstructor diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQEntry.java b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQEntry.java index 9caaa9a7..c80c58ed 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQEntry.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQEntry.java @@ -12,23 +12,26 @@ package com.cloudera.cyber.enrichment.threatq; +import static com.cloudera.cyber.AvroTypes.toListOf; +import static com.cloudera.cyber.AvroTypes.utf8toStringMap; + import com.cloudera.cyber.EnrichmentEntry; import com.fasterxml.jackson.annotation.JsonFormat; import com.google.common.base.Joiner; import com.hortonworks.registries.schemaregistry.serdes.avro.exceptions.AvroException; -import lombok.*; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.specific.SpecificRecordBase; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; - import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; - -import static com.cloudera.cyber.AvroTypes.toListOf; -import static com.cloudera.cyber.AvroTypes.utf8toStringMap; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.specific.SpecificRecordBase; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; @EqualsAndHashCode(callSuper = true) @Data @@ -36,20 +39,21 @@ @AllArgsConstructor @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) +@SuppressWarnings("checkstyle:MemberName") public class ThreatQEntry extends SpecificRecordBase { private String indicator; private List tq_sources; @JsonFormat - (shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd hh:mm:ss") + (shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd hh:mm:ss") private Date tq_created_at; private Float tq_score; private String tq_type; private String tq_saved_search; @JsonFormat - (shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd hh:mm:ss") + (shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd hh:mm:ss") private Date tq_updated_at; @JsonFormat - (shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd hh:mm:ss") + (shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd hh:mm:ss") private Date tq_touched_at; private Long tq_id; @@ -64,49 +68,52 @@ public static String createKey(String indicatorType, String indicator) { public static EnrichmentEntry toEnrichmentEntry(ThreatQEntry threatQEntry) { return EnrichmentEntry.builder() - .ts(threatQEntry.tq_updated_at.getTime()) - .key(createKey(threatQEntry.getTq_type(), threatQEntry.getIndicator())) - .type("threatq") - .entries(new HashMap() {{ - putAll(threatQEntry.getTq_attributes()); - put("id", threatQEntry.getTq_id().toString()); - put("createdAt",Long.toString(threatQEntry.getTq_created_at().getTime())); - put("updatedAt", Long.toString(threatQEntry.getTq_updated_at().getTime())); - put("touchedAt", Long.toString(threatQEntry.getTq_touched_at().getTime())); + .ts(threatQEntry.tq_updated_at.getTime()) + .key(createKey(threatQEntry.getTq_type(), threatQEntry.getIndicator())) + .type("threatq") + .entries(new HashMap() { + { + putAll(threatQEntry.getTq_attributes()); + put("id", threatQEntry.getTq_id().toString()); + put("createdAt", Long.toString(threatQEntry.getTq_created_at().getTime())); + put("updatedAt", Long.toString(threatQEntry.getTq_updated_at().getTime())); + put("touchedAt", Long.toString(threatQEntry.getTq_touched_at().getTime())); - put("status", threatQEntry.getTq_status()); - put("url", threatQEntry.getTq_url()); - if (threatQEntry.getTq_tags() != null && threatQEntry.getTq_tags().size() > 0) { - put("tags", Joiner.on(",").join(threatQEntry.getTq_tags())); - } - put("type", threatQEntry.getTq_type()); - put("savedSearch", threatQEntry.getTq_saved_search()); - if (threatQEntry.getTq_sources() != null && threatQEntry.getTq_sources().size() > 0) { - put("sources", threatQEntry.getTq_sources().toString()); - } - put("score", threatQEntry.getTq_score().toString()); - }}) - .build(); + put("status", threatQEntry.getTq_status()); + put("url", threatQEntry.getTq_url()); + if (threatQEntry.getTq_tags() != null && threatQEntry.getTq_tags().size() > 0) { + put("tags", Joiner.on(",").join(threatQEntry.getTq_tags())); + } + put("type", threatQEntry.getTq_type()); + put("savedSearch", threatQEntry.getTq_saved_search()); + if (threatQEntry.getTq_sources() != null + && threatQEntry.getTq_sources().size() > 0) { + put("sources", threatQEntry.getTq_sources().toString()); + } + put("score", threatQEntry.getTq_score().toString()); + } + }) + .build(); } public static Schema SCHEMA$ = SchemaBuilder - .record(ThreatQEntry.class.getName()) - .namespace(ThreatQEntry.class.getPackage().getName()) - .fields() - .requiredString("indicator") - .name("tq_sources").type(SchemaBuilder.array().items(SchemaBuilder.builder().stringType())).noDefault() - .requiredLong("tq_created_at") - .requiredFloat("tq_score") - .requiredString("tq_type") - .requiredString("tq_saved_search") - .requiredLong("tq_updated_at") - .requiredLong("tq_touched_at") - .requiredLong("tq_id") - .name("tq_attributes").type(SchemaBuilder.map().values(SchemaBuilder.builder().stringType())).noDefault() - .requiredString("tq_status") - .requiredString("tq_url") - .name("tq_tags").type(SchemaBuilder.array().items(SchemaBuilder.builder().stringType())).noDefault() - .endRecord(); + .record(ThreatQEntry.class.getName()) + .namespace(ThreatQEntry.class.getPackage().getName()) + .fields() + .requiredString("indicator") + .name("tq_sources").type(SchemaBuilder.array().items(SchemaBuilder.builder().stringType())).noDefault() + .requiredLong("tq_created_at") + .requiredFloat("tq_score") + .requiredString("tq_type") + .requiredString("tq_saved_search") + .requiredLong("tq_updated_at") + .requiredLong("tq_touched_at") + .requiredLong("tq_id") + .name("tq_attributes").type(SchemaBuilder.map().values(SchemaBuilder.builder().stringType())).noDefault() + .requiredString("tq_status") + .requiredString("tq_url") + .name("tq_tags").type(SchemaBuilder.array().items(SchemaBuilder.builder().stringType())).noDefault() + .endRecord(); @Override public Schema getSchema() { @@ -115,41 +122,82 @@ public Schema getSchema() { @Override public Object get(int field) { - switch(field) { - case 0: return indicator; - case 1: return tq_sources; - case 2: return tq_created_at; - case 3: return tq_score; - case 4: return tq_type; - case 5: return tq_saved_search; - case 6: return tq_updated_at; - case 7: return tq_touched_at; - case 8: return tq_id; - case 9: return tq_attributes; - case 10: return tq_status; - case 11: return tq_url; - case 12: return tq_tags; - default: throw new AvroException("Bad Index"); + switch (field) { + case 0: + return indicator; + case 1: + return tq_sources; + case 2: + return tq_created_at; + case 3: + return tq_score; + case 4: + return tq_type; + case 5: + return tq_saved_search; + case 6: + return tq_updated_at; + case 7: + return tq_touched_at; + case 8: + return tq_id; + case 9: + return tq_attributes; + case 10: + return tq_status; + case 11: + return tq_url; + case 12: + return tq_tags; + default: + throw new AvroException("Bad Index"); } } @Override public void put(int field, Object value) { - switch(field) { - case 0: this.indicator = value.toString(); break; - case 1: this.tq_sources = toListOf(String.class, value); break; - case 2: this.tq_created_at = value instanceof Date ? (Date) value : new Date((long)value); break; - case 3: this.tq_score = (Float) value; break; - case 4: this.tq_type = value.toString(); break; - case 5: this.tq_saved_search = value.toString(); break; - case 6: this.tq_updated_at = value instanceof Date ? (Date) value : new Date((long)value); break; - case 7: this.tq_touched_at = value instanceof Date ? (Date) value : new Date((long)value); break; - case 8: this.tq_id = (Long) value; break; - case 9: this.tq_attributes = utf8toStringMap(value); break; - case 10: this.tq_status = value.toString(); break; - case 11: this.tq_url = value.toString(); break; - case 12: this.tq_tags = toListOf(String.class, value); break; - default: throw new AvroException("Bad Index"); + switch (field) { + case 0: + this.indicator = value.toString(); + break; + case 1: + this.tq_sources = toListOf(String.class, value); + break; + case 2: + this.tq_created_at = value instanceof Date ? (Date) value : new Date((long) value); + break; + case 3: + this.tq_score = (Float) value; + break; + case 4: + this.tq_type = value.toString(); + break; + case 5: + this.tq_saved_search = value.toString(); + break; + case 6: + this.tq_updated_at = value instanceof Date ? (Date) value : new Date((long) value); + break; + case 7: + this.tq_touched_at = value instanceof Date ? (Date) value : new Date((long) value); + break; + case 8: + this.tq_id = (Long) value; + break; + case 9: + this.tq_attributes = utf8toStringMap(value); + break; + case 10: + this.tq_status = value.toString(); + break; + case 11: + this.tq_url = value.toString(); + break; + case 12: + this.tq_tags = toListOf(String.class, value); + break; + default: + throw new AvroException("Bad Index"); } } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQHBaseMap.java b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQHBaseMap.java index aad26da4..84039ff9 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQHBaseMap.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQHBaseMap.java @@ -12,6 +12,8 @@ package com.cloudera.cyber.enrichment.threatq; +import static java.util.stream.Collectors.toMap; + import com.cloudera.cyber.Message; import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfig; @@ -19,14 +21,11 @@ import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig; import com.cloudera.cyber.hbase.AbstractHbaseMapFunction; import com.cloudera.cyber.hbase.LookupKey; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.util.Collector; - import java.util.Collections; import java.util.List; import java.util.Map; - -import static java.util.stream.Collectors.toMap; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.util.Collector; @Slf4j public class ThreatQHBaseMap extends AbstractHbaseMapFunction { @@ -38,7 +37,8 @@ public ThreatQHBaseMap(List configs, EnrichmentsConfig enrichment super(); this.configs = configs; log.info("ThreatQ Configuration: {}", configs); - this.threatqStorage = enrichmentStorageConfig.getStorageForEnrichmentType(EnrichmentFieldsConfig.THREATQ_ENRICHMENT_NAME); + this.threatqStorage = + enrichmentStorageConfig.getStorageForEnrichmentType(EnrichmentFieldsConfig.THREATQ_ENRICHMENT_NAME); } @Override @@ -47,17 +47,21 @@ public void processElement(Message message, Context context, Collector collector.collect(message); } else { Map results = configs.stream() - .map(config -> { - String f = config.getField(); - if (!message.getExtensions().containsKey(f)) { - return Collections.emptyMap(); - } - String k = ThreatQEntry.createKey(config.getIndicatorType(), message.getExtensions().get(f)); - LookupKey lookup = threatqStorage.getFormat().getLookupBuilder().build(threatqStorage, EnrichmentFieldsConfig.THREATQ_ENRICHMENT_NAME, k); - return hbaseLookup(message.getTs(), lookup, f + ".threatq"); - }) - .flatMap(m -> m.entrySet().stream()) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + .map(config -> { + String f = config.getField(); + if (!message.getExtensions().containsKey(f)) { + return Collections.emptyMap(); + } + String k = ThreatQEntry.createKey(config.getIndicatorType(), + message.getExtensions().get(f)); + LookupKey lookup = threatqStorage.getFormat().getLookupBuilder() + .build(threatqStorage, + EnrichmentFieldsConfig.THREATQ_ENRICHMENT_NAME, + k); + return hbaseLookup(message.getTs(), lookup, f + ".threatq"); + }) + .flatMap(m -> m.entrySet().stream()) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); collector.collect(MessageUtils.addFields(message, results)); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQJob.java b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQJob.java index ed5cb68c..b9bc34d7 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQJob.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQJob.java @@ -12,10 +12,18 @@ package com.cloudera.cyber.enrichment.threatq; +import static com.cloudera.cyber.enrichment.ConfigUtils.PARAMS_CONFIG_FILE; +import static com.cloudera.cyber.enrichment.hbase.HbaseJob.PARAMS_ENRICHMENT_CONFIG; + import com.cloudera.cyber.Message; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig; import com.cloudera.cyber.flink.FlinkUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.datastream.DataStream; @@ -23,15 +31,6 @@ import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Collections; -import java.util.List; - -import static com.cloudera.cyber.enrichment.ConfigUtils.PARAMS_CONFIG_FILE; -import static com.cloudera.cyber.enrichment.hbase.HbaseJob.PARAMS_ENRICHMENT_CONFIG; - @Slf4j public abstract class ThreatQJob { @@ -39,16 +38,18 @@ public static DataStream enrich(DataStream source, List configs, EnrichmentsConfig enrichmentStorageConfig ) { - return source.process(new ThreatQHBaseMap(configs, enrichmentStorageConfig)).name("Apply ThreatQ").uid("threatq-enrich"); + return source.process(new ThreatQHBaseMap(configs, enrichmentStorageConfig)).name("Apply ThreatQ") + .uid("threatq-enrich"); } public static List parseConfigs(byte[] configJson) throws IOException { return new ObjectMapper().readValue( - configJson, - new TypeReference>() { - }); + configJson, + new TypeReference>() { + }); } + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") public StreamExecutionEnvironment createPipeline(ParameterTool params) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FlinkUtils.setupEnv(env, params); @@ -60,29 +61,33 @@ public StreamExecutionEnvironment createPipeline(ParameterTool params) throws Ex List configs = parseConfigs(configJson); log.info("ThreatQ Configs {}", configs); - EnrichmentsConfig enrichmentsStorageConfig = EnrichmentsConfig.load(params.getRequired(PARAMS_ENRICHMENT_CONFIG)); + EnrichmentsConfig enrichmentsStorageConfig = + EnrichmentsConfig.load(params.getRequired(PARAMS_ENRICHMENT_CONFIG)); writeEnrichments(enrichmentSource, enrichmentsStorageConfig, params); String enrichmentsStorageConfigFileName = params.get(PARAMS_ENRICHMENT_CONFIG); - EnrichmentsConfig enrichmentStorageConfig = new EnrichmentsConfig(Collections.emptyMap(), Collections.emptyMap()); + EnrichmentsConfig enrichmentStorageConfig = + new EnrichmentsConfig(Collections.emptyMap(), Collections.emptyMap()); if (enrichmentsStorageConfigFileName != null) { enrichmentStorageConfig = EnrichmentsConfig.load(enrichmentsStorageConfigFileName); } - DataStream pipeline = enrich(source,configs, enrichmentStorageConfig); + DataStream pipeline = enrich(source, configs, enrichmentStorageConfig); writeResults(env, params, pipeline); return env; } - protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream reduction); + protected abstract void writeResults(StreamExecutionEnvironment env, ParameterTool params, + DataStream reduction); public abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); - protected abstract DataStream createEnrichmentSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract DataStream createEnrichmentSource(StreamExecutionEnvironment env, + ParameterTool params); public abstract void writeEnrichments(DataStream enrichmentSource, - EnrichmentsConfig enrichmentsConfig, - ParameterTool params); + EnrichmentsConfig enrichmentsConfig, + ParameterTool params); } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQJobKafka.java b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQJobKafka.java index 18063c4e..c18d1aa3 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQJobKafka.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQJobKafka.java @@ -12,6 +12,10 @@ package com.cloudera.cyber.enrichment.threatq; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; +import static com.cloudera.cyber.flink.Utils.readKafkaProperties; + import com.cloudera.cyber.Message; import com.cloudera.cyber.commands.EnrichmentCommand; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfig; @@ -19,6 +23,7 @@ import com.cloudera.cyber.enrichment.hbase.writer.HbaseEnrichmentCommandSink; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; +import java.util.Properties; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.kafka.source.KafkaSource; @@ -26,12 +31,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.Properties; - -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_OUTPUT; -import static com.cloudera.cyber.flink.Utils.readKafkaProperties; - public class ThreatQJobKafka extends ThreatQJob { private static final String PARAMS_TOPIC_ENRICHMENT_INPUT = "enrichment.topic.input"; @@ -45,36 +44,39 @@ public static void main(String[] args) throws Exception { @Override protected void writeResults(StreamExecutionEnvironment env, ParameterTool params, DataStream reduction) { - reduction.sinkTo(new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), THREATQ_ENRICHMENT_GROUP_ID,params)) - .name("Kafka Sink").uid("kafka-sink"); + reduction.sinkTo(new FlinkUtils<>(Message.class).createKafkaSink(params.getRequired(PARAMS_TOPIC_OUTPUT), + THREATQ_ENRICHMENT_GROUP_ID, params)) + .name("Kafka Sink").uid("kafka-sink"); } @Override public DataStream createSource(StreamExecutionEnvironment env, ParameterTool params) { return env.fromSource( - FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, THREATQ_ENRICHMENT_GROUP_ID), - WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); + FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, THREATQ_ENRICHMENT_GROUP_ID), + WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); } @Override - protected DataStream createEnrichmentSource(StreamExecutionEnvironment env, ParameterTool params) { + protected DataStream createEnrichmentSource(StreamExecutionEnvironment env, + ParameterTool params) { String topic = params.getRequired(PARAMS_TOPIC_ENRICHMENT_INPUT); Properties kafkaProperties = readKafkaProperties(params, THREATQ_PARSER_GROUP_ID, true); KafkaSource source = FlinkUtils.createKafkaStringSource(topic, kafkaProperties); - return env.fromSource(source, WatermarkStrategy.noWatermarks(), "ThreatQ Input"). - uid("kafka-enrichment-source"). - flatMap(new ThreatQParserFlatMap()); + return env.fromSource(source, WatermarkStrategy.noWatermarks(), "ThreatQ Input") + .uid("kafka-enrichment-source") + .flatMap(new ThreatQParserFlatMap()); } public void writeEnrichments(DataStream enrichmentSource, - EnrichmentsConfig enrichmentsConfig, - ParameterTool params) { - String tableName = enrichmentsConfig.getStorageForEnrichmentType(EnrichmentFieldsConfig.THREATQ_ENRICHMENT_NAME).getHbaseTableName(); + EnrichmentsConfig enrichmentsConfig, + ParameterTool params) { + String tableName = enrichmentsConfig.getStorageForEnrichmentType(EnrichmentFieldsConfig.THREATQ_ENRICHMENT_NAME) + .getHbaseTableName(); - enrichmentSource.addSink(new HbaseEnrichmentCommandSink(tableName, enrichmentsConfig, params)). - name("ThreatQ HBase Writer").uid("threatq-hbase"); + enrichmentSource.addSink(new HbaseEnrichmentCommandSink(tableName, enrichmentsConfig, params)) + .name("ThreatQ HBase Writer").uid("threatq-hbase"); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQParser.java b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQParser.java index f849ae69..c1cc60b7 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQParser.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQParser.java @@ -14,8 +14,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -24,11 +22,12 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; @Slf4j public class ThreatQParser { - private static DateFormat df = new SimpleDateFormat("dd-MM-yyyy hh:mm:ss"); - private static ObjectMapper om; + private static final DateFormat df = new SimpleDateFormat("dd-MM-yyyy hh:mm:ss"); + private static final ObjectMapper om; static { om = new ObjectMapper(); @@ -36,25 +35,23 @@ public class ThreatQParser { } /** - * The threatQ sample is almost JSON, but broken in a variety of ways + * The threatQ sample is almost JSON, but broken in a variety of ways. * - * @param input - * @return */ public static Stream parse(InputStream input) throws IOException { Stream output = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)).lines(); return output.map(str -> str - .replace("'", "\"") - .replace(": None,", ":null,")) - .map(str -> { - try { - return om.readValue(str, ThreatQEntry.class); - } catch (JsonProcessingException e) { - log.error("Parsing failed", e); - return null; - } - }) - .filter(r -> r != null); + .replace("'", "\"") + .replace(": None,", ":null,")) + .map(str -> { + try { + return om.readValue(str, ThreatQEntry.class); + } catch (JsonProcessingException e) { + log.error("Parsing failed", e); + return null; + } + }) + .filter(r -> r != null); } } diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQParserFlatMap.java b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQParserFlatMap.java index 45345caf..a8a81392 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQParserFlatMap.java +++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/src/main/java/com/cloudera/cyber/enrichment/threatq/ThreatQParserFlatMap.java @@ -14,19 +14,18 @@ import com.cloudera.cyber.commands.CommandType; import com.cloudera.cyber.commands.EnrichmentCommand; -import org.apache.flink.api.common.functions.FlatMapFunction; -import org.apache.flink.util.Collector; - import java.io.ByteArrayInputStream; import java.util.Collections; +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.util.Collector; public class ThreatQParserFlatMap implements FlatMapFunction { @Override public void flatMap(String s, Collector collector) throws Exception { ThreatQParser.parse(new ByteArrayInputStream(s.getBytes())).forEach(out -> - collector.collect(EnrichmentCommand.builder().headers(Collections.emptyMap()). - type(CommandType.ADD). - payload(ThreatQEntry.toEnrichmentEntry(out)).build())); + collector.collect(EnrichmentCommand.builder().headers(Collections.emptyMap()) + .type(CommandType.ADD) + .payload(ThreatQEntry.toEnrichmentEntry(out)).build())); } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java index 485f87ed..16e3866e 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,30 +22,28 @@ import java.io.Serializable; import java.util.Map; - import org.apache.metron.enrichment.cache.CacheKey; +import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; +public abstract class AbstractCIFAdapter implements EnrichmentAdapter, Serializable { + + private static final long serialVersionUID = -5040559164824221816L; + protected static final Logger LOG = LoggerFactory + .getLogger(AbstractCIFAdapter.class); + + @Override + public abstract boolean initializeAdapter(Map config); + + public abstract String enrichByIP(String metadata); + + public abstract String enrichByDomain(String metadata); + + public abstract String enrichByEmail(String metadata); + + @Override + public void cleanup() { -public abstract class AbstractCIFAdapter implements EnrichmentAdapter,Serializable{ - - /** - * - */ - private static final long serialVersionUID = -5040559164824221816L; - protected static final Logger LOG = LoggerFactory - .getLogger(AbstractCIFAdapter.class); - - @Override - abstract public boolean initializeAdapter(Map config); - abstract public String enrichByIP(String metadata); - abstract public String enrichByDomain(String metadata); - abstract public String enrichByEmail(String metadata); - - @Override - public void cleanup() { - - } + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java index dc4b8b0c..8a28f246 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,18 +20,15 @@ package org.apache.metron.enrichment.adapters.cif; +import com.cloudera.cyber.hbase.HbaseConfiguration; import java.io.IOException; import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; - -import com.cloudera.cyber.hbase.HbaseConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -43,109 +42,111 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@SuppressWarnings("unchecked") -public class CIFHbaseAdapter implements EnrichmentAdapter,Serializable { - private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final long serialVersionUID = 1L; - private String _tableName; - private Table table; - private String _quorum; - private String _port; +@SuppressWarnings({"unchecked", "checkstyle:MemberName"}) +public class CIFHbaseAdapter implements EnrichmentAdapter, Serializable { + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final long serialVersionUID = 1L; + private final String _tableName; + private Table table; + private final String _quorum; + private final String _port; - public CIFHbaseAdapter(String quorum, String port, String tableName) { - _quorum = quorum; - _port = port; - _tableName = tableName; - } + public CIFHbaseAdapter(String quorum, String port, String tableName) { + _quorum = quorum; + _port = port; + _tableName = tableName; + } - @Override - public void logAccess(CacheKey value) { + @Override + public void logAccess(CacheKey value) { - } + } - @Override - public JSONObject enrich(CacheKey k) { - String metadata = k.coerceValue(String.class); - JSONObject output = new JSONObject(); - LOGGER.debug("=======Looking Up For: {}", metadata); - output.putAll(getCIFObject(metadata)); + @Override + public JSONObject enrich(CacheKey k) { + String metadata = k.coerceValue(String.class); + JSONObject output = new JSONObject(); + LOGGER.debug("=======Looking Up For: {}", metadata); + output.putAll(getCIFObject(metadata)); - return output; - } + return output; + } - @SuppressWarnings({ "rawtypes", "deprecation" }) - protected Map getCIFObject(String key) { + @SuppressWarnings({"rawtypes"}) + protected Map getCIFObject(String key) { - LOGGER.debug("=======Pinging HBase For: {}", key); + LOGGER.debug("=======Pinging HBase For: {}", key); - Get get = new Get(key.getBytes(StandardCharsets.UTF_8)); - Result rs; - Map output = new HashMap(); + Get get = new Get(key.getBytes(StandardCharsets.UTF_8)); + Result rs; + Map output = new HashMap(); - try { - rs = table.get(get); + try { + rs = table.get(get); - for (Cell cell : rs.rawCells()) { - output.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()), "Y"); - } + for (Cell cell : rs.rawCells()) { + output.put( + Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()), + "Y"); + } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return output; - } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return output; + } - @Override - public boolean initializeAdapter(Map config) { + @Override + public boolean initializeAdapter(Map config) { - // Initialize HBase Table - Configuration conf = null; - conf = HbaseConfiguration.configureHbase(); - conf.set("hbase.zookeeper.quorum", _quorum); - conf.set("hbase.zookeeper.property.clientPort", _port); + // Initialize HBase Table + Configuration conf = null; + conf = HbaseConfiguration.configureHbase(); + conf.set("hbase.zookeeper.quorum", _quorum); + conf.set("hbase.zookeeper.property.clientPort", _port); - try { - LOGGER.debug("=======Connecting to HBASE==========="); - LOGGER.debug("=======ZOOKEEPER = {}", conf.get("hbase.zookeeper.quorum")); - Connection connection = ConnectionFactory.createConnection(conf); - table = connection.getTable(TableName.valueOf(_tableName)); - return true; - } catch (IOException e) { - LOGGER.debug("=======Unable to Connect to HBASE==========="); - e.printStackTrace(); - } + try { + LOGGER.debug("=======Connecting to HBASE==========="); + LOGGER.debug("=======ZOOKEEPER = {}", conf.get("hbase.zookeeper.quorum")); + Connection connection = ConnectionFactory.createConnection(conf); + table = connection.getTable(TableName.valueOf(_tableName)); + return true; + } catch (IOException e) { + LOGGER.debug("=======Unable to Connect to HBASE==========="); + e.printStackTrace(); + } - return false; - } + return false; + } - @Override - public void updateAdapter(Map config) { - } + @Override + public void updateAdapter(Map config) { + } - public String enrichByIP(String metadata) { - return null; - } + public String enrichByIP(String metadata) { + return null; + } - public String enrichByDomain(String metadata) { - return null; - } + public String enrichByDomain(String metadata) { + return null; + } - public String enrichByEmail(String metadata) { - return null; - } + public String enrichByEmail(String metadata) { + return null; + } - @Override - public void cleanup() { + @Override + public void cleanup() { - } + } - @Override - public String getOutputPrefix(CacheKey value) { - return value.getField(); - } + @Override + public String getOutputPrefix(CacheKey value) { + return value.getField(); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java index 243d1f6e..55ec655b 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.geo; import java.io.Serializable; @@ -29,42 +32,42 @@ import org.slf4j.LoggerFactory; public class GeoAdapter implements EnrichmentAdapter, Serializable { - protected static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @Override - public void logAccess(CacheKey value) { - } + protected static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @Override - public String getOutputPrefix(CacheKey value) { - return value.getField(); - } + @Override + public void logAccess(CacheKey value) { + } - @Override - public JSONObject enrich(CacheKey value) { - JSONObject enriched = new JSONObject(); - Optional> result = GeoLiteCityDatabase.INSTANCE.get(value.coerceValue(String.class)); - if(!result.isPresent()) { - return new JSONObject(); + @Override + public String getOutputPrefix(CacheKey value) { + return value.getField(); } - enriched = new JSONObject(result.get()); - _LOG.trace("GEO Enrichment success: {}", enriched); - return enriched; - } + @Override + public JSONObject enrich(CacheKey value) { + JSONObject enriched = new JSONObject(); + Optional> result = GeoLiteCityDatabase.INSTANCE.get(value.coerceValue(String.class)); + if (!result.isPresent()) { + return new JSONObject(); + } + + enriched = new JSONObject(result.get()); + _LOG.trace("GEO Enrichment success: {}", enriched); + return enriched; + } - @Override - public boolean initializeAdapter(Map config) { - GeoLiteCityDatabase.INSTANCE.update((String)config.get(GeoLiteCityDatabase.GEO_HDFS_FILE)); - return true; - } + @Override + public boolean initializeAdapter(Map config) { + GeoLiteCityDatabase.INSTANCE.update((String) config.get(GeoLiteCityDatabase.GEO_HDFS_FILE)); + return true; + } - @Override - public void updateAdapter(Map config) { - GeoLiteCityDatabase.INSTANCE.updateIfNecessary(config); - } + @Override + public void updateAdapter(Map config) { + GeoLiteCityDatabase.INSTANCE.updateIfNecessary(config); + } - @Override - public void cleanup() { - } + @Override + public void cleanup() { + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java index 0efc1fea..65ce1fe5 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,31 +22,30 @@ import java.io.Serializable; import java.util.Map; - import org.apache.metron.enrichment.cache.CacheKey; +import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; - public abstract class AbstractHostAdapter implements EnrichmentAdapter, - Serializable{ - - /** - * Adapter to attach reputation information to the telemetry message - */ - private static final long serialVersionUID = 8280523289446309728L; - protected static final Logger LOG = LoggerFactory - .getLogger(AbstractHostAdapter.class); - - @Override - abstract public boolean initializeAdapter(Map config); - @Override - abstract public JSONObject enrich(CacheKey metadata); - - @Override - public void cleanup() { - - } + Serializable { + + /** + * Adapter to attach reputation information to the telemetry message. + */ + private static final long serialVersionUID = 8280523289446309728L; + protected static final Logger LOG = LoggerFactory + .getLogger(AbstractHostAdapter.class); + + @Override + public abstract boolean initializeAdapter(Map config); + + @Override + public abstract JSONObject enrich(CacheKey metadata); + + @Override + public void cleanup() { + + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java index 9631fa4f..7eb44181 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java @@ -7,78 +7,78 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.host; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; import org.apache.metron.enrichment.cache.CacheKey; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.JSONValue; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - public class HostFromJSONListAdapter extends AbstractHostAdapter { - Map _known_hosts = new HashMap<>(); + @SuppressWarnings("checkstyle:MemberName") + Map _known_hosts = new HashMap<>(); - public HostFromJSONListAdapter(String jsonList) { - JSONArray jsonArray = (JSONArray) JSONValue.parse(jsonList); - Iterator jsonArrayIterator = jsonArray.iterator(); - while(jsonArrayIterator.hasNext()) { - JSONObject jsonObject = (JSONObject) jsonArrayIterator.next(); - String host = (String) jsonObject.remove("ip"); - _known_hosts.put(host, jsonObject); + public HostFromJSONListAdapter(String jsonList) { + JSONArray jsonArray = (JSONArray) JSONValue.parse(jsonList); + Iterator jsonArrayIterator = jsonArray.iterator(); + while (jsonArrayIterator.hasNext()) { + JSONObject jsonObject = (JSONObject) jsonArrayIterator.next(); + String host = (String) jsonObject.remove("ip"); + _known_hosts.put(host, jsonObject); + } } - } - @Override - public String getOutputPrefix(CacheKey value) { - return value.getField(); - } + @Override + public String getOutputPrefix(CacheKey value) { + return value.getField(); + } - @Override - public boolean initializeAdapter(Map config) - { + @Override + public boolean initializeAdapter(Map config) { - if(_known_hosts.size() > 0) - return true; - else - return false; - } + return _known_hosts.size() > 0; + } - @Override - public void updateAdapter(Map config) { - } + @Override + public void updateAdapter(Map config) { + } - @Override - public void logAccess(CacheKey value) { + @Override + public void logAccess(CacheKey value) { - } + } - @SuppressWarnings("unchecked") - @Override - public JSONObject enrich(CacheKey k) { - String metadata = k.coerceValue(String.class); + @SuppressWarnings("unchecked") + @Override + public JSONObject enrich(CacheKey k) { + String metadata = k.coerceValue(String.class); - if(!_known_hosts.containsKey(metadata)) - return new JSONObject(); + if (!_known_hosts.containsKey(metadata)) { + return new JSONObject(); + } - JSONObject enrichment = new JSONObject(); - String prefix = "known_info."; - JSONObject knownInfo = _known_hosts.get(metadata); - for(Object key: knownInfo.keySet()) { - enrichment.put(prefix + key, knownInfo.get(key)); + JSONObject enrichment = new JSONObject(); + String prefix = "known_info."; + JSONObject knownInfo = _known_hosts.get(metadata); + for (Object key : knownInfo.keySet()) { + enrichment.put(prefix + key, knownInfo.get(key)); + } + //enrichment.put("known_info", _known_hosts.get(metadata)); + return enrichment; } - //enrichment.put("known_info", _known_hosts.get(metadata)); - return enrichment; - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java index 5b3c2d82..b94a897f 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,56 +21,52 @@ package org.apache.metron.enrichment.adapters.host; import java.util.Map; - import org.apache.metron.enrichment.cache.CacheKey; import org.json.simple.JSONObject; @SuppressWarnings("serial") public class HostFromPropertiesFileAdapter extends AbstractHostAdapter { - - Map _known_hosts; - - public HostFromPropertiesFileAdapter(Map known_hosts) - { - _known_hosts = known_hosts; - } - - @Override - public boolean initializeAdapter(Map config) - { - - if(_known_hosts.size() > 0) - return true; - else - return false; - } - - @Override - public void updateAdapter(Map config) { - } - - @Override - public String getOutputPrefix(CacheKey value) { - return value.getField(); - } - - @Override - public void logAccess(CacheKey value) { - - } - - @SuppressWarnings("unchecked") + + @SuppressWarnings("checkstyle:MemberName") + Map _known_hosts; + + public HostFromPropertiesFileAdapter(Map knownHosts) { + _known_hosts = knownHosts; + } + + @Override + public boolean initializeAdapter(Map config) { + + return _known_hosts.size() > 0; + } + @Override - public JSONObject enrich(CacheKey metadata) { - - - if(!_known_hosts.containsKey(metadata.getValue())) - return new JSONObject(); - - JSONObject enrichment = new JSONObject(); - enrichment.put("known_info", (JSONObject) _known_hosts.get(metadata.getValue())); - return enrichment; - } - - + public void updateAdapter(Map config) { + } + + @Override + public String getOutputPrefix(CacheKey value) { + return value.getField(); + } + + @Override + public void logAccess(CacheKey value) { + + } + + @SuppressWarnings("unchecked") + @Override + public JSONObject enrich(CacheKey metadata) { + + + if (!_known_hosts.containsKey(metadata.getValue())) { + return new JSONObject(); + } + + JSONObject enrichment = new JSONObject(); + enrichment.put("known_info", _known_hosts.get(metadata.getValue())); + return enrichment; + } + + } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/BaseJdbcConfig.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/BaseJdbcConfig.java index e2e26cce..45158e87 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/BaseJdbcConfig.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/BaseJdbcConfig.java @@ -7,64 +7,67 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.jdbc; import java.io.Serializable; public abstract class BaseJdbcConfig implements JdbcConfig, Serializable { - protected String host; - protected int port = -1; - protected String username; - protected String password; - protected String table = ""; + protected String host; + protected int port = -1; + protected String username; + protected String password; + protected String table = ""; - @Override - public String getHost() { - return host; - } + @Override + public String getHost() { + return host; + } - public void setHost(String host) { - this.host = host; - } + public void setHost(String host) { + this.host = host; + } - public int getPort() { - return port; - } + public int getPort() { + return port; + } - public void setPort(int port) { - this.port = port; - } + public void setPort(int port) { + this.port = port; + } - public String getUsername() { - return username; - } + public String getUsername() { + return username; + } - public void setUsername(String username) { - this.username = username; - } + public void setUsername(String username) { + this.username = username; + } - public String getPassword() { - return password; - } + public String getPassword() { + return password; + } - public void setPassword(String password) { - this.password = password; - } + public void setPassword(String password) { + this.password = password; + } - public String getTable() { - return table; - } + public String getTable() { + return table; + } - public void setTable(String table) { - this.table = table; - } + public void setTable(String table) { + this.table = table; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java index edbe8919..3c72cff2 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java @@ -7,106 +7,112 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.jdbc; +import java.io.Serializable; +import java.net.InetAddress; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; import org.apache.metron.enrichment.cache.CacheKey; import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; -import java.net.InetAddress; -import java.sql.*; -import java.util.Map; - public abstract class JdbcAdapter implements EnrichmentAdapter, Serializable { - protected static final Logger _LOG = LoggerFactory + protected static final Logger _LOG = LoggerFactory .getLogger(JdbcAdapter.class); - protected Connection connection; - protected Statement statement; + protected Connection connection; + protected Statement statement; - private JdbcConfig config; - private String host; + private JdbcConfig config; + private String host; - protected boolean isConnectionClosed() { - boolean isClosed = statement == null || connection == null; - if(!isClosed) { - try { - isClosed = statement.isClosed() || connection.isClosed(); - } catch (SQLException e) { - _LOG.error("Unable to maintain open JDBC connection: {}", e.getMessage(), e); - isClosed = true; - } + protected boolean isConnectionClosed() { + boolean isClosed = statement == null || connection == null; + if (!isClosed) { + try { + isClosed = statement.isClosed() || connection.isClosed(); + } catch (SQLException e) { + _LOG.error("Unable to maintain open JDBC connection: {}", e.getMessage(), e); + isClosed = true; + } + } + return isClosed; } - return isClosed; - } - protected boolean resetConnectionIfNecessary() { - if(isConnectionClosed()) { - this.cleanup(); - return this.initializeAdapter(null); + protected boolean resetConnectionIfNecessary() { + if (isConnectionClosed()) { + this.cleanup(); + return this.initializeAdapter(null); + } + return true; } - return true; - } - public void setStatement(Statement statement) { - this.statement = statement; - } - - public JdbcAdapter withJdbcConfig(JdbcConfig config) { - this.config = config; - this.host = config.getHost(); - return this; - } - - @Override - public boolean initializeAdapter(Map config) { - try { - if (!InetAddress.getByName(host).isReachable(500)) { - throw new Exception("Unable to reach host " + host); - } - Class.forName(this.config.getClassName()); - connection = DriverManager.getConnection(this.config.getJdbcUrl()); - connection.setReadOnly(true); - if (!connection.isValid(0)) { - throw new Exception("Invalid connection string...."); - } - statement = connection.createStatement( - ResultSet.TYPE_SCROLL_INSENSITIVE, - ResultSet.CONCUR_READ_ONLY); - return true; - } catch (Exception e) { - _LOG.error("[Metron] JDBC connection failed....", e); - return false; + public void setStatement(Statement statement) { + this.statement = statement; } - } - @Override - public void cleanup() { - try { - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - _LOG.error("[Metron] JDBC statement close failed....", e); + public JdbcAdapter withJdbcConfig(JdbcConfig config) { + this.config = config; + this.host = config.getHost(); + return this; } - try { - if (connection != null) { - connection.close(); - } + + + @Override + public boolean initializeAdapter(Map config) { + try { + if (!InetAddress.getByName(host).isReachable(500)) { + throw new Exception("Unable to reach host " + host); + } + Class.forName(this.config.getClassName()); + connection = DriverManager.getConnection(this.config.getJdbcUrl()); + connection.setReadOnly(true); + if (!connection.isValid(0)) { + throw new Exception("Invalid connection string...."); + } + statement = connection.createStatement( + ResultSet.TYPE_SCROLL_INSENSITIVE, + ResultSet.CONCUR_READ_ONLY); + return true; + } catch (Exception e) { + _LOG.error("[Metron] JDBC connection failed....", e); + return false; + } } - catch(SQLException e) { - _LOG.error("[Metron] JDBC connection close failed....", e); + + @Override + public void cleanup() { + try { + if (statement != null) { + statement.close(); + } + } catch (SQLException e) { + _LOG.error("[Metron] JDBC statement close failed....", e); + } + try { + if (connection != null) { + connection.close(); + } + } catch (SQLException e) { + _LOG.error("[Metron] JDBC connection close failed....", e); + } } - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcConfig.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcConfig.java index f88cebe8..e161fb9c 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcConfig.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcConfig.java @@ -6,21 +6,26 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.jdbc; public interface JdbcConfig { - public String getClassName(); - public String getJdbcUrl(); - public String getHost(); + String getClassName(); + + String getJdbcUrl(); + + String getHost(); } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfig.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfig.java index 1dbe0051..0dd4cb7b 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfig.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfig.java @@ -6,34 +6,37 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.jdbc; public class MySqlConfig extends BaseJdbcConfig { - @Override - public String getClassName() { - return "com.mysql.jdbc.Driver"; - } + @Override + public String getClassName() { + return "com.mysql.jdbc.Driver"; + } - @Override - public String getJdbcUrl() { - StringBuilder url = new StringBuilder(); - url.append("jdbc:mysql://").append(host); - if (port > 0) { - url.append(":").append(port); + @Override + public String getJdbcUrl() { + StringBuilder url = new StringBuilder(); + url.append("jdbc:mysql://").append(host); + if (port > 0) { + url.append(":").append(port); + } + url.append("/").append(table); + url.append("?user=").append(username); + url.append("&password=").append(password); + return url.toString(); } - url.append("/").append(table); - url.append("?user=").append(username); - url.append("&password=").append(password); - return url.toString(); - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDatabase.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDatabase.java index 148431b2..011c1084 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDatabase.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDatabase.java @@ -10,7 +10,9 @@ * limitations governing your use of the file. */ -package org.apache.metron.enrichment.adapters.maxmind;/* +package org.apache.metron.enrichment.adapters.maxmind; + +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -43,103 +45,107 @@ import org.slf4j.LoggerFactory; public interface MaxMindDatabase { - Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - String EXTENSION_MMDB = ".mmdb"; - String EXTENSION_TAR_GZ = ".tar.gz"; - String EXTENSION_MMDB_GZ = ".mmdb.gz"; + Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + String EXTENSION_MMDB = ".mmdb"; + String EXTENSION_TAR_GZ = ".tar.gz"; + String EXTENSION_MMDB_GZ = ".mmdb.gz"; - /** - * Retrieves the configuration key that holds the HDFS database file location - * @return The configuration key - */ - String getHdfsFileConfig(); + /** + * Retrieves the configuration key that holds the HDFS database file location. + * + * @return The configuration key + */ + String getHdfsFileConfig(); - /** - * Retrieves the default HDFS database file location - * @return The HDFS database file location - */ - String getHdfsFileDefault(); + /** + * Retrieves the default HDFS database file location. + * + * @return The HDFS database file location + */ + String getHdfsFileDefault(); - /** - * Locks any underlying resources to ensure they are smoothly updated without disruption. - * Any callers implementing an update() function should lock during the update to ensure uninterrupted querying. - */ - void lockIfNecessary(); + /** + * Locks any underlying resources to ensure they are smoothly updated without disruption. + * Any callers implementing an update() function should lock during the update to ensure uninterrupted querying. + */ + void lockIfNecessary(); - /** - * Unlocks any underlying resources to ensure the lock is released after an update. - * Any callers implementing an update() function should ensure they've unlocked post update. - */ - void unlockIfNecessary(); + /** + * Unlocks any underlying resources to ensure the lock is released after an update. + * Any callers implementing an update() function should ensure they've unlocked post update. + */ + void unlockIfNecessary(); - /** - * Gets the appropriate database reader for the underlying database that's been loaded - * @return The DatabaseReader for the MaxMind database. - */ - DatabaseReader getReader(); - void setReader(DatabaseReader reader); + /** + * Gets the appropriate database reader for the underlying database that's been loaded. + * + * @return The DatabaseReader for the MaxMind database. + */ + DatabaseReader getReader(); - /** - * Updates the database file, if the configuration points to a new file. - * Implementations may need to be synchronized to avoid issues querying during updates. - * - * @param globalConfig The global configuration that will be used to determine if an update is necessary. - */ - void updateIfNecessary(Map globalConfig); + void setReader(DatabaseReader reader); - /** - * Update the database being queried to one backed by the provided HDFS file. - * Access to the database should be guarded by read locks to avoid disruption while updates are occurring. - * @param hdfsFile The HDFS file path to be used for new queries. - */ - default void update(String hdfsFile) { - // If nothing is set (or it's been unset, use the defaults) - if (hdfsFile == null || hdfsFile.isEmpty()) { - LOG.debug("Using default for {}: {}", getHdfsFileConfig(), getHdfsFileDefault()); - hdfsFile = getHdfsFileDefault(); - } + /** + * Updates the database file, if the configuration points to a new file. + * Implementations may need to be synchronized to avoid issues querying during updates. + * + * @param globalConfig The global configuration that will be used to determine if an update is necessary. + */ + void updateIfNecessary(Map globalConfig); - FileSystem fs = MaxMindDbUtilities.getFileSystem(); + /** + * Update the database being queried to one backed by the provided HDFS file. + * Access to the database should be guarded by read locks to avoid disruption while updates are occurring. + * + * @param hdfsFile The HDFS file path to be used for new queries. + */ + default void update(String hdfsFile) { + // If nothing is set (or it's been unset, use the defaults) + if (hdfsFile == null || hdfsFile.isEmpty()) { + LOG.debug("Using default for {}: {}", getHdfsFileConfig(), getHdfsFileDefault()); + hdfsFile = getHdfsFileDefault(); + } - if (hdfsFile.endsWith(MaxMindDatabase.EXTENSION_MMDB)) { - lockIfNecessary(); - try (BufferedInputStream is = new BufferedInputStream(fs.open(new Path(hdfsFile)))) { - setReader(MaxMindDbUtilities.readNewDatabase(getReader(), hdfsFile, is)); - } catch (IOException e) { - MaxMindDbUtilities.handleDatabaseIOException(hdfsFile, e); - } finally { - unlockIfNecessary(); - } - } else if (hdfsFile.endsWith(MaxMindDatabase.EXTENSION_MMDB_GZ)) { - lockIfNecessary(); - try (GZIPInputStream is = new GZIPInputStream(fs.open(new Path(hdfsFile)))) { - setReader(MaxMindDbUtilities.readNewDatabase(getReader(), hdfsFile, is)); - } catch (IOException e) { - MaxMindDbUtilities.handleDatabaseIOException(hdfsFile, e); - } finally { - unlockIfNecessary(); - } - } else if (hdfsFile.endsWith(MaxMindDatabase.EXTENSION_TAR_GZ)) { - lockIfNecessary(); - try (TarArchiveInputStream is = new TarArchiveInputStream( - new GZIPInputStream(fs.open(new Path(hdfsFile))))) { - // Need to find the mmdb entry. - TarArchiveEntry entry = is.getNextTarEntry(); - while (entry != null) { - if (entry.isFile() && entry.getName().endsWith(MaxMindDatabase.EXTENSION_MMDB)) { - try(InputStream mmdb = new BufferedInputStream(is)) - { // Read directly from tarInput - setReader(MaxMindDbUtilities.readNewDatabase(getReader(), hdfsFile, mmdb)); - break; // Don't care about the other entries, leave immediately + FileSystem fs = MaxMindDbUtilities.getFileSystem(); + + if (hdfsFile.endsWith(MaxMindDatabase.EXTENSION_MMDB)) { + lockIfNecessary(); + try (BufferedInputStream is = new BufferedInputStream(fs.open(new Path(hdfsFile)))) { + setReader(MaxMindDbUtilities.readNewDatabase(getReader(), hdfsFile, is)); + } catch (IOException e) { + MaxMindDbUtilities.handleDatabaseIOException(hdfsFile, e); + } finally { + unlockIfNecessary(); + } + } else if (hdfsFile.endsWith(MaxMindDatabase.EXTENSION_MMDB_GZ)) { + lockIfNecessary(); + try (GZIPInputStream is = new GZIPInputStream(fs.open(new Path(hdfsFile)))) { + setReader(MaxMindDbUtilities.readNewDatabase(getReader(), hdfsFile, is)); + } catch (IOException e) { + MaxMindDbUtilities.handleDatabaseIOException(hdfsFile, e); + } finally { + unlockIfNecessary(); + } + } else if (hdfsFile.endsWith(MaxMindDatabase.EXTENSION_TAR_GZ)) { + lockIfNecessary(); + try (TarArchiveInputStream is = new TarArchiveInputStream( + new GZIPInputStream(fs.open(new Path(hdfsFile))))) { + // Need to find the mmdb entry. + TarArchiveEntry entry = is.getNextTarEntry(); + while (entry != null) { + if (entry.isFile() && entry.getName().endsWith(MaxMindDatabase.EXTENSION_MMDB)) { + try (InputStream mmdb = new BufferedInputStream(is)) { // Read directly from tarInput + setReader(MaxMindDbUtilities.readNewDatabase(getReader(), hdfsFile, mmdb)); + break; // Don't care about the other entries, leave immediately + } + } + entry = is.getNextTarEntry(); + } + } catch (IOException e) { + MaxMindDbUtilities.handleDatabaseIOException(hdfsFile, e); + } finally { + unlockIfNecessary(); } - } - entry = is.getNextTarEntry(); } - } catch (IOException e) { - MaxMindDbUtilities.handleDatabaseIOException(hdfsFile, e); - } finally { - unlockIfNecessary(); - } } - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDbUtilities.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDbUtilities.java index 5ce6ef0a..8c3eb408 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDbUtilities.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/MaxMindDbUtilities.java @@ -10,7 +10,9 @@ * limitations governing your use of the file. */ -package org.apache.metron.enrichment.adapters.maxmind;/* +package org.apache.metron.enrichment.adapters.maxmind; + +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -45,95 +47,103 @@ * A utilities class for working with MaxMind GeoLite2 databases. In particular, when the DB is stored on HDFS. */ public enum MaxMindDbUtilities { - INSTANCE; + INSTANCE; - static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - static InetAddressValidator ipvalidator = new InetAddressValidator(); + static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + static InetAddressValidator ipvalidator = new InetAddressValidator(); - /** - * Determines if an IP is ineligible. In particular, this is used to filter before querying the database, as that requires a readlock to be set. - * @param ip The IP to be tested - * @return true if invalid, false otherwise - */ - public static boolean invalidIp(String ip) { - LOG.trace("Called validateIp({})", ip); - InetAddress addr; - try { - addr = InetAddress.getByName(ip); - } catch (UnknownHostException e) { - LOG.warn("No result found for IP {}", ip, e); - return true; - } - if (isIneligibleAddress(ip, addr)) { - LOG.debug("IP ineligible for lookup {}", ip); - return true; + /** + * Determines if an IP is ineligible. In particular, this is used to filter + * before querying the database, as that requires a readlock to be set. + * + * @param ip The IP to be tested + * @return true if invalid, false otherwise + */ + public static boolean invalidIp(String ip) { + LOG.trace("Called validateIp({})", ip); + InetAddress addr; + try { + addr = InetAddress.getByName(ip); + } catch (UnknownHostException e) { + LOG.warn("No result found for IP {}", ip, e); + return true; + } + if (isIneligibleAddress(ip, addr)) { + LOG.debug("IP ineligible for lookup {}", ip); + return true; + } + return false; } - return false; - } - /** - * Determines if an address isn't eligible for getting appropriate results from the underlying database. - * @param ipStr The IP String - * @param addr The addr to be tested - * @return True if ineligible, false otherwise - */ - public static boolean isIneligibleAddress(String ipStr, InetAddress addr) { - return addr.isAnyLocalAddress() || addr.isLoopbackAddress() - || addr.isSiteLocalAddress() || addr.isMulticastAddress() - || !ipvalidator.isValidInet4Address(ipStr); - } + /** + * Determines if an address isn't eligible for getting appropriate results from the underlying database. + * + * @param ipStr The IP String + * @param addr The addr to be tested + * @return True if ineligible, false otherwise + */ + public static boolean isIneligibleAddress(String ipStr, InetAddress addr) { + return addr.isAnyLocalAddress() || addr.isLoopbackAddress() + || addr.isSiteLocalAddress() || addr.isMulticastAddress() + || !ipvalidator.isValidInet4Address(ipStr); + } - /** - * Logs and rethrows an IOException in a common way across implementations. - * @param hdfsFile The hdfsFile we were trying to read from - * @param e The exception we saw - */ - public static void handleDatabaseIOException(String hdfsFile, IOException e) { - LOG.error("Unable to open new database file {}", hdfsFile, e); - throw new IllegalStateException("Unable to update MaxMind database"); - } + /** + * Logs and rethrows an IOException in a common way across implementations. + * + * @param hdfsFile The hdfsFile we were trying to read from + * @param e The exception we saw + */ + public static void handleDatabaseIOException(String hdfsFile, IOException e) { + LOG.error("Unable to open new database file {}", hdfsFile, e); + throw new IllegalStateException("Unable to update MaxMind database"); + } - /** - * Reads a new Database from a given HDFS file - * @param reader The DatabaseReader to use to read the file - * @param hdfsFile The HDFS file to read - * @param is An InputStream for use with the reader - * @return The DatabaseReader that is set up with the new file - * @throws IOException If there is an issue reading the file. - */ - public static DatabaseReader readNewDatabase(DatabaseReader reader, String hdfsFile, InputStream is) throws IOException { - LOG.info("Update to GeoIP data started with {}", hdfsFile); - // InputStream based DatabaseReaders are always in memory. - DatabaseReader newReader = new DatabaseReader.Builder(is).withCache(new CHMCache()).build(); - // If we've never set a reader, don't close the old one - if (reader != null) { - reader.close(); + /** + * Reads a new Database from a given HDFS file. + * + * @param reader The DatabaseReader to use to read the file + * @param hdfsFile The HDFS file to read + * @param is An InputStream for use with the reader + * @return The DatabaseReader that is set up with the new file + * @throws IOException If there is an issue reading the file. + */ + public static DatabaseReader readNewDatabase(DatabaseReader reader, String hdfsFile, InputStream is) + throws IOException { + LOG.info("Update to GeoIP data started with {}", hdfsFile); + // InputStream based DatabaseReaders are always in memory. + DatabaseReader newReader = new DatabaseReader.Builder(is).withCache(new CHMCache()).build(); + // If we've never set a reader, don't close the old one + if (reader != null) { + reader.close(); + } + LOG.info("Finished update to GeoIP data started with {}", hdfsFile); + return newReader; } - LOG.info("Finished update to GeoIP data started with {}", hdfsFile); - return newReader; - } - /** - * Retrieves the FileSystem - * @return A FileSystem object used to retrieve the the database - */ - public static FileSystem getFileSystem() { - FileSystem fs; - try { - fs = FileSystem.get(new Configuration()); - } catch (IOException e) { - LOG.error("Unable to retrieve get HDFS FileSystem"); - throw new IllegalStateException("Unable to get HDFS FileSystem"); + /** + * Retrieves the FileSystem. + * + * @return A FileSystem object used to retrieve the the database + */ + public static FileSystem getFileSystem() { + FileSystem fs; + try { + fs = FileSystem.get(new Configuration()); + } catch (IOException e) { + LOG.error("Unable to retrieve get HDFS FileSystem"); + throw new IllegalStateException("Unable to get HDFS FileSystem"); + } + return fs; } - return fs; - } - /** - * Converts null to empty string - * @param raw The raw object - * @return Empty string if null, or the String value if not - */ - public static String convertNullToEmptyString(Object raw) { - return raw == null ? "" : String.valueOf(raw); - } + /** + * Converts null to empty string. + * + * @param raw The raw object + * @return Empty string if null, or the String value if not + */ + public static String convertNullToEmptyString(Object raw) { + return raw == null ? "" : String.valueOf(raw); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabase.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabase.java index 8d1bc923..228dbbfd 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabase.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabase.java @@ -10,7 +10,9 @@ * limitations governing your use of the file. */ -package org.apache.metron.enrichment.adapters.maxmind.asn;/* +package org.apache.metron.enrichment.adapters.maxmind.asn; + +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -51,126 +53,128 @@ * Manages querying and updating of an Autonomous System Number (ASN) database provided by MaxMind. */ public enum GeoLiteAsnDatabase implements MaxMindDatabase { - INSTANCE; - - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String ASN_HDFS_FILE = "asn.hdfs.file"; - public static final String ASN_HDFS_FILE_DEFAULT = "/apps/metron/asn/default/GeoLite2-ASN.tar.gz"; - - private static ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - private static final Lock readLock = lock.readLock(); - private static final Lock writeLock = lock.writeLock(); - private static volatile String hdfsLoc = ASN_HDFS_FILE_DEFAULT; - private static DatabaseReader reader = null; - - public enum AsnProps { - NETWORK("network"), - ASN("autonomous_system_number"), - ASO("autonomous_system_organization"); - Function, Object> getter; - String simpleName; - - AsnProps(String simpleName) { - this(simpleName, m -> m.get(simpleName)); + INSTANCE; + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String ASN_HDFS_FILE = "asn.hdfs.file"; + public static final String ASN_HDFS_FILE_DEFAULT = "/apps/metron/asn/default/GeoLite2-ASN.tar.gz"; + + private static final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private static final Lock readLock = lock.readLock(); + private static final Lock writeLock = lock.writeLock(); + private static volatile String hdfsLoc = ASN_HDFS_FILE_DEFAULT; + private static DatabaseReader reader = null; + + public enum AsnProps { + NETWORK("network"), + ASN("autonomous_system_number"), + ASO("autonomous_system_organization"); + Function, Object> getter; + String simpleName; + + AsnProps(String simpleName) { + this(simpleName, m -> m.get(simpleName)); + } + + AsnProps(String simpleName, + Function, Object> getter + ) { + this.simpleName = simpleName; + this.getter = getter; + } + + public String getSimpleName() { + return simpleName; + } + + public Object get(Map map) { + return getter.apply(map); + } + + public void set(Map map, Object val) { + map.put(simpleName, val); + } } - AsnProps(String simpleName, - Function, Object> getter - ) { - this.simpleName = simpleName; - this.getter = getter; + @Override + public String getHdfsFileConfig() { + return ASN_HDFS_FILE; } - public String getSimpleName() { - return simpleName; + @Override + public String getHdfsFileDefault() { + return ASN_HDFS_FILE_DEFAULT; } - public Object get(Map map) { - return getter.apply(map); + @Override + public void lockIfNecessary() { + writeLock.lock(); } - public void set(Map map, Object val) { - map.put(simpleName, val); + @Override + public void unlockIfNecessary() { + writeLock.unlock(); } - } - - @Override - public String getHdfsFileConfig() { - return ASN_HDFS_FILE; - } - - @Override - public String getHdfsFileDefault() { - return ASN_HDFS_FILE_DEFAULT; - } - - @Override - public void lockIfNecessary() { - writeLock.lock(); - } - - @Override - public void unlockIfNecessary() { - writeLock.unlock(); - } - - @Override - public DatabaseReader getReader() { - return reader; - } - - @Override - public void setReader(DatabaseReader reader) { - GeoLiteAsnDatabase.reader = reader; - } - - public synchronized void updateIfNecessary(Map globalConfig) { - // Reload database if necessary (file changes on HDFS) - LOG.trace("Determining if GeoLiteAsnDatabase update required"); - String hdfsFile = ASN_HDFS_FILE_DEFAULT; - if (globalConfig != null) { - hdfsFile = (String) globalConfig.getOrDefault(ASN_HDFS_FILE, ASN_HDFS_FILE_DEFAULT); + + @Override + public DatabaseReader getReader() { + return reader; } - // Always update if we don't have a DatabaseReader - if (reader == null || !hdfsLoc.equals(hdfsFile)) { - // Update - hdfsLoc = hdfsFile; - update(hdfsFile); - } else { - LOG.trace("Update to GeoLiteAsnDatabase unnecessary"); + @Override + public void setReader(DatabaseReader reader) { + GeoLiteAsnDatabase.reader = reader; } - } - - /** - * Retrieves the result fields based on the incoming IP address - * @param ip The IP to lookup in the database - * @return Optional.empty() if the IP address is invalid or not in the database. - */ - public Optional> get(String ip) { - if (MaxMindDbUtilities.invalidIp(ip)) { - return Optional.empty(); + + public synchronized void updateIfNecessary(Map globalConfig) { + // Reload database if necessary (file changes on HDFS) + LOG.trace("Determining if GeoLiteAsnDatabase update required"); + String hdfsFile = ASN_HDFS_FILE_DEFAULT; + if (globalConfig != null) { + hdfsFile = (String) globalConfig.getOrDefault(ASN_HDFS_FILE, ASN_HDFS_FILE_DEFAULT); + } + + // Always update if we don't have a DatabaseReader + if (reader == null || !hdfsLoc.equals(hdfsFile)) { + // Update + hdfsLoc = hdfsFile; + update(hdfsFile); + } else { + LOG.trace("Update to GeoLiteAsnDatabase unnecessary"); + } } - try { - readLock.lock(); - InetAddress addr = InetAddress.getByName(ip); - AsnResponse asnResponse = reader.asn(addr); - HashMap asnInfo = new HashMap<>(); - AsnProps.ASN.set(asnInfo, asnResponse.getAutonomousSystemNumber()); - AsnProps.ASO - .set(asnInfo, MaxMindDbUtilities.convertNullToEmptyString(asnResponse.getAutonomousSystemOrganization())); - AsnProps.NETWORK - .set(asnInfo, MaxMindDbUtilities.convertNullToEmptyString(asnResponse.getIpAddress())); - - return Optional.of(asnInfo); - } catch (UnknownHostException | AddressNotFoundException e) { - LOG.debug("No result found for IP {}", ip); - } catch (GeoIp2Exception | IOException e) { - LOG.warn("GeoLite2 ASN DB encountered an error", e); - } finally { - readLock.unlock(); + /** + * Retrieves the result fields based on the incoming IP address. + * + * @param ip The IP to lookup in the database + * @return Optional.empty() if the IP address is invalid or not in the database. + */ + public Optional> get(String ip) { + if (MaxMindDbUtilities.invalidIp(ip)) { + return Optional.empty(); + } + + try { + readLock.lock(); + InetAddress addr = InetAddress.getByName(ip); + AsnResponse asnResponse = reader.asn(addr); + HashMap asnInfo = new HashMap<>(); + AsnProps.ASN.set(asnInfo, asnResponse.getAutonomousSystemNumber()); + AsnProps.ASO + .set(asnInfo, + MaxMindDbUtilities.convertNullToEmptyString(asnResponse.getAutonomousSystemOrganization())); + AsnProps.NETWORK + .set(asnInfo, MaxMindDbUtilities.convertNullToEmptyString(asnResponse.getIpAddress())); + + return Optional.of(asnInfo); + } catch (UnknownHostException | AddressNotFoundException e) { + LOG.debug("No result found for IP {}", ip); + } catch (GeoIp2Exception | IOException e) { + LOG.warn("GeoLite2 ASN DB encountered an error", e); + } finally { + readLock.unlock(); + } + return Optional.empty(); } - return Optional.empty(); - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabase.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabase.java index 42570b7a..e54adb76 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabase.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabase.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.maxmind.geo; import ch.hsr.geohash.WGS84Point; @@ -47,196 +48,201 @@ * Manages querying and updating of an GeoLite2 City database provided by MaxMind. */ public enum GeoLiteCityDatabase implements MaxMindDatabase { - INSTANCE; - - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String GEO_HDFS_FILE = "geo.hdfs.file"; - public static final String GEO_HDFS_FILE_DEFAULT = "/tmp/flink-cyber/reference-data/GeoLite2-City.mmdb"; - public static final String GEO_HDFS_FILE_DEFAULT_FALLBACK = "/tmp/flink-cyber/reference-data/GeoLite2-City.mmdb"; - - private static ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - private static final Lock readLock = lock.readLock(); - private static final Lock writeLock = lock.writeLock(); - private static volatile String hdfsLoc = GEO_HDFS_FILE_DEFAULT; - private static DatabaseReader reader = null; - - public enum GeoProps { - LOC_ID("locID"), - COUNTRY("country"), - CITY("city"), - POSTAL_CODE("postalCode"), - DMA_CODE("dmaCode"), - LATITUDE("latitude"), - LONGITUDE("longitude"), - LOCATION_POINT("location_point"), - ; - Function, String> getter; - String simpleName; - - GeoProps(String simpleName) { - this(simpleName, m -> m.get(simpleName)); + INSTANCE; + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String GEO_HDFS_FILE = "geo.hdfs.file"; + public static final String GEO_HDFS_FILE_DEFAULT = "/tmp/flink-cyber/reference-data/GeoLite2-City.mmdb"; + public static final String GEO_HDFS_FILE_DEFAULT_FALLBACK = "/tmp/flink-cyber/reference-data/GeoLite2-City.mmdb"; + + private static final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private static final Lock readLock = lock.readLock(); + private static final Lock writeLock = lock.writeLock(); + private static volatile String hdfsLoc = GEO_HDFS_FILE_DEFAULT; + private static DatabaseReader reader = null; + + public enum GeoProps { + LOC_ID("locID"), + COUNTRY("country"), + CITY("city"), + POSTAL_CODE("postalCode"), + DMA_CODE("dmaCode"), + LATITUDE("latitude"), + LONGITUDE("longitude"), + LOCATION_POINT("location_point"), + ; + Function, String> getter; + String simpleName; + + GeoProps(String simpleName) { + this(simpleName, m -> m.get(simpleName)); + } + + GeoProps(String simpleName, + Function, String> getter + ) { + this.simpleName = simpleName; + this.getter = getter; + } + + public String getSimpleName() { + return simpleName; + } + + public String get(Map map) { + return getter.apply(map); + } + + public void set(Map map, String val) { + map.put(simpleName, val); + } } - GeoProps(String simpleName, - Function, String> getter - ) { - this.simpleName = simpleName; - this.getter = getter; + @Override + public String getHdfsFileConfig() { + return GEO_HDFS_FILE; } - public String getSimpleName() { - return simpleName; + + @Override + public String getHdfsFileDefault() { + return GEO_HDFS_FILE_DEFAULT; } - public String get(Map map) { - return getter.apply(map); + @Override + public void lockIfNecessary() { + writeLock.lock(); } - public void set(Map map, String val) { - map.put(simpleName, val); + @Override + public void unlockIfNecessary() { + writeLock.unlock(); } - } - - @Override - public String getHdfsFileConfig() { - return GEO_HDFS_FILE; - } - - @Override - public String getHdfsFileDefault() { - return GEO_HDFS_FILE_DEFAULT; - } - - @Override - public void lockIfNecessary() { - writeLock.lock(); - } - - @Override - public void unlockIfNecessary() { - writeLock.unlock(); - } - - @Override - public DatabaseReader getReader() { - return reader; - } - - @Override - public void setReader(DatabaseReader reader) { - GeoLiteCityDatabase.reader = reader; - } - - public synchronized void updateIfNecessary(Map globalConfig) { - // Reload database if necessary (file changes on HDFS) - LOG.trace("Determining if GeoIpDatabase update required"); - String hdfsFile = GEO_HDFS_FILE_DEFAULT; - if (globalConfig != null) { - hdfsFile = (String) globalConfig.getOrDefault(GEO_HDFS_FILE, GEO_HDFS_FILE_DEFAULT); - hdfsFile = determineHdfsDirWithFallback(globalConfig, hdfsFile, GEO_HDFS_FILE_DEFAULT_FALLBACK); + + @Override + public DatabaseReader getReader() { + return reader; } - // Always update if we don't have a DatabaseReader - if (reader == null || !hdfsLoc.equals(hdfsFile)) { - // Update - hdfsLoc = hdfsFile; - update(hdfsFile); - } else { - LOG.trace("Update to GeoLiteCity2Database unnecessary"); + @Override + public void setReader(DatabaseReader reader) { + GeoLiteCityDatabase.reader = reader; } - } - - protected String determineHdfsDirWithFallback(Map globalConfig, String hdfsFile, String hdfsFallbackFile) { - // GeoLite2 City has the case where our new default isn't the old, but we want to fallback if needed. - // Only consider fallback if the user hasn't specified a location via config. - if (!globalConfig.containsKey(GEO_HDFS_FILE)) { - FileSystem fs = MaxMindDbUtilities.getFileSystem(); - try { - // Want to fallback under two conditions here - // 1. The default file doesn't exist. If it wasn't in the global config, it has to be the default. - // 2. The fallback exists. - // Otherwise, we'll leave it as the base default (which will cause issues later, but ensures logging encourages use of new database). - // Note that hdfsFile will be GEO_HDFS_FILE_DEFAULT if we even made it here. - if (hdfsPathsExist(fs, hdfsFile, hdfsFallbackFile)) { - hdfsFile = hdfsFallbackFile; + + public synchronized void updateIfNecessary(Map globalConfig) { + // Reload database if necessary (file changes on HDFS) + LOG.trace("Determining if GeoIpDatabase update required"); + String hdfsFile = GEO_HDFS_FILE_DEFAULT; + if (globalConfig != null) { + hdfsFile = (String) globalConfig.getOrDefault(GEO_HDFS_FILE, GEO_HDFS_FILE_DEFAULT); + hdfsFile = determineHdfsDirWithFallback(globalConfig, hdfsFile, GEO_HDFS_FILE_DEFAULT_FALLBACK); + } + + // Always update if we don't have a DatabaseReader + if (reader == null || !hdfsLoc.equals(hdfsFile)) { + // Update + hdfsLoc = hdfsFile; + update(hdfsFile); + } else { + LOG.trace("Update to GeoLiteCity2Database unnecessary"); } - } catch (IOException e) { - LOG.warn("Issue validating database HDFS fallback locations", e); - // Do nothing else - } } - return hdfsFile; - } - - protected boolean hdfsPathsExist(FileSystem fs, String hdfsFile, String fallbackFile) throws IOException { - return !fs.exists(new Path(hdfsFile)) && fs.exists(new Path(fallbackFile)); - } - - /** - * Retrieves the result fields based on the incoming IP address - * @param ip The IP to lookup in the database - * @return Optional.empty() if the IP address is invalid or not in the database. - */ - public Optional> get(String ip) { - if (MaxMindDbUtilities.invalidIp(ip)) { - return Optional.empty(); + + String determineHdfsDirWithFallback(Map globalConfig, String hdfsFile, + String hdfsFallbackFile) { + // GeoLite2 City has the case where our new default isn't the old, but we want to fallback if needed. + // Only consider fallback if the user hasn't specified a location via config. + if (!globalConfig.containsKey(GEO_HDFS_FILE)) { + FileSystem fs = MaxMindDbUtilities.getFileSystem(); + try { + // Want to fallback under two conditions here + // 1. The default file doesn't exist. If it wasn't in the global config, it has to be the default. + // 2. The fallback exists. + // Otherwise, we'll leave it as the base default (which will cause issues later, + // but ensures logging encourages use of new database). + // Note that hdfsFile will be GEO_HDFS_FILE_DEFAULT if we even made it here. + if (hdfsPathsExist(fs, hdfsFile, hdfsFallbackFile)) { + hdfsFile = hdfsFallbackFile; + } + } catch (IOException e) { + LOG.warn("Issue validating database HDFS fallback locations", e); + // Do nothing else + } + } + return hdfsFile; } - try { - readLock.lock(); - InetAddress addr = InetAddress.getByName(ip); - CityResponse cityResponse = reader.city(addr); - HashMap geoInfo = new HashMap<>(); - - Country country = cityResponse.getCountry(); - City city = cityResponse.getCity(); - Postal postal = cityResponse.getPostal(); - Location location = cityResponse.getLocation(); - - GeoProps.LOC_ID.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(city.getGeoNameId())); - GeoProps.COUNTRY.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(country.getIsoCode())); - GeoProps.CITY.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(city.getName())); - GeoProps.POSTAL_CODE.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(postal.getCode())); - GeoProps.DMA_CODE.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(location.getMetroCode())); - - Double latitudeRaw = location.getLatitude(); - String latitude = MaxMindDbUtilities.convertNullToEmptyString(latitudeRaw); - GeoProps.LATITUDE.set(geoInfo, latitude); - - Double longitudeRaw = location.getLongitude(); - String longitude = MaxMindDbUtilities.convertNullToEmptyString(longitudeRaw); - GeoProps.LONGITUDE.set(geoInfo, longitude); - - if (latitudeRaw == null || longitudeRaw == null) { - GeoProps.LOCATION_POINT.set(geoInfo, ""); - } else { - GeoProps.LOCATION_POINT.set(geoInfo, latitude + "," + longitude); - } - - return Optional.of(geoInfo); - } catch (UnknownHostException | AddressNotFoundException e) { - LOG.debug("No result found for IP {}", ip); - } catch (GeoIp2Exception | IOException e) { - LOG.warn("GeoLite2 City DB encountered an error", e); - } finally { - readLock.unlock(); + private boolean hdfsPathsExist(FileSystem fs, String hdfsFile, String fallbackFile) throws IOException { + return !fs.exists(new Path(hdfsFile)) && fs.exists(new Path(fallbackFile)); } - return Optional.empty(); - } - - public Optional toPoint(Map geoInfo) { - String latitude = GeoProps.LATITUDE.get(geoInfo); - String longitude = GeoProps.LONGITUDE.get(geoInfo); - if(latitude == null || longitude == null) { - return Optional.empty(); + + /** + * Retrieves the result fields based on the incoming IP address. + * + * @param ip The IP to lookup in the database + * @return Optional.empty() if the IP address is invalid or not in the database. + */ + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") + public Optional> get(String ip) { + if (MaxMindDbUtilities.invalidIp(ip)) { + return Optional.empty(); + } + + try { + readLock.lock(); + InetAddress addr = InetAddress.getByName(ip); + CityResponse cityResponse = reader.city(addr); + HashMap geoInfo = new HashMap<>(); + + Country country = cityResponse.getCountry(); + City city = cityResponse.getCity(); + Postal postal = cityResponse.getPostal(); + Location location = cityResponse.getLocation(); + + GeoProps.LOC_ID.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(city.getGeoNameId())); + GeoProps.COUNTRY.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(country.getIsoCode())); + GeoProps.CITY.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(city.getName())); + GeoProps.POSTAL_CODE.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(postal.getCode())); + GeoProps.DMA_CODE.set(geoInfo, MaxMindDbUtilities.convertNullToEmptyString(location.getMetroCode())); + + Double latitudeRaw = location.getLatitude(); + String latitude = MaxMindDbUtilities.convertNullToEmptyString(latitudeRaw); + GeoProps.LATITUDE.set(geoInfo, latitude); + + Double longitudeRaw = location.getLongitude(); + String longitude = MaxMindDbUtilities.convertNullToEmptyString(longitudeRaw); + GeoProps.LONGITUDE.set(geoInfo, longitude); + + if (latitudeRaw == null || longitudeRaw == null) { + GeoProps.LOCATION_POINT.set(geoInfo, ""); + } else { + GeoProps.LOCATION_POINT.set(geoInfo, latitude + "," + longitude); + } + + return Optional.of(geoInfo); + } catch (UnknownHostException | AddressNotFoundException e) { + LOG.debug("No result found for IP {}", ip); + } catch (GeoIp2Exception | IOException e) { + LOG.warn("GeoLite2 City DB encountered an error", e); + } finally { + readLock.unlock(); + } + return Optional.empty(); } - try { - double latD = Double.parseDouble(latitude); - double longD = Double.parseDouble(longitude); - return Optional.of(new WGS84Point(latD, longD)); - } catch (NumberFormatException nfe) { - LOG.warn(String.format("Invalid lat/long: %s/%s: %s", latitude, longitude, nfe.getMessage()), nfe); - return Optional.empty(); + public Optional toPoint(Map geoInfo) { + String latitude = GeoProps.LATITUDE.get(geoInfo); + String longitude = GeoProps.LONGITUDE.get(geoInfo); + if (latitude == null || longitude == null) { + return Optional.empty(); + } + + try { + double latD = Double.parseDouble(latitude); + double longD = Double.parseDouble(longitude); + return Optional.of(new WGS84Point(latD, longD)); + } catch (NumberFormatException nfe) { + LOG.warn(String.format("Invalid lat/long: %s/%s: %s", latitude, longitude, nfe.getMessage()), nfe); + return Optional.empty(); + } } - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategies.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategies.java index d0dd7d2a..dcda8875 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategies.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategies.java @@ -15,32 +15,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.maxmind.geo.hash; import ch.hsr.geohash.WGS84Point; import org.locationtech.spatial4j.distance.DistanceUtils; public enum DistanceStrategies implements DistanceStrategy { - HAVERSINE((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distHaversineRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude()) - , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude()) - ) - ), - LAW_OF_COSINES((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distLawOfCosinesRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude()) - , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude()) - ) - ), - VICENTY((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distVincentyRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude()) - , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude()) - ) - ) - ; - DistanceStrategy strat; - DistanceStrategies(DistanceStrategy strat) { - this.strat = strat; - } + HAVERSINE((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM + * DistanceUtils.distHaversineRAD(Math.toRadians(p1.getLatitude()), + Math.toRadians(p1.getLongitude()), Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude()) + ) + ), + LAW_OF_COSINES((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM + * DistanceUtils.distLawOfCosinesRAD(Math.toRadians(p1.getLatitude()), + Math.toRadians(p1.getLongitude()), Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude()) + ) + ), + VICENTY((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM + * DistanceUtils.distVincentyRAD(Math.toRadians(p1.getLatitude()), + Math.toRadians(p1.getLongitude()), Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())) + ); + DistanceStrategy strat; + + DistanceStrategies(DistanceStrategy strat) { + this.strat = strat; + } - @Override - public double distance(WGS84Point point1, WGS84Point point2) { - return strat.distance(point1, point2); - } + @Override + public double distance(WGS84Point point1, WGS84Point point2) { + return strat.distance(point1, point2); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategy.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategy.java index 9b9eac5e..c027a48f 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategy.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/DistanceStrategy.java @@ -15,10 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.maxmind.geo.hash; import ch.hsr.geohash.WGS84Point; public interface DistanceStrategy { - public double distance(WGS84Point point1, WGS84Point point2); + double distance(WGS84Point point1, WGS84Point point2); } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/GeoHashUtil.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/GeoHashUtil.java index 10b90404..95c5cee3 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/GeoHashUtil.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/maxmind/geo/hash/GeoHashUtil.java @@ -15,175 +15,171 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.maxmind.geo.hash; import ch.hsr.geohash.GeoHash; import ch.hsr.geohash.WGS84Point; import com.google.common.collect.Iterables; -import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase; - import java.util.AbstractMap; import java.util.Map; import java.util.Optional; +import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase; public enum GeoHashUtil { - INSTANCE; + INSTANCE; - public Optional computeHash(Double latitude, Double longitude, int precision) { - if(latitude == null || longitude == null) { - return Optional.empty(); + public Optional computeHash(Double latitude, Double longitude, int precision) { + if (latitude == null || longitude == null) { + return Optional.empty(); + } + return computeHash(new WGS84Point(latitude, longitude), precision); } - return computeHash(new WGS84Point(latitude, longitude), precision); - } - - public Optional computeHash(WGS84Point point, int precision) { - GeoHash hash = GeoHash.withCharacterPrecision(point.getLatitude(), point.getLongitude(), precision); - return Optional.of(hash.toBase32()); - } - - public Optional computeHash(Map geoLoc, int precision) { - Optional point = GeoLiteCityDatabase.INSTANCE.toPoint(geoLoc); - if(point.isPresent()) { - return computeHash(point.get(), precision); + + public Optional computeHash(WGS84Point point, int precision) { + GeoHash hash = GeoHash.withCharacterPrecision(point.getLatitude(), point.getLongitude(), precision); + return Optional.of(hash.toBase32()); } - else { - return Optional.empty(); + + public Optional computeHash(Map geoLoc, int precision) { + Optional point = GeoLiteCityDatabase.INSTANCE.toPoint(geoLoc); + if (point.isPresent()) { + return computeHash(point.get(), precision); + } else { + return Optional.empty(); + } + } + + public Optional toPoint(String hash) { + if (hash == null) { + return Optional.empty(); + } + GeoHash h = GeoHash.fromGeohashString(hash); + return Optional.ofNullable(h == null ? null : h.getPoint()); + } + + public double distance(WGS84Point point1, WGS84Point point2, DistanceStrategy strategy) { + return strategy.distance(point1, point2); } - } - public Optional toPoint(String hash) { - if(hash == null) { - return Optional.empty(); + public WGS84Point centroidOfHashes(Iterable hashes) { + Iterable points = Iterables.transform(hashes, h -> toPoint(h).orElse(null)); + return centroidOfPoints(points); } - GeoHash h = GeoHash.fromGeohashString(hash); - return Optional.ofNullable(h == null?null:h.getPoint()); - } - - public double distance(WGS84Point point1, WGS84Point point2, DistanceStrategy strategy) { - return strategy.distance(point1, point2); - } - - public WGS84Point centroidOfHashes(Iterable hashes) { - Iterable points = Iterables.transform(hashes, h -> toPoint(h).orElse(null)); - return centroidOfPoints(points); - } - - public WGS84Point centroidOfPoints(Iterable points) { - Iterable nonNullPoints = Iterables.filter(points, p -> p != null); - return centroid(Iterables.transform(nonNullPoints - , p -> new AbstractMap.SimpleImmutableEntry<>(p, 1) - ) - ); - } - - public WGS84Point centroidOfWeightedPoints(Map points) { - - Iterable> weightedPoints = Iterables.transform(points.entrySet() - , kv -> { - WGS84Point pt = toPoint(kv.getKey()).orElse(null); - return new AbstractMap.SimpleImmutableEntry<>(pt, kv.getValue()); - }); - return centroid(Iterables.filter(weightedPoints, kv -> kv.getKey() != null)); - } - - /** - * Find the equilibrium point of a weighted set of lat/long geo points. - * @param points The points and their weights (e.g. multiplicity) - * @return - */ - private WGS84Point centroid(Iterable> points) { - double x = 0d - , y = 0d - , z = 0d - , totalWeight = 0d - ; - int n = 0; + + public WGS84Point centroidOfPoints(Iterable points) { + Iterable nonNullPoints = Iterables.filter(points, p -> p != null); + return centroid(Iterables.transform(nonNullPoints, + p -> new AbstractMap.SimpleImmutableEntry<>(p, 1)) + ); + } + + public WGS84Point centroidOfWeightedPoints(Map points) { + + Iterable> weightedPoints = Iterables.transform(points.entrySet(), + kv -> { + WGS84Point pt = toPoint(kv.getKey()).orElse(null); + return new AbstractMap.SimpleImmutableEntry<>(pt, kv.getValue()); + }); + return centroid(Iterables.filter(weightedPoints, kv -> kv.getKey() != null)); + } + /** - * So, it's first important to realize that long/lat are not cartesian, so simple weighted averaging - * is insufficient here as it denies the fact that we're not living on a flat square, but rather the surface of - * an ellipsoid. A crow, for instance, does not fly a straight line to an observer outside of Earth, but - * rather flies across the arc tracing the surface of earth, or a "great-earth arc". When computing the centroid - * you want to find the centroid of the points with distance defined as the great-earth arc. + * Find the equilibrium point of a weighted set of lat/long geo points. * - * The general strategy is to: - * 1. Change coordinate systems from degrees on a WGS84 projection (e.g. lat/long) - * to a 3 dimensional cartesian surface atop a sphere approximating the earth. - * 2. Compute a weighted average of the cartesian coordinates - * 3. Change coordinate systems of the resulting centroid in cartesian space back to lat/long - * - * This is generally detailed at http://www.geomidpoint.com/example.html + * @param points The points and their weights (e.g. multiplicity) */ - for(Map.Entry weightedPoint : points) { - WGS84Point pt = weightedPoint.getKey(); - if(pt == null) { - continue; - } - double latRad = Math.toRadians(pt.getLatitude()); - double longRad = Math.toRadians(pt.getLongitude()); - double cosLat = Math.cos(latRad); - /* - Convert from lat/long coordinates to cartesian coordinates. The cartesian coordinate system is a right-hand, - rectangular, three-dimensional, earth-fixed coordinate system - with an origin at (0, 0, 0). The Z-axis, is parrallel to the axis of rotation of the earth. The Z-coordinate - is positive toward the North pole. The X-Y plane lies in the equatorial plane. The X-axis lies along the - intersection of the plane containing the prime meridian and the equatorial plane. The X-coordinate is positive - toward the intersection of the prime meridian and equator. - - Please see https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates - for more information about this coordinate transformation. - */ - double ptX = cosLat * Math.cos(longRad); - double ptY = cosLat * Math.sin(longRad); - double ptZ = Math.sin(latRad); - double weight = weightedPoint.getValue().doubleValue(); - x += ptX*weight; - y += ptY*weight; - z += ptZ*weight; - n++; - totalWeight += weight; + private WGS84Point centroid(Iterable> points) { + double x = 0d; + double y = 0d; + double z = 0d; + double totalWeight = 0d; + int n = 0; + /* + * So, it's first important to realize that long/lat are not cartesian, so simple weighted averaging + * is insufficient here as it denies the fact that we're not living on a flat square, but rather the surface of + * an ellipsoid. A crow, for instance, does not fly a straight line to an observer outside of Earth, but + * rather flies across the arc tracing the surface of earth, or a "great-earth arc". When computing the centroid + * you want to find the centroid of the points with distance defined as the great-earth arc. + * + * The general strategy is to: + * 1. Change coordinate systems from degrees on a WGS84 projection (e.g. lat/long) + * to a 3 dimensional cartesian surface atop a sphere approximating the earth. + * 2. Compute a weighted average of the cartesian coordinates + * 3. Change coordinate systems of the resulting centroid in cartesian space back to lat/long + * + * This is generally detailed at http://www.geomidpoint.com/example.html + */ + for (Map.Entry weightedPoint : points) { + WGS84Point pt = weightedPoint.getKey(); + if (pt == null) { + continue; + } + double latRad = Math.toRadians(pt.getLatitude()); + double longRad = Math.toRadians(pt.getLongitude()); + double cosLat = Math.cos(latRad); + /* + Convert from lat/long coordinates to cartesian coordinates. The cartesian coordinate system is a right-hand, + rectangular, three-dimensional, earth-fixed coordinate system + with an origin at (0, 0, 0). The Z-axis, is parrallel to the axis of rotation of the earth. The Z-coordinate + is positive toward the North pole. The X-Y plane lies in the equatorial plane. The X-axis lies along the + intersection of the plane containing the prime meridian and the equatorial plane. The X-coordinate is positive + toward the intersection of the prime meridian and equator. + + Please see https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates + for more information about this coordinate transformation. + */ + double ptX = cosLat * Math.cos(longRad); + double ptY = cosLat * Math.sin(longRad); + double ptZ = Math.sin(latRad); + double weight = weightedPoint.getValue().doubleValue(); + x += ptX * weight; + y += ptY * weight; + z += ptZ * weight; + n++; + totalWeight += weight; + } + if (n == 0) { + return null; + } + //average the vector representation in cartesian space, forming the center of gravity in cartesian space + x /= totalWeight; + y /= totalWeight; + z /= totalWeight; + + //convert the cartesian representation back to radians + double longitude = Math.atan2(y, x); + double hypotenuse = Math.sqrt(x * x + y * y); + double latitude = Math.atan2(z, hypotenuse); + + //convert the radians back to degrees latitude and longitude. + return new WGS84Point(Math.toDegrees(latitude), Math.toDegrees(longitude)); } - if(n == 0) { - return null; + + public double maxDistanceHashes(Iterable hashes, DistanceStrategy strategy) { + Iterable points = Iterables.transform(hashes, s -> toPoint(s).orElse(null)); + return maxDistancePoints(Iterables.filter(points, p -> p != null), strategy); } - //average the vector representation in cartesian space, forming the center of gravity in cartesian space - x /= totalWeight; - y /= totalWeight; - z /= totalWeight; - - //convert the cartesian representation back to radians - double longitude = Math.atan2(y, x); - double hypotenuse = Math.sqrt(x*x + y*y); - double latitude = Math.atan2(z, hypotenuse); - - //convert the radians back to degrees latitude and longitude. - return new WGS84Point(Math.toDegrees(latitude), Math.toDegrees(longitude)); - } - - public double maxDistanceHashes(Iterable hashes, DistanceStrategy strategy) { - Iterable points = Iterables.transform(hashes, s -> toPoint(s).orElse(null)); - return maxDistancePoints(Iterables.filter(points, p -> p != null), strategy); - } - - public double maxDistancePoints(Iterable points, DistanceStrategy strategy) { - //Note: because distance is commutative, we only need search the upper triangle - int i = 0; - double max = Double.NaN; - for(WGS84Point pt1 : points) { - int j = 0; - for(WGS84Point pt2 : points) { - if(j <= i) { - double d = strategy.distance(pt1, pt2); - if(Double.isNaN(max)|| d > max) { - max = d; - } - j++; - } - else { - break; + + public double maxDistancePoints(Iterable points, DistanceStrategy strategy) { + //Note: because distance is commutative, we only need search the upper triangle + int i = 0; + double max = Double.NaN; + for (WGS84Point pt1 : points) { + int j = 0; + for (WGS84Point pt2 : points) { + if (j <= i) { + double d = strategy.distance(pt1, pt2); + if (Double.isNaN(max) || d > max) { + max = d; + } + j++; + } else { + break; + } + } + i++; } - } - i++; + return max; } - return max; - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java index 43578ece..ab6ee9c4 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +27,6 @@ import java.lang.invoke.MethodHandles; import java.util.List; import java.util.Map; - import org.apache.hadoop.hbase.client.Connection; import org.apache.metron.common.utils.LazyLogger; import org.apache.metron.common.utils.LazyLoggerFactory; @@ -39,100 +40,98 @@ import org.apache.metron.enrichment.utils.EnrichmentUtils; import org.json.simple.JSONObject; -public class SimpleHBaseAdapter implements EnrichmentAdapter,Serializable { - protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - protected SimpleHBaseConfig config; - protected EnrichmentLookup lookup; - protected Connection connection; +public class SimpleHBaseAdapter implements EnrichmentAdapter, Serializable { + protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected SimpleHBaseConfig config; + protected EnrichmentLookup lookup; + protected Connection connection; + + public SimpleHBaseAdapter() { + } - public SimpleHBaseAdapter() { - } - public SimpleHBaseAdapter(SimpleHBaseConfig config) { - withConfig(config); - } + public SimpleHBaseAdapter(SimpleHBaseConfig config) { + withConfig(config); + } - public SimpleHBaseAdapter withConfig(SimpleHBaseConfig config) { - this.config = config; - return this; - } + public SimpleHBaseAdapter withConfig(SimpleHBaseConfig config) { + this.config = config; + return this; + } - @Override - public void logAccess(CacheKey value) { - } + @Override + public void logAccess(CacheKey value) { + } - public boolean isInitialized() { - return lookup != null && lookup.getTable() != null; - } - @Override - public JSONObject enrich(CacheKey value) { - JSONObject enriched = new JSONObject(); - if(!isInitialized()) { - initializeAdapter(null); + public boolean isInitialized() { + return lookup != null && lookup.getTable() != null; } - List enrichmentTypes = value.getConfig() - .getEnrichment().getFieldToTypeMap() - .get(EnrichmentUtils.toTopLevelField(value.getField())); - if(isInitialized() && enrichmentTypes != null && value.getValue() != null) { - try { - for (LookupKV kv : - lookup.get(Iterables.transform(enrichmentTypes - , new EnrichmentUtils.TypeToKey( value.coerceValue(String.class) - , lookup.getTable() - , value.getConfig().getEnrichment() - ) - ) - , false - ) - ) - { - if (kv != null && kv.getValue() != null && kv.getValue().getMetadata() != null) { - for (Map.Entry values : kv.getValue().getMetadata().entrySet()) { - enriched.put(kv.getKey().type + "." + values.getKey(), values.getValue()); + + @Override + public JSONObject enrich(CacheKey value) { + JSONObject enriched = new JSONObject(); + if (!isInitialized()) { + initializeAdapter(null); + } + List enrichmentTypes = value.getConfig() + .getEnrichment().getFieldToTypeMap() + .get(EnrichmentUtils.toTopLevelField(value.getField())); + if (isInitialized() && enrichmentTypes != null && value.getValue() != null) { + try { + for (LookupKV kv : + lookup.get(Iterables.transform(enrichmentTypes, + new EnrichmentUtils.TypeToKey(value.coerceValue(String.class), + lookup.getTable(), + value.getConfig().getEnrichment()) + ), + false) + ) { + if (kv != null && kv.getValue() != null && kv.getValue().getMetadata() != null) { + for (Map.Entry values : kv.getValue().getMetadata().entrySet()) { + enriched.put(kv.getKey().type + "." + values.getKey(), values.getValue()); + } + LOG.trace("Enriched type {} => {}", () -> kv.getKey().type, () -> enriched); + } + } + } catch (IOException e) { + LOG.error("Unable to retrieve value: {}", e.getMessage(), e); + initializeAdapter(null); + throw new RuntimeException("Unable to retrieve value: " + e.getMessage(), e); } - LOG.trace("Enriched type {} => {}", () -> kv.getKey().type, ()->enriched); - } } - } - catch (IOException e) { - LOG.error("Unable to retrieve value: {}", e.getMessage(), e); - initializeAdapter(null); - throw new RuntimeException("Unable to retrieve value: " + e.getMessage(), e); - } + LOG.trace("SimpleHBaseAdapter succeeded: {}", enriched); + return enriched; } - LOG.trace("SimpleHBaseAdapter succeeded: {}", enriched); - return enriched; - } - @Override - public boolean initializeAdapter(Map configuration) { - String hbaseTable = config.getHBaseTable(); - try { - lookup = new EnrichmentLookup(config.getProvider().getTable(config.getHbaseConfig(), hbaseTable), - config.getHBaseCF(), - new NoopAccessTracker()); - } catch (IOException e) { - LOG.error("Unable to initialize adapter: {}", e.getMessage(), e); - return false; + @Override + public boolean initializeAdapter(Map configuration) { + String hbaseTable = config.getHBaseTable(); + try { + lookup = new EnrichmentLookup(config.getProvider().getTable(config.getHbaseConfig(), hbaseTable), + config.getHBaseCF(), + new NoopAccessTracker()); + } catch (IOException e) { + LOG.error("Unable to initialize adapter: {}", e.getMessage(), e); + return false; + } + return true; } - return true; - } - @Override - public void updateAdapter(Map config) { - } + @Override + public void updateAdapter(Map config) { + } - @Override - public void cleanup() { - try { - lookup.close(); - } catch (Exception e) { - LOG.error("Unable to cleanup access tracker", e); + @Override + public void cleanup() { + try { + lookup.close(); + } catch (Exception e) { + LOG.error("Unable to cleanup access tracker", e); + } } - } - @Override - public String getOutputPrefix(CacheKey value) { - return value.getField(); - } + @Override + public String getOutputPrefix(CacheKey value) { + return value.getField(); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfig.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfig.java index fe6c69fe..e0cc666a 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfig.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfig.java @@ -7,61 +7,66 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.simplehbase; +import java.io.Serializable; import org.apache.hadoop.conf.Configuration; import org.apache.metron.enrichment.utils.EnrichmentUtils; import org.apache.metron.hbase.HTableProvider; import org.apache.metron.hbase.TableProvider; -import java.io.Serializable; - public class SimpleHBaseConfig implements Serializable { - private String hBaseTable; - private String hBaseCF; - private TableProvider provider = new HTableProvider(); + private String hbaseTable; + private String hbaseCF; + private TableProvider provider = new HTableProvider(); + + private Configuration hbaseConfig; + + public String getHBaseTable() { + return hbaseTable; + } + + public String getHBaseCF() { + return hbaseCF; + } - private Configuration hbaseConfig; - public String getHBaseTable() { - return hBaseTable; - } - public String getHBaseCF() { - return hBaseCF; - } + public TableProvider getProvider() { + return provider; + } - public TableProvider getProvider() { - return provider; - } + public Configuration getHbaseConfig() { + return hbaseConfig; + } - public Configuration getHbaseConfig() { - return hbaseConfig; - } + public SimpleHBaseConfig withProviderImpl(String connectorImpl) { + provider = EnrichmentUtils.getTableProvider(connectorImpl, new HTableProvider()); + return this; + } - public SimpleHBaseConfig withProviderImpl(String connectorImpl) { - provider = EnrichmentUtils.getTableProvider(connectorImpl, new HTableProvider()); - return this; - } - public SimpleHBaseConfig withHBaseTable(String hBaseTable) { - this.hBaseTable = hBaseTable; - return this; - } + public SimpleHBaseConfig withHBaseTable(String hbaseTable) { + this.hbaseTable = hbaseTable; + return this; + } - public SimpleHBaseConfig withHBaseCF(String cf) { - this.hBaseCF= cf; - return this; - } + public SimpleHBaseConfig withHBaseCF(String cf) { + this.hbaseCF = cf; + return this; + } - public SimpleHBaseConfig withHbaseConfig(Configuration config) { - this.hbaseConfig = config; - return this; - } + public SimpleHBaseConfig withHbaseConfig(Configuration config) { + this.hbaseConfig = config; + return this; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java index a3fc8b88..aa7fd184 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java @@ -7,19 +7,21 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.stellar; import static org.apache.metron.stellar.common.Constants.STELLAR_CONTEXT_CONF; - import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -27,7 +29,6 @@ import java.util.List; import java.util.Map; import java.util.function.Function; - import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; import org.apache.metron.enrichment.cache.CacheKey; @@ -42,177 +43,181 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StellarAdapter implements EnrichmentAdapter,Serializable { - public static class Perf {} - protected static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - protected static final Logger _PERF_LOG = LoggerFactory.getLogger(Perf.class); - public static final String STELLAR_SLOW_LOG = "stellar.slow.threshold.ms"; - public static final Long STELLAR_SLOW_LOG_DEFAULT = 1000l; - - private enum EnrichmentType implements Function{ - ENRICHMENT(config -> config.getEnrichment().getEnrichmentConfigs().get("stellar")) - ,THREAT_INTEL(config -> config.getThreatIntel().getEnrichmentConfigs().get("stellar")) - ; - Function func; - EnrichmentType(Function func) { - this.func = func; +public class StellarAdapter implements EnrichmentAdapter, Serializable { + public static class Perf { + } + + protected static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected static final Logger _PERF_LOG = LoggerFactory.getLogger(Perf.class); + public static final String STELLAR_SLOW_LOG = "stellar.slow.threshold.ms"; + public static final Long STELLAR_SLOW_LOG_DEFAULT = 1000L; + + private enum EnrichmentType implements Function { + ENRICHMENT(config -> config.getEnrichment().getEnrichmentConfigs().get("stellar")), + THREAT_INTEL(config -> config.getThreatIntel().getEnrichmentConfigs().get("stellar")); + Function func; + + EnrichmentType(Function func) { + this.func = func; + } + + @Override + public ConfigHandler apply(SensorEnrichmentConfig cacheKey) { + return func.apply(cacheKey); + } + } + + transient Function getHandler; + private String enrichmentType; + + public StellarAdapter ofType(String enrichmentType) { + this.enrichmentType = enrichmentType; + return this; } @Override - public ConfigHandler apply(SensorEnrichmentConfig cacheKey) { - return func.apply(cacheKey); + public String getOutputPrefix(CacheKey value) { + return ""; } - } - transient Function getHandler; - private String enrichmentType; - public StellarAdapter ofType(String enrichmentType) { - this.enrichmentType = enrichmentType; - return this; - } - @Override - public String getOutputPrefix(CacheKey value) { - return ""; - } - - @Override - public void logAccess(CacheKey value) { - - } - - @Override - public String getStreamSubGroup(String enrichmentType, String field) { - return field; - } - - public static Iterable> getStellarStatements(ConfigHandler handler, String field) { - if(field.length() == 0) { - return handler.getType().toConfig(handler.getConfig()); + + @Override + public void logAccess(CacheKey value) { + } - else { - Map groupStatements = (Map)handler.getConfig(); - return handler.getType().toConfig(groupStatements.get(field)); + + @Override + public String getStreamSubGroup(String enrichmentType, String field) { + return field; } - } - - - public static JSONObject process( Map message - , ConfigHandler handler - , String field - , Long slowLogThreshold - , StellarProcessor processor - , VariableResolver resolver - , Context stellarContext - ) - { - JSONObject ret = new JSONObject(); - Iterable> stellarStatements = getStellarStatements(handler, field); - - _LOG.debug("message := {}", message); - if(stellarStatements != null) { - List mapEntries = new ArrayList<>(); - for (Map.Entry kv : stellarStatements) { - if(kv.getKey() != null && kv.getValue() != null) { - if (kv.getValue() instanceof String) { - long startTime = System.currentTimeMillis(); - String stellarStatement = (String) kv.getValue(); - Object o = null; - try { - o = processor.parse(stellarStatement, resolver, StellarFunctions.FUNCTION_RESOLVER(), stellarContext); - } - catch(Exception e) { - _LOG.error(e.getMessage(), e); - throw e; - } - if (slowLogThreshold != null && _PERF_LOG.isDebugEnabled()) { - long duration = System.currentTimeMillis() - startTime; - if (duration > slowLogThreshold) { - _PERF_LOG.debug("SLOW LOG: {} took {} ms",stellarStatement,duration); - } - } - _LOG.debug("{} := {} yields {}", kv.getKey(), stellarStatement , o); - if (o != null && o instanceof Map) { - mapEntries.add(kv.getKey()); + + public static Iterable> getStellarStatements(ConfigHandler handler, String field) { + if (field.length() == 0) { + return handler.getType().toConfig(handler.getConfig()); + } else { + Map groupStatements = (Map) handler.getConfig(); + return handler.getType().toConfig(groupStatements.get(field)); + } + } + + + public static JSONObject process(Map message, + ConfigHandler handler, + String field, + Long slowLogThreshold, + StellarProcessor processor, + VariableResolver resolver, + Context stellarContext + ) { + JSONObject ret = new JSONObject(); + Iterable> stellarStatements = getStellarStatements(handler, field); + + _LOG.debug("message := {}", message); + if (stellarStatements != null) { + List mapEntries = new ArrayList<>(); + for (Map.Entry kv : stellarStatements) { + if (kv.getKey() != null && kv.getValue() != null) { + if (kv.getValue() instanceof String) { + long startTime = System.currentTimeMillis(); + String stellarStatement = (String) kv.getValue(); + Object o = null; + try { + o = processor.parse(stellarStatement, resolver, StellarFunctions.FUNCTION_RESOLVER(), + stellarContext); + } catch (Exception e) { + _LOG.error(e.getMessage(), e); + throw e; + } + if (slowLogThreshold != null && _PERF_LOG.isDebugEnabled()) { + long duration = System.currentTimeMillis() - startTime; + if (duration > slowLogThreshold) { + _PERF_LOG.debug("SLOW LOG: {} took {} ms", stellarStatement, duration); + } + } + _LOG.debug("{} := {} yields {}", kv.getKey(), stellarStatement, o); + if (o != null && o instanceof Map) { + mapEntries.add(kv.getKey()); + } + if (o == null) { + message.remove(kv.getKey()); + ret.remove(kv.getKey()); + } else { + message.put(kv.getKey(), o); + ret.put(kv.getKey(), o); + } + } + } } - if(o == null) { - message.remove(kv.getKey()); - ret.remove(kv.getKey()); + /* + We need to handle the map entries separately now. + We want to explode them out, so if "var" is + { + "foo" : "bar" } - else { - message.put(kv.getKey(), o); - ret.put(kv.getKey(), o); + then we want "var.foo" == "bar" + and no "var" + */ + for (String mapEntry : mapEntries) { + String key = mapEntry; + Map value = (Map) ret.get(key); + if (value != null) { + _LOG.debug("Exploding map: {} == {}", key, value); + for (Map.Entry valueKv : value.entrySet()) { + String newKey = ((key.length() > 0) ? key + "." : "") + valueKv.getKey(); + ret.put(newKey, valueKv.getValue()); + } + //removing the map from downstream + ret.remove(key); + } } - } } - } - /* - We need to handle the map entries separately now. - We want to explode them out, so if "var" is - { - "foo" : "bar" - } - then we want "var.foo" == "bar" - and no "var" - */ - for(String mapEntry : mapEntries) { - String key = mapEntry; - Map value = (Map) ret.get(key); - if(value != null) { - _LOG.debug("Exploding map: {} == {}", key, value); - for (Map.Entry valueKv : value.entrySet()) { - String newKey = ((key.length() > 0) ? key + "." : "") + valueKv.getKey(); - ret.put(newKey, valueKv.getValue()); - } - //removing the map from downstream - ret.remove(key); + return ret; + } + + @Override + public JSONObject enrich(CacheKey value) { + Context stellarContext = (Context) value.getConfig().getConfiguration().get(STELLAR_CONTEXT_CONF); + ConfigHandler handler = getHandler.apply(value.getConfig()); + Map globalConfig = value.getConfig().getConfiguration(); + Map sensorConfig = value.getConfig().getEnrichment().getConfig(); + if (handler == null) { + _LOG.trace("Stellar ConfigHandler is null."); + return new JSONObject(); } - } + Long slowLogThreshold = null; + if (_PERF_LOG.isDebugEnabled()) { + slowLogThreshold = + ConversionUtils.convert(globalConfig.getOrDefault(STELLAR_SLOW_LOG, STELLAR_SLOW_LOG_DEFAULT), + Long.class); + } + //Ensure that you clone the message, because process will modify the message. If the message object is modified + //then cache misses will happen because the cache will be modified. + Map message = new HashMap<>(value.getValue(Map.class)); + VariableResolver resolver = new MapVariableResolver(message, sensorConfig, globalConfig); + StellarProcessor processor = new StellarProcessor(); + JSONObject enriched = process(message, + handler, + value.getField(), + slowLogThreshold, + processor, + resolver, + stellarContext + ); + _LOG.trace("Stellar Enrichment Success: {}", enriched); + return enriched; } - return ret; - } - - @Override - public JSONObject enrich(CacheKey value) { - Context stellarContext = (Context) value.getConfig().getConfiguration().get(STELLAR_CONTEXT_CONF); - ConfigHandler handler = getHandler.apply(value.getConfig()); - Map globalConfig = value.getConfig().getConfiguration(); - Map sensorConfig = value.getConfig().getEnrichment().getConfig(); - if(handler == null) { - _LOG.trace("Stellar ConfigHandler is null."); - return new JSONObject(); + + @Override + public boolean initializeAdapter(Map config) { + getHandler = EnrichmentType.valueOf(enrichmentType); + return true; } - Long slowLogThreshold = null; - if(_PERF_LOG.isDebugEnabled()) { - slowLogThreshold = ConversionUtils.convert(globalConfig.getOrDefault(STELLAR_SLOW_LOG, STELLAR_SLOW_LOG_DEFAULT), Long.class); + + @Override + public void updateAdapter(Map config) { + } + + @Override + public void cleanup() { + } - //Ensure that you clone the message, because process will modify the message. If the message object is modified - //then cache misses will happen because the cache will be modified. - Map message = new HashMap<>(value.getValue(Map.class)); - VariableResolver resolver = new MapVariableResolver(message, sensorConfig, globalConfig); - StellarProcessor processor = new StellarProcessor(); - JSONObject enriched = process(message - , handler - , value.getField() - , slowLogThreshold - , processor - , resolver - , stellarContext - ); - _LOG.trace("Stellar Enrichment Success: {}", enriched); - return enriched; - } - - @Override - public boolean initializeAdapter(Map config) { - getHandler = EnrichmentType.valueOf(enrichmentType); - return true; - } - - @Override - public void updateAdapter(Map config) { - } - - @Override - public void cleanup() { - - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java index 619fe824..63247f9f 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.threatintel; import com.google.common.collect.Iterables; @@ -25,7 +28,6 @@ import java.util.Map; import java.util.UUID; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Connection; import org.apache.metron.enrichment.cache.CacheKey; import org.apache.metron.enrichment.converter.EnrichmentKey; @@ -38,121 +40,119 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ThreatIntelAdapter implements EnrichmentAdapter,Serializable { - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - protected ThreatIntelConfig config; - protected EnrichmentLookup lookup; - protected Connection connection; - - public ThreatIntelAdapter() { - } - public ThreatIntelAdapter(ThreatIntelConfig config) { - withConfig(config); - } +public class ThreatIntelAdapter implements EnrichmentAdapter, Serializable { + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected ThreatIntelConfig config; + protected EnrichmentLookup lookup; + protected Connection connection; - public ThreatIntelAdapter withConfig(ThreatIntelConfig config) { - this.config = config; - return this; - } - - @Override - public void logAccess(CacheKey value) { - List enrichmentTypes = value.getConfig().getThreatIntel().getFieldToTypeMap().get(value.getField()); - if(enrichmentTypes != null) { - for(String enrichmentType : enrichmentTypes) { - lookup.getAccessTracker().logAccess(new EnrichmentKey(enrichmentType, value.coerceValue(String.class))); - } + public ThreatIntelAdapter() { } - } + public ThreatIntelAdapter(ThreatIntelConfig config) { + withConfig(config); + } - @Override - public JSONObject enrich(CacheKey value) { - if(!isInitialized()) { - initializeAdapter(null); + public ThreatIntelAdapter withConfig(ThreatIntelConfig config) { + this.config = config; + return this; } - JSONObject enriched = new JSONObject(); - List enrichmentTypes = value.getConfig() - .getThreatIntel().getFieldToTypeMap() - .get(EnrichmentUtils.toTopLevelField(value.getField())); - if(isInitialized() && enrichmentTypes != null) { - int i = 0; - try { - for (Boolean isThreat : - lookup.exists(Iterables.transform(enrichmentTypes - , new EnrichmentUtils.TypeToKey(value.coerceValue(String.class) - , lookup.getTable() - , value.getConfig().getThreatIntel() - ) - ) - , false - ) - ) - { - String enrichmentType = enrichmentTypes.get(i++); - if (isThreat) { - enriched.put(enrichmentType, "alert"); - LOG.trace("Theat Intel Enriched value => {}", enriched); - } + + @Override + public void logAccess(CacheKey value) { + List enrichmentTypes = value.getConfig().getThreatIntel().getFieldToTypeMap().get(value.getField()); + if (enrichmentTypes != null) { + for (String enrichmentType : enrichmentTypes) { + lookup.getAccessTracker().logAccess(new EnrichmentKey(enrichmentType, value.coerceValue(String.class))); + } } - } - catch(IOException e) { - LOG.error("Unable to retrieve value: {}", e.getMessage(), e); - initializeAdapter(null); - throw new RuntimeException("Theat Intel Unable to retrieve value", e); - } } - LOG.trace("Threat Intel Enrichment Success: {}", enriched); - return enriched; - } - public boolean isInitialized() { - return lookup != null && lookup.getTable() != null; - } - @Override - public boolean initializeAdapter(Map configuration) { - PersistentAccessTracker accessTracker; - String hbaseTable = config.getHBaseTable(); - int expectedInsertions = config.getExpectedInsertions(); - double falsePositives = config.getFalsePositiveRate(); - String trackerHBaseTable = config.getTrackerHBaseTable(); - String trackerHBaseCF = config.getTrackerHBaseCF(); - long millisecondsBetweenPersist = config.getMillisecondsBetweenPersists(); - BloomAccessTracker bat = new BloomAccessTracker(hbaseTable, expectedInsertions, falsePositives); - Configuration hbaseConfig = config.getHbaseConfig(); - try { - accessTracker = new PersistentAccessTracker( hbaseTable - , UUID.randomUUID().toString() - , config.getProvider().getTable(hbaseConfig, trackerHBaseTable) - , trackerHBaseCF - , bat - , millisecondsBetweenPersist - ); - lookup = new EnrichmentLookup(config.getProvider().getTable(hbaseConfig, hbaseTable), config.getHBaseCF(), accessTracker); - } catch (IOException e) { - LOG.error("Unable to initialize ThreatIntelAdapter", e); - return false; + @Override + public JSONObject enrich(CacheKey value) { + if (!isInitialized()) { + initializeAdapter(null); + } + JSONObject enriched = new JSONObject(); + List enrichmentTypes = value.getConfig() + .getThreatIntel().getFieldToTypeMap() + .get(EnrichmentUtils.toTopLevelField(value.getField())); + if (isInitialized() && enrichmentTypes != null) { + int i = 0; + try { + for (Boolean isThreat : + lookup.exists(Iterables.transform(enrichmentTypes, + new EnrichmentUtils.TypeToKey(value.coerceValue(String.class), + lookup.getTable(), + value.getConfig().getThreatIntel()) + ), + false + ) + ) { + String enrichmentType = enrichmentTypes.get(i++); + if (isThreat) { + enriched.put(enrichmentType, "alert"); + LOG.trace("Theat Intel Enriched value => {}", enriched); + } + } + } catch (IOException e) { + LOG.error("Unable to retrieve value: {}", e.getMessage(), e); + initializeAdapter(null); + throw new RuntimeException("Theat Intel Unable to retrieve value", e); + } + } + LOG.trace("Threat Intel Enrichment Success: {}", enriched); + return enriched; } - return true; - } + public boolean isInitialized() { + return lookup != null && lookup.getTable() != null; + } - @Override - public void updateAdapter(Map config) { - } + @Override + public boolean initializeAdapter(Map configuration) { + PersistentAccessTracker accessTracker; + String hbaseTable = config.getHBaseTable(); + int expectedInsertions = config.getExpectedInsertions(); + double falsePositives = config.getFalsePositiveRate(); + String trackerHBaseTable = config.getTrackerHBaseTable(); + String trackerHBaseCF = config.getTrackerHBaseCF(); + long millisecondsBetweenPersist = config.getMillisecondsBetweenPersists(); + BloomAccessTracker bat = new BloomAccessTracker(hbaseTable, expectedInsertions, falsePositives); + Configuration hbaseConfig = config.getHbaseConfig(); + try { + accessTracker = new PersistentAccessTracker(hbaseTable, + UUID.randomUUID().toString(), + config.getProvider().getTable(hbaseConfig, trackerHBaseTable), + trackerHBaseCF, + bat, + millisecondsBetweenPersist); + lookup = new EnrichmentLookup(config.getProvider().getTable(hbaseConfig, hbaseTable), config.getHBaseCF(), + accessTracker); + } catch (IOException e) { + LOG.error("Unable to initialize ThreatIntelAdapter", e); + return false; + } + + return true; + } - @Override - public void cleanup() { - try { - lookup.close(); - } catch (Exception e) { - throw new RuntimeException("Unable to cleanup access tracker", e); + @Override + public void updateAdapter(Map config) { } - } - @Override - public String getOutputPrefix(CacheKey value) { - return value.getField(); - } + @Override + public void cleanup() { + try { + lookup.close(); + } catch (Exception e) { + throw new RuntimeException("Unable to cleanup access tracker", e); + } + } + + @Override + public String getOutputPrefix(CacheKey value) { + return value.getField(); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfig.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfig.java index a160c1c0..dc18fa69 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfig.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfig.java @@ -7,115 +7,118 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.adapters.threatintel; +import java.io.Serializable; import org.apache.hadoop.conf.Configuration; import org.apache.metron.enrichment.utils.EnrichmentUtils; import org.apache.metron.hbase.HTableProvider; import org.apache.metron.hbase.TableProvider; -import java.io.Serializable; - public class ThreatIntelConfig implements Serializable { - public static final long MS_IN_HOUR = 10000*60*60; - private String hBaseTable; - private String hBaseCF; - - private Configuration hbaseConfig; - - private double falsePositiveRate = 0.03; - private int expectedInsertions = 100000; - private String trackerHBaseTable; - private String trackerHBaseCF; - private long millisecondsBetweenPersists = 2*MS_IN_HOUR; - private TableProvider provider = new HTableProvider(); - - public String getHBaseTable() { - return hBaseTable; - } - - public int getExpectedInsertions() { - return expectedInsertions; - } - - public Configuration getHbaseConfig() { - return hbaseConfig; - } - - public double getFalsePositiveRate() { - return falsePositiveRate; - } - - public String getTrackerHBaseTable() { - return trackerHBaseTable; - } - - public String getTrackerHBaseCF() { - return trackerHBaseCF; - } - - public long getMillisecondsBetweenPersists() { - return millisecondsBetweenPersists; - } - - public String getHBaseCF() { - return hBaseCF; - } - - public TableProvider getProvider() { - return provider; - } - - public ThreatIntelConfig withProviderImpl(String connectorImpl) { - provider = EnrichmentUtils.getTableProvider(connectorImpl, new HTableProvider()); - return this; - } - - public ThreatIntelConfig withTrackerHBaseTable(String hBaseTable) { - this.trackerHBaseTable = hBaseTable; - return this; - } - - public ThreatIntelConfig withTrackerHBaseCF(String cf) { - this.trackerHBaseCF = cf; - return this; - } - public ThreatIntelConfig withHBaseTable(String hBaseTable) { - this.hBaseTable = hBaseTable; - return this; - } - - public ThreatIntelConfig withHBaseCF(String cf) { - this.hBaseCF= cf; - return this; - } - - public ThreatIntelConfig withFalsePositiveRate(double falsePositiveRate) { - this.falsePositiveRate = falsePositiveRate; - return this; - } - - public ThreatIntelConfig withExpectedInsertions(int expectedInsertions) { - this.expectedInsertions = expectedInsertions; - return this; - } - - public ThreatIntelConfig withMillisecondsBetweenPersists(long millisecondsBetweenPersists) { - this.millisecondsBetweenPersists = millisecondsBetweenPersists; - return this; - } - - public ThreatIntelConfig withHbaseConfig(Configuration hbaseConfig) { - this.hbaseConfig = hbaseConfig; - return this; - - } + public static final long MS_IN_HOUR = 10000 * 60 * 60; + private String hbaseTable; + private String hbaseCF; + + private Configuration hbaseConfig; + + private double falsePositiveRate = 0.03; + private int expectedInsertions = 100000; + private String trackerHBaseTable; + private String trackerHBaseCF; + private long millisecondsBetweenPersists = 2 * MS_IN_HOUR; + private TableProvider provider = new HTableProvider(); + + public String getHBaseTable() { + return hbaseTable; + } + + public int getExpectedInsertions() { + return expectedInsertions; + } + + public Configuration getHbaseConfig() { + return hbaseConfig; + } + + public double getFalsePositiveRate() { + return falsePositiveRate; + } + + public String getTrackerHBaseTable() { + return trackerHBaseTable; + } + + public String getTrackerHBaseCF() { + return trackerHBaseCF; + } + + public long getMillisecondsBetweenPersists() { + return millisecondsBetweenPersists; + } + + public String getHBaseCF() { + return hbaseCF; + } + + public TableProvider getProvider() { + return provider; + } + + public ThreatIntelConfig withProviderImpl(String connectorImpl) { + provider = EnrichmentUtils.getTableProvider(connectorImpl, new HTableProvider()); + return this; + } + + public ThreatIntelConfig withTrackerHBaseTable(String hbaseTable) { + this.trackerHBaseTable = hbaseTable; + return this; + } + + public ThreatIntelConfig withTrackerHBaseCF(String cf) { + this.trackerHBaseCF = cf; + return this; + } + + public ThreatIntelConfig withHBaseTable(String hbaseTable) { + this.hbaseTable = hbaseTable; + return this; + } + + public ThreatIntelConfig withHBaseCF(String cf) { + this.hbaseCF = cf; + return this; + } + + public ThreatIntelConfig withFalsePositiveRate(double falsePositiveRate) { + this.falsePositiveRate = falsePositiveRate; + return this; + } + + public ThreatIntelConfig withExpectedInsertions(int expectedInsertions) { + this.expectedInsertions = expectedInsertions; + return this; + } + + public ThreatIntelConfig withMillisecondsBetweenPersists(long millisecondsBetweenPersists) { + this.millisecondsBetweenPersists = millisecondsBetweenPersists; + return this; + } + + public ThreatIntelConfig withHbaseConfig(Configuration hbaseConfig) { + this.hbaseConfig = hbaseConfig; + return this; + + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/CacheKey.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/CacheKey.java index 6be87902..0628ebf4 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/CacheKey.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/CacheKey.java @@ -6,76 +6,89 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.cache; +import java.util.Objects; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.stellar.common.utils.ConversionUtils; public class CacheKey { - private String field; - private Object value; - private SensorEnrichmentConfig config; + private final String field; + private final Object value; + private final SensorEnrichmentConfig config; + + public CacheKey(String field, Object value, SensorEnrichmentConfig config) { + this.field = field; + this.value = value; + this.config = config; + } - public CacheKey(String field, Object value, SensorEnrichmentConfig config) { - this.field = field; - this.value = value; - this.config = config; - } + public String getField() { + return field; + } - public String getField() { - return field; - } + public Object getValue() { + return value; + } - public Object getValue() { - return value; - } + public T getValue(Class clazz) { + return clazz.cast(getValue()); + } - public T getValue(Class clazz) { - return clazz.cast(getValue()); - } + public T coerceValue(Class clazz) { + return ConversionUtils.convert(getValue(), clazz); + } - public T coerceValue(Class clazz) { - return ConversionUtils.convert(getValue(), clazz); - } - public SensorEnrichmentConfig getConfig() { - return config; - } + public SensorEnrichmentConfig getConfig() { + return config; + } - @Override - public String toString() { - return "CacheKey{" + - "field='" + field + '\'' + - ", value='" + value + '\'' + - '}'; - } + @Override + public String toString() { + return "CacheKey{" + + "field='" + field + '\'' + + ", value='" + value + '\'' + + '}'; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - CacheKey cacheKey = (CacheKey) o; + CacheKey cacheKey = (CacheKey) o; - if (getField() != null ? !getField().equals(cacheKey.getField()) : cacheKey.getField() != null) return false; - if (getValue() != null ? !getValue().equals(cacheKey.getValue()) : cacheKey.getValue() != null) return false; - return config != null ? config.equals(cacheKey.config) : cacheKey.config == null; + if (getField() != null ? !getField().equals(cacheKey.getField()) : cacheKey.getField() != null) { + return false; + } + if (getValue() != null ? !getValue().equals(cacheKey.getValue()) : cacheKey.getValue() != null) { + return false; + } + return Objects.equals(config, cacheKey.config); - } + } - @Override - public int hashCode() { - int result = getField() != null ? getField().hashCode() : 0; - result = 31 * result + (getValue() != null ? getValue().hashCode() : 0); - result = 31 * result + (config != null ? config.hashCode() : 0); - return result; - } + @Override + public int hashCode() { + int result = getField() != null ? getField().hashCode() : 0; + result = 31 * result + (getValue() != null ? getValue().hashCode() : 0); + result = 31 * result + (config != null ? config.hashCode() : 0); + return result; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCache.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCache.java index f8af8326..5137b3c8 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCache.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCache.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,9 +20,15 @@ package org.apache.metron.enrichment.cache; -import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.CacheLoader; +import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.invoke.MethodHandles; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -30,94 +38,89 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - public class ObjectCache { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - protected LoadingCache cache; - private static ReadWriteLock lock = new ReentrantReadWriteLock(); + protected LoadingCache cache; + private static final ReadWriteLock lock = new ReentrantReadWriteLock(); - public class Loader implements CacheLoader { - FileSystem fs; - ObjectCacheConfig objectCacheConfig; + public class Loader implements CacheLoader { + FileSystem fs; + ObjectCacheConfig objectCacheConfig; - public Loader(Configuration hadoopConfig, ObjectCacheConfig objectCacheConfig) throws IOException { - this.fs = FileSystem.get(hadoopConfig); - this.objectCacheConfig = objectCacheConfig; - } + public Loader(Configuration hadoopConfig, ObjectCacheConfig objectCacheConfig) throws IOException { + this.fs = FileSystem.get(hadoopConfig); + this.objectCacheConfig = objectCacheConfig; + } - @Override - public Object load(String s) throws Exception { - LOG.debug("Loading object from path '{}'", s); - if (StringUtils.isEmpty(s)) { - throw new IllegalArgumentException("Path cannot be empty"); - } - Object object = null; - Path p = new Path(s); - if (fs.exists(p)) { - if (fs.getFileStatus(p).getLen() <= objectCacheConfig.getMaxFileSize()) { - try (InputStream is = new BufferedInputStream(fs.open(p))) { - byte[] serialized = IOUtils.toByteArray(is); - if (serialized.length > 0) { - object = SerDeUtils.fromBytes(serialized, Object.class); + @Override + public Object load(String s) throws Exception { + LOG.debug("Loading object from path '{}'", s); + if (StringUtils.isEmpty(s)) { + throw new IllegalArgumentException("Path cannot be empty"); + } + Object object = null; + Path p = new Path(s); + if (fs.exists(p)) { + if (fs.getFileStatus(p).getLen() <= objectCacheConfig.getMaxFileSize()) { + try (InputStream is = new BufferedInputStream(fs.open(p))) { + byte[] serialized = IOUtils.toByteArray(is); + if (serialized.length > 0) { + object = SerDeUtils.fromBytes(serialized, Object.class); + } + } + } else { + throw new IllegalArgumentException( + String.format("File at path '%s' is larger than the configured max file size of %s", p, + objectCacheConfig.getMaxFileSize())); + } + } else { + throw new IllegalArgumentException(String.format("Path '%s' could not be found in HDFS", s)); } - } - } else { - throw new IllegalArgumentException(String.format("File at path '%s' is larger than the configured max file size of %s", p, objectCacheConfig.getMaxFileSize())); + return object; } - } else { - throw new IllegalArgumentException(String.format("Path '%s' could not be found in HDFS", s)); - } - return object; } - } - public Object get(String path) { - return cache.get(path); - } + public Object get(String path) { + return cache.get(path); + } - public void initialize(ObjectCacheConfig config) { - try { - lock.writeLock().lock(); + public void initialize(ObjectCacheConfig config) { + try { + lock.writeLock().lock(); - cache = setupCache(config); - } catch (IOException e) { - throw new IllegalStateException("Unable to initialize: " + e.getMessage(), e); - } finally { - lock.writeLock().unlock(); + cache = setupCache(config); + } catch (IOException e) { + throw new IllegalStateException("Unable to initialize: " + e.getMessage(), e); + } finally { + lock.writeLock().unlock(); + } } - } - public boolean isInitialized() { - try { - lock.readLock().lock(); - return cache != null; - } finally { - lock.readLock().unlock(); + public boolean isInitialized() { + try { + lock.readLock().lock(); + return cache != null; + } finally { + lock.readLock().unlock(); + } } - } - protected LoadingCache setupCache(ObjectCacheConfig config) throws IOException { - LOG.info("Building ObjectCache with {}", config); - return Caffeine.newBuilder().maximumSize(config.getCacheSize()) - .expireAfterWrite(config.getCacheExpiration(), config.getTimeUnit()) - .removalListener((path, value, removalCause) -> { - LOG.debug("Object retrieved from path '{}' was removed with cause {}", path, removalCause); - }) - .build(new Loader(new Configuration(), config)); - } + protected LoadingCache setupCache(ObjectCacheConfig config) throws IOException { + LOG.info("Building ObjectCache with {}", config); + return Caffeine.newBuilder().maximumSize(config.getCacheSize()) + .expireAfterWrite(config.getCacheExpiration(), config.getTimeUnit()) + .removalListener((path, value, removalCause) -> { + LOG.debug("Object retrieved from path '{}' was removed with cause {}", path, removalCause); + }) + .build(new Loader(new Configuration(), config)); + } - public boolean isEmpty() { - return cache == null || cache.estimatedSize() == 0; - } + public boolean isEmpty() { + return cache == null || cache.estimatedSize() == 0; + } - public boolean containsKey(String key) { - return cache != null && cache.asMap().containsKey(key); - } + public boolean containsKey(String key) { + return cache != null && cache.asMap().containsKey(key); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCacheConfig.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCacheConfig.java index 2c0c97e8..c7cf0078 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCacheConfig.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCacheConfig.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,98 +20,106 @@ package org.apache.metron.enrichment.cache; -import org.apache.metron.stellar.common.utils.ConversionUtils; - import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; +import org.apache.metron.stellar.common.utils.ConversionUtils; public class ObjectCacheConfig { - public static final String OBJECT_CACHE_SIZE_KEY = "object.cache.size"; - public static final String OBJECT_CACHE_EXPIRATION_MINUTES_KEY = "object.cache.expiration.minutes"; - public static final String OBJECT_CACHE_EXPIRATION_KEY = "object.cache.expiration"; - public static final String OBJECT_CACHE_TIME_UNIT_KEY = "object.cache.time.unit"; - public static final String OBJECT_CACHE_MAX_FILE_SIZE_KEY = "object.cache.max.file.size"; - public static final long OBJECT_CACHE_SIZE_DEFAULT = 1000; - public static final long OBJECT_CACHE_EXPIRATION_MIN_DEFAULT = 1440; - public static final TimeUnit OBJECT_CACHE_TIME_UNIT_DEFAULT = TimeUnit.MINUTES; - public static final long OBJECT_CACHE_MAX_FILE_SIZE_DEFAULT = 1048576; // default to 1 mb - - private long cacheSize; - private long cacheExpiration; - private TimeUnit timeUnit; - private long maxFileSize; - - public ObjectCacheConfig(Map config) { - cacheSize = ConversionUtils.convert(config.getOrDefault(OBJECT_CACHE_SIZE_KEY, OBJECT_CACHE_SIZE_DEFAULT), Long.class); - if (config.containsKey(OBJECT_CACHE_EXPIRATION_MINUTES_KEY)) { - cacheExpiration = ConversionUtils.convert(config.getOrDefault(OBJECT_CACHE_EXPIRATION_MINUTES_KEY, OBJECT_CACHE_EXPIRATION_MIN_DEFAULT), Long.class); - timeUnit = OBJECT_CACHE_TIME_UNIT_DEFAULT; - } else { - cacheExpiration = ConversionUtils.convert(config.getOrDefault(OBJECT_CACHE_EXPIRATION_KEY, OBJECT_CACHE_EXPIRATION_MIN_DEFAULT), Long.class); - timeUnit = config.containsKey(OBJECT_CACHE_TIME_UNIT_KEY) ? - TimeUnit.valueOf((String) config.get(OBJECT_CACHE_TIME_UNIT_KEY)) : OBJECT_CACHE_TIME_UNIT_DEFAULT; - } - maxFileSize = ConversionUtils.convert(config.getOrDefault(OBJECT_CACHE_MAX_FILE_SIZE_KEY, OBJECT_CACHE_MAX_FILE_SIZE_DEFAULT), Long.class); - } - - public long getCacheSize() { - return cacheSize; - } - - public void setCacheSize(long cacheSize) { - this.cacheSize = cacheSize; - } - - public long getCacheExpiration() { - return cacheExpiration; - } - - public void setCacheExpiration(long cacheExpiration) { - this.cacheExpiration = cacheExpiration; - } - - public TimeUnit getTimeUnit() { - return timeUnit; - } - - public void setTimeUnit(TimeUnit timeUnit) { - this.timeUnit = timeUnit; - } - - public long getMaxFileSize() { - return maxFileSize; - } - - public void setMaxFileSize(long maxFileSize) { - this.maxFileSize = maxFileSize; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ObjectCacheConfig that = (ObjectCacheConfig) o; - return cacheSize == that.cacheSize && - cacheExpiration == that.cacheExpiration && - timeUnit == that.timeUnit && - maxFileSize == that.maxFileSize; - } - - @Override - public int hashCode() { - - return Objects.hash(cacheSize, cacheExpiration, timeUnit, maxFileSize); - } - - @Override - public String toString() { - return "ObjectCacheConfig{" + - "cacheSize=" + cacheSize + - ", cacheExpiration=" + cacheExpiration + - ", timeUnit=" + timeUnit + - ", maxFileSize=" + maxFileSize + - '}'; - } + public static final String OBJECT_CACHE_SIZE_KEY = "object.cache.size"; + public static final String OBJECT_CACHE_EXPIRATION_MINUTES_KEY = "object.cache.expiration.minutes"; + public static final String OBJECT_CACHE_EXPIRATION_KEY = "object.cache.expiration"; + public static final String OBJECT_CACHE_TIME_UNIT_KEY = "object.cache.time.unit"; + public static final String OBJECT_CACHE_MAX_FILE_SIZE_KEY = "object.cache.max.file.size"; + public static final long OBJECT_CACHE_SIZE_DEFAULT = 1000; + public static final long OBJECT_CACHE_EXPIRATION_MIN_DEFAULT = 1440; + public static final TimeUnit OBJECT_CACHE_TIME_UNIT_DEFAULT = TimeUnit.MINUTES; + public static final long OBJECT_CACHE_MAX_FILE_SIZE_DEFAULT = 1048576; // default to 1 mb + + private long cacheSize; + private long cacheExpiration; + private TimeUnit timeUnit; + private long maxFileSize; + + public ObjectCacheConfig(Map config) { + cacheSize = ConversionUtils.convert(config.getOrDefault(OBJECT_CACHE_SIZE_KEY, OBJECT_CACHE_SIZE_DEFAULT), + Long.class); + if (config.containsKey(OBJECT_CACHE_EXPIRATION_MINUTES_KEY)) { + cacheExpiration = ConversionUtils.convert( + config.getOrDefault(OBJECT_CACHE_EXPIRATION_MINUTES_KEY, OBJECT_CACHE_EXPIRATION_MIN_DEFAULT), + Long.class); + timeUnit = OBJECT_CACHE_TIME_UNIT_DEFAULT; + } else { + cacheExpiration = ConversionUtils.convert( + config.getOrDefault(OBJECT_CACHE_EXPIRATION_KEY, OBJECT_CACHE_EXPIRATION_MIN_DEFAULT), Long.class); + timeUnit = config.containsKey(OBJECT_CACHE_TIME_UNIT_KEY) + ? TimeUnit.valueOf((String) config.get(OBJECT_CACHE_TIME_UNIT_KEY)) : OBJECT_CACHE_TIME_UNIT_DEFAULT; + } + maxFileSize = ConversionUtils.convert( + config.getOrDefault(OBJECT_CACHE_MAX_FILE_SIZE_KEY, OBJECT_CACHE_MAX_FILE_SIZE_DEFAULT), Long.class); + } + + public long getCacheSize() { + return cacheSize; + } + + public void setCacheSize(long cacheSize) { + this.cacheSize = cacheSize; + } + + public long getCacheExpiration() { + return cacheExpiration; + } + + public void setCacheExpiration(long cacheExpiration) { + this.cacheExpiration = cacheExpiration; + } + + public TimeUnit getTimeUnit() { + return timeUnit; + } + + public void setTimeUnit(TimeUnit timeUnit) { + this.timeUnit = timeUnit; + } + + public long getMaxFileSize() { + return maxFileSize; + } + + public void setMaxFileSize(long maxFileSize) { + this.maxFileSize = maxFileSize; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ObjectCacheConfig that = (ObjectCacheConfig) o; + return cacheSize == that.cacheSize + && cacheExpiration == that.cacheExpiration + && timeUnit == that.timeUnit + && maxFileSize == that.maxFileSize; + } + + @Override + public int hashCode() { + + return Objects.hash(cacheSize, cacheExpiration, timeUnit, maxFileSize); + } + + @Override + public String toString() { + return "ObjectCacheConfig{" + + "cacheSize=" + cacheSize + + ", cacheExpiration=" + cacheExpiration + + ", timeUnit=" + timeUnit + + ", maxFileSize=" + maxFileSize + + '}'; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java index e96ea399..dc60b351 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java @@ -6,185 +6,206 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.cli; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; -import java.nio.charset.StandardCharsets; -import org.apache.commons.cli.*; -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; -import org.apache.metron.common.utils.JSONUtils; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintStream; -import java.util.*; +import java.nio.charset.StandardCharsets; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.apache.metron.common.utils.JSONUtils; public class LatencySummarizer { - public static class Pair extends AbstractMap.SimpleEntry { - public Pair(String key, String value) { - super(key, value); + public static class Pair extends AbstractMap.SimpleEntry { + public Pair(String key, String value) { + super(key, value); + } } - } - public static class LatencyStats { - private NavigableMap> depthMap = new TreeMap<>(); - private List metrics; - public void updateMetrics(List metrics) { - this.metrics = metrics; - } - public Map getStatsMap(int depth) { - Map statsMap = depthMap.get(depth); - if(statsMap == null) { - statsMap = new HashMap<>(); - depthMap.put(depth, statsMap); - } - return statsMap; - } - public DescriptiveStatistics getStats( int depth, Pair p) { - Map statsMap = getStatsMap(depth); - DescriptiveStatistics stats = statsMap.get(p); - if(stats == null) { - stats = new DescriptiveStatistics(); - statsMap.put(p, stats); - } - return stats; - } - public void put(int depth, Pair p, double val) { - getStats(depth, p).addValue(val); - } + public static class LatencyStats { + private final NavigableMap> depthMap = new TreeMap<>(); + private List metrics; - public static void summary(String title, DescriptiveStatistics statistics, PrintStream pw, boolean meanOnly) { - if(meanOnly) { - pw.println(title + ": " - + "\n\tMean: " + statistics.getMean() - ); - } - else { - pw.println(title + ": " - + "\n\tMean: " + statistics.getMean() - + "\n\tMin: " + statistics.getMin() - + "\n\t1th: " + statistics.getPercentile(1) - + "\n\t5th: " + statistics.getPercentile(5) - + "\n\t10th: " + statistics.getPercentile(10) - + "\n\t25th: " + statistics.getPercentile(25) - + "\n\t50th: " + statistics.getPercentile(50) - + "\n\t90th: " + statistics.getPercentile(90) - + "\n\t95th: " + statistics.getPercentile(95) - + "\n\t99th: " + statistics.getPercentile(99) - + "\n\tMax: " + statistics.getMax() - + "\n\tStdDev: " + statistics.getStandardDeviation() - ); - } - } - public void printDepthSummary(int depth, boolean meanOnly) { - Map statsMap = depthMap.get(depth); - System.out.println("\nDistance " + depth); - System.out.println("----------------\n"); - List> sortedStats = new ArrayList<>(); - for(Map.Entry stats : statsMap.entrySet()) { - sortedStats.add(stats); - } - Collections.sort(sortedStats, new Comparator>() { - @Override - public int compare(Map.Entry o1, Map.Entry o2) { - return -1*Double.compare(o1.getValue().getMean(), o2.getValue().getMean()); - } - }); - for(Map.Entry stats : sortedStats) { - summary(stats.getKey().getKey() + " -> " + stats.getKey().getValue(), stats.getValue(), System.out, meanOnly); - } - } - public void printSummary(boolean meanOnly) { - System.out.println("Flow:"); - System.out.println("\t" + Joiner.on(" -> ").join(metrics)); - System.out.println("\nSUMMARY BY DISTANCE\n--------------------------"); - for(int depth : depthMap.keySet()) { - printDepthSummary(depth, meanOnly); - } - } + public void updateMetrics(List metrics) { + this.metrics = metrics; + } - } - - public static String getBaseMetric(String s) { - Iterable tokenIt = Splitter.on('.').split(s); - int num = Iterables.size(tokenIt); - return Joiner.on('.').join(Iterables.limit(tokenIt, num-1)); - } - - public static void updateStats(LatencyStats stats, Map doc) { - Map latencyMap = new HashMap<>(); - NavigableMap latencyInvMap = new TreeMap<>(); - for(Map.Entry kv : doc.entrySet()) { - if(kv.getKey().endsWith(".ts")) { - String base = getBaseMetric(kv.getKey()); - long latency = Long.parseLong(kv.getValue().toString()); - latencyInvMap.put(latency, base); - latencyMap.put( base, latency); - } - } - List metrics = new ArrayList<>(); - for(Map.Entry kv : latencyInvMap.entrySet()) { - metrics.add(kv.getValue()); - } - stats.updateMetrics(metrics); - for(int i = 0;i < metrics.size();++i) { - for(int j = i+1;j < metrics.size();++j) { - Pair p = new Pair(metrics.get(i), metrics.get(j)); - long ms = latencyMap.get(metrics.get(j)) - latencyMap.get(metrics.get(i)); - stats.put(j-i, p, ms); - } - } - } + public Map getStatsMap(int depth) { + Map statsMap = depthMap.get(depth); + if (statsMap == null) { + statsMap = new HashMap<>(); + depthMap.put(depth, statsMap); + } + return statsMap; + } + public DescriptiveStatistics getStats(int depth, Pair p) { + Map statsMap = getStatsMap(depth); + DescriptiveStatistics stats = statsMap.get(p); + if (stats == null) { + stats = new DescriptiveStatistics(); + statsMap.put(p, stats); + } + return stats; + } + public void put(int depth, Pair p, double val) { + getStats(depth, p).addValue(val); + } + + public static void summary(String title, DescriptiveStatistics statistics, PrintStream pw, boolean meanOnly) { + if (meanOnly) { + pw.println(title + ": " + + "\n\tMean: " + statistics.getMean() + ); + } else { + pw.println(title + ": " + + "\n\tMean: " + statistics.getMean() + + "\n\tMin: " + statistics.getMin() + + "\n\t1th: " + statistics.getPercentile(1) + + "\n\t5th: " + statistics.getPercentile(5) + + "\n\t10th: " + statistics.getPercentile(10) + + "\n\t25th: " + statistics.getPercentile(25) + + "\n\t50th: " + statistics.getPercentile(50) + + "\n\t90th: " + statistics.getPercentile(90) + + "\n\t95th: " + statistics.getPercentile(95) + + "\n\t99th: " + statistics.getPercentile(99) + + "\n\tMax: " + statistics.getMax() + + "\n\tStdDev: " + statistics.getStandardDeviation() + ); + } + } + + public void printDepthSummary(int depth, boolean meanOnly) { + Map statsMap = depthMap.get(depth); + System.out.println("\nDistance " + depth); + System.out.println("----------------\n"); + List> sortedStats = new ArrayList<>(); + for (Map.Entry stats : statsMap.entrySet()) { + sortedStats.add(stats); + } + Collections.sort(sortedStats, new Comparator>() { + @Override + public int compare(Map.Entry o1, + Map.Entry o2) { + return -1 * Double.compare(o1.getValue().getMean(), o2.getValue().getMean()); + } + }); + for (Map.Entry stats : sortedStats) { + summary(stats.getKey().getKey() + " -> " + stats.getKey().getValue(), stats.getValue(), System.out, + meanOnly); + } + } + + public void printSummary(boolean meanOnly) { + System.out.println("Flow:"); + System.out.println("\t" + Joiner.on(" -> ").join(metrics)); + System.out.println("\nSUMMARY BY DISTANCE\n--------------------------"); + for (int depth : depthMap.keySet()) { + printDepthSummary(depth, meanOnly); + } + } - public static void main(String... argv) throws IOException { - Options options = new Options(); - { - Option o = new Option("h", "help", false, "This screen"); - o.setRequired(false); - options.addOption(o); - } - { - Option o = new Option("m", "mean_only", false, "Print the mean only when we summarize"); - o.setRequired(false); - options.addOption(o); - } - CommandLineParser parser = new PosixParser(); - CommandLine cmd = null; - try { - cmd = parser.parse(options, argv); } - catch(ParseException pe) { - pe.printStackTrace(); - final HelpFormatter usageFormatter = new HelpFormatter(); - usageFormatter.printHelp(LatencySummarizer.class.getSimpleName().toLowerCase(), null, options, null, true); - System.exit(-1); + + public static String getBaseMetric(String s) { + Iterable tokenIt = Splitter.on('.').split(s); + int num = Iterables.size(tokenIt); + return Joiner.on('.').join(Iterables.limit(tokenIt, num - 1)); } - if( cmd.hasOption("h") ){ - final HelpFormatter usageFormatter = new HelpFormatter(); - usageFormatter.printHelp(LatencySummarizer.class.getSimpleName().toLowerCase(), null, options, null, true); - System.exit(0); + + public static void updateStats(LatencyStats stats, Map doc) { + Map latencyMap = new HashMap<>(); + NavigableMap latencyInvMap = new TreeMap<>(); + for (Map.Entry kv : doc.entrySet()) { + if (kv.getKey().endsWith(".ts")) { + String base = getBaseMetric(kv.getKey()); + long latency = Long.parseLong(kv.getValue().toString()); + latencyInvMap.put(latency, base); + latencyMap.put(base, latency); + } + } + List metrics = new ArrayList<>(); + for (Map.Entry kv : latencyInvMap.entrySet()) { + metrics.add(kv.getValue()); + } + stats.updateMetrics(metrics); + for (int i = 0; i < metrics.size(); ++i) { + for (int j = i + 1; j < metrics.size(); ++j) { + Pair p = new Pair(metrics.get(i), metrics.get(j)); + long ms = latencyMap.get(metrics.get(j)) - latencyMap.get(metrics.get(i)); + stats.put(j - i, p, ms); + } + } } - LatencyStats statsMap = new LatencyStats(); - BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, - StandardCharsets.UTF_8)); - for(String line = null;(line = reader.readLine()) != null;) { - Map doc = JSONUtils.INSTANCE.load(line, JSONUtils.MAP_SUPPLIER); - updateStats(statsMap, doc); + + + public static void main(String... argv) throws IOException { + Options options = new Options(); + { + Option o = new Option("h", "help", false, "This screen"); + o.setRequired(false); + options.addOption(o); + } + { + Option o = new Option("m", "mean_only", false, "Print the mean only when we summarize"); + o.setRequired(false); + options.addOption(o); + } + CommandLineParser parser = new PosixParser(); + CommandLine cmd = null; + try { + cmd = parser.parse(options, argv); + } catch (ParseException pe) { + pe.printStackTrace(); + final HelpFormatter usageFormatter = new HelpFormatter(); + usageFormatter.printHelp(LatencySummarizer.class.getSimpleName().toLowerCase(), null, options, null, true); + System.exit(-1); + } + if (cmd.hasOption("h")) { + final HelpFormatter usageFormatter = new HelpFormatter(); + usageFormatter.printHelp(LatencySummarizer.class.getSimpleName().toLowerCase(), null, options, null, true); + System.exit(0); + } + LatencyStats statsMap = new LatencyStats(); + BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, + StandardCharsets.UTF_8)); + for (String line = null; (line = reader.readLine()) != null; ) { + Map doc = JSONUtils.INSTANCE.load(line, JSONUtils.MAP_SUPPLIER); + updateStats(statsMap, doc); + } + statsMap.printSummary(cmd.hasOption('m')); } - statsMap.printSummary(cmd.hasOption('m')); - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/configuration/Enrichment.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/configuration/Enrichment.java index 736a9110..432c1600 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/configuration/Enrichment.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/configuration/Enrichment.java @@ -6,57 +6,60 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.enrichment.configuration; -import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; +package org.apache.metron.enrichment.configuration; import java.io.Serializable; import java.util.List; +import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; public class Enrichment implements Serializable { - private String type; - private List fields; - private T adapter; + private String type; + private List fields; + private T adapter; - public Enrichment() {} + public Enrichment() { + } - public Enrichment(String type, T adapter) { - this.type = type; - this.adapter = adapter; - } + public Enrichment(String type, T adapter) { + this.type = type; + this.adapter = adapter; + } - public List getFields() { - return fields; - } + public List getFields() { + return fields; + } - public void setFields(List fields) { - this.fields = fields; - } + public void setFields(List fields) { + this.fields = fields; + } - public String getType() { - return type; - } + public String getType() { + return type; + } - public void setType(String type) { - this.type = type; - } + public void setType(String type) { + this.type = type; + } - public T getAdapter() { - return adapter; - } + public T getAdapter() { + return adapter; + } - public void setAdapter(T adapter) { - this.adapter = adapter; - } + public void setAdapter(T adapter) { + this.adapter = adapter; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/AbstractConverter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/AbstractConverter.java index 16172565..91af0623 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/AbstractConverter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/AbstractConverter.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

* http://www.apache.org/licenses/LICENSE-2.0 + * *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,6 +20,17 @@ package org.apache.metron.enrichment.converter; +import java.io.IOException; +import java.io.Serializable; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; @@ -27,28 +40,25 @@ import org.apache.metron.enrichment.lookup.LookupKey; import org.apache.metron.enrichment.lookup.LookupValue; -import javax.annotation.Nullable; -import java.io.IOException; -import java.io.Serializable; -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; - -public abstract class AbstractConverter implements HbaseConverter, Serializable { - public static final Function> CELL_TO_ENTRY = new Function>() { +public abstract class AbstractConverter + implements HbaseConverter, Serializable { + public static final Function> CELL_TO_ENTRY = + new Function>() { - @Override - public Map.Entry apply(@Nullable Cell cell) { - if (cell != null) { - byte[] qualifier = Arrays.copyOfRange(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierOffset() + cell.getQualifierLength()); - byte[] value = Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), cell.getValueOffset() + cell.getValueLength()); - return new AbstractMap.SimpleEntry<>(qualifier, value); - } else { - return new AbstractMap.SimpleEntry<>(null, null); - } - } - }; + @Override + public Map.Entry apply(@Nullable Cell cell) { + if (cell != null) { + byte[] qualifier = Arrays.copyOfRange(cell.getQualifierArray(), cell.getQualifierOffset(), + cell.getQualifierOffset() + cell.getQualifierLength()); + byte[] value = Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), + cell.getValueOffset() + cell.getValueLength()); + return new AbstractMap.SimpleEntry<>(qualifier, value); + } else { + return new AbstractMap.SimpleEntry<>(null, null); + } + } + }; @Override public Put toPut(String columnFamily, KEY_T key, VALUE_T values) { @@ -73,7 +83,8 @@ public Result toResult(String columnFamily, KEY_T key, VALUE_T values) throws IO return Result.create(put.getFamilyCellMap().get(Bytes.toBytes(columnFamily))); } - public LookupKV fromResult(Result result, String columnFamily, KEY_T key, VALUE_T value) throws IOException { + public LookupKV fromResult(Result result, String columnFamily, KEY_T key, VALUE_T value) + throws IOException { if (result == null || result.getRow() == null) { return null; } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentConverter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentConverter.java index 6f19781d..eba2cf11 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentConverter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentConverter.java @@ -6,32 +6,34 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.converter; +import java.io.IOException; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.metron.enrichment.lookup.LookupKV; -import java.io.IOException; - public class EnrichmentConverter extends AbstractConverter { - @Override - public LookupKV fromPut(Put put, String columnFamily) throws IOException { - return fromPut(put, columnFamily, new EnrichmentKey(), new EnrichmentValue()); - } + @Override + public LookupKV fromPut(Put put, String columnFamily) throws IOException { + return fromPut(put, columnFamily, new EnrichmentKey(), new EnrichmentValue()); + } - @Override - public LookupKV fromResult(Result result, String columnFamily) throws IOException { - return fromResult(result, columnFamily, new EnrichmentKey(), new EnrichmentValue()); - } + @Override + public LookupKV fromResult(Result result, String columnFamily) throws IOException { + return fromResult(result, columnFamily, new EnrichmentKey(), new EnrichmentValue()); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentHelper.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentHelper.java index b8198dfc..5ee6d618 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentHelper.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentHelper.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.converter; import java.io.IOException; @@ -26,8 +29,9 @@ public enum EnrichmentHelper { INSTANCE; EnrichmentConverter converter = new EnrichmentConverter(); - public void load(Table table, String cf, Iterable> results) throws IOException { - for(LookupKV result : results) { + public void load(Table table, String cf, Iterable> results) + throws IOException { + for (LookupKV result : results) { Put put = converter.toPut(cf, result.getKey(), result.getValue()); table.put(put); } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentKey.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentKey.java index 60812602..2c732d02 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentKey.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentKey.java @@ -7,106 +7,116 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.enrichment.converter; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.metron.common.utils.KeyUtil; -import org.apache.metron.enrichment.lookup.LookupKey; +package org.apache.metron.enrichment.converter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.util.Objects; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.metron.common.utils.KeyUtil; +import org.apache.metron.enrichment.lookup.LookupKey; public class EnrichmentKey implements LookupKey { - public String indicator; - public String type; - - public EnrichmentKey() { - - } - public EnrichmentKey(String type, String indicator) { - this.indicator = indicator; - this.type = type; - } - - private byte[] typedIndicatorToBytes() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream w = new DataOutputStream(baos); - w.writeUTF(type); - w.writeUTF(indicator); - w.flush(); - return baos.toByteArray(); - } - - @Override - public byte[] toBytes() { - byte[] indicatorBytes = new byte[0]; - try { - indicatorBytes = typedIndicatorToBytes(); - } catch (IOException e) { - throw new RuntimeException("Unable to convert type and indicator to bytes", e); + public String indicator; + public String type; + + public EnrichmentKey() { + + } + + public EnrichmentKey(String type, String indicator) { + this.indicator = indicator; + this.type = type; + } + + private byte[] typedIndicatorToBytes() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream w = new DataOutputStream(baos); + w.writeUTF(type); + w.writeUTF(indicator); + w.flush(); + return baos.toByteArray(); + } + + @Override + public byte[] toBytes() { + byte[] indicatorBytes = new byte[0]; + try { + indicatorBytes = typedIndicatorToBytes(); + } catch (IOException e) { + throw new RuntimeException("Unable to convert type and indicator to bytes", e); + } + byte[] prefix = KeyUtil.INSTANCE.getPrefix(Bytes.toBytes(indicator)); + return KeyUtil.INSTANCE.merge(prefix, indicatorBytes); + } + + @Override + public void fromBytes(byte[] row) { + ByteArrayInputStream baos = new ByteArrayInputStream(row); + baos.skip(KeyUtil.HASH_PREFIX_SIZE); + DataInputStream w = new DataInputStream(baos); + try { + type = w.readUTF(); + indicator = w.readUTF(); + } catch (IOException e) { + throw new RuntimeException("Unable to convert type and indicator from bytes", e); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + EnrichmentKey that = (EnrichmentKey) o; + + if (!Objects.equals(indicator, that.indicator)) { + return false; + } + return Objects.equals(type, that.type); + } - byte[] prefix = KeyUtil.INSTANCE.getPrefix(Bytes.toBytes(indicator)); - return KeyUtil.INSTANCE.merge(prefix, indicatorBytes); - } - - @Override - public void fromBytes(byte[] row) { - ByteArrayInputStream baos = new ByteArrayInputStream(row); - baos.skip(KeyUtil.HASH_PREFIX_SIZE); - DataInputStream w = new DataInputStream(baos); - try { - type = w.readUTF(); - indicator = w.readUTF(); - } catch (IOException e) { - throw new RuntimeException("Unable to convert type and indicator from bytes", e); + + @Override + public int hashCode() { + int result = indicator != null ? indicator.hashCode() : 0; + result = 31 * result + (type != null ? type.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "EnrichmentKey{" + + "indicator='" + indicator + '\'' + + ", type='" + type + '\'' + + '}'; + } + + @Override + public String getIndicator() { + return indicator; + } + + @Override + public void setIndicator(String indicator) { + this.indicator = indicator; } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - EnrichmentKey that = (EnrichmentKey) o; - - if (indicator != null ? !indicator.equals(that.indicator) : that.indicator != null) return false; - return type != null ? type.equals(that.type) : that.type == null; - - } - - @Override - public int hashCode() { - int result = indicator != null ? indicator.hashCode() : 0; - result = 31 * result + (type != null ? type.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "EnrichmentKey{" + - "indicator='" + indicator + '\'' + - ", type='" + type + '\'' + - '}'; - } - - @Override - public String getIndicator() { - return indicator; - } - - @Override - public void setIndicator(String indicator) { - this.indicator = indicator; - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java index bd019a0f..8fb14efd 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java @@ -6,39 +6,40 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.converter; +import java.io.IOException; +import java.util.Map; import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.enrichment.lookup.LookupValue; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; -import java.io.IOException; -import java.util.Map; - public class EnrichmentValue implements LookupValue { - private static final ThreadLocal _mapper = new ThreadLocal() { - @Override - protected ObjectMapper initialValue() { - return new ObjectMapper(); - } + private static final ThreadLocal _mapper = new ThreadLocal() { + @Override + protected ObjectMapper initialValue() { + return new ObjectMapper(); + } }; public static final String VALUE_COLUMN_NAME = "v"; public static final byte[] VALUE_COLUMN_NAME_B = Bytes.toBytes(VALUE_COLUMN_NAME); private Map metadata = null; - public EnrichmentValue() - { + public EnrichmentValue() { } @@ -47,32 +48,34 @@ public EnrichmentValue(Map metadata) { } - public Map getMetadata() { return metadata; } @Override public Iterable> toColumns() { - return AbstractConverter.toEntries( VALUE_COLUMN_NAME_B, Bytes.toBytes(valueToString(metadata)) - ); + return AbstractConverter.toEntries(VALUE_COLUMN_NAME_B, Bytes.toBytes(valueToString(metadata)) + ); } @Override public void fromColumns(Iterable> values) { - for(Map.Entry cell : values) { - if(Bytes.equals(cell.getKey(), VALUE_COLUMN_NAME_B)) { + for (Map.Entry cell : values) { + if (Bytes.equals(cell.getKey(), VALUE_COLUMN_NAME_B)) { metadata = stringToValue(Bytes.toString(cell.getValue())); } } } - public Map stringToValue(String s){ + + public Map stringToValue(String s) { try { - return _mapper.get().readValue(s, new TypeReference>(){}); + return _mapper.get().readValue(s, new TypeReference>() { + }); } catch (IOException e) { throw new RuntimeException("Unable to convert string to metadata: " + s); } } + public String valueToString(Map value) { try { return _mapper.get().writeValueAsString(value); @@ -83,8 +86,12 @@ public String valueToString(Map value) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } EnrichmentValue that = (EnrichmentValue) o; @@ -99,8 +106,8 @@ public int hashCode() { @Override public String toString() { - return "EnrichmentValue{" + - "metadata=" + metadata + - '}'; + return "EnrichmentValue{" + + "metadata=" + metadata + + '}'; } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/HbaseConverter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/HbaseConverter.java index 7300b76e..988c6bcb 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/HbaseConverter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/converter/HbaseConverter.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +20,7 @@ package org.apache.metron.enrichment.converter; +import java.io.IOException; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -25,8 +28,7 @@ import org.apache.metron.enrichment.lookup.LookupKey; import org.apache.metron.enrichment.lookup.LookupValue; -import java.io.IOException; - +@SuppressWarnings("checkstyle:InterfaceTypeParameterName") public interface HbaseConverter { Put toPut(String columnFamily, KEY_T key, VALUE_T values) throws IOException; diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/interfaces/EnrichmentAdapter.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/interfaces/EnrichmentAdapter.java index 1ee6ade8..1c27bdf4 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/interfaces/EnrichmentAdapter.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/interfaces/EnrichmentAdapter.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,19 +20,23 @@ package org.apache.metron.enrichment.interfaces; +import java.util.Map; import org.json.simple.JSONObject; -import java.util.Map; +public interface EnrichmentAdapter { + void logAccess(T value); + + JSONObject enrich(T value); + + boolean initializeAdapter(Map config); + + void updateAdapter(Map config); + + void cleanup(); + + String getOutputPrefix(T value); -public interface EnrichmentAdapter -{ - void logAccess(T value); - JSONObject enrich(T value); - boolean initializeAdapter(Map config); - void updateAdapter(Map config); - void cleanup(); - String getOutputPrefix(T value); - default String getStreamSubGroup(String enrichmentType, String field) { - return ""; - } + default String getStreamSubGroup(String enrichmentType, String field) { + return ""; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/EnrichmentLookup.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/EnrichmentLookup.java index c148e9ec..c4a05332 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/EnrichmentLookup.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/EnrichmentLookup.java @@ -7,128 +7,142 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.lookup; import com.google.common.collect.Iterables; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.Result; -import org.apache.metron.enrichment.converter.HbaseConverter; +import org.apache.hadoop.hbase.client.Table; import org.apache.metron.enrichment.converter.EnrichmentConverter; import org.apache.metron.enrichment.converter.EnrichmentKey; import org.apache.metron.enrichment.converter.EnrichmentValue; +import org.apache.metron.enrichment.converter.HbaseConverter; import org.apache.metron.enrichment.lookup.accesstracker.AccessTracker; import org.apache.metron.enrichment.lookup.handler.KeyWithContext; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; +public class EnrichmentLookup + extends Lookup> + implements AutoCloseable { -public class EnrichmentLookup extends Lookup> implements AutoCloseable { + public static class HBaseContext { + private final Table table; + private final String columnFamily; - public static class HBaseContext { - private Table table; - private String columnFamily; - public HBaseContext(Table table, String columnFamily) { - this.table = table; - this.columnFamily = columnFamily; - } + public HBaseContext(Table table, String columnFamily) { + this.table = table; + this.columnFamily = columnFamily; + } - public Table getTable() { return table; } - public String getColumnFamily() { return columnFamily; } - } + public Table getTable() { + return table; + } - public static class Handler implements org.apache.metron.enrichment.lookup.handler.Handler> { - String columnFamily; - HbaseConverter converter = new EnrichmentConverter(); - public Handler(String columnFamily) { - this.columnFamily = columnFamily; + public String getColumnFamily() { + return columnFamily; + } } - private String getColumnFamily(HBaseContext context) { - return context.getColumnFamily() == null?columnFamily:context.getColumnFamily(); + public static class Handler implements + org.apache.metron.enrichment.lookup.handler.Handler> { + String columnFamily; + HbaseConverter converter = new EnrichmentConverter(); + + public Handler(String columnFamily) { + this.columnFamily = columnFamily; + } + + private String getColumnFamily(HBaseContext context) { + return context.getColumnFamily() == null ? columnFamily : context.getColumnFamily(); + } + + @Override + public boolean exists(EnrichmentKey key, HBaseContext context, boolean logAccess) throws IOException { + return context.getTable().exists(converter.toGet(getColumnFamily(context), key)); + } + + @Override + public Iterable exists(Iterable> key, boolean logAccess) + throws IOException { + List ret = new ArrayList<>(); + if (Iterables.isEmpty(key)) { + return Collections.emptyList(); + } + Table table = Iterables.getFirst(key, null).getContext().getTable(); + for (boolean b : table.existsAll(keysToGets(key))) { + ret.add(b); + } + return ret; + } + + @Override + public LookupKV get(EnrichmentKey key, HBaseContext context, boolean logAccess) + throws IOException { + return converter.fromResult(context.getTable().get(converter.toGet(getColumnFamily(context), key)), + getColumnFamily(context)); + } + + @Override + public Iterable> get( + Iterable> keys, boolean logAccess) throws IOException { + if (Iterables.isEmpty(keys)) { + return Collections.emptyList(); + } + Table table = Iterables.getFirst(keys, null).getContext().getTable(); + List> ret = new ArrayList<>(); + Iterator> keyWithContextIterator = keys.iterator(); + for (Result result : table.get(keysToGets(keys))) { + HBaseContext context = keyWithContextIterator.next().getContext(); + ret.add(converter.fromResult(result, getColumnFamily(context))); + } + return ret; + } + + private List keysToGets(Iterable> keys) { + List ret = new ArrayList<>(); + for (KeyWithContext key : keys) { + ret.add(converter.toGet(getColumnFamily(key.getContext()), key.getKey())); + } + return ret; + } + + @Override + public void close() throws Exception { + + } } - @Override - public boolean exists(EnrichmentKey key, HBaseContext context, boolean logAccess) throws IOException { - return context.getTable().exists(converter.toGet(getColumnFamily(context), key)); - } + private final Table table; - @Override - public LookupKV get(EnrichmentKey key, HBaseContext context, boolean logAccess) throws IOException { - return converter.fromResult(context.getTable().get(converter.toGet(getColumnFamily(context), key)), getColumnFamily(context)); + public EnrichmentLookup(Table table, String columnFamily, AccessTracker tracker) { + this.table = table; + this.setLookupHandler(new Handler(columnFamily)); + this.setAccessTracker(tracker); } - private List keysToGets(Iterable> keys) { - List ret = new ArrayList<>(); - for(KeyWithContext key : keys) { - ret.add(converter.toGet(getColumnFamily(key.getContext()), key.getKey())); - } - return ret; + public Table getTable() { + return table; } - @Override - public Iterable exists(Iterable> key, boolean logAccess) throws IOException { - List ret = new ArrayList<>(); - if(Iterables.isEmpty(key)) { - return Collections.emptyList(); - } - Table table = Iterables.getFirst(key, null).getContext().getTable(); - for(boolean b : table.existsAll(keysToGets(key))) { - ret.add(b); - } - return ret; - } - - @Override - public Iterable> get( Iterable> keys - , boolean logAccess - ) throws IOException - { - if(Iterables.isEmpty(keys)) { - return Collections.emptyList(); - } - Table table = Iterables.getFirst(keys, null).getContext().getTable(); - List> ret = new ArrayList<>(); - Iterator> keyWithContextIterator = keys.iterator(); - for(Result result : table.get(keysToGets(keys))) { - HBaseContext context = keyWithContextIterator.next().getContext(); - ret.add(converter.fromResult(result, getColumnFamily(context))); - } - return ret; - } - - @Override public void close() throws Exception { - + super.close(); + table.close(); } - } - private Table table; - public EnrichmentLookup(Table table, String columnFamily, AccessTracker tracker) { - this.table = table; - this.setLookupHandler(new Handler(columnFamily)); - this.setAccessTracker(tracker); - } - - public Table getTable() { - return table; - } - - @Override - public void close() throws Exception { - super.close(); - table.close(); - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/Lookup.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/Lookup.java index 1ec8c957..f357910c 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/Lookup.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/Lookup.java @@ -6,92 +6,96 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.lookup; +import java.io.IOException; import org.apache.metron.enrichment.lookup.accesstracker.AccessTracker; import org.apache.metron.enrichment.lookup.handler.Handler; import org.apache.metron.enrichment.lookup.handler.KeyWithContext; -import java.io.IOException; - public class Lookup implements Handler { - private String name; - private AccessTracker accessTracker; - private Handler lookupHandler; + private String name; + private AccessTracker accessTracker; + private Handler lookupHandler; - public String getName() { - return name; - } + public String getName() { + return name; + } - public void setName(String name) { - this.name = name; - } + public void setName(String name) { + this.name = name; + } - public AccessTracker getAccessTracker() { - return accessTracker; - } + public AccessTracker getAccessTracker() { + return accessTracker; + } - public void setAccessTracker(AccessTracker accessTracker) { - this.accessTracker = accessTracker; - } + public void setAccessTracker(AccessTracker accessTracker) { + this.accessTracker = accessTracker; + } - public Handler< CONTEXT_T, KEY_T, RESULT_T > getLookupHandler() { - return lookupHandler; - } + public Handler getLookupHandler() { + return lookupHandler; + } - public void setLookupHandler(Handler< CONTEXT_T, KEY_T, RESULT_T > lookupHandler) { - this.lookupHandler = lookupHandler; - } + public void setLookupHandler(Handler lookupHandler) { + this.lookupHandler = lookupHandler; + } - @Override - public boolean exists(KEY_T key, CONTEXT_T context, boolean logAccess) throws IOException { - if(logAccess) { - accessTracker.logAccess(key); + @Override + public boolean exists(KEY_T key, CONTEXT_T context, boolean logAccess) throws IOException { + if (logAccess) { + accessTracker.logAccess(key); + } + return lookupHandler.exists(key, context, logAccess); } - return lookupHandler.exists(key, context, logAccess); - } - @Override - public RESULT_T get(KEY_T key, CONTEXT_T context, boolean logAccess) throws IOException { - if(logAccess) { - accessTracker.logAccess(key); + @Override + public Iterable exists(Iterable> key, boolean logAccess) + throws IOException { + if (logAccess) { + for (KeyWithContext k : key) { + accessTracker.logAccess(k.getKey()); + } + } + return lookupHandler.exists(key, logAccess); } - return lookupHandler.get(key, context, logAccess); - } - @Override - public Iterable exists(Iterable> key, boolean logAccess) throws IOException { - if(logAccess) { - for (KeyWithContext k : key) { - accessTracker.logAccess(k.getKey()); - } + @Override + public RESULT_T get(KEY_T key, CONTEXT_T context, boolean logAccess) throws IOException { + if (logAccess) { + accessTracker.logAccess(key); + } + return lookupHandler.get(key, context, logAccess); } - return lookupHandler.exists(key, logAccess); - } - @Override - public Iterable get(Iterable> key, boolean logAccess) throws IOException { - if(logAccess) { - for (KeyWithContext k : key) { - accessTracker.logAccess(k.getKey()); - } + @Override + public Iterable get(Iterable> key, boolean logAccess) + throws IOException { + if (logAccess) { + for (KeyWithContext k : key) { + accessTracker.logAccess(k.getKey()); + } + } + return lookupHandler.get(key, logAccess); } - return lookupHandler.get(key, logAccess); - } - @Override - public void close() throws Exception { - accessTracker.cleanup(); - lookupHandler.close(); - } + @Override + public void close() throws Exception { + accessTracker.cleanup(); + lookupHandler.close(); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKV.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKV.java index 3538aabb..c1f32b0e 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKV.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKV.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,10 +21,12 @@ package org.apache.metron.enrichment.lookup; import java.io.Serializable; +import java.util.Objects; public class LookupKV implements Serializable { - private KEY_T key; - private VALUE_T value; + private final KEY_T key; + private final VALUE_T value; + public LookupKV(KEY_T key, VALUE_T value) { this.key = key; this.value = value; @@ -38,13 +42,19 @@ public VALUE_T getValue() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } LookupKV lookupKV = (LookupKV) o; - if (key != null ? !key.equals(lookupKV.key) : lookupKV.key != null) return false; - return value != null ? value.equals(lookupKV.value) : lookupKV.value == null; + if (!Objects.equals(key, lookupKV.key)) { + return false; + } + return Objects.equals(value, lookupKV.value); } @@ -57,9 +67,9 @@ public int hashCode() { @Override public String toString() { - return "LookupKV{" + - "key=" + key + - ", value=" + value + - '}'; + return "LookupKV{" + + "key=" + key + + ", value=" + value + + '}'; } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKey.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKey.java index 5a258e7c..383761a0 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKey.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupKey.java @@ -7,19 +7,25 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.lookup; public interface LookupKey { - byte[] toBytes(); - void fromBytes(byte[] in); - String getIndicator(); - void setIndicator(String indicator); + byte[] toBytes(); + + void fromBytes(byte[] in); + + String getIndicator(); + + void setIndicator(String indicator); } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupValue.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupValue.java index 6cbad028..792d08b7 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupValue.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/LookupValue.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,6 +25,8 @@ public interface LookupValue { Iterable> toColumns(); + void fromColumns(Iterable> values); + Map getMetadata(); } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTracker.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTracker.java index bde6604e..04d725bc 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTracker.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTracker.java @@ -7,29 +7,38 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.enrichment.lookup.accesstracker; -import org.apache.metron.enrichment.lookup.LookupKey; +package org.apache.metron.enrichment.lookup.accesstracker; import java.io.IOException; import java.io.Serializable; import java.util.Map; +import org.apache.metron.enrichment.lookup.LookupKey; -public interface AccessTracker extends Serializable{ +public interface AccessTracker extends Serializable { void logAccess(LookupKey key); + void configure(Map config); + boolean hasSeen(LookupKey key); + String getName(); + AccessTracker union(AccessTracker tracker); + void reset(); + boolean isFull(); + void cleanup() throws IOException; } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerCreator.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerCreator.java index f4d0c4cd..8fe387b3 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerCreator.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerCreator.java @@ -6,22 +6,24 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.enrichment.lookup.accesstracker; -import org.apache.metron.hbase.TableProvider; +package org.apache.metron.enrichment.lookup.accesstracker; import java.io.IOException; import java.util.Map; +import org.apache.metron.hbase.TableProvider; public interface AccessTrackerCreator { - public AccessTracker create(Map config, TableProvider provider) throws IOException; + AccessTracker create(Map config, TableProvider provider) throws IOException; } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerUtil.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerUtil.java index 7339fd97..87f82768 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerUtil.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackerUtil.java @@ -6,24 +6,34 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.lookup.accesstracker; import com.google.common.base.Function; import com.google.common.collect.Iterables; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; - +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import javax.annotation.Nullable; -import java.io.*; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; public enum AccessTrackerUtil { INSTANCE; @@ -34,6 +44,7 @@ public AccessTracker deserializeTracker(byte[] bytes) throws IOException, ClassN ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); return (AccessTracker) ois.readObject(); } + public byte[] serializeTracker(AccessTracker tracker) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(bos); @@ -44,13 +55,16 @@ public byte[] serializeTracker(AccessTracker tracker) throws IOException { } - public void persistTracker(Table accessTrackerTable, String columnFamily, PersistentAccessTracker.AccessTrackerKey key, AccessTracker underlyingTracker) throws IOException { + public void persistTracker(Table accessTrackerTable, String columnFamily, + PersistentAccessTracker.AccessTrackerKey key, AccessTracker underlyingTracker) + throws IOException { Put put = new Put(key.toRowKey()); put.addColumn(Bytes.toBytes(columnFamily), COLUMN, serializeTracker(underlyingTracker)); accessTrackerTable.put(put); } - public Iterable loadAll(Table accessTrackerTable, final String columnFamily, final String name, final long earliest) throws IOException { + public Iterable loadAll(Table accessTrackerTable, final String columnFamily, final String name, + final long earliest) throws IOException { Scan scan = new Scan(PersistentAccessTracker.AccessTrackerKey.getTimestampScanKey(name, earliest)); ResultScanner scanner = accessTrackerTable.getScanner(scan); return Iterables.transform(scanner, new Function() { @@ -70,11 +84,10 @@ public AccessTracker apply(@Nullable Result result) { public AccessTracker loadAll(Iterable trackers) throws IOException, ClassNotFoundException { AccessTracker tracker = null; - for(AccessTracker t : trackers) { - if(tracker == null) { + for (AccessTracker t : trackers) { + if (tracker == null) { tracker = t; - } - else { + } else { tracker = tracker.union(t); } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackers.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackers.java index 7dad6ebc..b41371e7 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackers.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/AccessTrackers.java @@ -7,33 +7,35 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.enrichment.lookup.accesstracker; +package org.apache.metron.enrichment.lookup.accesstracker; -import org.apache.metron.hbase.TableProvider; import java.io.IOException; import java.util.Map; +import org.apache.metron.hbase.TableProvider; public enum AccessTrackers implements AccessTrackerCreator { - NOOP((config, provider) -> new NoopAccessTracker()) - ,PERSISTENT_BLOOM( new PersistentBloomTrackerCreator()); - AccessTrackerCreator creator; - AccessTrackers(AccessTrackerCreator creator) { - this.creator = creator; - } + NOOP((config, provider) -> new NoopAccessTracker()), PERSISTENT_BLOOM(new PersistentBloomTrackerCreator()); + AccessTrackerCreator creator; + + AccessTrackers(AccessTrackerCreator creator) { + this.creator = creator; + } - @Override - public AccessTracker create(Map config, TableProvider provider) throws IOException { - return creator.create(config, provider); - } + @Override + public AccessTracker create(Map config, TableProvider provider) throws IOException { + return creator.create(config, provider); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java index d21638d9..71ea0006 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java @@ -6,23 +6,26 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.enrichment.lookup.accesstracker; -import org.apache.metron.stellar.common.utils.BloomFilter; -import org.apache.metron.enrichment.lookup.LookupKey; +package org.apache.metron.enrichment.lookup.accesstracker; -import java.io.*; +import java.io.IOException; +import java.io.Serializable; import java.util.Map; import java.util.function.Function; +import org.apache.metron.enrichment.lookup.LookupKey; +import org.apache.metron.stellar.common.utils.BloomFilter; public class BloomAccessTracker implements AccessTracker { private static final long serialVersionUID = 1L; @@ -50,7 +53,10 @@ public BloomAccessTracker(String name, int expectedInsertions, double falsePosit this.falsePositiveRate = falsePositiveRate; filter = new BloomFilter(new LookupKeySerializer(), expectedInsertions, falsePositiveRate); } - public BloomAccessTracker() {} + + public BloomAccessTracker() { + } + public BloomAccessTracker(Map config) { configure(config); } @@ -58,6 +64,7 @@ public BloomAccessTracker(Map config) { protected BloomFilter getFilter() { return filter; } + @Override public void logAccess(LookupKey key) { numInsertions++; @@ -83,24 +90,21 @@ public void reset() { } private static double toDouble(Object o) { - if(o instanceof String) { - return Double.parseDouble((String)o); - } - else if(o instanceof Number) { + if (o instanceof String) { + return Double.parseDouble((String) o); + } else if (o instanceof Number) { return ((Number) o).doubleValue(); - } - else { + } else { throw new IllegalStateException("Unable to convert " + o + " to a double."); } } + private static int toInt(Object o) { - if(o instanceof String) { - return Integer.parseInt((String)o); - } - else if(o instanceof Number) { + if (o instanceof String) { + return Integer.parseInt((String) o); + } else if (o instanceof Number) { return ((Number) o).intValue(); - } - else { + } else { throw new IllegalStateException("Unable to convert " + o + " to a double."); } } @@ -113,15 +117,15 @@ public String getName() { @Override public AccessTracker union(AccessTracker tracker) { - if(filter == null) { + if (filter == null) { throw new IllegalStateException("Unable to union access tracker, because this tracker is not initialized."); } - if(tracker instanceof BloomAccessTracker ) { - filter.merge(((BloomAccessTracker)tracker).getFilter()); + if (tracker instanceof BloomAccessTracker) { + filter.merge(((BloomAccessTracker) tracker).getFilter()); return this; - } - else { - throw new IllegalStateException("Unable to union access tracker, because it's not of the right type (BloomAccessTracker)"); + } else { + throw new IllegalStateException( + "Unable to union access tracker, because it's not of the right type (BloomAccessTracker)"); } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/NoopAccessTracker.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/NoopAccessTracker.java index 18cad3c3..5a1ec8db 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/NoopAccessTracker.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/NoopAccessTracker.java @@ -6,60 +6,62 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.enrichment.lookup.accesstracker; -import org.apache.metron.enrichment.lookup.LookupKey; +package org.apache.metron.enrichment.lookup.accesstracker; import java.io.IOException; import java.util.Map; +import org.apache.metron.enrichment.lookup.LookupKey; public class NoopAccessTracker implements AccessTracker { - @Override - public void logAccess(LookupKey key) { + @Override + public void logAccess(LookupKey key) { - } + } - @Override - public void configure(Map config) { + @Override + public void configure(Map config) { - } + } - @Override - public boolean hasSeen(LookupKey key) { - return false; - } + @Override + public boolean hasSeen(LookupKey key) { + return false; + } - @Override - public String getName() { - return "noop"; - } + @Override + public String getName() { + return "noop"; + } - @Override - public AccessTracker union(AccessTracker tracker) { - return null; - } + @Override + public AccessTracker union(AccessTracker tracker) { + return null; + } - @Override - public void reset() { + @Override + public void reset() { - } + } - @Override - public boolean isFull() { - return false; - } + @Override + public boolean isFull() { + return false; + } - @Override - public void cleanup() throws IOException { + @Override + public void cleanup() throws IOException { - } + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentAccessTracker.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentAccessTracker.java index 2c63ea7e..91cd83d6 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentAccessTracker.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentAccessTracker.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.lookup.accesstracker; import java.io.ByteArrayInputStream; @@ -35,10 +38,11 @@ public class PersistentAccessTracker implements AccessTracker { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final long serialVersionUID = 1L; - public static class AccessTrackerKey { + public static class AccessTrackerKey { String name; String containerName; long timestamp; + public AccessTrackerKey(String name, String containerName, long timestamp) { this.name = name; this.containerName = containerName; @@ -67,7 +71,7 @@ public static byte[] getTimestampScanKey(String name, long timestamp) { dos.writeUTF(name); dos.writeLong(timestamp); } catch (IOException e) { - throw new RuntimeException("Unable to create scan key " , e); + throw new RuntimeException("Unable to create scan key ", e); } return os.toByteArray(); @@ -89,9 +93,11 @@ public static AccessTrackerKey fromRowKey(byte[] rowKey) { private static class Persister extends TimerTask { PersistentAccessTracker tracker; + public Persister(PersistentAccessTracker tracker) { this.tracker = tracker; } + /** * The action to be performed by this timer task. */ @@ -108,17 +114,15 @@ public void run() { long timestamp = System.currentTimeMillis(); String name; String containerName; - private Timer timer; + private final Timer timer; long maxMillisecondsBetweenPersists; - public PersistentAccessTracker( String name - , String containerName - , Table accessTrackerTable - , String columnFamily - , AccessTracker underlyingTracker - , long maxMillisecondsBetweenPersists - ) - { + public PersistentAccessTracker(String name, + String containerName, + Table accessTrackerTable, + String columnFamily, + AccessTracker underlyingTracker, + long maxMillisecondsBetweenPersists) { this.containerName = containerName; this.accessTrackerTable = accessTrackerTable; this.name = name; @@ -126,17 +130,19 @@ public PersistentAccessTracker( String name this.underlyingTracker = underlyingTracker; this.maxMillisecondsBetweenPersists = maxMillisecondsBetweenPersists; timer = new Timer(); - if(maxMillisecondsBetweenPersists > 0) { - timer.scheduleAtFixedRate(new Persister(this), maxMillisecondsBetweenPersists, maxMillisecondsBetweenPersists); + if (maxMillisecondsBetweenPersists > 0) { + timer.scheduleAtFixedRate(new Persister(this), maxMillisecondsBetweenPersists, + maxMillisecondsBetweenPersists); } } public void persist(boolean force) { - synchronized(sync) { - if(force || (System.currentTimeMillis() - timestamp) >= maxMillisecondsBetweenPersists) { + synchronized (sync) { + if (force || (System.currentTimeMillis() - timestamp) >= maxMillisecondsBetweenPersists) { //persist try { - AccessTrackerUtil.INSTANCE.persistTracker(accessTrackerTable, accessTrackerColumnFamily, new AccessTrackerKey(name, containerName, timestamp), underlyingTracker); + AccessTrackerUtil.INSTANCE.persistTracker(accessTrackerTable, accessTrackerColumnFamily, + new AccessTrackerKey(name, containerName, timestamp), underlyingTracker); timestamp = System.currentTimeMillis(); reset(); } catch (IOException e) { @@ -163,7 +169,7 @@ public void configure(Map config) { @Override public boolean hasSeen(LookupKey key) { - synchronized(sync) { + synchronized (sync) { return underlyingTracker.hasSeen(key); } } @@ -175,14 +181,14 @@ public String getName() { @Override public AccessTracker union(AccessTracker tracker) { - PersistentAccessTracker t1 = (PersistentAccessTracker)tracker; + PersistentAccessTracker t1 = (PersistentAccessTracker) tracker; underlyingTracker = underlyingTracker.union(t1.underlyingTracker); return this; } @Override public void reset() { - synchronized(sync) { + synchronized (sync) { underlyingTracker.reset(); } } @@ -196,11 +202,10 @@ public boolean isFull() { @Override public void cleanup() throws IOException { - synchronized(sync) { + synchronized (sync) { try { persist(true); - } - catch(Throwable t) { + } catch (Throwable t) { LOG.error("Unable to persist underlying tracker", t); } underlyingTracker.cleanup(); diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentBloomTrackerCreator.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentBloomTrackerCreator.java index cfbcd616..ddb5653a 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentBloomTrackerCreator.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/PersistentBloomTrackerCreator.java @@ -7,103 +7,105 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.lookup.accesstracker; +import com.cloudera.cyber.hbase.HbaseConfiguration; import java.io.IOException; import java.util.Map; import java.util.UUID; - -import com.cloudera.cyber.hbase.HbaseConfiguration; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.metron.hbase.TableProvider; import org.apache.metron.stellar.common.utils.ConversionUtils; public class PersistentBloomTrackerCreator implements AccessTrackerCreator { - public static class Config { - public static final String PERSISTENT_BLOOM_TABLE = "pbat.table"; - public static final String PERSISTENT_BLOOM_CF = "pbat.cf"; - public static final String PERSISTENT_BLOOM_FP = "pbat.false_positive"; - public static final String PERSISTENT_BLOOM_EI = "pbat.expected_insertions"; - public static final String PERSISTENT_BLOOM_MS_BETWEEN_PERSISTS = "pbat.ms_between_persists"; - public static final long MS_IN_HOUR = 10000*60*60; - private String hBaseTable; - private String hBaseCF; + public static class Config { + public static final String PERSISTENT_BLOOM_TABLE = "pbat.table"; + public static final String PERSISTENT_BLOOM_CF = "pbat.cf"; + public static final String PERSISTENT_BLOOM_FP = "pbat.false_positive"; + public static final String PERSISTENT_BLOOM_EI = "pbat.expected_insertions"; + public static final String PERSISTENT_BLOOM_MS_BETWEEN_PERSISTS = "pbat.ms_between_persists"; + public static final long MS_IN_HOUR = 10000 * 60 * 60; + private final String hbaseTable; + private final String hbaseCF; - private Configuration hbaseConfig; + private final Configuration hbaseConfig; - private double falsePositiveRate = 0.03; - private int expectedInsertions = 100000; - private long millisecondsBetweenPersists = 2*MS_IN_HOUR; - public Config(Map config) { - hBaseTable = (String) config.get(PERSISTENT_BLOOM_TABLE); - hBaseCF = (String) config.get(PERSISTENT_BLOOM_CF); - hbaseConfig = (Configuration) config.getOrDefault(HbaseConfiguration.HBASE_CONFIG_NAME, HbaseConfiguration.configureHbase()); - Object fpObj = config.get(PERSISTENT_BLOOM_FP); - if (fpObj != null) { - falsePositiveRate = ConversionUtils.convert(fpObj, Double.class); - } - Object eiObj = config.get(PERSISTENT_BLOOM_EI); - if (eiObj != null) { - expectedInsertions = ConversionUtils.convert(eiObj, Integer.class); - } - Object msObj = config.get(PERSISTENT_BLOOM_MS_BETWEEN_PERSISTS); - if(msObj != null) { - millisecondsBetweenPersists = ConversionUtils.convert(msObj, Long.class); - } - } + private double falsePositiveRate = 0.03; + private int expectedInsertions = 100000; + private long millisecondsBetweenPersists = 2 * MS_IN_HOUR; - public String getHBaseTable() { - return hBaseTable; - } + public Config(Map config) { + hbaseTable = (String) config.get(PERSISTENT_BLOOM_TABLE); + hbaseCF = (String) config.get(PERSISTENT_BLOOM_CF); + hbaseConfig = (Configuration) config.getOrDefault(HbaseConfiguration.HBASE_CONFIG_NAME, + HbaseConfiguration.configureHbase()); + Object fpObj = config.get(PERSISTENT_BLOOM_FP); + if (fpObj != null) { + falsePositiveRate = ConversionUtils.convert(fpObj, Double.class); + } + Object eiObj = config.get(PERSISTENT_BLOOM_EI); + if (eiObj != null) { + expectedInsertions = ConversionUtils.convert(eiObj, Integer.class); + } + Object msObj = config.get(PERSISTENT_BLOOM_MS_BETWEEN_PERSISTS); + if (msObj != null) { + millisecondsBetweenPersists = ConversionUtils.convert(msObj, Long.class); + } + } - public String getHBaseCF() { - return hBaseCF; - } + public String getHBaseTable() { + return hbaseTable; + } - public double getFalsePositiveRate() { - return falsePositiveRate; - } + public String getHBaseCF() { + return hbaseCF; + } - public int getExpectedInsertions() { - return expectedInsertions; - } + public double getFalsePositiveRate() { + return falsePositiveRate; + } - public Configuration getHbaseConfig() { - return hbaseConfig; - } + public int getExpectedInsertions() { + return expectedInsertions; + } + + public Configuration getHbaseConfig() { + return hbaseConfig; + } - public long getMillisecondsBetweenPersists() { - return millisecondsBetweenPersists; + public long getMillisecondsBetweenPersists() { + return millisecondsBetweenPersists; + } } - } - @Override - public AccessTracker create(Map config, TableProvider provider) throws IOException { - Config patConfig = new Config(config); - String hbaseTable = patConfig.getHBaseTable(); - int expectedInsertions = patConfig.getExpectedInsertions(); - double falsePositives = patConfig.getFalsePositiveRate(); - long millisecondsBetweenPersist = patConfig.getMillisecondsBetweenPersists(); - BloomAccessTracker bat = new BloomAccessTracker(hbaseTable, expectedInsertions, falsePositives); - Configuration hbaseConfig = patConfig.getHbaseConfig(); + @Override + public AccessTracker create(Map config, TableProvider provider) throws IOException { + Config patConfig = new Config(config); + String hbaseTable = patConfig.getHBaseTable(); + int expectedInsertions = patConfig.getExpectedInsertions(); + double falsePositives = patConfig.getFalsePositiveRate(); + long millisecondsBetweenPersist = patConfig.getMillisecondsBetweenPersists(); + BloomAccessTracker bat = new BloomAccessTracker(hbaseTable, expectedInsertions, falsePositives); + Configuration hbaseConfig = patConfig.getHbaseConfig(); - AccessTracker ret = new PersistentAccessTracker( hbaseTable - , UUID.randomUUID().toString() - , provider.getTable(hbaseConfig, hbaseTable) - , patConfig.getHBaseCF() - , bat - , millisecondsBetweenPersist - ); - return ret; - } + AccessTracker ret = new PersistentAccessTracker(hbaseTable, + UUID.randomUUID().toString(), + provider.getTable(hbaseConfig, hbaseTable), + patConfig.getHBaseCF(), + bat, + millisecondsBetweenPersist); + return ret; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/Handler.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/Handler.java index 0477a749..0ac10a5a 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/Handler.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/Handler.java @@ -6,24 +6,30 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.lookup.handler; +import java.io.IOException; import org.apache.metron.enrichment.lookup.LookupKey; -import java.io.IOException; +@SuppressWarnings("checkstyle:InterfaceTypeParameterName") +public interface Handler extends AutoCloseable { + boolean exists(KEY_T key, CONTEXT_T context, boolean logAccess) throws IOException; + + Iterable exists(Iterable> key, boolean logAccess) throws IOException; + + RESULT_T get(KEY_T key, CONTEXT_T context, boolean logAccess) throws IOException; -public interface Handler extends AutoCloseable{ - boolean exists(KEY_T key, CONTEXT_T context, boolean logAccess) throws IOException; - RESULT_T get(KEY_T key, CONTEXT_T context, boolean logAccess) throws IOException; - Iterable exists(Iterable> key, boolean logAccess) throws IOException; - Iterable get(Iterable> key, boolean logAccess) throws IOException; + Iterable get(Iterable> key, boolean logAccess) throws IOException; } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/KeyWithContext.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/KeyWithContext.java index 0150711e..780ac5d7 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/KeyWithContext.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/lookup/handler/KeyWithContext.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,12 +21,19 @@ package org.apache.metron.enrichment.lookup.handler; public class KeyWithContext { - private KEY_T key; - private CONTEXT_T context; - public KeyWithContext(KEY_T key, CONTEXT_T context) { - this.key = key; - this.context = context; - } - public KEY_T getKey() { return key; } - public CONTEXT_T getContext() { return context; } + private final KEY_T key; + private final CONTEXT_T context; + + public KeyWithContext(KEY_T key, CONTEXT_T context) { + this.key = key; + this.context = context; + } + + public KEY_T getKey() { + return key; + } + + public CONTEXT_T getContext() { + return context; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java index 41c05951..caf46848 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentCallable.java @@ -7,60 +7,59 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.parallel; +import java.util.concurrent.Callable; +import java.util.function.Function; import org.apache.metron.enrichment.cache.CacheKey; import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; import org.json.simple.JSONObject; -import java.util.concurrent.Callable; -import java.util.function.Function; - /** * Enrich based on a key and enrichment adapter. The CacheKey contains all necessary input information for an enrichment. */ public class EnrichmentCallable implements Callable, Function { - CacheKey key; - EnrichmentAdapter adapter; + CacheKey key; + EnrichmentAdapter adapter; - public EnrichmentCallable( CacheKey key - , EnrichmentAdapter adapter - ) - { - this.key = key; - this.adapter = adapter; - } + public EnrichmentCallable(CacheKey key, EnrichmentAdapter adapter) { + this.key = key; + this.adapter = adapter; + } - /** - * Computes a result, or throws an exception if unable to do so. - * - * @return computed result - * @throws Exception if unable to compute a result - */ - @Override - public JSONObject call() throws Exception { - //Log access for this key. - adapter.logAccess(key); - return adapter.enrich(key); - } + /** + * Computes a result, or throws an exception if unable to do so. + * + * @return computed result + * @throws Exception if unable to compute a result + */ + @Override + public JSONObject call() throws Exception { + //Log access for this key. + adapter.logAccess(key); + return adapter.enrich(key); + } - /** - * Applies this function to the given argument. - * - * @param cacheKey the function argument - * @return the function result - */ - @Override - public JSONObject apply(CacheKey cacheKey) { - adapter.logAccess(key); - return adapter.enrich(cacheKey); - } + /** + * Applies this function to the given argument. + * + * @param cacheKey the function argument + * @return the function result + */ + @Override + public JSONObject apply(CacheKey cacheKey) { + adapter.logAccess(key); + return adapter.enrich(cacheKey); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentContext.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentContext.java index d2a9fe73..b71663c2 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentContext.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentContext.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.parallel; import org.apache.metron.stellar.dsl.Context; @@ -25,19 +28,19 @@ * environment (e.g. a storm bolt) to the set of storm independent enrichment infrastructure. */ public class EnrichmentContext { - private FunctionResolver functionResolver; - private Context stellarContext; + private final FunctionResolver functionResolver; + private final Context stellarContext; - public EnrichmentContext(FunctionResolver functionResolver, Context stellarContext) { - this.functionResolver = functionResolver; - this.stellarContext = stellarContext; - } + public EnrichmentContext(FunctionResolver functionResolver, Context stellarContext) { + this.functionResolver = functionResolver; + this.stellarContext = stellarContext; + } - public FunctionResolver getFunctionResolver() { - return functionResolver; - } + public FunctionResolver getFunctionResolver() { + return functionResolver; + } - public Context getStellarContext() { - return stellarContext; - } + public Context getStellarContext() { + return stellarContext; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategies.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategies.java index 906e5185..45d46e28 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategies.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategies.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.parallel; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; @@ -32,77 +35,76 @@ * rather than bind the abstraction to Storm, our distributed processing engine. */ public enum EnrichmentStrategies implements EnrichmentStrategy { - /** - * Interact with the enrichment portion of the enrichment config - */ - ENRICHMENT(new EnrichmentStrategy() { - @Override - public EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config) { - return config.getEnrichment(); - } + /** + * Interact with the enrichment portion of the enrichment config. + */ + ENRICHMENT(new EnrichmentStrategy() { + @Override + public EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config) { + return config.getEnrichment(); + } - @Override - public Constants.ErrorType getErrorType() { - return Constants.ErrorType.ENRICHMENT_ERROR; - } + @Override + public Constants.ErrorType getErrorType() { + return Constants.ErrorType.ENRICHMENT_ERROR; + } - @Override - public String fieldToEnrichmentKey(String type, String field) { - return EnrichmentUtils.getEnrichmentKey(type, field); + @Override + public String fieldToEnrichmentKey(String type, String field) { + return EnrichmentUtils.getEnrichmentKey(type, field); + } + }), + /** + * Interact with the threat intel portion of the enrichment config. + */ + THREAT_INTEL(new EnrichmentStrategy() { + @Override + public EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config) { + return config.getThreatIntel(); + } + + @Override + public Constants.ErrorType getErrorType() { + return Constants.ErrorType.THREAT_INTEL_ERROR; + } + + @Override + public String fieldToEnrichmentKey(String type, String field) { + return ThreatIntelUtils.getThreatIntelKey(type, field); + } + + @Override + public JSONObject postProcess(JSONObject message, SensorEnrichmentConfig config, EnrichmentContext context) { + return ThreatIntelUtils.triage(message, config, context.getFunctionResolver(), context.getStellarContext()); + } + }); + + EnrichmentStrategy enrichmentStrategy; + + EnrichmentStrategies(EnrichmentStrategy enrichmentStrategy) { + this.enrichmentStrategy = enrichmentStrategy; } - }), - /** - * Interact with the threat intel portion of the enrichment config. - */ - THREAT_INTEL(new EnrichmentStrategy() { + + /** + * Get the underlying enrichment config. If this is provided, then we need not retrieve. + */ @Override public EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config) { - return config.getThreatIntel(); + return enrichmentStrategy.getUnderlyingConfig(config); } - @Override - public Constants.ErrorType getErrorType() { - return Constants.ErrorType.THREAT_INTEL_ERROR; + public String fieldToEnrichmentKey(String type, String field) { + return enrichmentStrategy.fieldToEnrichmentKey(type, field); } - @Override - public String fieldToEnrichmentKey(String type, String field) { - return ThreatIntelUtils.getThreatIntelKey(type, field); + + public JSONObject postProcess(JSONObject message, SensorEnrichmentConfig config, EnrichmentContext context) { + return enrichmentStrategy.postProcess(message, config, context); } @Override - public JSONObject postProcess(JSONObject message, SensorEnrichmentConfig config, EnrichmentContext context) { - return ThreatIntelUtils.triage(message, config, context.getFunctionResolver(), context.getStellarContext()); + public Constants.ErrorType getErrorType() { + return enrichmentStrategy.getErrorType(); } - }) - ; - - EnrichmentStrategy enrichmentStrategy; - EnrichmentStrategies(EnrichmentStrategy enrichmentStrategy) { - this.enrichmentStrategy = enrichmentStrategy; - } - - /** - * Get the underlying enrichment config. If this is provided, then we need not retrieve - * @return - */ - @Override - public EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config) { - return enrichmentStrategy.getUnderlyingConfig(config); - } - - public String fieldToEnrichmentKey(String type, String field) { - return enrichmentStrategy.fieldToEnrichmentKey(type, field); - } - - - public JSONObject postProcess(JSONObject message, SensorEnrichmentConfig config, EnrichmentContext context) { - return enrichmentStrategy.postProcess(message, config, context); - } - - @Override - public Constants.ErrorType getErrorType() { - return enrichmentStrategy.getErrorType(); - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategy.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategy.java index 4e43d656..3deb562b 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategy.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/EnrichmentStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.parallel; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; @@ -26,6 +29,7 @@ * Enrichment strategy. This interface provides a mechanism to interface with the enrichment config and any * post processing steps that are needed to be done after-the-fact. * + *

* The reasoning behind this is that the key difference between enrichments and threat intel is that they pull * their configurations from different parts of the SensorEnrichmentConfig object and as a post-join step, they differ * slightly. @@ -33,35 +37,34 @@ */ public interface EnrichmentStrategy { - /** - * Get the underlying configuration for this phase from the sensor enrichment config. - * @return - */ - EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config); + /** + * Get the underlying configuration for this phase from the sensor enrichment config. + */ + EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config); - /** - * Retrieves the error type, so that error messages can be constructed appropriately. - */ - Constants.ErrorType getErrorType(); + /** + * Retrieves the error type, so that error messages can be constructed appropriately. + */ + Constants.ErrorType getErrorType(); - /** - * Takes the enrichment type and the field and returns a unique key to prefix the output of the enrichment. For - * less adaptable enrichments than Stellar, this is important to allow for namespacing in the new fields created. - * @param type The enrichment type name - * @param field The input field - * @return - */ - String fieldToEnrichmentKey(String type, String field); + /** + * Takes the enrichment type and the field and returns a unique key to prefix the output of the enrichment. For + * less adaptable enrichments than Stellar, this is important to allow for namespacing in the new fields created. + * + * @param type The enrichment type name + * @param field The input field + */ + String fieldToEnrichmentKey(String type, String field); - /** - * Post-process callback after messages are enriched and joined. By default, this is noop. - * @param message The input message. - * @param config The enrichment configuration - * @param context The enrichment context - * @return - */ - default JSONObject postProcess(JSONObject message, SensorEnrichmentConfig config, EnrichmentContext context) { - return message; - } + /** + * Post-process callback after messages are enriched and joined. By default, this is noop. + * + * @param message The input message. + * @param config The enrichment configuration + * @param context The enrichment context + */ + default JSONObject postProcess(JSONObject message, SensorEnrichmentConfig config, EnrichmentContext context) { + return message; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java index 5f82b1c4..0f49162c 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.parallel; import java.util.concurrent.ExecutorService; @@ -25,21 +28,21 @@ * The strategy to use to construct the thread pool. */ public enum WorkerPoolStrategies { - /** - * Fixed thread pool - */ - FIXED(numThreads -> Executors.newFixedThreadPool(numThreads)), - /** - * Work stealing thread pool. - */ - WORK_STEALING(numThreads -> Executors.newWorkStealingPool(numThreads)) - ; - Function creator; - WorkerPoolStrategies(Function creator) { - this.creator = creator; - } + /** + * Fixed thread pool. + */ + FIXED(numThreads -> Executors.newFixedThreadPool(numThreads)), + /** + * Work stealing thread pool. + */ + WORK_STEALING(numThreads -> Executors.newWorkStealingPool(numThreads)); + Function creator; + + WorkerPoolStrategies(Function creator) { + this.creator = creator; + } - public ExecutorService create(int numThreads) { - return creator.apply(numThreads); - } + public ExecutorService create(int numThreads) { + return creator.apply(numThreads); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java index 2c4b3727..f5463e46 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/AsnEnrichmentFunctions.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.stellar; import java.lang.invoke.MethodHandles; @@ -26,85 +27,84 @@ import org.apache.metron.enrichment.adapters.maxmind.asn.GeoLiteAsnDatabase; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class AsnEnrichmentFunctions { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static class AsnGet implements StellarFunction { + public static class AsnGet implements StellarFunction { - boolean initialized = false; + boolean initialized = false; - @Override - public Object apply(List args, Context context) throws ParseException { - if (!initialized) { - return null; - } - if (args.size() > 2) { - throw new IllegalArgumentException( - "ASN_GET received more arguments than expected: " + args.size()); - } + @Override + public Object apply(List args, Context context) throws ParseException { + if (!initialized) { + return null; + } + if (args.size() > 2) { + throw new IllegalArgumentException( + "ASN_GET received more arguments than expected: " + args.size()); + } - if (args.size() == 1 && args.get(0) instanceof String) { - // If no fields are provided, return everything - String ip = (String) args.get(0); - if (ip == null || ip.trim().isEmpty()) { - LOG.debug("No IP provided, returning null"); - return null; - } + if (args.size() == 1 && args.get(0) instanceof String) { + // If no fields are provided, return everything + String ip = (String) args.get(0); + if (ip == null || ip.trim().isEmpty()) { + LOG.debug("No IP provided, returning null"); + return null; + } - Optional> result = GeoLiteAsnDatabase.INSTANCE.get(ip); - return result.orElse(Collections.EMPTY_MAP); - } else if (args.size() == 2 && args.get(1) instanceof List) { - // If fields are provided, return just those fields. - String ip = (String) args.get(0); - @SuppressWarnings("unchecked") - List fields = (List) args.get(1); - Optional> result = GeoLiteAsnDatabase.INSTANCE.get(ip); + Optional> result = GeoLiteAsnDatabase.INSTANCE.get(ip); + return result.orElse(Collections.EMPTY_MAP); + } else if (args.size() == 2 && args.get(1) instanceof List) { + // If fields are provided, return just those fields. + String ip = (String) args.get(0); + @SuppressWarnings("unchecked") + List fields = (List) args.get(1); + Optional> result = GeoLiteAsnDatabase.INSTANCE.get(ip); + + // If only one field is requested, just return it directly + if (fields.size() == 1 && result.isPresent()) { + if (!result.get().containsKey(fields.get(0))) { + return null; + } + return result.get().get(fields.get(0)); + } else if (result.isPresent()) { + // If multiple fields are requested, return all of them + Map filteredInfo = new HashMap<>(); + for (String field : fields) { + Map asnInfo = result.get(); + filteredInfo.put(field, asnInfo.get(field)); + } + return filteredInfo; + } + } - // If only one field is requested, just return it directly - if (fields.size() == 1 && result.isPresent()) { - if (!result.get().containsKey(fields.get(0))) { return null; - } - return result.get().get(fields.get(0)); - } else if (result.isPresent()) { - // If multiple fields are requested, return all of them - Map filteredInfo = new HashMap<>(); - for (String field : fields) { - Map asnInfo = result.get(); - filteredInfo.put(field, asnInfo.get(field)); - } - return filteredInfo; } - } - return null; - } + @Override + public void initialize(Context context) { + LOG.info("Initializing AsnEnrichmentFunctions"); + Map config = getConfig(context); + String hdfsDir = (String) config.get(GeoLiteAsnDatabase.ASN_HDFS_FILE); + GeoLiteAsnDatabase.INSTANCE.update(hdfsDir); + initialized = true; + } - @Override - public void initialize(Context context) { - LOG.info("Initializing AsnEnrichmentFunctions"); - Map config = getConfig(context); - String hdfsDir = (String) config.get(GeoLiteAsnDatabase.ASN_HDFS_FILE); - GeoLiteAsnDatabase.INSTANCE.update(hdfsDir); - initialized = true; - } + @SuppressWarnings("unchecked") + private static Map getConfig(Context context) { + return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false) + .orElse(new HashMap<>()); + } - @SuppressWarnings("unchecked") - private static Map getConfig(Context context) { - return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false) - .orElse(new HashMap<>()); - } + @Override + public boolean isInitialized() { + return initialized; + } - @Override - public boolean isInitialized() { - return initialized; } - - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/EnrichmentObjectGet.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/EnrichmentObjectGet.java index 8e7cff15..02797797 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/EnrichmentObjectGet.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/EnrichmentObjectGet.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +20,12 @@ package org.apache.metron.enrichment.stellar; +import static org.apache.metron.enrichment.stellar.EnrichmentObjectGet.ENRICHMENT_OBJECT_GET_SETTINGS; + +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.metron.enrichment.cache.ObjectCache; import org.apache.metron.enrichment.cache.ObjectCacheConfig; import org.apache.metron.stellar.dsl.Context; @@ -27,81 +35,82 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.metron.enrichment.stellar.EnrichmentObjectGet.ENRICHMENT_OBJECT_GET_SETTINGS; - -@Stellar(namespace="ENRICHMENT" - ,name="OBJECT_GET" - ,description="Retrieve and deserialize a serialized object from HDFS and stores it in the ObjectCache, " + - "then returns the value associated with the indicator." + - "The cache can be specified via three properties in the global config: " + - "\"" + ObjectCacheConfig.OBJECT_CACHE_SIZE_KEY + "\" (default " + ObjectCacheConfig.OBJECT_CACHE_SIZE_DEFAULT + ")," + - "\"" + ObjectCacheConfig.OBJECT_CACHE_EXPIRATION_KEY + "\" (default " + ObjectCacheConfig.OBJECT_CACHE_EXPIRATION_MIN_DEFAULT + ")," + - "\"" + ObjectCacheConfig.OBJECT_CACHE_TIME_UNIT_KEY+ "\" (default MINUTES)." + - "Cache settings that apply only to this function can also be specified in the global config by nesting the settings above under the " + ENRICHMENT_OBJECT_GET_SETTINGS + " key." + - "Note, if these are changed in global config, topology restart is required." - , params = { - "path - The path in HDFS to the serialized object" + - "indicator - The string indicator to look up" - } - , returns="Value associated with the indicator." +@Stellar(namespace = "ENRICHMENT", + name = "OBJECT_GET", + description = "Retrieve and deserialize a serialized object from HDFS and stores it in the ObjectCache, " + + "then returns the value associated with the indicator." + + "The cache can be specified via three properties in the global config: " + + "\"" + ObjectCacheConfig.OBJECT_CACHE_SIZE_KEY + "\" (default " + + ObjectCacheConfig.OBJECT_CACHE_SIZE_DEFAULT + ")," + + "\"" + ObjectCacheConfig.OBJECT_CACHE_EXPIRATION_KEY + "\" (default " + + ObjectCacheConfig.OBJECT_CACHE_EXPIRATION_MIN_DEFAULT + ")," + + "\"" + ObjectCacheConfig.OBJECT_CACHE_TIME_UNIT_KEY + "\" (default MINUTES)." + + + "Cache settings that apply only to this function can also be specified in the " + + "global config by nesting the settings above under the " + + ENRICHMENT_OBJECT_GET_SETTINGS + " key." + + "Note, if these are changed in global config, topology restart is required.", + params = { + "path - The path in HDFS to the serialized object" + + "indicator - The string indicator to look up" + }, + returns = "Value associated with the indicator." ) public class EnrichmentObjectGet implements StellarFunction { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public final static String ENRICHMENT_OBJECT_GET_SETTINGS = "enrichment.object.get.settings"; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String ENRICHMENT_OBJECT_GET_SETTINGS = "enrichment.object.get.settings"; - private ObjectCache objectCache; + private ObjectCache objectCache; - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() != 2) { - throw new IllegalArgumentException("All parameters are mandatory, submit 'hdfs path', 'indicator'"); - } - if(!isInitialized()) { - return null; - } + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() != 2) { + throw new IllegalArgumentException("All parameters are mandatory, submit 'hdfs path', 'indicator'"); + } + if (!isInitialized()) { + return null; + } - String path = (String) args.get(0); - String indicator = (String) args.get(1); - if(path == null || indicator == null) { - return null; - } + String path = (String) args.get(0); + String indicator = (String) args.get(1); + if (path == null || indicator == null) { + return null; + } - Object value; - try { - Map cachedMap = (Map) objectCache.get(path); - LOG.debug("Looking up value from object at path '{}' using indicator {}", path, indicator); - value = cachedMap.get(indicator); - } catch(ClassCastException e) { - throw new ClassCastException(String.format("The object stored in HDFS at '%s' must be serialized in JSON format.", path)); - } + Object value; + try { + Map cachedMap = (Map) objectCache.get(path); + LOG.debug("Looking up value from object at path '{}' using indicator {}", path, indicator); + value = cachedMap.get(indicator); + } catch (ClassCastException e) { + throw new ClassCastException( + String.format("The object stored in HDFS at '%s' must be serialized in JSON format.", path)); + } - return value; - } + return value; + } - @SuppressWarnings("unchecked") - @Override - public void initialize(Context context) { - Map config = (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false) - .orElse(new HashMap<>()); - Map enrichmentGetConfig = (Map) config.getOrDefault(ENRICHMENT_OBJECT_GET_SETTINGS, new HashMap<>()); - ObjectCacheConfig objectCacheConfig = new ObjectCacheConfig(enrichmentGetConfig); - objectCache = new ObjectCache(); - objectCache.initialize(objectCacheConfig); - } + @SuppressWarnings("unchecked") + @Override + public void initialize(Context context) { + Map config = + (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false) + .orElse(new HashMap<>()); + Map enrichmentGetConfig = + (Map) config.getOrDefault(ENRICHMENT_OBJECT_GET_SETTINGS, new HashMap<>()); + ObjectCacheConfig objectCacheConfig = new ObjectCacheConfig(enrichmentGetConfig); + objectCache = new ObjectCache(); + objectCache.initialize(objectCacheConfig); + } - // Exposed for testing - protected void initialize(ObjectCache objectCache) { - this.objectCache = objectCache; - } + // Exposed for testing + protected void initialize(ObjectCache objectCache) { + this.objectCache = objectCache; + } - @Override - public boolean isInitialized() { - return objectCache != null && objectCache.isInitialized(); - } + @Override + public boolean isInitialized() { + return objectCache != null && objectCache.isInitialized(); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java index 5039bbe6..a924d35b 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctions.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.stellar; import java.lang.invoke.MethodHandles; @@ -26,76 +27,76 @@ import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class GeoEnrichmentFunctions { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static class GeoGet implements StellarFunction { - boolean initialized = false; + public static class GeoGet implements StellarFunction { + boolean initialized = false; - @Override - public Object apply(List args, Context context) throws ParseException { - if(!initialized) { - return null; - } - if(args.size() > 2) { - throw new IllegalArgumentException("GEO_GET received more arguments than expected: " + args.size()); - } + @Override + public Object apply(List args, Context context) throws ParseException { + if (!initialized) { + return null; + } + if (args.size() > 2) { + throw new IllegalArgumentException("GEO_GET received more arguments than expected: " + args.size()); + } - if(args.size() == 1 && args.get(0) instanceof String) { - // If no fields are provided, return everything - String ip = (String) args.get(0); - if(ip == null || ip.trim().isEmpty()) { - return null; - } + if (args.size() == 1 && args.get(0) instanceof String) { + // If no fields are provided, return everything + String ip = (String) args.get(0); + if (ip == null || ip.trim().isEmpty()) { + return null; + } + + Optional> result = GeoLiteCityDatabase.INSTANCE.get(ip); + return result.orElse(Collections.emptyMap()); + } else if (args.size() == 2 && args.get(1) instanceof List) { + // If fields are provided, return just those fields. + String ip = (String) args.get(0); + @SuppressWarnings("unchecked") + List fields = (List) args.get(1); + Optional> result = GeoLiteCityDatabase.INSTANCE.get(ip); - Optional> result = GeoLiteCityDatabase.INSTANCE.get(ip); - return result.orElse(Collections.emptyMap()); - } else if (args.size() == 2 && args.get(1) instanceof List) { - // If fields are provided, return just those fields. - String ip = (String) args.get(0); - @SuppressWarnings("unchecked") - List fields = (List) args.get(1); - Optional> result = GeoLiteCityDatabase.INSTANCE.get(ip); + // If only one field is requested, just return it directly + if (fields.size() == 1 && result.isPresent()) { + return result.get().get(fields.get(0)); + } else if (result.isPresent()) { + // If multiple fields are requested, return all of them + Map filteredInfo = new HashMap<>(); + for (String field : fields) { + Map geoInfo = result.get(); + filteredInfo.put(field, geoInfo.get(field)); + } + return filteredInfo; + } + } - // If only one field is requested, just return it directly - if(fields.size() == 1 && result.isPresent()) { - return result.get().get(fields.get(0)); - } else if (result.isPresent()) { - // If multiple fields are requested, return all of them - Map filteredInfo = new HashMap<>(); - for(String field : fields) { - Map geoInfo = result.get(); - filteredInfo.put(field, geoInfo.get(field)); - } - return filteredInfo; + return null; } - } - return null; - } + @Override + public void initialize(Context context) { + LOG.info("Initializing GeoEnrichmentFunctions"); + Map config = getConfig(context); + String hdfsDir = (String) config.get(GeoLiteCityDatabase.GEO_HDFS_FILE); + GeoLiteCityDatabase.INSTANCE.update(hdfsDir); + initialized = true; + } - @Override - public void initialize(Context context) { - LOG.info("Initializing GeoEnrichmentFunctions"); - Map config = getConfig(context); - String hdfsDir = (String) config.get(GeoLiteCityDatabase.GEO_HDFS_FILE); - GeoLiteCityDatabase.INSTANCE.update(hdfsDir); - initialized = true; - } + private static Map getConfig(Context context) { + return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false) + .orElse(new HashMap<>()); + } - private static Map getConfig(Context context) { - return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false).orElse(new HashMap<>()); - } + @Override + public boolean isInitialized() { + return initialized; + } - @Override - public boolean isInitialized() { - return initialized; } - - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java index 99c99cdf..878a2d98 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java @@ -15,9 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.stellar; import ch.hsr.geohash.WGS84Point; +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.metron.enrichment.adapters.maxmind.geo.GeoLiteCityDatabase; import org.apache.metron.enrichment.adapters.maxmind.geo.hash.DistanceStrategies; import org.apache.metron.enrichment.adapters.maxmind.geo.hash.DistanceStrategy; @@ -30,270 +36,263 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - public class GeoHashFunctions { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @Stellar(name="TO_LATLONG" - ,namespace="GEOHASH" - ,description="Compute the lat/long of a given [geohash](https://en.wikipedia.org/wiki/Geohash)" - ,params = { - "hash - The [geohash](https://en.wikipedia.org/wiki/Geohash)" - } - ,returns = "A map containing the latitude and longitude of the hash (keys \"latitude\" and \"longitude\")" - ) - public static class ToLatLong implements StellarFunction { - - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() < 1) { - return null; - } - String hash = (String)args.get(0); - if(hash == null) { - return null; - } - - Optional point = GeoHashUtil.INSTANCE.toPoint(hash); - if(point.isPresent()) { - Map ret = new HashMap<>(); - ret.put(GeoLiteCityDatabase.GeoProps.LONGITUDE.getSimpleName(), point.get().getLongitude()); - ret.put(GeoLiteCityDatabase.GeoProps.LATITUDE.getSimpleName(), point.get().getLatitude()); - return ret; - } - return null; - } - - @Override - public void initialize(Context context) { - - } - - @Override - public boolean isInitialized() { - return true; - } - } - - @Stellar(name="FROM_LATLONG" - ,namespace="GEOHASH" - ,description="Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a lat/long" - ,params = { - "latitude - The latitude", - "longitude - The longitude", - "character_precision? - The number of characters to use in the hash. Default is 12" - } - ,returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the lat/long" - ) - public static class FromLatLong implements StellarFunction { - - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() < 2) { - return null; - } - Object latObj = args.get(0); - Object longObj = args.get(1); - if(latObj == null || longObj == null) { - return null; - } - Double latitude = ConversionUtils.convert(latObj, Double.class); - Double longitude = ConversionUtils.convert(longObj, Double.class); - int charPrecision = 12; - if(args.size() > 2) { - charPrecision = ConversionUtils.convert(args.get(2), Integer.class); - } - Optional ret = GeoHashUtil.INSTANCE.computeHash(latitude, longitude, charPrecision); - return ret.orElse(null); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Stellar(name = "TO_LATLONG", + namespace = "GEOHASH", + description = "Compute the lat/long of a given [geohash](https://en.wikipedia.org/wiki/Geohash)", + params = { + "hash - The [geohash](https://en.wikipedia.org/wiki/Geohash)" + }, + returns = "A map containing the latitude and longitude of the hash (keys \"latitude\" and \"longitude\")" + ) + public static class ToLatLong implements StellarFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() < 1) { + return null; + } + String hash = (String) args.get(0); + if (hash == null) { + return null; + } + + Optional point = GeoHashUtil.INSTANCE.toPoint(hash); + if (point.isPresent()) { + Map ret = new HashMap<>(); + ret.put(GeoLiteCityDatabase.GeoProps.LONGITUDE.getSimpleName(), point.get().getLongitude()); + ret.put(GeoLiteCityDatabase.GeoProps.LATITUDE.getSimpleName(), point.get().getLatitude()); + return ret; + } + return null; + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } } - @Override - public void initialize(Context context) { - - } - - @Override - public boolean isInitialized() { - return true; - } - } - - @Stellar(name="FROM_LOC" - ,namespace="GEOHASH" - ,description="Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a geo enrichment location" - ,params = { - "map - the latitude and logitude in a map (the output of GEO_GET)", - "character_precision? - The number of characters to use in the hash. Default is 12" - } - ,returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the location" - ) - public static class FromLoc implements StellarFunction { - - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() < 1) { - return null; - } - Map map = (Map) args.get(0); - if(map == null) { - return null; - } - int charPrecision = 12; - if(args.size() > 1) { - charPrecision = ConversionUtils.convert(args.get(1), Integer.class); - } - Optional ret = GeoHashUtil.INSTANCE.computeHash(map, charPrecision); - return ret.orElse(null); - } - - @Override - public void initialize(Context context) { - - } - - @Override - public boolean isInitialized() { - return true; + @Stellar(name = "FROM_LATLONG", + namespace = "GEOHASH", + description = "Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a lat/long", + params = { + "latitude - The latitude", + "longitude - The longitude", + "character_precision? - The number of characters to use in the hash. Default is 12" + }, + returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the lat/long" + ) + public static class FromLatLong implements StellarFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() < 2) { + return null; + } + Object latObj = args.get(0); + Object longObj = args.get(1); + if (latObj == null || longObj == null) { + return null; + } + Double latitude = ConversionUtils.convert(latObj, Double.class); + Double longitude = ConversionUtils.convert(longObj, Double.class); + int charPrecision = 12; + if (args.size() > 2) { + charPrecision = ConversionUtils.convert(args.get(2), Integer.class); + } + Optional ret = GeoHashUtil.INSTANCE.computeHash(latitude, longitude, charPrecision); + return ret.orElse(null); + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } } - } - - - @Stellar(name="DIST" - ,namespace="GEOHASH" - ,description="Compute the distance between [geohashes](https://en.wikipedia.org/wiki/Geohash)" - ,params = { - "hash1 - The first location as a geohash", - "hash2 - The second location as a geohash", - "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default." - } - ,returns = "The distance in kilometers between the hashes" - ) - public static class Dist implements StellarFunction { - - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() < 2) { - return null; - } - String hash1 = (String)args.get(0); - if(hash1 == null) { - return null; - } - Optional pt1 = GeoHashUtil.INSTANCE.toPoint(hash1); - String hash2 = (String)args.get(1); - if(hash2 == null) { - return null; - } - Optional pt2 = GeoHashUtil.INSTANCE.toPoint(hash2); - DistanceStrategy strat = DistanceStrategies.HAVERSINE; - if(args.size() > 2) { - strat = DistanceStrategies.valueOf((String) args.get(2)); - } - if(pt1.isPresent() && pt2.isPresent()) { - return GeoHashUtil.INSTANCE.distance(pt1.get(), pt2.get(), strat); - } - return Double.NaN; - } - - @Override - public void initialize(Context context) { + @Stellar(name = "FROM_LOC", + namespace = "GEOHASH", + description = "Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a geo enrichment location", + params = { + "map - the latitude and logitude in a map (the output of GEO_GET)", + "character_precision? - The number of characters to use in the hash. Default is 12" + }, + returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the location" + ) + public static class FromLoc implements StellarFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() < 1) { + return null; + } + Map map = (Map) args.get(0); + if (map == null) { + return null; + } + int charPrecision = 12; + if (args.size() > 1) { + charPrecision = ConversionUtils.convert(args.get(1), Integer.class); + } + Optional ret = GeoHashUtil.INSTANCE.computeHash(map, charPrecision); + return ret.orElse(null); + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } } - @Override - public boolean isInitialized() { - return true; - } - } - - @Stellar(name="MAX_DIST" - ,namespace="GEOHASH" - ,description="Compute the maximum distance among a list of [geohashes](https://en.wikipedia.org/wiki/Geohash)" - ,params = { - "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash)", - "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default." - } - ,returns = "The maximum distance in kilometers between any two locations" - ) - public static class MaxDist implements StellarFunction { - - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() < 1) { - return null; - } - Iterable hashes = (Iterable)args.get(0); - if(hashes == null) { - return null; - } - DistanceStrategy strat = DistanceStrategies.HAVERSINE; - if(args.size() > 1) { - strat = DistanceStrategies.valueOf((String) args.get(1)); - } - return GeoHashUtil.INSTANCE.maxDistanceHashes(hashes, strat); - } - - @Override - public void initialize(Context context) { + @Stellar(name = "DIST", + namespace = "GEOHASH", + description = "Compute the distance between [geohashes](https://en.wikipedia.org/wiki/Geohash)", + params = { + "hash1 - The first location as a geohash", + "hash2 - The second location as a geohash", + "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default." + }, + returns = "The distance in kilometers between the hashes" + ) + public static class Dist implements StellarFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() < 2) { + return null; + } + String hash1 = (String) args.get(0); + if (hash1 == null) { + return null; + } + Optional pt1 = GeoHashUtil.INSTANCE.toPoint(hash1); + String hash2 = (String) args.get(1); + if (hash2 == null) { + return null; + } + Optional pt2 = GeoHashUtil.INSTANCE.toPoint(hash2); + DistanceStrategy strat = DistanceStrategies.HAVERSINE; + if (args.size() > 2) { + strat = DistanceStrategies.valueOf((String) args.get(2)); + } + if (pt1.isPresent() && pt2.isPresent()) { + return GeoHashUtil.INSTANCE.distance(pt1.get(), pt2.get(), strat); + } + return Double.NaN; + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } } - @Override - public boolean isInitialized() { - return true; - } - } - - @Stellar(name="CENTROID" - ,namespace="GEOHASH" - ,description="Compute the centroid (geographic midpoint or center of gravity) of a set of [geohashes](https://en.wikipedia.org/wiki/Geohash)" - ,params = { - "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash) or a map associating geohashes to numeric weights" - ,"character_precision? - The number of characters to use in the hash. Default is 12" - } - ,returns = "The geohash of the centroid" - ) - public static class Centroid implements StellarFunction { - - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() < 1) { - return null; - } - Object o1 = args.get(0); - if(o1 == null) { - return null; - } - WGS84Point centroid = null; - if(o1 instanceof Map) { - centroid = GeoHashUtil.INSTANCE.centroidOfWeightedPoints((Map)o1); - } - else if(o1 instanceof Iterable) { - centroid = GeoHashUtil.INSTANCE.centroidOfHashes((Iterable)o1); - } - if(centroid == null) { - return null; - } - Integer precision = 12; - if(args.size() > 1) { - precision = (Integer)args.get(1); - } - return GeoHashUtil.INSTANCE.computeHash(centroid, precision).orElse(null); - } - - @Override - public void initialize(Context context) { - + @Stellar(name = "MAX_DIST", + namespace = "GEOHASH", + description = "Compute the maximum distance among a list of [geohashes](https://en.wikipedia.org/wiki/Geohash)", + params = { + "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash)", + "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default." + }, + returns = "The maximum distance in kilometers between any two locations" + ) + public static class MaxDist implements StellarFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() < 1) { + return null; + } + Iterable hashes = (Iterable) args.get(0); + if (hashes == null) { + return null; + } + DistanceStrategy strat = DistanceStrategies.HAVERSINE; + if (args.size() > 1) { + strat = DistanceStrategies.valueOf((String) args.get(1)); + } + return GeoHashUtil.INSTANCE.maxDistanceHashes(hashes, strat); + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } } - @Override - public boolean isInitialized() { - return true; + @Stellar(name = "CENTROID", + namespace = "GEOHASH", + description = "Compute the centroid (geographic midpoint or center of gravity) of a set of [geohashes](https://en.wikipedia.org/wiki/Geohash)", + params = { + "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash) or a map associating geohashes to numeric weights", + "character_precision? - The number of characters to use in the hash. Default is 12" + }, + returns = "The geohash of the centroid" + ) + public static class Centroid implements StellarFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() < 1) { + return null; + } + Object o1 = args.get(0); + if (o1 == null) { + return null; + } + WGS84Point centroid = null; + if (o1 instanceof Map) { + centroid = GeoHashUtil.INSTANCE.centroidOfWeightedPoints((Map) o1); + } else if (o1 instanceof Iterable) { + centroid = GeoHashUtil.INSTANCE.centroidOfHashes((Iterable) o1); + } + if (centroid == null) { + return null; + } + Integer precision = 12; + if (args.size() > 1) { + precision = (Integer) args.get(1); + } + return GeoHashUtil.INSTANCE.computeHash(centroid, precision).orElse(null); + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } } - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java index f6536eb7..edccd6d9 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/ObjectGet.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +20,10 @@ package org.apache.metron.enrichment.stellar; +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.metron.enrichment.cache.ObjectCache; import org.apache.metron.enrichment.cache.ObjectCacheConfig; import org.apache.metron.stellar.dsl.Context; @@ -27,72 +33,69 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Stellar(namespace="OBJECT" - ,name="GET" - ,description="Retrieve and deserialize a serialized object from HDFS. " + - "The cache can be specified via three properties in the global config: " + - "\"" + ObjectCacheConfig.OBJECT_CACHE_SIZE_KEY + "\" (default " + ObjectCacheConfig.OBJECT_CACHE_SIZE_DEFAULT + ")," + - "\"" + ObjectCacheConfig.OBJECT_CACHE_EXPIRATION_KEY + "\" (default " + ObjectCacheConfig.OBJECT_CACHE_EXPIRATION_MIN_DEFAULT + ")," + - "\"" + ObjectCacheConfig.OBJECT_CACHE_TIME_UNIT_KEY+ "\" (default MINUTES)." + - "Note, if these are changed in global config, topology restart is required." - , params = { +@Stellar(namespace = "OBJECT", + name = "GET", + description = "Retrieve and deserialize a serialized object from HDFS. " + + "The cache can be specified via three properties in the global config: " + + "\"" + ObjectCacheConfig.OBJECT_CACHE_SIZE_KEY + "\" (default " + + ObjectCacheConfig.OBJECT_CACHE_SIZE_DEFAULT + ")," + + "\"" + ObjectCacheConfig.OBJECT_CACHE_EXPIRATION_KEY + "\" (default " + + ObjectCacheConfig.OBJECT_CACHE_EXPIRATION_MIN_DEFAULT + ")," + + "\"" + ObjectCacheConfig.OBJECT_CACHE_TIME_UNIT_KEY + "\" (default MINUTES)." + + "Note, if these are changed in global config, topology restart is required.", + params = { "path - The path in HDFS to the serialized object" - } - , returns="The deserialized object." + }, + returns = "The deserialized object." ) public class ObjectGet implements StellarFunction { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private ObjectCache objectCache; + private ObjectCache objectCache; - @Override - public Object apply(List args, Context context) throws ParseException { - if(!isInitialized()) { - return null; + @Override + public Object apply(List args, Context context) throws ParseException { + if (!isInitialized()) { + return null; + } + if (args.size() < 1) { + return null; + } + Object o = args.get(0); + if (o == null) { + return null; + } + if (o instanceof String) { + return objectCache.get((String) o); + } else { + throw new IllegalStateException("Unable to retrieve " + o + " as it is not a path"); + } } - if(args.size() < 1) { - return null; - } - Object o = args.get(0); - if(o == null) { - return null; - } - if(o instanceof String) { - return objectCache.get((String) o); - } - else { - throw new IllegalStateException("Unable to retrieve " + o + " as it is not a path"); - } - } - @Override - public void initialize(Context context) { - Map config = getConfig(context); - objectCache = new ObjectCache(); - objectCache.initialize(new ObjectCacheConfig(config)); - } + @Override + public void initialize(Context context) { + Map config = getConfig(context); + objectCache = new ObjectCache(); + objectCache.initialize(new ObjectCacheConfig(config)); + } - // Exposed for testing - protected void initialize(ObjectCache objectCache) { - this.objectCache = objectCache; - } + // Exposed for testing + protected void initialize(ObjectCache objectCache) { + this.objectCache = objectCache; + } - @Override - public boolean isInitialized() { - return objectCache != null && objectCache.isInitialized(); - } + @Override + public boolean isInitialized() { + return objectCache != null && objectCache.isInitialized(); + } - protected Map getConfig(Context context) { - return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false).orElse(new HashMap<>()); + protected Map getConfig(Context context) { + return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG, false) + .orElse(new HashMap<>()); } - // exposed for testing - protected ObjectCache getObjectCache() { - return objectCache; - } + // exposed for testing + protected ObjectCache getObjectCache() { + return objectCache; + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java index d2f460f0..c2655098 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java @@ -7,19 +7,29 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.stellar; import com.cloudera.cyber.hbase.HbaseConfiguration; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ExecutionException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Table; import org.apache.metron.enrichment.converter.EnrichmentKey; @@ -37,247 +47,254 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; - public class SimpleHBaseEnrichmentFunctions { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String ACCESS_TRACKER_TYPE_CONF = "accessTracker"; - public static final String TABLE_PROVIDER_TYPE_CONF = "tableProviderImpl"; - private static AccessTracker tracker; - private static TableProvider provider; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String ACCESS_TRACKER_TYPE_CONF = "accessTracker"; + public static final String TABLE_PROVIDER_TYPE_CONF = "tableProviderImpl"; + private static AccessTracker tracker; + private static TableProvider provider; - private static Configuration hBaseConfiguration; + private static Configuration hBaseConfiguration; - private static class WrapperTable { - String name; - String columnFamily; + private static class WrapperTable { + String name; + String columnFamily; - public WrapperTable(String name, String columnFamily) { - this.name = name; - this.columnFamily = columnFamily; - } + public WrapperTable(String name, String columnFamily) { + this.name = name; + this.columnFamily = columnFamily; + } - @Override - public String toString() { - return "Table{" + - "name='" + name + '\'' + - ", columnFamily='" + columnFamily + '\'' + - '}'; - } + @Override + public String toString() { + return "Table{" + + "name='" + name + '\'' + + ", columnFamily='" + columnFamily + '\'' + + '}'; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - WrapperTable table = (WrapperTable) o; + WrapperTable table = (WrapperTable) o; - if (name != null ? !name.equals(table.name) : table.name != null) return false; - return columnFamily != null ? columnFamily.equals(table.columnFamily) : table.columnFamily == null; + if (!Objects.equals(name, table.name)) { + return false; + } + return Objects.equals(columnFamily, table.columnFamily); - } + } - @Override - public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + (columnFamily != null ? columnFamily.hashCode() : 0); - return result; + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (columnFamily != null ? columnFamily.hashCode() : 0); + return result; + } } - } - - private static Map getConfig(Context context) { - return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG).orElse(new HashMap<>()); - } - private static synchronized void initializeTracker(Map config, TableProvider provider) throws IOException { - if(tracker == null) { - String accessTrackerType = (String) config.getOrDefault(ACCESS_TRACKER_TYPE_CONF, AccessTrackers.NOOP.toString()); - AccessTrackers trackers = AccessTrackers.valueOf(accessTrackerType); - tracker = trackers.create(config, provider); + private static Map getConfig(Context context) { + return (Map) context.getCapability(Context.Capabilities.GLOBAL_CONFIG).orElse(new HashMap<>()); } - } - private static TableProvider createProvider(String tableProviderClass) { - try { - Class providerClazz = (Class) Class.forName(tableProviderClass); - return providerClazz.getConstructor().newInstance(); - } catch (Exception e) { - return new HTableProvider(); + private static synchronized void initializeTracker(Map config, TableProvider provider) + throws IOException { + if (tracker == null) { + String accessTrackerType = + (String) config.getOrDefault(ACCESS_TRACKER_TYPE_CONF, AccessTrackers.NOOP.toString()); + AccessTrackers trackers = AccessTrackers.valueOf(accessTrackerType); + tracker = trackers.create(config, provider); + } } - } - private static synchronized void initializeProvider(Map config) { - if(provider == null) { - String tableProviderClass = (String) config.getOrDefault(TABLE_PROVIDER_TYPE_CONF, HTableProvider.class.getName()); - provider = createProvider(tableProviderClass); + private static TableProvider createProvider(String tableProviderClass) { + try { + Class providerClazz = + (Class) Class.forName(tableProviderClass); + return providerClazz.getConstructor().newInstance(); + } catch (Exception e) { + return new HTableProvider(); + } } - } - private static void initializeHbaseConfiguration(Map config) { - if (hBaseConfiguration == null) { - hBaseConfiguration = (Configuration) config.getOrDefault(HbaseConfiguration.HBASE_CONFIG_NAME, HbaseConfiguration.configureHbase()); + private static synchronized void initializeProvider(Map config) { + if (provider == null) { + String tableProviderClass = + (String) config.getOrDefault(TABLE_PROVIDER_TYPE_CONF, HTableProvider.class.getName()); + provider = createProvider(tableProviderClass); + } } - } - @Stellar(name="EXISTS" - ,namespace="ENRICHMENT" - ,description="Interrogates the HBase table holding the simple hbase enrichment data and returns whether the" + - " enrichment type and indicator are in the table." - ,params = { - "enrichment_type - The enrichment type" - ,"indicator - The string indicator to look up" - ,"nosql_table - The NoSQL Table to use" - ,"column_family - The Column Family to use" - } - ,returns = "True if the enrichment indicator exists and false otherwise" - ) - public static class EnrichmentExists implements StellarFunction { - boolean initialized = false; - private static Cache enrichmentCollateralCache = CacheBuilder.newBuilder() - .build(); - @Override - public Object apply(List args, Context context) throws ParseException { - if(!initialized) { - return false; - } - if(args.size() != 4) { - throw new IllegalStateException("All parameters are mandatory, submit 'enrichment type', 'indicator', 'nosql_table' and 'column_family'"); - } - int i = 0; - String enrichmentType = (String) args.get(i++); - String indicator = (String) args.get(i++); - String table = (String) args.get(i++); - String cf = (String) args.get(i++); - if(enrichmentType == null || indicator == null) { - return false; - } - final WrapperTable key = new WrapperTable(table, cf); - EnrichmentLookup lookup = null; - try { - lookup = enrichmentCollateralCache.get(key, () -> { - Table hTable = provider.getTable(hBaseConfiguration, key.name); - return new EnrichmentLookup(hTable, key.columnFamily, tracker); - } - ); - } catch (ExecutionException e) { - LOG.error("Unable to retrieve enrichmentLookup: {}", e.getMessage(), e); - return false; - } - EnrichmentLookup.HBaseContext hbaseContext = new EnrichmentLookup.HBaseContext(lookup.getTable(), cf); - try { - return lookup.exists(new EnrichmentKey(enrichmentType, indicator), hbaseContext, true); - } catch (IOException e) { - LOG.error("Unable to call exists: {}", e.getMessage(), e); - return false; - } + private static void initializeHbaseConfiguration(Map config) { + if (hBaseConfiguration == null) { + hBaseConfiguration = (Configuration) config.getOrDefault(HbaseConfiguration.HBASE_CONFIG_NAME, + HbaseConfiguration.configureHbase()); + } } - @Override - public void initialize(Context context) { - try { - Map config = getConfig(context); - initializeProvider(config); - initializeTracker(config, provider); - } catch (IOException e) { - LOG.error("Unable to initialize ENRICHMENT.EXISTS: {}", e.getMessage(), e); - } - finally{ - initialized = true; - } + @Stellar(name = "EXISTS", + namespace = "ENRICHMENT", + description = + "Interrogates the HBase table holding the simple hbase enrichment data and returns whether the" + + " enrichment type and indicator are in the table.", + params = { + "enrichment_type - The enrichment type", + "indicator - The string indicator to look up", + "nosql_table - The NoSQL Table to use", + "column_family - The Column Family to use" + }, + returns = "True if the enrichment indicator exists and false otherwise" + ) + public static class EnrichmentExists implements StellarFunction { + boolean initialized = false; + private static final Cache enrichmentCollateralCache = CacheBuilder.newBuilder() + .build(); - } + @Override + public Object apply(List args, Context context) throws ParseException { + if (!initialized) { + return false; + } + if (args.size() != 4) { + throw new IllegalStateException( + "All parameters are mandatory, submit 'enrichment type', 'indicator', 'nosql_table' and 'column_family'"); + } + int i = 0; + String enrichmentType = (String) args.get(i++); + String indicator = (String) args.get(i++); + String table = (String) args.get(i++); + String cf = (String) args.get(i++); + if (enrichmentType == null || indicator == null) { + return false; + } + final WrapperTable key = new WrapperTable(table, cf); + EnrichmentLookup lookup = null; + try { + lookup = enrichmentCollateralCache.get(key, () -> { + Table hbaseTable = provider.getTable(hBaseConfiguration, key.name); + return new EnrichmentLookup(hbaseTable, key.columnFamily, tracker); + } + ); + } catch (ExecutionException e) { + LOG.error("Unable to retrieve enrichmentLookup: {}", e.getMessage(), e); + return false; + } + EnrichmentLookup.HBaseContext hbaseContext = new EnrichmentLookup.HBaseContext(lookup.getTable(), cf); + try { + return lookup.exists(new EnrichmentKey(enrichmentType, indicator), hbaseContext, true); + } catch (IOException e) { + LOG.error("Unable to call exists: {}", e.getMessage(), e); + return false; + } + } - @Override - public boolean isInitialized() { - return initialized; - } + @Override + public void initialize(Context context) { + try { + Map config = getConfig(context); + initializeProvider(config); + initializeTracker(config, provider); + } catch (IOException e) { + LOG.error("Unable to initialize ENRICHMENT.EXISTS: {}", e.getMessage(), e); + } finally { + initialized = true; + } - } + } + + @Override + public boolean isInitialized() { + return initialized; + } + + } + @Stellar(name = "GET", + namespace = "ENRICHMENT", + description = "Interrogates the HBase table holding the simple hbase enrichment data and retrieves the " + + "tabular value associated with the enrichment type and indicator.", + params = { + "enrichment_type - The enrichment type", + "indicator - The string indicator to look up", + "nosql_table - The NoSQL Table to use", + "column_family - The Column Family to use" + }, + returns = "A Map associated with the indicator and enrichment type. Empty otherwise." + ) + public static class EnrichmentGet implements StellarFunction { + boolean initialized = false; + private static final Cache enrichmentCollateralCache = CacheBuilder.newBuilder() + .build(); - @Stellar(name="GET" - ,namespace="ENRICHMENT" - ,description="Interrogates the HBase table holding the simple hbase enrichment data and retrieves the " + - "tabular value associated with the enrichment type and indicator." - ,params = { - "enrichment_type - The enrichment type" - ,"indicator - The string indicator to look up" - ,"nosql_table - The NoSQL Table to use" - ,"column_family - The Column Family to use" - } - ,returns = "A Map associated with the indicator and enrichment type. Empty otherwise." - ) - public static class EnrichmentGet implements StellarFunction { - boolean initialized = false; - private static Cache enrichmentCollateralCache = CacheBuilder.newBuilder() - .build(); - @Override - public Object apply(List args, Context context) throws ParseException { - if(!initialized) { - return false; - } - if(args.size() != 4) { - throw new IllegalStateException("All parameters are mandatory, submit 'enrichment type', 'indicator', 'nosql_table' and 'column_family'"); - } - int i = 0; - String enrichmentType = (String) args.get(i++); - String indicator = (String) args.get(i++); - String table = (String) args.get(i++); - String cf = (String) args.get(i++); - if(enrichmentType == null || indicator == null) { - return new HashMap(); - } - final WrapperTable key = new WrapperTable(table, cf); - EnrichmentLookup lookup = null; - try { - lookup = enrichmentCollateralCache.get(key, () -> { - Table hTable = provider.getTable(hBaseConfiguration, key.name); - return new EnrichmentLookup(hTable, key.columnFamily, tracker); + @Override + public Object apply(List args, Context context) throws ParseException { + if (!initialized) { + return false; + } + if (args.size() != 4) { + throw new IllegalStateException( + "All parameters are mandatory, submit 'enrichment type', 'indicator', 'nosql_table' and 'column_family'"); + } + int i = 0; + String enrichmentType = (String) args.get(i++); + String indicator = (String) args.get(i++); + String table = (String) args.get(i++); + String cf = (String) args.get(i++); + if (enrichmentType == null || indicator == null) { + return new HashMap(); + } + final WrapperTable key = new WrapperTable(table, cf); + EnrichmentLookup lookup = null; + try { + lookup = enrichmentCollateralCache.get(key, () -> { + Table hbaseTable = provider.getTable(hBaseConfiguration, key.name); + return new EnrichmentLookup(hbaseTable, key.columnFamily, tracker); + } + ); + } catch (ExecutionException e) { + LOG.error("Unable to retrieve enrichmentLookup: {}", e.getMessage(), e); + return new HashMap(); + } + EnrichmentLookup.HBaseContext hbaseContext = new EnrichmentLookup.HBaseContext(lookup.getTable(), cf); + try { + LookupKV kv = + lookup.get(new EnrichmentKey(enrichmentType, indicator), hbaseContext, true); + if (kv != null && kv.getValue() != null && kv.getValue().getMetadata() != null) { + return kv.getValue().getMetadata(); } - ); - } catch (ExecutionException e) { - LOG.error("Unable to retrieve enrichmentLookup: {}", e.getMessage(), e); - return new HashMap(); - } - EnrichmentLookup.HBaseContext hbaseContext = new EnrichmentLookup.HBaseContext(lookup.getTable(), cf); - try { - LookupKV kv = lookup.get(new EnrichmentKey(enrichmentType, indicator), hbaseContext, true); - if (kv != null && kv.getValue() != null && kv.getValue().getMetadata() != null) { - return kv.getValue().getMetadata(); + return new HashMap(); + } catch (IOException e) { + LOG.error("Unable to call exists: {}", e.getMessage(), e); + return new HashMap(); + } } - return new HashMap(); - } catch (IOException e) { - LOG.error("Unable to call exists: {}", e.getMessage(), e); - return new HashMap(); - } - } - @Override - public void initialize(Context context) { - try { - Map config = getConfig(context); - initializeProvider(config); - initializeTracker(config, provider); - initializeHbaseConfiguration(config); - } catch (IOException e) { - LOG.error("Unable to initialize ENRICHMENT.GET: {}", e.getMessage(), e); - } - finally{ - initialized = true; - } - } + @Override + public void initialize(Context context) { + try { + Map config = getConfig(context); + initializeProvider(config); + initializeTracker(config, provider); + initializeHbaseConfiguration(config); + } catch (IOException e) { + LOG.error("Unable to initialize ENRICHMENT.GET: {}", e.getMessage(), e); + } finally { + initialized = true; + } + } - @Override - public boolean isInitialized() { - return initialized; + @Override + public boolean isInitialized() { + return initialized; + } } - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java index cec99581..ae58c572 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.utils; import com.google.common.base.Function; @@ -36,99 +39,104 @@ public class EnrichmentUtils { - public static final String KEY_PREFIX = "enrichments"; - - public static String getEnrichmentKey(String enrichmentName, String field) { - return Joiner.on(".").join(new String[]{KEY_PREFIX, enrichmentName, field}); - } + public static final String KEY_PREFIX = "enrichments"; - public static class TypeToKey implements Function> { - private final String indicator; - private final EnrichmentConfig config; - private final Table table; - public TypeToKey(String indicator, Table table, EnrichmentConfig config) { - this.indicator = indicator; - this.config = config; - this.table = table; - } - @Nullable - @Override - public KeyWithContext apply(@Nullable String enrichmentType) { - EnrichmentKey key = new EnrichmentKey(enrichmentType, indicator); - EnrichmentLookup.HBaseContext context = new EnrichmentLookup.HBaseContext(table, getColumnFamily(enrichmentType, config)); - return new KeyWithContext<>(key, context); - } - } - private static ThreadLocal>> typeToCFs = new ThreadLocal>>() { - @Override - protected Map>initialValue() { - return new HashMap<>(); + public static String getEnrichmentKey(String enrichmentName, String field) { + return Joiner.on(".").join(new String[] {KEY_PREFIX, enrichmentName, field}); } - }; - public static final String TYPE_TO_COLUMN_FAMILY_CONF = "typeToColumnFamily"; - public static String getColumnFamily(String enrichmentType, EnrichmentConfig config) { - Object o = config.getConfig().get(TYPE_TO_COLUMN_FAMILY_CONF); - if(o == null) { - return null; - } - else { - Map cfMap = typeToCFs.get().get(o); - if(cfMap == null) { - cfMap = new HashMap<>(); - if(o instanceof Map) { - Map map = (Map) o; - for(Object key : map.keySet()) { - cfMap.put(key.toString(), map.get(key).toString()); - } + public static class TypeToKey + implements Function> { + private final String indicator; + private final EnrichmentConfig config; + private final Table table; + + public TypeToKey(String indicator, Table table, EnrichmentConfig config) { + this.indicator = indicator; + this.config = config; + this.table = table; } - typeToCFs.get().put(o, cfMap); - } - return cfMap.get(enrichmentType); - } - } - public static String toTopLevelField(String field) { - if(field == null) { - return null; + @Nullable + @Override + public KeyWithContext apply(@Nullable String enrichmentType) { + EnrichmentKey key = new EnrichmentKey(enrichmentType, indicator); + EnrichmentLookup.HBaseContext context = + new EnrichmentLookup.HBaseContext(table, getColumnFamily(enrichmentType, config)); + return new KeyWithContext<>(key, context); + } } - return Iterables.getLast(Splitter.on('.').split(field)); - } - public static TableProvider getTableProvider(String connectorImpl, TableProvider defaultImpl) { - if(connectorImpl == null || connectorImpl.length() == 0 || connectorImpl.charAt(0) == '$') { - return defaultImpl; + private static final ThreadLocal>> typeToCFs = + new ThreadLocal>>() { + @Override + protected Map> initialValue() { + return new HashMap<>(); + } + }; + + public static final String TYPE_TO_COLUMN_FAMILY_CONF = "typeToColumnFamily"; + + public static String getColumnFamily(String enrichmentType, EnrichmentConfig config) { + Object o = config.getConfig().get(TYPE_TO_COLUMN_FAMILY_CONF); + if (o == null) { + return null; + } else { + Map cfMap = typeToCFs.get().get(o); + if (cfMap == null) { + cfMap = new HashMap<>(); + if (o instanceof Map) { + Map map = (Map) o; + for (Object key : map.keySet()) { + cfMap.put(key.toString(), map.get(key).toString()); + } + } + typeToCFs.get().put(o, cfMap); + } + return cfMap.get(enrichmentType); + } } - else { - try { - Class clazz = (Class) Class.forName(connectorImpl); - return clazz.getConstructor().newInstance(); - } catch (InstantiationException e) { - throw new IllegalStateException("Unable to instantiate connector.", e); - } catch (IllegalAccessException e) { - throw new IllegalStateException("Unable to instantiate connector: illegal access", e); - } catch (InvocationTargetException e) { - throw new IllegalStateException("Unable to instantiate connector", e); - } catch (NoSuchMethodException e) { - throw new IllegalStateException("Unable to instantiate connector: no such method", e); - } catch (ClassNotFoundException e) { - throw new IllegalStateException("Unable to instantiate connector: class not found", e); - } + + public static String toTopLevelField(String field) { + if (field == null) { + return null; + } + return Iterables.getLast(Splitter.on('.').split(field)); } - } - public static JSONObject adjustKeys(JSONObject enrichedMessage, JSONObject enrichedField, String field, String prefix) { - if ( !enrichedField.isEmpty()) { - for (Object enrichedKey : enrichedField.keySet()) { - if(!StringUtils.isEmpty(prefix)) { - enrichedMessage.put(field + "." + enrichedKey, enrichedField.get(enrichedKey)); + public static TableProvider getTableProvider(String connectorImpl, TableProvider defaultImpl) { + if (connectorImpl == null || connectorImpl.length() == 0 || connectorImpl.charAt(0) == '$') { + return defaultImpl; + } else { + try { + Class clazz = (Class) Class.forName(connectorImpl); + return clazz.getConstructor().newInstance(); + } catch (InstantiationException e) { + throw new IllegalStateException("Unable to instantiate connector.", e); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Unable to instantiate connector: illegal access", e); + } catch (InvocationTargetException e) { + throw new IllegalStateException("Unable to instantiate connector", e); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to instantiate connector: no such method", e); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Unable to instantiate connector: class not found", e); + } } - else { - enrichedMessage.put(enrichedKey, enrichedField.get(enrichedKey)); + } + + public static JSONObject adjustKeys(JSONObject enrichedMessage, JSONObject enrichedField, String field, + String prefix) { + if (!enrichedField.isEmpty()) { + for (Object enrichedKey : enrichedField.keySet()) { + if (!StringUtils.isEmpty(prefix)) { + enrichedMessage.put(field + "." + enrichedKey, enrichedField.get(enrichedKey)); + } else { + enrichedMessage.put(enrichedKey, enrichedField.get(enrichedKey)); + } + } } - } + return enrichedMessage; } - return enrichedMessage; - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java index 646547e5..6db01a26 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java @@ -6,18 +6,22 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.enrichment.utils; import com.google.common.base.Joiner; +import java.lang.invoke.MethodHandles; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RuleScore; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatScore; @@ -31,129 +35,130 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; - public class ThreatIntelUtils { - public static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String KEY_PREFIX = "threatintels"; - /** - * The message key under which the overall threat triage score is stored. - */ - public static final String THREAT_TRIAGE_SCORE_KEY = "threat.triage.score"; - - /** - * The prefix of the message keys that record the threat triage rules that fired. - */ - public static final String THREAT_TRIAGE_RULES_KEY = "threat.triage.rules"; - - /** - * The portion of the message key used to record the 'name' field of a rule. - */ - public static final String THREAT_TRIAGE_RULE_NAME = "name"; - - /** - * The portion of the message key used to record the 'comment' field of a rule. - */ - public static final String THREAT_TRIAGE_RULE_COMMENT = "comment"; - - /** - * The portion of the message key used to record the 'score' field of a rule. - */ - public static final String THREAT_TRIAGE_RULE_SCORE = "score"; - - /** - * The portion of the message key used to record the 'reason' field of a rule. - */ - public static final String THREAT_TRIAGE_RULE_REASON = "reason"; - - - public static String getThreatIntelKey(String threatIntelName, String field) { - return Joiner.on(".").join(new String[]{KEY_PREFIX, threatIntelName, field}); - } - -public static JSONObject triage(JSONObject ret, SensorEnrichmentConfig config, FunctionResolver functionResolver, Context stellarContext) { - LOG.trace("Received joined messages: {}", ret); - boolean isAlert = ret.containsKey("is_alert"); - if(!isAlert) { - for (Object key : ret.keySet()) { - if (key.toString().startsWith("threatintels") && !key.toString().endsWith(".ts")) { - isAlert = true; - break; - } - } - } - else { - Object isAlertObj = ret.get("is_alert"); - isAlert = ConversionUtils.convert(isAlertObj, Boolean.class); - if(!isAlert) { - ret.remove("is_alert"); - } + public static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String KEY_PREFIX = "threatintels"; + /** + * The message key under which the overall threat triage score is stored. + */ + public static final String THREAT_TRIAGE_SCORE_KEY = "threat.triage.score"; + + /** + * The prefix of the message keys that record the threat triage rules that fired. + */ + public static final String THREAT_TRIAGE_RULES_KEY = "threat.triage.rules"; + + /** + * The portion of the message key used to record the 'name' field of a rule. + */ + public static final String THREAT_TRIAGE_RULE_NAME = "name"; + + /** + * The portion of the message key used to record the 'comment' field of a rule. + */ + public static final String THREAT_TRIAGE_RULE_COMMENT = "comment"; + + /** + * The portion of the message key used to record the 'score' field of a rule. + */ + public static final String THREAT_TRIAGE_RULE_SCORE = "score"; + + /** + * The portion of the message key used to record the 'reason' field of a rule. + */ + public static final String THREAT_TRIAGE_RULE_REASON = "reason"; + + + public static String getThreatIntelKey(String threatIntelName, String field) { + return Joiner.on(".").join(new String[] {KEY_PREFIX, threatIntelName, field}); } - if(isAlert) { - ret.put("is_alert" , "true"); - String sourceType = MessageUtils.getSensorType(ret); - ThreatTriageConfig triageConfig = null; - if(config != null) { - triageConfig = config.getThreatIntel().getTriageConfig(); - if(LOG.isDebugEnabled()) { - LOG.debug("{}: Found sensor enrichment config.", sourceType); - } - } - else { - LOG.debug("{}: Unable to find threat config.", sourceType ); - } - if(triageConfig != null) { - if(LOG.isDebugEnabled()) { - LOG.debug("{}: Found threat triage config: {}", sourceType, triageConfig); - } - if(LOG.isDebugEnabled() && (triageConfig.getRiskLevelRules() == null || triageConfig.getRiskLevelRules().isEmpty())) { - LOG.debug("{}: Empty rules!", sourceType); + public static JSONObject triage(JSONObject ret, SensorEnrichmentConfig config, FunctionResolver functionResolver, + Context stellarContext) { + LOG.trace("Received joined messages: {}", ret); + boolean isAlert = ret.containsKey("is_alert"); + if (!isAlert) { + for (Object key : ret.keySet()) { + if (key.toString().startsWith("threatintels") && !key.toString().endsWith(".ts")) { + isAlert = true; + break; + } + } + } else { + Object isAlertObj = ret.get("is_alert"); + isAlert = ConversionUtils.convert(isAlertObj, Boolean.class); + if (!isAlert) { + ret.remove("is_alert"); + } } - - // triage the threat - ThreatTriageProcessor threatTriageProcessor = new ThreatTriageProcessor(config, functionResolver, stellarContext); - ThreatScore score = threatTriageProcessor.apply(ret); - - if(LOG.isDebugEnabled()) { - String rules = Joiner.on('\n').join(triageConfig.getRiskLevelRules()); - LOG.debug("Marked {} as triage level {} with rules {}", sourceType, score.getScore(), - rules); + if (isAlert) { + ret.put("is_alert", "true"); + String sourceType = MessageUtils.getSensorType(ret); + ThreatTriageConfig triageConfig = null; + if (config != null) { + triageConfig = config.getThreatIntel().getTriageConfig(); + if (LOG.isDebugEnabled()) { + LOG.debug("{}: Found sensor enrichment config.", sourceType); + } + } else { + LOG.debug("{}: Unable to find threat config.", sourceType); + } + if (triageConfig != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("{}: Found threat triage config: {}", sourceType, triageConfig); + } + + if (LOG.isDebugEnabled() + && (triageConfig.getRiskLevelRules() == null || triageConfig.getRiskLevelRules().isEmpty())) { + LOG.debug("{}: Empty rules!", sourceType); + } + + // triage the threat + ThreatTriageProcessor threatTriageProcessor = + new ThreatTriageProcessor(config, functionResolver, stellarContext); + ThreatScore score = threatTriageProcessor.apply(ret); + + if (LOG.isDebugEnabled()) { + String rules = Joiner.on('\n').join(triageConfig.getRiskLevelRules()); + LOG.debug("Marked {} as triage level {} with rules {}", sourceType, score.getScore(), + rules); + } + + // attach the triage threat score to the message + if (score.getRuleScores().size() > 0) { + appendThreatScore(score, ret); + } + } else { + LOG.debug("{}: Unable to find threat triage config!", sourceType); + } } - // attach the triage threat score to the message - if(score.getRuleScores().size() > 0) { - appendThreatScore(score, ret); - } - } - else { - LOG.debug("{}: Unable to find threat triage config!", sourceType); - } + return ret; } - return ret; - } - - /** - * Appends the threat score to the telemetry message. - * @param threatScore The threat triage score - * @param message The telemetry message being triaged. - */ - private static void appendThreatScore(ThreatScore threatScore, JSONObject message) { - - // append the overall threat score - message.put(THREAT_TRIAGE_SCORE_KEY, threatScore.getScore()); - - // append each of the rules - each rule is 'flat' - Joiner joiner = Joiner.on("."); - int i = 0; - for(RuleScore score: threatScore.getRuleScores()) { - message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i, THREAT_TRIAGE_RULE_NAME), score.getRule().getName()); - message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i, THREAT_TRIAGE_RULE_COMMENT), score.getRule().getComment()); - message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i, THREAT_TRIAGE_RULE_SCORE), score.getRule().getScoreExpression()); - message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i++, THREAT_TRIAGE_RULE_REASON), score.getReason()); + /** + * Appends the threat score to the telemetry message. + * + * @param threatScore The threat triage score + * @param message The telemetry message being triaged. + */ + private static void appendThreatScore(ThreatScore threatScore, JSONObject message) { + + // append the overall threat score + message.put(THREAT_TRIAGE_SCORE_KEY, threatScore.getScore()); + + // append each of the rules - each rule is 'flat' + Joiner joiner = Joiner.on("."); + int i = 0; + for (RuleScore score : threatScore.getRuleScores()) { + message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i, THREAT_TRIAGE_RULE_NAME), score.getRule().getName()); + message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i, THREAT_TRIAGE_RULE_COMMENT), + score.getRule().getComment()); + message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i, THREAT_TRIAGE_RULE_SCORE), + score.getRule().getScoreExpression()); + message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i++, THREAT_TRIAGE_RULE_REASON), score.getReason()); + } } - } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java index 64a2ae62..896d2211 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +21,10 @@ package org.apache.metron.threatintel.triage; import com.google.common.base.Function; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; import org.apache.metron.common.aggregator.Aggregators; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; @@ -34,94 +40,92 @@ import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - /** * Applies the threat triage rules to an alert and produces a threat score that is * attached to the alert. * + *

* The goal of threat triage is to prioritize the alerts that pose the greatest * threat and thus need urgent attention. To perform threat triage, a set of rules * are applied to each message. Each rule has a predicate to determine if the rule * applies or not. The threat score from each applied rule is aggregated into a single * threat triage score that can be used to prioritize high risk threats. * + *

* Tuning the threat triage process involves creating one or more rules, adjusting * the score of each rule, and changing the way that each rule's score is aggregated. */ public class ThreatTriageProcessor implements Function { - private SensorEnrichmentConfig sensorConfig; - private ThreatIntelConfig threatIntelConfig; - private ThreatTriageConfig threatTriageConfig; - private Context context; - private FunctionResolver functionResolver; - - public ThreatTriageProcessor( SensorEnrichmentConfig config - , FunctionResolver functionResolver - , Context context - ) - { - this.threatIntelConfig = config.getThreatIntel(); - this.sensorConfig = config; - this.threatTriageConfig = config.getThreatIntel().getTriageConfig(); - this.functionResolver = functionResolver; - this.context = context; - } - - /** - * @param message The message being triaged. - */ - @Nullable - @Override - public ThreatScore apply(@Nullable Map message) { - - ThreatScore threatScore = new ThreatScore(); - StellarPredicateProcessor predicateProcessor = new StellarPredicateProcessor(); - StellarProcessor processor = new StellarProcessor(); - VariableResolver variableResolver = new MapVariableResolver(message, sensorConfig.getConfiguration(), threatIntelConfig.getConfig()); - - // attempt to apply each rule to the threat - for(RiskLevelRule rule : threatTriageConfig.getRiskLevelRules()) { - if(predicateProcessor.parse(rule.getRule(), variableResolver, functionResolver, context)) { - - // add the rule's score to the overall threat score - String reason = execute(rule.getReason(), processor, variableResolver, String.class); - Double score = execute(rule.getScoreExpression(), processor, variableResolver, Double.class); - threatScore.addRuleScore(new RuleScore(rule, reason, score)); - } + private final SensorEnrichmentConfig sensorConfig; + private final ThreatIntelConfig threatIntelConfig; + private final ThreatTriageConfig threatTriageConfig; + private final Context context; + private final FunctionResolver functionResolver; + + public ThreatTriageProcessor(SensorEnrichmentConfig config, + FunctionResolver functionResolver, + Context context) { + this.threatIntelConfig = config.getThreatIntel(); + this.sensorConfig = config; + this.threatTriageConfig = config.getThreatIntel().getTriageConfig(); + this.functionResolver = functionResolver; + this.context = context; } - // calculate the aggregate threat score - List ruleScores = new ArrayList<>(); - for(RuleScore ruleScore: threatScore.getRuleScores()) { - ruleScores.add(ruleScore.getScore()); + /** + * Method to apply processing to a message. + * + * @param message The message being triaged. + */ + @Nullable + @Override + public ThreatScore apply(@Nullable Map message) { + + ThreatScore threatScore = new ThreatScore(); + StellarPredicateProcessor predicateProcessor = new StellarPredicateProcessor(); + StellarProcessor processor = new StellarProcessor(); + VariableResolver variableResolver = + new MapVariableResolver(message, sensorConfig.getConfiguration(), threatIntelConfig.getConfig()); + + // attempt to apply each rule to the threat + for (RiskLevelRule rule : threatTriageConfig.getRiskLevelRules()) { + if (predicateProcessor.parse(rule.getRule(), variableResolver, functionResolver, context)) { + + // add the rule's score to the overall threat score + String reason = execute(rule.getReason(), processor, variableResolver, String.class); + Double score = execute(rule.getScoreExpression(), processor, variableResolver, Double.class); + threatScore.addRuleScore(new RuleScore(rule, reason, score)); + } + } + + // calculate the aggregate threat score + List ruleScores = new ArrayList<>(); + for (RuleScore ruleScore : threatScore.getRuleScores()) { + ruleScores.add(ruleScore.getScore()); + } + Aggregators aggregators = threatTriageConfig.getAggregator(); + Double aggregateScore = aggregators.aggregate(ruleScores, threatTriageConfig.getAggregationConfig()); + threatScore.setScore(aggregateScore); + + return threatScore; } - Aggregators aggregators = threatTriageConfig.getAggregator(); - Double aggregateScore = aggregators.aggregate(ruleScores, threatTriageConfig.getAggregationConfig()); - threatScore.setScore(aggregateScore); - - return threatScore; - } - private T execute(String expression, StellarProcessor processor, VariableResolver resolver, Class clazz) { - Object result = processor.parse(expression, resolver, functionResolver, context); - return ConversionUtils.convert(result, clazz); - } + private T execute(String expression, StellarProcessor processor, VariableResolver resolver, Class clazz) { + Object result = processor.parse(expression, resolver, functionResolver, context); + return ConversionUtils.convert(result, clazz); + } - public List getRiskLevelRules() { - return threatTriageConfig.getRiskLevelRules(); - } + public List getRiskLevelRules() { + return threatTriageConfig.getRiskLevelRules(); + } - public SensorEnrichmentConfig getSensorConfig() { - return sensorConfig; - } + public SensorEnrichmentConfig getSensorConfig() { + return sensorConfig; + } - @Override - public String toString() { - return String.format("ThreatTriage{%d rule(s)}", threatTriageConfig.getRiskLevelRules().size()); - } + @Override + public String toString() { + return String.format("ThreatTriage{%d rule(s)}", threatTriageConfig.getRiskLevelRules().size()); + } } diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabaseTest.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabaseTest.java index 643bf3af..3161f2aa 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabaseTest.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/asn/GeoLiteAsnDatabaseTest.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

* http://www.apache.org/licenses/LICENSE-2.0 + * *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,8 +19,17 @@ */ package org.apache.metron.enrichment.adapters.maxmind.asn; +import static org.apache.metron.enrichment.adapters.maxmind.MaxMindDatabase.EXTENSION_TAR_GZ; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + import com.cloudera.cyber.TestUtils; import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.apache.commons.io.FileUtils; import org.apache.metron.stellar.dsl.Context; import org.json.simple.JSONObject; @@ -30,16 +41,6 @@ import org.junit.jupiter.migrationsupport.rules.EnableRuleMigrationSupport; import org.junit.rules.TemporaryFolder; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.enrichment.adapters.maxmind.MaxMindDatabase.EXTENSION_TAR_GZ; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; - @EnableRuleMigrationSupport public class GeoLiteAsnDatabaseTest { diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabaseTest.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabaseTest.java index 9ed46f51..57ee1597 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabaseTest.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/adapters/maxmind/geo/GeoLiteCityDatabaseTest.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

* http://www.apache.org/licenses/LICENSE-2.0 + * *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,8 +19,19 @@ */ package org.apache.metron.enrichment.adapters.maxmind.geo; +import static org.apache.metron.enrichment.adapters.maxmind.MaxMindDatabase.EXTENSION_MMDB_GZ; +import static org.apache.metron.enrichment.adapters.maxmind.MaxMindDatabase.EXTENSION_TAR_GZ; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + import com.cloudera.cyber.TestUtils; import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -35,18 +48,6 @@ import org.junit.jupiter.migrationsupport.rules.EnableRuleMigrationSupport; import org.junit.rules.TemporaryFolder; -import java.io.File; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.enrichment.adapters.maxmind.MaxMindDatabase.EXTENSION_MMDB_GZ; -import static org.apache.metron.enrichment.adapters.maxmind.MaxMindDatabase.EXTENSION_TAR_GZ; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; - @EnableRuleMigrationSupport public class GeoLiteCityDatabaseTest { diff --git a/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java b/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java index a810761f..7c575e25 100644 --- a/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java +++ b/flink-cyber/flink-enrichment/metron-enrichment-common/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

* http://www.apache.org/licenses/LICENSE-2.0 + * *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,7 +20,16 @@ package org.apache.metron.enrichment.stellar; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.metron.enrichment.converter.EnrichmentHelper; import org.apache.metron.enrichment.converter.EnrichmentKey; import org.apache.metron.enrichment.converter.EnrichmentValue; @@ -33,13 +44,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; - public class SimpleHBaseEnrichmentFunctionsTest { private final String hbaseTableName = "enrichments"; private static final String ENRICHMENT_TYPE = "et"; diff --git a/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/AbstractHbaseMapFunction.java b/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/AbstractHbaseMapFunction.java index d7d1fb0a..025237e2 100644 --- a/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/AbstractHbaseMapFunction.java +++ b/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/AbstractHbaseMapFunction.java @@ -12,9 +12,16 @@ package com.cloudera.cyber.hbase; +import static java.util.stream.Collectors.toMap; + import com.cloudera.cyber.flink.CacheMetrics; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.hbase.util.HBaseConfigurationUtil; @@ -22,15 +29,11 @@ import org.apache.flink.metrics.MetricGroup; import org.apache.flink.streaming.api.functions.ProcessFunction; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import static java.util.stream.Collectors.toMap; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; @Slf4j public abstract class AbstractHbaseMapFunction extends ProcessFunction { @@ -65,7 +68,7 @@ protected Map fetch(LookupKey key) { } } - public AbstractHbaseMapFunction() { + public AbstractHbaseMapFunction() { serializedHbaseConfig = HBaseConfigurationUtil.serializeConfiguration(HbaseConfiguration.configureHbase()); } @@ -81,18 +84,20 @@ public final void open(Configuration parameters) throws Exception { connectHbase(); cache = Caffeine.newBuilder() - .expireAfterAccess(60, TimeUnit.SECONDS) - .maximumSize(1000) - .recordStats(() -> new CacheMetrics(metricsGroup)) - .build(); + .expireAfterAccess(60, TimeUnit.SECONDS) + .maximumSize(1000) + .recordStats(() -> new CacheMetrics(metricsGroup)) + .build(); } protected void connectHbase() throws Exception { - org.apache.hadoop.conf.Configuration hbaseConfig = HBaseConfigurationUtil.deserializeConfiguration(serializedHbaseConfig, HbaseConfiguration.configureHbase()); - log.info("Start connection to hbase zookeeper quorum: {}",hbaseConfig.get("hbase.zookeeper.quorum")); + org.apache.hadoop.conf.Configuration hbaseConfig = + HBaseConfigurationUtil.deserializeConfiguration(serializedHbaseConfig, + HbaseConfiguration.configureHbase()); + log.info("Start connection to hbase zookeeper quorum: {}", hbaseConfig.get("hbase.zookeeper.quorum")); connection = ConnectionFactory.createConnection(hbaseConfig); - log.info("Connected to hbase zookeeper quorum: {}",hbaseConfig.get("hbase.zookeeper.quorum")); + log.info("Connected to hbase zookeeper quorum: {}", hbaseConfig.get("hbase.zookeeper.quorum")); } protected final Map notFound() { @@ -105,10 +110,10 @@ private static String objectToString(Object objectValue) { public final Map hbaseLookup(long ts, LookupKey key, String prefix) { return Objects.requireNonNull(cache.get(key, this::fetch)).entrySet().stream() - .collect(toMap( - k -> prefix + "." + k.getKey(), - v -> objectToString(v.getValue())) - ); + .collect(toMap( + k -> prefix + "." + k.getKey(), + v -> objectToString(v.getValue())) + ); } } diff --git a/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/AbstractHbaseSinkFunction.java b/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/AbstractHbaseSinkFunction.java index fe069af6..acc75a01 100644 --- a/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/AbstractHbaseSinkFunction.java +++ b/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/AbstractHbaseSinkFunction.java @@ -25,10 +25,12 @@ public class AbstractHbaseSinkFunction extends HBaseSinkFunction { private transient Counter hbaseWriteCounter; - public AbstractHbaseSinkFunction(String hTableName, HBaseMutationConverter mutationConverter, ParameterTool params, String counterName) { - super(hTableName, HbaseConfiguration.configureHbase(), mutationConverter, params.getLong("hbase.sink.buffer-flush.max-size", 2), - params.getLong("hbase.sink.buffer-flush.max-rows", 1000), - params.getLong("hbase.sink.buffer-flush.interval", 1000)); + public AbstractHbaseSinkFunction(String htableName, HBaseMutationConverter mutationConverter, + ParameterTool params, String counterName) { + super(htableName, HbaseConfiguration.configureHbase(), mutationConverter, + params.getLong("hbase.sink.buffer-flush.max-size", 2), + params.getLong("hbase.sink.buffer-flush.max-rows", 1000), + params.getLong("hbase.sink.buffer-flush.interval", 1000)); this.counterName = counterName; } diff --git a/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/HbaseConfiguration.java b/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/HbaseConfiguration.java index bca29db2..f38c1593 100644 --- a/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/HbaseConfiguration.java +++ b/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/HbaseConfiguration.java @@ -13,16 +13,16 @@ package com.cloudera.cyber.hbase; import com.google.common.collect.Lists; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.flink.connector.hbase.util.HBaseConfigurationUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import java.util.List; - @Slf4j public class HbaseConfiguration { - private static final List HBASE_CONFIG_FILES = Lists.newArrayList("core-site.xml", "hdfs-site.xml", "hbase-site.xml" ); + private static final List HBASE_CONFIG_FILES = + Lists.newArrayList("core-site.xml", "hdfs-site.xml", "hbase-site.xml"); public static final String HBASE_CONFIG_NAME = "hbase-config"; public static Configuration configureHbase() { @@ -33,7 +33,7 @@ public static Configuration configureHbase() { if (!"localhost".equals(hbaseClientConf.get("hbase.zookeeper.quorum"))) { log.warn("Using local configs"); } else { - hbaseClientConf = HBaseConfigurationUtil.getHBaseConfiguration(); + hbaseClientConf = HBaseConfigurationUtil.getHBaseConfiguration(); } log.info("HbaseConfiguration configure hbase end"); return hbaseClientConf; diff --git a/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/LookupKey.java b/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/LookupKey.java index 4111a357..8732c529 100644 --- a/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/LookupKey.java +++ b/flink-cyber/flink-hbase-common/src/main/java/com/cloudera/cyber/hbase/LookupKey.java @@ -12,13 +12,14 @@ package com.cloudera.cyber.hbase; -import lombok.*; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; import lombok.experimental.SuperBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; -import java.util.Map; - @Getter @SuperBuilder @EqualsAndHashCode @@ -29,5 +30,6 @@ public abstract class LookupKey { private String key; public abstract Get toGet(); + public abstract Map resultToMap(Result hbaseResult); } \ No newline at end of file diff --git a/flink-cyber/flink-indexing/flink-indexing-hive-partitions/src/main/java/com/cloudera/cyber/indexing/hive/FieldExtractor.java b/flink-cyber/flink-indexing/flink-indexing-hive-partitions/src/main/java/com/cloudera/cyber/indexing/hive/FieldExtractor.java index d968a785..e54592fb 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive-partitions/src/main/java/com/cloudera/cyber/indexing/hive/FieldExtractor.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive-partitions/src/main/java/com/cloudera/cyber/indexing/hive/FieldExtractor.java @@ -26,8 +26,8 @@ public class FieldExtractor extends RichMapFunction { public FieldExtractor(List listFields) { this.listFields = listFields.stream().filter(n -> !( - FieldExtractor.headerFields.contains(n) || - FieldExtractor.footerFields.contains(n)) + FieldExtractor.headerFields.contains(n) + || FieldExtractor.footerFields.contains(n)) ).collect(toList()); } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/FilterMapFunction.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/FilterMapFunction.java index 6609f994..255f0f6e 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/FilterMapFunction.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/FilterMapFunction.java @@ -1,21 +1,20 @@ package com.cloudera.cyber.indexing.hive; -import org.apache.flink.table.functions.ScalarFunction; - import java.util.Arrays; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.apache.flink.table.functions.ScalarFunction; public class FilterMapFunction extends ScalarFunction { public Map eval(Map map, String... keys) { - if (keys.length < 1){ + if (keys.length < 1) { return map; } final Set fieldsToIgnore = Arrays.stream(keys).collect(Collectors.toSet()); return map.entrySet().stream() - .filter(e -> e.getKey() != null && !fieldsToIgnore.contains(e.getKey().toLowerCase())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .filter(e -> e.getKey() != null && !fieldsToIgnore.contains(e.getKey().toLowerCase())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveJob.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveJob.java index 2e15292a..f40786d7 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveJob.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveJob.java @@ -35,7 +35,8 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) { if (params.get("flink.writer", "").equalsIgnoreCase("TableAPI")) { try { final String connectorName = params.get("flink.output-connector", "hive"); - final TableApiAbstractJob job = TableApiJobFactory.getJobByConnectorName(connectorName, params, env, source); + final TableApiAbstractJob job = + TableApiJobFactory.getJobByConnectorName(connectorName, params, env, source); return job.startJob(); } catch (Exception e) { throw new RuntimeException(e); @@ -46,5 +47,6 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) { return env; } - protected abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, + ParameterTool params); } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveJobKafka.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveJobKafka.java index 5b1d01ab..a81d90aa 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveJobKafka.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveJobKafka.java @@ -12,6 +12,8 @@ package com.cloudera.cyber.indexing.hive; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; + import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; import com.cloudera.cyber.scoring.ScoredMessage; @@ -22,8 +24,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; - @Slf4j public class HiveJobKafka extends HiveJob { @@ -40,10 +40,12 @@ public static void main(String[] args) throws Exception { } @Override - protected SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, ParameterTool params) { + protected SingleOutputStreamOperator createSource(StreamExecutionEnvironment env, + ParameterTool params) { return env.fromSource( - new FlinkUtils<>(ScoredMessage.class).createKafkaGenericSource(params.getRequired(PARAMS_TOPIC_INPUT), params, params.get(PARAMS_GROUP_ID, DEFAULT_GROUP_ID)), - WatermarkStrategy.noWatermarks(), "Kafka Source"). - uid("kafka-source"); + new FlinkUtils<>(ScoredMessage.class).createKafkaGenericSource(params.getRequired(PARAMS_TOPIC_INPUT), + params, params.get(PARAMS_GROUP_ID, DEFAULT_GROUP_ID)), + WatermarkStrategy.noWatermarks(), "Kafka Source") + .uid("kafka-source"); } } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveStreamingMessageWriter.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveStreamingMessageWriter.java index 864e79cc..f721c50f 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveStreamingMessageWriter.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/HiveStreamingMessageWriter.java @@ -12,9 +12,23 @@ package com.cloudera.cyber.indexing.hive; +import static java.util.stream.Collectors.toMap; + import com.cloudera.cyber.Message; import com.cloudera.cyber.scoring.ScoredMessage; import com.cloudera.cyber.scoring.Scores; +import java.net.MalformedURLException; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nullable; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -36,77 +50,110 @@ import org.apache.hive.streaming.StrictJsonWriter; import org.apache.thrift.TException; -import javax.annotation.Nullable; -import java.net.MalformedURLException; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.time.Instant; -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.toMap; - /** * Stores events to a Hive table using Hive Streaming Data Ingest V2. * + *

* https://cwiki.apache.org/confluence/display/Hive/Streaming+Data+Ingest+V2 */ @Slf4j @Data @NoArgsConstructor -public class HiveStreamingMessageWriter { +public class HiveStreamingMessageWriter { private static final String FIELDS_COLUMN_NAME = "fields"; - private static final String[] HIVE_SPECIAL_COLUMN_CHARACTERS = new String[] { ".", ":"}; + private static final String[] HIVE_SPECIAL_COLUMN_CHARACTERS = new String[] {".", ":"}; private static final String[] HIVE_SPECIAL_COLUMN_REPLACEMENT_CHARACTERS = new String[] {"_", "_"}; - protected static final String DEFAULT_TIMESTAMP_FORMATS = "yyyy-MM-dd HH:mm:ss.SSSSSS,yyyy-MM-dd'T'HH:mm:ss.SSS'Z',yyyy-MM-dd'T'HH:mm:ss.SS'Z',yyyy-MM-dd'T'HH:mm:ss.S'Z',yyyy-MM-dd'T'HH:mm:ss'Z'"; + protected static final String DEFAULT_TIMESTAMP_FORMATS = + "yyyy-MM-dd HH:mm:ss.SSSSSS," + + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'," + + "yyyy-MM-dd'T'HH:mm:ss.SS'Z'," + + "yyyy-MM-dd'T'HH:mm:ss.S'Z'," + + "yyyy-MM-dd'T'HH:mm:ss'Z'"; protected static final String HIVE_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; - /** Hive database containing the table to write events to **/ + /** + * Hive database containing the table to write events to. + **/ private String databaseName; - /** Hive table to write events to **/ + /** + * Hive table to write events to. + **/ private String tableName; - /** Hive streaming v2 connection batch size **/ + /** + * Hive streaming v2 connection batch size. + **/ private int batchSize; - /** Number of messages top send before ending the transaction */ + /** + * Number of messages top send before ending the transaction. + */ private int messagesPerTransaction; - /** directory containing the hive configuration files - hive-site.xml */ + /** + * directory containing the hive configuration files - hive-site.xml. + */ private String hiveConfDir; - /** default java timestamp format to use when converting a string to a Hive timestamp */ + /** + * default java timestamp format to use when converting a string to a Hive timestamp. + */ private String possibleTimestampFormats; - /** connection to hive for streaming updates **/ + /** + * connection to hive for streaming updates. + **/ private transient HiveStreamingConnection connection; - /** schema describing the columns in the hive table where the flink job will write events **/ + /** + * schema describing the columns in the hive table where the flink job will write events. + **/ private transient TableSchema tableSchema; - /** methods for mapping built in Message class fields into hive column values **/ + /** + * methods for mapping built in Message class fields into hive column values. + **/ private transient Map> builtinColumnExtractor; - /** Type information for converting a single message to a json byte array. The json is required by the StrictJsonWriter for the hive connection. **/ + /** + * Type information for converting a single message to a json byte array. + * The json is required by the StrictJsonWriter for the hive connection. + **/ private transient TypeInformation typeInfo; - /** methods for converting String extensions to the column values required by hive. **/ + /** + * methods for converting String extensions to the column values required by hive. + **/ private transient Map> columnConversion; - /** schema for serializing a hive row to json for use with the StrictJsonWriter */ + /** + * schema for serializing a hive row to json for use with the StrictJsonWriter. + */ private transient JsonRowSerializationSchema jsonRowSerializationSchema; - /** date format for extracting the year month and date required by hive partitions */ + /** + * date format for extracting the year month and date required by hive partitions. + */ private transient SimpleDateFormat dayFormat; - /** date format for extracting the hour required by hive partitions */ + /** + * date format for extracting the hour required by hive partitions. + */ private transient SimpleDateFormat hourFormat; - /** date format for extracting the entire event timestamp to the format required by hive */ + /** + * date format for extracting the entire event timestamp to the format required by hive. + */ private transient SimpleDateFormat hiveTimestampFormat; - /** number of messages in the current transaction. If set to zero, no transaction is open */ + /** + * number of messages in the current transaction. If set to zero, no transaction is open. + */ private transient int messagesInCurrentTransaction = 0; - /** names of the fields with type timestamp. Keep track of them since they will need to be normalized and stored as a string */ + /** + * names of the fields with type timestamp. + * Keep track of them since they will need to be normalized and stored as a string. + */ private transient List timestampFieldNames; - /** normalizer for non-built in timestamp fields */ + /** + * normalizer for non-built in timestamp fields. + */ private transient TimestampNormalizer timestampNormalizer; public HiveStreamingMessageWriter(ParameterTool params) { this.dayFormat = new SimpleDateFormat("yyyy-MM-dd"); this.hourFormat = new SimpleDateFormat("HH"); this.hiveTimestampFormat = new SimpleDateFormat(HIVE_DATE_FORMAT); - this.hiveConfDir = params.get("hive.confdir","/etc/hive/conf"); + this.hiveConfDir = params.get("hive.confdir", "/etc/hive/conf"); this.databaseName = params.get("hive.dbname", "cyber"); this.tableName = params.get("hive.table", "events"); @@ -120,42 +167,46 @@ public void connect() throws StreamingException, TException { StrictJsonWriter jsonWriter = StrictJsonWriter.newBuilder().build(); connection = HiveStreamingConnection.newBuilder() - .withDatabase(databaseName) - .withTable(tableName) - //.withStreamingOptimizations(true) - .withTransactionBatchSize(batchSize) - .withAgentInfo("flink-cyber") - .withRecordWriter(jsonWriter) - .withHiveConf(hiveConf) - .connect(); + .withDatabase(databaseName) + .withTable(tableName) + //.withStreamingOptimizations(true) + .withTransactionBatchSize(batchSize) + .withAgentInfo("flink-cyber") + .withRecordWriter(jsonWriter) + .withHiveConf(hiveConf) + .connect(); log.info("Hive Connection made {}", connection); tableSchema = getHiveTable(hiveConf, databaseName, tableName); - builtinColumnExtractor = new HashMap>() {{ - put("originalsource_topic", (m) -> m.getMessage().getOriginalSource().getTopic()); - put("originalsource_partition", (m) -> m.getMessage().getOriginalSource().getPartition()); - put("originalsource_offset", (m) -> m.getMessage().getOriginalSource().getOffset()); - put("originalsource_signature", (m) -> m.getMessage().getOriginalSource().getSignature()); - put("id", (m) -> m.getMessage().getId()); - put("ts", (m) -> formatMessageTimestamp(m.getMessage(), hiveTimestampFormat)); - put("message", (m) -> m.getMessage().getMessage()); - put("source", (m) -> m.getMessage().getSource()); - put("dt", (m) -> formatMessageTimestamp(m.getMessage(), dayFormat)); - put("hr", (m) -> formatMessageTimestamp(m.getMessage(), hourFormat)); - put("cyberscore", (m) -> m.getCyberScore().floatValue()); - put("cyberscore_details", HiveStreamingMessageWriter::convertCyberScoreDetailsToRow); - }}; - - columnConversion = new HashMap>() {{ - put(DataTypes.DOUBLE(), Double::parseDouble); - put(DataTypes.FLOAT(), Float::parseFloat); - put(DataTypes.BIGINT(), Long::parseLong); - put(DataTypes.INT(), Integer::parseInt); - put(DataTypes.STRING(), (s) -> s); - put(DataTypes.BOOLEAN(), Boolean::parseBoolean); - }}; + builtinColumnExtractor = new HashMap>() { + { + put("originalsource_topic", (m) -> m.getMessage().getOriginalSource().getTopic()); + put("originalsource_partition", (m) -> m.getMessage().getOriginalSource().getPartition()); + put("originalsource_offset", (m) -> m.getMessage().getOriginalSource().getOffset()); + put("originalsource_signature", (m) -> m.getMessage().getOriginalSource().getSignature()); + put("id", (m) -> m.getMessage().getId()); + put("ts", (m) -> formatMessageTimestamp(m.getMessage(), hiveTimestampFormat)); + put("message", (m) -> m.getMessage().getMessage()); + put("source", (m) -> m.getMessage().getSource()); + put("dt", (m) -> formatMessageTimestamp(m.getMessage(), dayFormat)); + put("hr", (m) -> formatMessageTimestamp(m.getMessage(), hourFormat)); + put("cyberscore", (m) -> m.getCyberScore().floatValue()); + put("cyberscore_details", HiveStreamingMessageWriter::convertCyberScoreDetailsToRow); + } + }; + + columnConversion = new HashMap>() { + { + put(DataTypes.DOUBLE(), Double::parseDouble); + put(DataTypes.FLOAT(), Float::parseFloat); + put(DataTypes.BIGINT(), Long::parseLong); + put(DataTypes.INT(), Integer::parseInt); + put(DataTypes.STRING(), (s) -> s); + put(DataTypes.BOOLEAN(), Boolean::parseBoolean); + } + }; typeInfo = new RowTypeInfo(tableSchema.getFieldTypes(), tableSchema.getFieldNames()); @@ -166,7 +217,7 @@ public void connect() throws StreamingException, TException { private static Row[] convertCyberScoreDetailsToRow(ScoredMessage scoredMessage) { Row[] rows = new Row[scoredMessage.getCyberScoresDetails().size()]; int index = 0; - for(Scores scoreDetail : scoredMessage.getCyberScoresDetails()) { + for (Scores scoreDetail : scoredMessage.getCyberScoresDetails()) { Row scoreDetailRow = new Row(3); scoreDetailRow.setField(0, scoreDetail.getRuleId()); scoreDetailRow.setField(1, scoreDetail.getScore().floatValue()); @@ -179,26 +230,29 @@ private static Row[] convertCyberScoreDetailsToRow(ScoredMessage scoredMessage) private static HiveConf createHiveConf(@Nullable String hiveConfDir) { log.info("Setting hive conf dir as {}", hiveConfDir); org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration(); - hadoopConf.addResource(hiveConfDir +"/hive-site.xml"); + hadoopConf.addResource(hiveConfDir + "/hive-site.xml"); hadoopConf.addResource(hiveConfDir + "/core-site.xml"); try { - HiveConf.setHiveSiteLocation(hiveConfDir == null ? null : Paths.get(hiveConfDir, "hive-site.xml").toUri().toURL()); + HiveConf.setHiveSiteLocation( + hiveConfDir == null ? null : Paths.get(hiveConfDir, "hive-site.xml").toUri().toURL()); } catch (MalformedURLException e) { log.error("Cannot load Hive Config", e); } return new HiveConf(hadoopConf, HiveConf.class); } - private TableSchema getHiveTable( HiveConf hiveConf, String schema, String table) throws TException { + private TableSchema getHiveTable(HiveConf hiveConf, String schema, String table) throws TException { HiveMetaStoreClient hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf); List hiveFields = hiveMetaStoreClient.getSchema(schema, table); log.info("Read hive fields {}", hiveFields); // find all the timestamp fields that are not the built in timestamp field - timestampFieldNames = hiveFields.stream().filter(field -> !field.getName().equals("ts") && field.getType().equalsIgnoreCase("timestamp")). - map(FieldSchema::getName).collect(Collectors.toList()); + timestampFieldNames = hiveFields.stream() + .filter(field -> !field.getName().equals("ts") + && field.getType().equalsIgnoreCase("timestamp")) + .map(FieldSchema::getName).collect(Collectors.toList()); return hiveFields.stream().reduce(TableSchema.builder(), - (build, field) -> build.field(field.getName(), hiveTypeToDataType(field.getType())), - (a, b) -> a + (build, field) -> build.field(field.getName(), hiveTypeToDataType(field.getType())), + (a, b) -> a ).build(); } @@ -220,20 +274,22 @@ public static DataType hiveTypeToDataType(String type) { return DataTypes.FLOAT(); case "array>": return DataTypes.ARRAY( - DataTypes.ROW(DataTypes.FIELD("ruleid", DataTypes.STRING()), - DataTypes.FIELD("score", DataTypes.FLOAT()), - DataTypes.FIELD("reason", DataTypes.STRING()))); + DataTypes.ROW(DataTypes.FIELD("ruleid", DataTypes.STRING()), + DataTypes.FIELD("score", DataTypes.FLOAT()), + DataTypes.FIELD("reason", DataTypes.STRING()))); case "string": case "timestamp": return DataTypes.STRING(); default: - throw new IllegalArgumentException(String.format("Data type '%s' in Hive schema is not supported.", type)); + throw new IllegalArgumentException( + String.format("Data type '%s' in Hive schema is not supported.", type)); } } byte[] serializeRow(Row row) { - if (jsonRowSerializationSchema == null) + if (jsonRowSerializationSchema == null) { jsonRowSerializationSchema = new JsonRowSerializationSchema.Builder(typeInfo).build(); + } try { jsonRowSerializationSchema.open(null); @@ -244,7 +300,8 @@ byte[] serializeRow(Row row) { } private static String toHiveColumnName(String messageExtensionName) { - return StringUtils.replaceEach(messageExtensionName, HIVE_SPECIAL_COLUMN_CHARACTERS, HIVE_SPECIAL_COLUMN_REPLACEMENT_CHARACTERS); + return StringUtils.replaceEach(messageExtensionName, HIVE_SPECIAL_COLUMN_CHARACTERS, + HIVE_SPECIAL_COLUMN_REPLACEMENT_CHARACTERS); } private static String formatMessageTimestamp(Message message, SimpleDateFormat dateFormat) { @@ -265,12 +322,13 @@ protected int addMessageToTransaction(ScoredMessage scoredMessage) throws Except Row newRow = new Row(tableSchema.getTableColumns().size()); int pos = 0; - Map extensions = message.getExtensions() != null ? message.getExtensions() : Collections.emptyMap(); + Map extensions = + message.getExtensions() != null ? message.getExtensions() : Collections.emptyMap(); // create legal hive column names from the field name extensions = extensions.entrySet().stream().collect( - toMap(e -> toHiveColumnName(e.getKey()), Map.Entry::getValue)); + toMap(e -> toHiveColumnName(e.getKey()), Map.Entry::getValue)); - for(TableColumn column : tableSchema.getTableColumns()) { + for (TableColumn column : tableSchema.getTableColumns()) { String columnName = column.getName(); Object columnValue = null; @@ -295,7 +353,9 @@ protected int addMessageToTransaction(ScoredMessage scoredMessage) throws Except extensions.remove(columnName); } catch (Exception e) { // leave value in field map - log.error(String.format("Could not convert value %s for column %s", extensionValue, columnName), e); + log.error( + String.format("Could not convert value %s for column %s", extensionValue, columnName), + e); } } } @@ -312,7 +372,7 @@ protected int addMessageToTransaction(ScoredMessage scoredMessage) throws Except messagesInCurrentTransaction++; return endTransaction(); - } + } protected void beginTransaction() throws Exception { if (messagesInCurrentTransaction == 0) { @@ -320,7 +380,7 @@ protected void beginTransaction() throws Exception { connection.beginTransaction(); log.info("Hive transaction created."); } - } + } protected int endTransaction() throws Exception { if (messagesInCurrentTransaction == messagesPerTransaction) { diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/StreamingHiveSink.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/StreamingHiveSink.java index a5789c32..d0ae6c69 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/StreamingHiveSink.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/StreamingHiveSink.java @@ -34,7 +34,7 @@ public StreamingHiveSink(ParameterTool params) { } public void open(Configuration parameters) throws Exception { - this.messageWriter = new HiveStreamingMessageWriter(params); + this.messageWriter = new HiveStreamingMessageWriter(params); messageWriter.connect(); this.counter = getRuntimeContext().getMetricGroup().counter("messagesToHive"); super.open(parameters); diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/TimestampNormalizer.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/TimestampNormalizer.java index 1d221132..a248520f 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/TimestampNormalizer.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/TimestampNormalizer.java @@ -12,8 +12,6 @@ package com.cloudera.cyber.indexing.hive; -import lombok.extern.slf4j.Slf4j; - import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; @@ -22,15 +20,16 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; @Slf4j public class TimestampNormalizer implements Function { - private List possibleTimestampFormats; - private SimpleDateFormat normalizedFormat; + private final List possibleTimestampFormats; + private final SimpleDateFormat normalizedFormat; public TimestampNormalizer(String possibleSourceTimestampFormatCSV, SimpleDateFormat normalizedFormat) { - this.possibleTimestampFormats = Stream.of(possibleSourceTimestampFormatCSV.split(",")). - map(SimpleDateFormat::new).collect(Collectors.toList()); + this.possibleTimestampFormats = Stream.of(possibleSourceTimestampFormatCSV.split(",")) + .map(SimpleDateFormat::new).collect(Collectors.toList()); this.normalizedFormat = normalizedFormat; } @@ -38,7 +37,7 @@ public TimestampNormalizer(String possibleSourceTimestampFormatCSV, SimpleDateFo public Object apply(String fieldValue) { Date parsedDate = tryEpochMillis(fieldValue); if (parsedDate == null) { - for(SimpleDateFormat nextFormat : possibleTimestampFormats) { + for (SimpleDateFormat nextFormat : possibleTimestampFormats) { try { parsedDate = nextFormat.parse(fieldValue); break; @@ -50,14 +49,16 @@ public Object apply(String fieldValue) { if (parsedDate != null) { return normalizedFormat.format(parsedDate); } else { - throw new IllegalStateException(String.format("Field value '%s' can't be normalized to a timestamp because it does not match any configured format.", fieldValue)); + throw new IllegalStateException(String.format( + "Field value '%s' can't be normalized to a timestamp because it does not match any configured format.", + fieldValue)); } } private Date tryEpochMillis(String fieldValue) { try { return Date.from(Instant.ofEpochMilli(Long.parseLong(fieldValue))); - } catch(NumberFormatException numberFormatException) { + } catch (NumberFormatException numberFormatException) { log.debug("Timestamp {} is not a long epoch millis.", fieldValue); // not a long, go on to the next possible formats. } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/TableApiAbstractJob.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/TableApiAbstractJob.java index ff41ba75..94cc113c 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/TableApiAbstractJob.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/TableApiAbstractJob.java @@ -10,8 +10,19 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.Streams; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.flink.api.java.utils.ParameterTool; @@ -19,7 +30,11 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.*; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.FormatDescriptor; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.SqlDialect; +import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.bridge.java.StreamStatementSet; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.catalog.Column; @@ -33,425 +48,447 @@ public abstract class TableApiAbstractJob { - private static final String TABLES_INIT_FILE_PARAM = "flink.tables-init-file"; - private static final String MAPPING_FILE_PARAM = "flink.mapping-file"; - protected static final String KAFKA_TABLE = "KafkaTempView"; - - protected static final String BASE_COLUMN_MAPPING_JSON = "base-column-mapping.json"; - - private final List defaultMappingList; - private final List defaultColumnList; - protected final DataStream source; - protected final StreamExecutionEnvironment env; - protected final ParameterTool params; - protected final String connectorName; - - public TableApiAbstractJob(ParameterTool params, StreamExecutionEnvironment env, DataStream source, - String connectorName, String baseTableJson) throws IOException { - this.params = params; - this.env = env; - this.source = source; - this.connectorName = connectorName; - defaultMappingList = Utils.readResourceFile(BASE_COLUMN_MAPPING_JSON, getClass(), - new TypeReference>() { - }); - defaultColumnList = Utils.readResourceFile(baseTableJson, getClass(), - new TypeReference>() { + private static final String TABLES_INIT_FILE_PARAM = "flink.tables-init-file"; + private static final String MAPPING_FILE_PARAM = "flink.mapping-file"; + protected static final String KAFKA_TABLE = "KafkaTempView"; + + protected static final String BASE_COLUMN_MAPPING_JSON = "base-column-mapping.json"; + + private final List defaultMappingList; + private final List defaultColumnList; + protected final DataStream source; + protected final StreamExecutionEnvironment env; + protected final ParameterTool params; + protected final String connectorName; + + public TableApiAbstractJob(ParameterTool params, StreamExecutionEnvironment env, DataStream source, + String connectorName, String baseTableJson) throws IOException { + this.params = params; + this.env = env; + this.source = source; + this.connectorName = connectorName; + defaultMappingList = Utils.readResourceFile(BASE_COLUMN_MAPPING_JSON, getClass(), + new TypeReference>() { + }); + defaultColumnList = Utils.readResourceFile(baseTableJson, getClass(), + new TypeReference>() { + }); + } + + public StreamExecutionEnvironment startJob() throws Exception { + System.out.println("Creating Table env..."); + final StreamTableEnvironment tableEnv = getTableEnvironment(); + + System.out.println("Configuring Table env..."); + configure(tableEnv); + + System.out.printf("Registering %s catalog... %s%n", connectorName, String.join(", ", tableEnv.listCatalogs())); + registerCatalog(tableEnv); + + System.out.println("Getting tables config..."); + final Map> rawTablesConfig = getRawTablesConfig(); + final Map> tablesConfig = appendDefaultTablesConfig(rawTablesConfig); + + System.out.println("Creating tables..."); + setConnectorDialect(tableEnv); + + final Map tableSchemaMap = createTables(tableEnv, tablesConfig); + + System.out.println("Getting topic mapping..."); + final Map topicMapping = getTopicMapping(); + + System.out.println("Validating output tables mappings..."); + validateMappings(tableSchemaMap, topicMapping); + + tableEnv.getConfig().setSqlDialect(SqlDialect.DEFAULT); + + System.out.println("Creating Kafka table..."); + createKafkaTable(tableEnv); + + System.out.printf("Executing %s insert...%n", connectorName); + executeInsert(tableEnv, topicMapping, tablesConfig, tableSchemaMap); + + System.out.println("TableApiJob is done!"); + return jobReturnValue(); + } + + protected boolean isNonDefaultColumn(String columnName) { + return defaultColumnList.stream().noneMatch(c -> c.getName().equals(columnName)); + } + + /** + * Creates tables in the tableEnv based on the tablesConfig. If table already exists, its schema is fetched. + * + * @param tableEnv is the Flink environment in which the tables are going to be created. + * @param tablesConfig modified version of rawTablesConfig that contains default and partition columns. + * @return Map with table name as a key and table schema as a value. + */ + private Map createTables(StreamTableEnvironment tableEnv, + Map> tablesConfig) { + final Set tableList = getExistingTableList(tableEnv); + + return tablesConfig.entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, + entry -> createTableIfNotExists(tableEnv, tableList, entry.getKey(), + entry.getValue()))); + } + + protected void validateMappings(Map tableSchemaMap, + Map topicMapping) { + for (Entry entry : topicMapping.entrySet()) { + final String source = entry.getKey(); + final MappingDto mappingDto = entry.getValue(); + final String tableName = mappingDto.getTableName(); + + if (!StringUtils.hasText(tableName)) { + throw new RuntimeException(String.format("Provided empty table name for the [%s] source!", source)); + } + + final ResolvedSchema schema = tableSchemaMap.get(tableName); + if (schema == null) { + throw new RuntimeException(String.format("Table [%s] is not found!", tableName)); + } + + final Set tableColumns = schema.getColumns().stream() + .map(Column::getName) + .collect(Collectors.toSet()); + + final List invalidColumnList = mappingDto.getColumnMapping().stream() + .map(MappingColumnDto::getName) + .filter(columnName -> !StringUtils.hasText(columnName) + || !tableColumns.contains(columnName)) + .collect(Collectors.toList()); + if (!invalidColumnList.isEmpty()) { + throw new RuntimeException( + String.format( + "Found invalid column mappings for source [%s]. " + + "Those columns are either not present in the table config or have empty names: %s", + source, invalidColumnList)); + } + + validateTransformations(source, schema, mappingDto); + } + } + + private void validateTransformations(String source, ResolvedSchema tableSchema, MappingDto mappingDto) { + + final Map tableColumnMap = tableSchema.getColumns().stream() + .collect(Collectors.toMap(Column::getName, + Column::getDataType)); + + final List columnListWithoutTransformation = mappingDto.getColumnMapping().stream() + .map(mapping -> { + final String columnName = mapping.getName(); + final DataType tableColumnDataType = + tableColumnMap.get(columnName); + + if (DataTypes.STRING() + .equals(tableColumnDataType)) { + return null; + } + + if (StringUtils.hasText( + getTransformation(tableColumnDataType, + mapping))) { + return null; + } + + return columnName; + }) + .filter(Objects::nonNull) + .filter(this::isNonDefaultColumn) + .collect(Collectors.toList()); + + if (!columnListWithoutTransformation.isEmpty()) { + throw new RuntimeException( + String.format( + "Found column mappings of non-string type without transformations for source [%s]: %s", + source, columnListWithoutTransformation)); + } + } + + + protected HashSet getExistingTableList(StreamTableEnvironment tableEnv) { + return new HashSet<>(Arrays.asList(tableEnv.listTables())); + } + + protected StreamExecutionEnvironment jobReturnValue() { + return env; + } + + protected void executeInsert(StreamTableEnvironment tableEnv, Map topicMapping, + Map> tablesConfig, + Map tableSchemaMap) { + System.out.printf("Filling Insert statement list...%s%n", Arrays.toString(tableEnv.listTables())); + final StreamStatementSet insertStatementSet = tableEnv.createStatementSet(); + + topicMapping.forEach((topic, mappingDto) -> { + ResolvedSchema tableSchema = tableSchemaMap.get(mappingDto.getTableName()); + final String insertSql = buildInsertSql(topic, mappingDto, tableSchema); + try { + insertStatementSet.addInsertSql(insertSql); + System.out.printf("Insert SQL added to the queue for the table: %s%nSQL: %s%n", + mappingDto.getTableName(), + insertSql); + } catch (Exception e) { + System.err.printf("Error adding insert to the statement set: %s%n", insertSql); + throw e; + } }); - } - - public StreamExecutionEnvironment startJob() throws Exception { - System.out.println("Creating Table env..."); - final StreamTableEnvironment tableEnv = getTableEnvironment(); - - System.out.println("Configuring Table env..."); - configure(tableEnv); - - System.out.printf("Registering %s catalog... %s%n", connectorName, String.join(", ", tableEnv.listCatalogs())); - registerCatalog(tableEnv); - - System.out.println("Getting tables config..."); - final Map> rawTablesConfig = getRawTablesConfig(); - final Map> tablesConfig = appendDefaultTablesConfig(rawTablesConfig); - - System.out.println("Creating tables..."); - setConnectorDialect(tableEnv); - - final Map tableSchemaMap = createTables(tableEnv, tablesConfig); - - System.out.println("Getting topic mapping..."); - final Map topicMapping = getTopicMapping(); - - System.out.println("Validating output tables mappings..."); - validateMappings(tableSchemaMap, topicMapping); - - tableEnv.getConfig().setSqlDialect(SqlDialect.DEFAULT); - - System.out.println("Creating Kafka table..."); - createKafkaTable(tableEnv); - - System.out.printf("Executing %s insert...%n", connectorName); - executeInsert(tableEnv, topicMapping, tablesConfig, tableSchemaMap); - - System.out.println("TableApiJob is done!"); - return jobReturnValue(); - } - - protected boolean isNonDefaultColumn(String columnName) { - return defaultColumnList.stream().noneMatch(c -> c.getName().equals(columnName)); - } - - /** - * Creates tables in the tableEnv based on the tablesConfig. If table already exists, its schema is fetched. - * - * @param tableEnv is the Flink environment in which the tables are going to be created. - * @param tablesConfig modified version of rawTablesConfig that contains default and partition columns. - * @return Map with table name as a key and table schema as a value. - */ - private Map createTables(StreamTableEnvironment tableEnv, - Map> tablesConfig) { - final Set tableList = getExistingTableList(tableEnv); - - return tablesConfig.entrySet().stream() - .collect(Collectors.toMap(Entry::getKey, - entry -> createTableIfNotExists(tableEnv, tableList, entry.getKey(), entry.getValue()))); - } - - protected void validateMappings(Map tableSchemaMap, - Map topicMapping) { - for (Entry entry : topicMapping.entrySet()) { - final String source = entry.getKey(); - final MappingDto mappingDto = entry.getValue(); - final String tableName = mappingDto.getTableName(); - - if (!StringUtils.hasText(tableName)) { - throw new RuntimeException(String.format("Provided empty table name for the [%s] source!", source)); - } - - final ResolvedSchema schema = tableSchemaMap.get(tableName); - if (schema == null) { - throw new RuntimeException(String.format("Table [%s] is not found!", tableName)); - } - - final Set tableColumns = schema.getColumns().stream() - .map(Column::getName) - .collect(Collectors.toSet()); - - final List invalidColumnList = mappingDto.getColumnMapping().stream() - .map(MappingColumnDto::getName) - .filter(columnName -> !StringUtils.hasText(columnName) || !tableColumns.contains(columnName)) - .collect(Collectors.toList()); - if (!invalidColumnList.isEmpty()) { - throw new RuntimeException( - String.format( - "Found invalid column mappings for source [%s]. Those columns are either not present in the table config or have empty names: %s", - source, invalidColumnList)); - } - - validateTransformations(source, schema, mappingDto); + + System.out.println("Executing Insert statement list..."); + insertStatementSet.execute(); + } + + protected abstract void registerCatalog(StreamTableEnvironment tableEnv); + + protected void setConnectorDialect(StreamTableEnvironment tableEnv) { + //to be overwritten if needed + } + + private ResolvedSchema createTableIfNotExists(StreamTableEnvironment tableEnv, Set tableList, + String tableName, List columnList) { + if (tableList.contains(tableName)) { + return handleExistingTable(tableEnv, tableName, columnList); + } + + return createTable(tableEnv, tableName, columnList); + } + + protected String getTransformation(DataType tableColumnDataType, MappingColumnDto mapping) { + String transformation = mapping.getTransformation(); + if (transformation == null && isNonDefaultColumn(mapping.getName()) + && (DataTypes.BOOLEAN().equals(tableColumnDataType) + || tableColumnDataType.getLogicalType().is(LogicalTypeFamily.NUMERIC))) { + transformation = + String.format("TRY_CAST(%%s AS %s)", tableColumnDataType.getLogicalType().getTypeRoot().name()); + } + + return transformation; + } + + protected ResolvedSchema createTable(StreamTableEnvironment tableEnv, String tableName, + List columnList) { + System.out.printf("Creating %s table %s...%n", connectorName, tableName); + final ResolvedSchema resolvedSchema = FlinkSchemaUtil.getResolvedSchema(columnList); + final Schema schema = FlinkSchemaUtil.buildSchema(resolvedSchema); + final TableDescriptor tableDescriptor = buildTableDescriptor(schema); + try { + System.out.printf("Creating %s table %s: %s%n", connectorName, tableName, tableDescriptor); + tableEnv.createTable(tableName, tableDescriptor); + } catch (Exception e) { + System.err.printf("Error creating the %s: %s%n", connectorName, tableDescriptor); + throw e; + } + System.out.printf("%s table created: %s%n", connectorName, tableName); + return resolvedSchema; + } + + protected ResolvedSchema handleExistingTable(StreamTableEnvironment tableEnv, String tableName, + List columnList) { + final ResolvedSchema existingTableSchema = tableEnv.from(tableName).getResolvedSchema(); + if (CollectionUtils.isEmpty(columnList)) { + System.out.printf("%s table [%s] already exists and no table config provided. Skipping its creation.%n", + connectorName, + tableName); + return existingTableSchema; + } + + final List existingColumns = existingTableSchema.getColumns(); + final List configColumns = FlinkSchemaUtil.getResolvedSchema(columnList).getColumns(); + for (Column configColumn : configColumns) { + final DataType dataType = configColumn.getDataType(); + final String columnName = configColumn.getName(); + final boolean matchingColumnExists = existingColumns.stream() + .anyMatch(c -> c.getName().equals(columnName) + && c.getDataType().equals(dataType)); + if (!matchingColumnExists) { + throw new RuntimeException(String.format( + "%s table [%s] already exists, but table config was provided for it as well. " + + "In this case all columns from table config should be present in the existing table, " + + "but we didn't find the [%s] column of type [%s].", + connectorName, tableName, columnName, dataType)); + } + } + System.out.printf("%s table [%s] already exists and provided table config matches it. Skipping its creation.%n", + connectorName, + tableName); + return existingTableSchema; } - } - private void validateTransformations(String source, ResolvedSchema tableSchema, MappingDto mappingDto) { + private TableDescriptor buildTableDescriptor(Schema schema) { + return fillTableOptions(TableDescriptor + .forConnector(getTableConnector()) + .schema(schema) + .partitionedBy("dt", "hr") + .format(getFormatDescriptor())) + .build(); + } + + protected abstract String getTableConnector(); - final Map tableColumnMap = tableSchema.getColumns().stream() - .collect(Collectors.toMap(Column::getName, Column::getDataType)); + protected abstract FormatDescriptor getFormatDescriptor(); - final List columnListWithoutTransformation = mappingDto.getColumnMapping().stream() - .map(mapping -> { - final String columnName = mapping.getName(); - final DataType tableColumnDataType = tableColumnMap.get(columnName); - if (DataTypes.STRING().equals(tableColumnDataType)) { - return null; - } + protected TableDescriptor.Builder fillTableOptions(TableDescriptor.Builder builder) { + return builder + .option("partition.time-extractor.timestamp-pattern", "$dt $hr:00:00") + .option("sink.partition-commit.trigger", "process-time") + .option("sink.partition-commit.delay", "1 h") + .option("sink.partition-commit.policy.kind", "metastore,success-file"); + } - if (StringUtils.hasText(getTransformation(tableColumnDataType, mapping))) { - return null; - } + private StreamTableEnvironment getTableEnvironment() { + return StreamTableEnvironment.create(env); + } - return columnName; - }) - .filter(Objects::nonNull) - .filter(this::isNonDefaultColumn) - .collect(Collectors.toList()); + private void createKafkaTable(StreamTableEnvironment tableEnv) { + final SingleOutputStreamOperator newSource = source.map(ScoredMessage::toRow, + ScoredMessage.FLINK_TYPE_INFO); + tableEnv.createTemporaryView(KAFKA_TABLE, newSource); + tableEnv.executeSql("DESCRIBE " + KAFKA_TABLE).print(); + } + + private void configure(StreamTableEnvironment tableEnv) { + final HashMap conf = new HashMap<>(); + conf.put("pipeline.name", params.get("flink.job.name", String.format("Indexing - %s TableApi", connectorName))); + tableEnv.getConfig().addConfiguration(Configuration.fromMap(conf)); + tableEnv.createTemporarySystemFunction("filterMap", FilterMapFunction.class); + } - if (!columnListWithoutTransformation.isEmpty()) { - throw new RuntimeException( - String.format( - "Found column mappings of non-string type without transformations for source [%s]: %s", - source, columnListWithoutTransformation)); + protected final String buildInsertSql(String topic, MappingDto mappingDto, ResolvedSchema tableSchema) { + return String.join("\n", + getInsertSqlPrefix() + " " + getTableName(topic, mappingDto) + "(" + getInsertColumns(mappingDto) + ") " + + getInsertSqlSuffix(), + " SELECT " + getFromColumns(mappingDto, tableSchema), + " from " + KAFKA_TABLE, + String.format(" where `source`='%s'", topic)); } - } - - - protected HashSet getExistingTableList(StreamTableEnvironment tableEnv) { - return new HashSet<>(Arrays.asList(tableEnv.listTables())); - } - - protected StreamExecutionEnvironment jobReturnValue() { - return env; - } - - protected void executeInsert(StreamTableEnvironment tableEnv, Map topicMapping, - Map> tablesConfig, Map tableSchemaMap) { - System.out.printf("Filling Insert statement list...%s%n", Arrays.toString(tableEnv.listTables())); - final StreamStatementSet insertStatementSet = tableEnv.createStatementSet(); - - topicMapping.forEach((topic, mappingDto) -> { - ResolvedSchema tableSchema = tableSchemaMap.get(mappingDto.getTableName()); - final String insertSql = buildInsertSql(topic, mappingDto, tableSchema); - try { - insertStatementSet.addInsertSql(insertSql); - System.out.printf("Insert SQL added to the queue for the table: %s%nSQL: %s%n", mappingDto.getTableName(), - insertSql); - } catch (Exception e) { - System.err.printf("Error adding insert to the statement set: %s%n", insertSql); - throw e; - } - }); - - System.out.println("Executing Insert statement list..."); - insertStatementSet.execute(); - } - - protected abstract void registerCatalog(StreamTableEnvironment tableEnv); - - protected void setConnectorDialect(StreamTableEnvironment tableEnv) { - //to be overwritten if needed - } - - private ResolvedSchema createTableIfNotExists(StreamTableEnvironment tableEnv, Set tableList, - String tableName, List columnList) { - if (tableList.contains(tableName)) { - return handleExistingTable(tableEnv, tableName, columnList); + + protected String getTableName(String source, MappingDto mappingDto) { + return mappingDto.getTableName(); } - return createTable(tableEnv, tableName, columnList); - } + protected String getInsertSqlPrefix() { + return "INSERT INTO "; + } - protected String getTransformation(DataType tableColumnDataType, MappingColumnDto mapping) { - String transformation = mapping.getTransformation(); - if (transformation == null && isNonDefaultColumn(mapping.getName()) && (DataTypes.BOOLEAN().equals(tableColumnDataType) || tableColumnDataType.getLogicalType().is(LogicalTypeFamily.NUMERIC))) { - transformation = String.format("TRY_CAST(%%s AS %s)", tableColumnDataType.getLogicalType().getTypeRoot().name()); + protected String getInsertSqlSuffix() { + return ""; } - return transformation; - } - - protected ResolvedSchema createTable(StreamTableEnvironment tableEnv, String tableName, - List columnList) { - System.out.printf("Creating %s table %s...%n", connectorName, tableName); - final ResolvedSchema resolvedSchema = FlinkSchemaUtil.getResolvedSchema(columnList); - final Schema schema = FlinkSchemaUtil.buildSchema(resolvedSchema); - final TableDescriptor tableDescriptor = buildTableDescriptor(schema); - try { - System.out.printf("Creating %s table %s: %s%n", connectorName, tableName, tableDescriptor); - tableEnv.createTable(tableName, tableDescriptor); - } catch (Exception e) { - System.err.printf("Error creating the %s: %s%n", connectorName, tableDescriptor); - throw e; + protected String getInsertColumns(MappingDto mappingDto) { + return mappingDto.getColumnMapping().stream() + .map(MappingColumnDto::getName) + .collect(Collectors.joining(", ", " ", " ")); } - System.out.printf("%s table created: %s%n", connectorName, tableName); - return resolvedSchema; - } - - protected ResolvedSchema handleExistingTable(StreamTableEnvironment tableEnv, String tableName, - List columnList) { - final ResolvedSchema existingTableSchema = tableEnv.from(tableName).getResolvedSchema(); - if (CollectionUtils.isEmpty(columnList)) { - System.out.printf("%s table [%s] already exists and no table config provided. Skipping its creation.%n", - connectorName, - tableName); - return existingTableSchema; + + private String getFromColumns(MappingDto mappingDto, ResolvedSchema tableSchema) { + return mappingDto.getColumnMapping().stream() + .map(mappingColumnDto -> { + final String kafkaName = mappingColumnDto.getKafkaName(); + final String path = mappingColumnDto.getPath(); + + String fullPath; + if (path.startsWith("..")) { + fullPath = path.substring(2); + } else { + fullPath = String.format("message.%s", path); + } + if (StringUtils.hasText(fullPath)) { + fullPath = String.join(".", fullPath.split("\\.")); + } + + fullPath = fullPath + kafkaName; + + Optional column = tableSchema.getColumn(mappingColumnDto.getName()); + final String transformation = + column.map(value -> getTransformation(value.getDataType(), mappingColumnDto)) + .orElse(""); + + return StringUtils.hasText(transformation) + ? String.format(transformation, "(" + fullPath + ")", + mappingDto.getIgnoreFields().stream() + .collect(Collectors.joining("','", "'", "'"))) + : fullPath; + }) + .collect(Collectors.joining(", ", " ", " ")); } - final List existingColumns = existingTableSchema.getColumns(); - final List configColumns = FlinkSchemaUtil.getResolvedSchema(columnList).getColumns(); - for (Column configColumn : configColumns) { - final DataType dataType = configColumn.getDataType(); - final String columnName = configColumn.getName(); - final boolean matchingColumnExists = existingColumns.stream() - .anyMatch(c -> c.getName().equals(columnName) && c.getDataType().equals(dataType)); - if (!matchingColumnExists) { - throw new RuntimeException(String.format( - "%s table [%s] already exists, but table config was provided for it as well. " - + "In this case all columns from table config should be present in the existing table, " - + "but we didn't find the [%s] column of type [%s].", - connectorName, tableName, columnName, dataType)); - } + protected Map getTopicMapping() throws IOException { + TypeReference> typeRef + = new TypeReference>() {}; + final HashMap columnMappingMap = Utils.readFile(params.getRequired(MAPPING_FILE_PARAM), + typeRef); + + //adding the default column mappings to each topic + columnMappingMap.values().forEach(mapping -> { + final List customMappings = Optional.ofNullable(mapping.getColumnMapping()) + .orElse(Collections.emptyList()); + final Collection columnMapping = + Streams.concat(customMappings.stream(), + defaultMappingList.stream()) + .collect( + Collectors.toMap(MappingColumnDto::getName, + Function.identity(), (f, s) -> f)) + .values(); + + mapping.setColumnMapping(new ArrayList<>(columnMapping)); + }); + return columnMappingMap; } - System.out.printf("%s table [%s] already exists and provided table config matches it. Skipping its creation.%n", - connectorName, - tableName); - return existingTableSchema; - } - - private TableDescriptor buildTableDescriptor(Schema schema) { - return fillTableOptions(TableDescriptor - .forConnector(getTableConnector()) - .schema(schema) - .partitionedBy("dt", "hr") - .format(getFormatDescriptor())) - .build(); - } - - protected abstract String getTableConnector(); - - protected abstract FormatDescriptor getFormatDescriptor(); - - - protected TableDescriptor.Builder fillTableOptions(TableDescriptor.Builder builder) { - return builder - .option("partition.time-extractor.timestamp-pattern", "$dt $hr:00:00") - .option("sink.partition-commit.trigger", "process-time") - .option("sink.partition-commit.delay", "1 h") - .option("sink.partition-commit.policy.kind", "metastore,success-file"); - } - - private StreamTableEnvironment getTableEnvironment() { - return StreamTableEnvironment.create(env); - } - - private void createKafkaTable(StreamTableEnvironment tableEnv) { - final SingleOutputStreamOperator newSource = source.map(ScoredMessage::toRow, - ScoredMessage.FLINK_TYPE_INFO); - tableEnv.createTemporaryView(KAFKA_TABLE, newSource); - tableEnv.executeSql("DESCRIBE " + KAFKA_TABLE).print(); - } - - private void configure(StreamTableEnvironment tableEnv) { - final HashMap conf = new HashMap<>(); - conf.put("pipeline.name", params.get("flink.job.name", String.format("Indexing - %s TableApi", connectorName))); - tableEnv.getConfig().addConfiguration(Configuration.fromMap(conf)); - tableEnv.createTemporarySystemFunction("filterMap", FilterMapFunction.class); - } - - protected final String buildInsertSql(String topic, MappingDto mappingDto, ResolvedSchema tableSchema) { - return String.join("\n", - getInsertSqlPrefix() + " " + getTableName(topic, mappingDto) + "(" + getInsertColumns(mappingDto) + ") " - + getInsertSqlSuffix(), - " SELECT " + getFromColumns(mappingDto, tableSchema), - " from " + KAFKA_TABLE, - String.format(" where `source`='%s'", topic)); - } - - protected String getTableName(String source, MappingDto mappingDto) { - return mappingDto.getTableName(); - } - - protected String getInsertSqlPrefix() { - return "INSERT INTO "; - } - - protected String getInsertSqlSuffix() { - return ""; - } - - protected String getInsertColumns(MappingDto mappingDto) { - return mappingDto.getColumnMapping().stream() - .map(MappingColumnDto::getName) - .collect(Collectors.joining(", ", " ", " ")); - } - - private String getFromColumns(MappingDto mappingDto, ResolvedSchema tableSchema) { - return mappingDto.getColumnMapping().stream() - .map(mappingColumnDto -> { - final String kafkaName = mappingColumnDto.getKafkaName(); - final String path = mappingColumnDto.getPath(); - - String fullPath; - if (path.startsWith("..")) { - fullPath = path.substring(2); - } else { - fullPath = String.format("message.%s", path); - } - if (StringUtils.hasText(fullPath)) { - fullPath = String.join(".", fullPath.split("\\.")); - } - - fullPath = fullPath + kafkaName; - - Optional column = tableSchema.getColumn(mappingColumnDto.getName()); - final String transformation = column.map(value -> getTransformation(value.getDataType(), mappingColumnDto)).orElse(""); - - return StringUtils.hasText(transformation) - ? String.format(transformation, "(" + fullPath + ")", mappingDto.getIgnoreFields().stream() - .collect(Collectors.joining("','", "'", "'"))) - : fullPath; - }) - .collect(Collectors.joining(", ", " ", " ")); - } - - protected Map getTopicMapping() throws IOException { - TypeReference> typeRef - = new TypeReference>() { - }; - final HashMap columnMappingMap = Utils.readFile(params.getRequired(MAPPING_FILE_PARAM), - typeRef); - - //adding the default column mappings to each topic - columnMappingMap.values().forEach(mapping -> { - final List customMappings = Optional.ofNullable(mapping.getColumnMapping()) - .orElse(Collections.emptyList()); - final Collection columnMapping = Streams.concat( - customMappings.stream(), - defaultMappingList.stream()) - .collect(Collectors.toMap(MappingColumnDto::getName, Function.identity(), (f, s) -> f)).values(); - - mapping.setColumnMapping(new ArrayList<>(columnMapping)); - }); - return columnMappingMap; - } - - /** - * Method provides contents of the tables config file. - * - * @return Map with table name as a key, and a list of columns as a value. - * @throws IOException in case it can't read the tables config file. - */ - protected Map> getRawTablesConfig() throws IOException { - TypeReference>> typeRef - = new TypeReference>>() { - }; - final String filePath = params.get(TABLES_INIT_FILE_PARAM); - if (filePath == null) { - return Collections.emptyMap(); + + /** + * Method provides contents of the tables config file. + * + * @return Map with table name as a key, and a list of columns as a value. + * @throws IOException in case it can't read the tables config file. + */ + protected Map> getRawTablesConfig() throws IOException { + TypeReference>> typeRef = + new TypeReference>>() { + }; + final String filePath = params.get(TABLES_INIT_FILE_PARAM); + if (filePath == null) { + return Collections.emptyMap(); + } + return Utils.readFile(filePath, typeRef); } - return Utils.readFile(filePath, typeRef); - } - - /** - * Method appends default and partition columns to the raw tables config. - * - * @return Map with table name as a key, and a list of columns as a value. - */ - protected Map> appendDefaultTablesConfig( - Map> rawTablesConfig) { - if (rawTablesConfig == null || rawTablesConfig.isEmpty()) { - return Collections.emptyMap(); + + /** + * Method appends default and partition columns to the raw tables config. + * + * @return Map with table name as a key, and a list of columns as a value. + */ + protected Map> appendDefaultTablesConfig( + Map> rawTablesConfig) { + if (rawTablesConfig == null || rawTablesConfig.isEmpty()) { + return Collections.emptyMap(); + } + final Map> columnMap = new HashMap<>(rawTablesConfig); + final List partitionColumns = Arrays.asList(TableColumnDto.builder() + .name("dt") + .type("string") + .build(), TableColumnDto.builder() + .name("hr") + .type("string") + .build()); + //adding the default columns to each table + columnMap.forEach((tableName, columnList) -> { + final List customColumns = Optional.ofNullable(columnList) + .orElse(Collections.emptyList()); + final Map combinedColumnMap = Streams.concat(customColumns.stream(), + defaultColumnList.stream(), partitionColumns.stream()) + .collect( + Collectors.toMap(TableColumnDto::getName, + Function.identity(), (f, s) -> f, + LinkedHashMap::new)); + //partition columns should be placed last + partitionColumns.forEach(col -> combinedColumnMap.put(col.getName(), col)); + + columnMap.put(tableName, new ArrayList<>(combinedColumnMap.values())); + }); + return columnMap; } - final Map> columnMap = new HashMap<>(rawTablesConfig); - final List partitionColumns = Arrays.asList(TableColumnDto.builder() - .name("dt") - .type("string") - .build(), TableColumnDto.builder() - .name("hr") - .type("string") - .build()); - //adding the default columns to each table - columnMap.forEach((tableName, columnList) -> { - final List customColumns = Optional.ofNullable(columnList) - .orElse(Collections.emptyList()); - final Map combinedColumnMap = Streams.concat(customColumns.stream(), - defaultColumnList.stream(), partitionColumns.stream()) - .collect(Collectors.toMap(TableColumnDto::getName, Function.identity(), (f, s) -> f, LinkedHashMap::new)); - //partition columns should be placed last - partitionColumns.forEach(col -> combinedColumnMap.put(col.getName(), col)); - - columnMap.put(tableName, new ArrayList<>(combinedColumnMap.values())); - }); - return columnMap; - } } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/TableApiJobFactory.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/TableApiJobFactory.java index dd502bed..538c3ed9 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/TableApiJobFactory.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/TableApiJobFactory.java @@ -3,15 +3,16 @@ import com.cloudera.cyber.indexing.hive.tableapi.impl.TableApiHiveJob; import com.cloudera.cyber.indexing.hive.tableapi.impl.TableApiKafkaJob; import com.cloudera.cyber.scoring.ScoredMessage; +import java.io.IOException; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import java.io.IOException; - public class TableApiJobFactory { - public static TableApiAbstractJob getJobByConnectorName(String typeName, ParameterTool params, StreamExecutionEnvironment env, DataStream source) throws IOException { + public static TableApiAbstractJob getJobByConnectorName(String typeName, ParameterTool params, + StreamExecutionEnvironment env, + DataStream source) throws IOException { if (typeName == null) { throw new RuntimeException("Null job type name provided while the Flink writer is selected as TableAPI"); } @@ -21,7 +22,8 @@ public static TableApiAbstractJob getJobByConnectorName(String typeName, Paramet case "kafka": return new TableApiKafkaJob(params, env, source); default: - throw new RuntimeException(String.format("Unknown job type name [%s] provided while the Flink writer is selected as TableAPI", typeName)); + throw new RuntimeException(String.format( + "Unknown job type name [%s] provided while the Flink writer is selected as TableAPI", typeName)); } } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java index 935c1c1b..cfcfc287 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java @@ -1,6 +1,7 @@ package com.cloudera.cyber.indexing.hive.tableapi.impl; import com.cloudera.cyber.indexing.hive.util.AvroSchemaUtil; +import java.util.Set; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -10,8 +11,6 @@ import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo; import org.apache.flink.types.Row; -import java.util.Set; - public class MapRowToAvro implements ResultTypeQueryable, MapFunction { private final GenericRecordAvroTypeInfo producedType; private final Schema schema; diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiHiveJob.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiHiveJob.java index e02251f0..057261dd 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiHiveJob.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiHiveJob.java @@ -2,6 +2,7 @@ import com.cloudera.cyber.indexing.hive.tableapi.TableApiAbstractJob; import com.cloudera.cyber.scoring.ScoredMessage; +import java.io.IOException; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -10,14 +11,13 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.catalog.hive.HiveCatalog; -import java.io.IOException; - public class TableApiHiveJob extends TableApiAbstractJob { private static final String BASE_TABLE_JSON = "base-hive-table.json"; - public TableApiHiveJob(ParameterTool params, StreamExecutionEnvironment env, DataStream source) throws IOException { + public TableApiHiveJob(ParameterTool params, StreamExecutionEnvironment env, DataStream source) + throws IOException { super(params, env, source, "Hive", BASE_TABLE_JSON); } @@ -50,7 +50,7 @@ protected void registerCatalog(StreamTableEnvironment tableEnv) { @Override protected TableDescriptor.Builder fillTableOptions(TableDescriptor.Builder builder) { return super.fillTableOptions(builder) - .option("hive.storage.file-format", "orc"); + .option("hive.storage.file-format", "orc"); } } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java index 20854531..785a8c64 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java @@ -11,9 +11,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.kafka.sink.KafkaSink; @@ -26,79 +23,82 @@ public class TableApiKafkaJob extends TableApiAbstractJob { - private static final String BASE_TABLE_JSON = "base-hive-table.json"; - - public TableApiKafkaJob(ParameterTool params, StreamExecutionEnvironment env, DataStream source) - throws IOException { - super(params, env, source, "Kafka", BASE_TABLE_JSON); - } - - @Override - protected ResolvedSchema createTable(StreamTableEnvironment tableEnv, String tableName, - List columnList) { - return FlinkSchemaUtil.getResolvedSchema(columnList); - } - - @Override - protected void executeInsert(StreamTableEnvironment tableEnv, Map topicMapping, - Map> tablesConfig, Map tableSchemaMap) { - topicMapping.forEach((topic, mappingDto) -> { - final String insertSql = buildInsertSql(topic, mappingDto, tableSchemaMap.get(mappingDto.getTableName())); - try { - //create view - tableEnv.executeSql(insertSql); - final KafkaSink kafkaSink = new FlinkUtils<>(GenericRecord.class).createKafkaSink( - mappingDto.getTableName(), "indexing-job", params); - - //read from view and write to kafka sink - final Table table = tableEnv.from(getTableName(topic, mappingDto)); - final String schemaString = AvroSchemaUtil.convertToAvro(tablesConfig.get(mappingDto.getTableName())) - .toString(); - - final DataStream stream = tableEnv.toDataStream(table).map(new MapRowToAvro(schemaString)); - stream.sinkTo(kafkaSink); - System.out.printf("Insert SQL added to the queue for the table: %s%nSQL: %s%n", mappingDto.getTableName(), - insertSql); - } catch (Exception e) { - System.err.printf("Error adding insert to the statement set: %s%n", insertSql); - throw e; - } - }); - } - - @Override - protected HashSet getExistingTableList(StreamTableEnvironment tableEnv) { - //Kafka tables are temporary, so no tables are present on the job creation - return new HashSet<>(); - } - - @Override - protected void registerCatalog(StreamTableEnvironment tableEnv) { - - } - - @Override - protected String getTableConnector() { - return "filesystem"; - } - - @Override - protected FormatDescriptor getFormatDescriptor() { - return null; - } - - @Override - protected String getTableName(String source, MappingDto mappingDto) { - return source.concat("_tmpview"); - } - - @Override - protected String getInsertSqlPrefix() { - return "CREATE TEMPORARY VIEW "; - } - - @Override - protected String getInsertSqlSuffix() { - return " AS "; - } + private static final String BASE_TABLE_JSON = "base-hive-table.json"; + + public TableApiKafkaJob(ParameterTool params, StreamExecutionEnvironment env, DataStream source) + throws IOException { + super(params, env, source, "Kafka", BASE_TABLE_JSON); + } + + @Override + protected ResolvedSchema createTable(StreamTableEnvironment tableEnv, String tableName, + List columnList) { + return FlinkSchemaUtil.getResolvedSchema(columnList); + } + + @Override + protected void executeInsert(StreamTableEnvironment tableEnv, Map topicMapping, + Map> tablesConfig, + Map tableSchemaMap) { + topicMapping.forEach((topic, mappingDto) -> { + final String insertSql = buildInsertSql(topic, mappingDto, tableSchemaMap.get(mappingDto.getTableName())); + try { + //create view + tableEnv.executeSql(insertSql); + final KafkaSink kafkaSink = new FlinkUtils<>(GenericRecord.class).createKafkaSink( + mappingDto.getTableName(), "indexing-job", params); + + //read from view and write to kafka sink + final Table table = tableEnv.from(getTableName(topic, mappingDto)); + final String schemaString = AvroSchemaUtil.convertToAvro(tablesConfig.get(mappingDto.getTableName())) + .toString(); + + final DataStream stream = + tableEnv.toDataStream(table).map(new MapRowToAvro(schemaString)); + stream.sinkTo(kafkaSink); + System.out.printf("Insert SQL added to the queue for the table: %s%nSQL: %s%n", + mappingDto.getTableName(), + insertSql); + } catch (Exception e) { + System.err.printf("Error adding insert to the statement set: %s%n", insertSql); + throw e; + } + }); + } + + @Override + protected HashSet getExistingTableList(StreamTableEnvironment tableEnv) { + //Kafka tables are temporary, so no tables are present on the job creation + return new HashSet<>(); + } + + @Override + protected void registerCatalog(StreamTableEnvironment tableEnv) { + + } + + @Override + protected String getTableConnector() { + return "filesystem"; + } + + @Override + protected FormatDescriptor getFormatDescriptor() { + return null; + } + + @Override + protected String getTableName(String source, MappingDto mappingDto) { + return source.concat("_tmpview"); + } + + @Override + protected String getInsertSqlPrefix() { + return "CREATE TEMPORARY VIEW "; + } + + @Override + protected String getInsertSqlSuffix() { + return " AS "; + } } diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/AvroSchemaUtil.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/AvroSchemaUtil.java index 16461b33..69688ad2 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/AvroSchemaUtil.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/AvroSchemaUtil.java @@ -2,6 +2,12 @@ import com.cloudera.cyber.avro.AvroSchemas; import com.cloudera.cyber.indexing.TableColumnDto; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; @@ -13,9 +19,6 @@ import org.apache.flink.table.types.logical.RowType; import org.apache.flink.types.Row; -import java.time.Instant; -import java.util.*; - public class AvroSchemaUtil { //method that converts from flink Schema to avro Schema @@ -25,11 +28,13 @@ public static Schema convertToAvro(List tableColumnList) { //method that converts from flink Schema to avro Schema public static Schema convertToAvro(ResolvedSchema schema) { - SchemaBuilder.FieldAssembler fieldAssembler = AvroSchemas.createRecordBuilder("com.cloudera.cyber","base") - .fields(); + SchemaBuilder.FieldAssembler fieldAssembler = + AvroSchemas.createRecordBuilder("com.cloudera.cyber", "base") + .fields(); for (Column col : schema.getColumns()) { - fieldAssembler = fieldAssembler.name(col.getName()).type().optional().type(AvroSchemaUtil.convertTypeToAvro(col.getName(), col.getDataType().getLogicalType())); + fieldAssembler = fieldAssembler.name(col.getName()).type().optional() + .type(AvroSchemaUtil.convertTypeToAvro(col.getName(), col.getDataType().getLogicalType())); } return fieldAssembler.endRecord(); } @@ -42,7 +47,8 @@ public static void putRowIntoAvro(Row row, GenericRecord record, String fieldNam value = null; } else { try { - value = convertToAvroObject(record.getSchema().getField(avroFieldName).schema(), row.getField(fieldName)); + value = + convertToAvroObject(record.getSchema().getField(avroFieldName).schema(), row.getField(fieldName)); } catch (Exception e) { throw new RuntimeException(String.format("Error converting avro field %s", avroFieldName), e); } @@ -62,7 +68,8 @@ private static Object convertToAvroObject(Schema fieldSchema, Object value) { if (nestedFieldNames != null) { for (String nestedFieldName : nestedFieldNames) { final String avroFieldName = nestedFieldName.toLowerCase(); - final Object nestedValue = convertToAvroObject(fieldSchema.getField(avroFieldName).schema(), nestedRow.getField(nestedFieldName)); + final Object nestedValue = convertToAvroObject(fieldSchema.getField(avroFieldName).schema(), + nestedRow.getField(nestedFieldName)); nestedRecord.put(avroFieldName, nestedValue); } } @@ -128,7 +135,8 @@ private static Schema convertTypeToAvro(String name, LogicalType dataType) { final List fieldList = ((RowType) dataType).getFields(); final SchemaBuilder.FieldAssembler fieldAssembler = SchemaBuilder.record(name).fields(); for (RowType.RowField field : fieldList) { - fieldAssembler.name(field.getName()).type().optional().type(convertTypeToAvro(name + "_" + field.getName(), field.getType())); + fieldAssembler.name(field.getName()).type().optional() + .type(convertTypeToAvro(name + "_" + field.getName(), field.getType())); } return fieldAssembler.endRecord(); default: diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java index 58c3856e..73e2934a 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java @@ -15,14 +15,14 @@ public class FlinkSchemaUtil { public static Schema buildSchema(ResolvedSchema resolvedSchema) { return Schema.newBuilder() - .fromResolvedSchema(resolvedSchema) - .build(); + .fromResolvedSchema(resolvedSchema) + .build(); } public static ResolvedSchema getResolvedSchema(List columnList) { final List flinkColumnList = columnList.stream() - .map(col -> Column.physical(col.getName(), getFlinkType(col.getType(), col.getNullable()))) - .collect(Collectors.toList()); + .map(col -> Column.physical(col.getName(), getFlinkType(col.getType(), col.getNullable()))) + .collect(Collectors.toList()); return ResolvedSchema.of(flinkColumnList); } @@ -35,8 +35,8 @@ private static DataType getFlinkType(String colType) { * * @param colType config column type. * Possible config column type values are: - * string, timestamp, date, int, bigint, float, double, boolean, bytes, null, - * array, map, struct + * {@code string, timestamp, date, int, bigint, float, double, boolean, bytes, null, + * array, map, struct} * @param nullable whether column is nullable or not * @return Flink DataType that describes provided column type */ @@ -81,7 +81,7 @@ private static DataType getFlinkType(String colType, Boolean nullable) { /** * Parses Array type from the config column type. * - * @param type config column type. Supported format is: array + * @param type config column type. Supported format is: {@code array} * @return Flink DataType that describes provided array type */ private static DataType parseArrayType(String type) { @@ -92,7 +92,7 @@ private static DataType parseArrayType(String type) { /** * Parses Map type from the config column type. * - * @param type config column type. Supported format is: map + * @param type config column type. Supported format is: {@code map} * @return Flink DataType that describes provided map type */ private static DataType parseMapType(String type) { @@ -107,7 +107,8 @@ private static DataType parseMapType(String type) { /** * Parses Struct type from the config column type. * - * @param type config column type. Supported format is: struct. The name can contain only alphanumeric characters and underscores. + * @param type config column type. Supported format is: {@code struct}. + * The name can contain only alphanumeric characters and underscores. * @return Flink DataType that describes provided struct type */ private static DataType parseStructType(String type) { diff --git a/flink-cyber/flink-indexing/flink-indexing-parquet/src/main/java/com/cloudera/cyber/indexing/ParquetJob.java b/flink-cyber/flink-indexing/flink-indexing-parquet/src/main/java/com/cloudera/cyber/indexing/ParquetJob.java index 051d2f84..f0f2dcf4 100644 --- a/flink-cyber/flink-indexing/flink-indexing-parquet/src/main/java/com/cloudera/cyber/indexing/ParquetJob.java +++ b/flink-cyber/flink-indexing/flink-indexing-parquet/src/main/java/com/cloudera/cyber/indexing/ParquetJob.java @@ -40,7 +40,10 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) { return env; } + protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract void writeResults(DataStream results, ParameterTool params); + protected abstract void writeTabularResults(DataStream results, String path, ParameterTool params); } diff --git a/flink-cyber/flink-indexing/flink-indexing-parquet/src/main/java/com/cloudera/cyber/indexing/ParquetJobKafka.java b/flink-cyber/flink-indexing/flink-indexing-parquet/src/main/java/com/cloudera/cyber/indexing/ParquetJobKafka.java index e0683ac1..e6d87e7f 100644 --- a/flink-cyber/flink-indexing/flink-indexing-parquet/src/main/java/com/cloudera/cyber/indexing/ParquetJobKafka.java +++ b/flink-cyber/flink-indexing/flink-indexing-parquet/src/main/java/com/cloudera/cyber/indexing/ParquetJobKafka.java @@ -12,6 +12,8 @@ package com.cloudera.cyber.indexing; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; + import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; @@ -24,8 +26,6 @@ import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; import org.apache.flink.util.Preconditions; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; - public class ParquetJobKafka extends ParquetJob { private static final String PARAMS_OUTPUT_BASEPATH = "index.basepath"; @@ -38,16 +38,17 @@ public static void main(String[] args) throws Exception { @Override public DataStream createSource(StreamExecutionEnvironment env, ParameterTool params) { return env.fromSource( - new FlinkUtils(Message.class).createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, "indexer-parquet"), - WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); + FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, + "indexer-parquet"), + WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); } @Override protected void writeResults(DataStream results, ParameterTool params) { final StreamingFileSink sink = StreamingFileSink - .forBulkFormat(new Path(params.getRequired(PARAMS_OUTPUT_BASEPATH)), - ParquetAvroWriters.forSpecificRecord(Message.class)) - .build(); + .forBulkFormat(new Path(params.getRequired(PARAMS_OUTPUT_BASEPATH)), + ParquetAvroWriters.forSpecificRecord(Message.class)) + .build(); results.addSink(sink).name("Parquet Sink").uid("parquet-sink"); } diff --git a/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/CollectionField.java b/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/CollectionField.java index f7cc4f86..9ca7ae56 100644 --- a/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/CollectionField.java +++ b/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/CollectionField.java @@ -12,14 +12,17 @@ package com.cloudera.cyber.indexing; -import lombok.*; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.specific.SpecificRecord; import org.apache.avro.specific.SpecificRecordBase; -import java.util.List; - @Data @EqualsAndHashCode @Builder @@ -30,13 +33,13 @@ public class CollectionField extends SpecificRecordBase implements SpecificRecor private List values; private static final Schema SCHEMA$ = SchemaBuilder - .record(CollectionField.class.getName()) - .namespace(CollectionField.class.getPackage().getName()) - .fields() - .requiredString("key") - .name("values") - .type(SchemaBuilder.array().items().stringType()).noDefault() - .endRecord(); + .record(CollectionField.class.getName()) + .namespace(CollectionField.class.getPackage().getName()) + .fields() + .requiredString("key") + .name("values") + .type(SchemaBuilder.array().items().stringType()).noDefault() + .endRecord(); @Override public Schema getSchema() { diff --git a/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/FilterStreamFieldsByConfig.java b/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/FilterStreamFieldsByConfig.java index 222bc8f3..08881f04 100644 --- a/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/FilterStreamFieldsByConfig.java +++ b/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/FilterStreamFieldsByConfig.java @@ -12,7 +12,13 @@ package com.cloudera.cyber.indexing; +import static java.util.stream.Collectors.toMap; + import com.cloudera.cyber.Message; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.state.MapStateDescriptor; import org.apache.flink.api.java.tuple.Tuple2; @@ -20,15 +26,9 @@ import org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction; import org.apache.flink.util.Collector; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toMap; - @Slf4j -public class FilterStreamFieldsByConfig extends KeyedBroadcastProcessFunction { +public class FilterStreamFieldsByConfig + extends KeyedBroadcastProcessFunction { private final MapStateDescriptor> broadcastState; @@ -43,7 +43,8 @@ public void open(Configuration parameters) throws Exception { } @Override - public void processElement(Message message, ReadOnlyContext readOnlyContext, Collector collector) throws Exception { + public void processElement(Message message, ReadOnlyContext readOnlyContext, Collector collector) + throws Exception { if (!readOnlyContext.getBroadcastState(broadcastState).contains(message.getSource())) { return; } @@ -51,46 +52,49 @@ public void processElement(Message message, ReadOnlyContext readOnlyContext, Col List fieldsRequired = readOnlyContext.getBroadcastState(broadcastState).get(message.getSource()); Stream> messageFields = message.getExtensions() == null ? Stream.empty() : - message.getExtensions().entrySet().stream() - .filter(f -> !f.getKey().equals("ts") && fieldsRequired.contains(f.getKey())) - .map(e -> Tuple2.of(e.getKey(), e.getValue())); + message.getExtensions().entrySet().stream() + .filter(f -> !f.getKey().equals("ts") && fieldsRequired.contains(f.getKey())) + .map(e -> Tuple2.of(e.getKey(), e.getValue())); Stream> threatFields = message.getThreats() == null ? Stream.empty() : - message.getThreats().entrySet().stream().flatMap(e -> - e.getValue().stream().flatMap(l -> - l.getFields().entrySet().stream().map(le -> Tuple2.of( - String.join(".", new String[]{e.getKey(), l.getObservableType(), l.getObservable(), e.getKey()}), - le.getValue()) - ) - ) - ); + message.getThreats().entrySet().stream().flatMap(e -> + e.getValue().stream().flatMap(l -> + l.getFields().entrySet().stream().map(le -> Tuple2.of( + String.join(".", + new String[] {e.getKey(), l.getObservableType(), l.getObservable(), e.getKey()}), + le.getValue()) + ) + ) + ); Stream> baseFields = Stream.of( - Tuple2.of("message", message.getMessage()) + Tuple2.of("message", message.getMessage()) ); Stream> allFields = Stream.of( - baseFields, - messageFields, - threatFields) - .flatMap(s->s) - .filter(r -> r != null && r.f1 != null); + baseFields, + messageFields, + threatFields) + .flatMap(s -> s) + .filter(r -> r != null && r.f1 != null); Map fields = allFields.collect(toMap( - e -> e.f0, - e -> e.f1) + e -> e.f0, + e -> e.f1) ); collector.collect(IndexEntry.builder() - .index(message.getSource()) - .id(message.getId()) - .timestamp(message.getTs()) - .fields(fields) - .build()); + .index(message.getSource()) + .id(message.getId()) + .timestamp(message.getTs()) + .fields(fields) + .build()); } @Override - public void processBroadcastElement(CollectionField collectionFields, Context context, Collector collector) throws Exception { + public void processBroadcastElement(CollectionField collectionFields, Context context, + Collector collector) throws Exception { log.info("Schema Update message {}", collectionFields.toString()); - context.getBroadcastState(broadcastState).put(collectionFields.getKey(), new ArrayList<>(collectionFields.getValues())); + context.getBroadcastState(broadcastState) + .put(collectionFields.getKey(), new ArrayList<>(collectionFields.getValues())); } } diff --git a/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/IndexEntry.java b/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/IndexEntry.java index 729b1c40..fe9b1e17 100644 --- a/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/IndexEntry.java +++ b/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/IndexEntry.java @@ -12,12 +12,11 @@ package com.cloudera.cyber.indexing; +import java.util.Map; import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import java.util.Map; - @Data @Slf4j @Builder diff --git a/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/SearchIndexJob.java b/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/SearchIndexJob.java index 293081b8..5b964d1c 100644 --- a/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/SearchIndexJob.java +++ b/flink-cyber/flink-indexing/flink-indexing-search/src/main/java/com/cloudera/cyber/indexing/SearchIndexJob.java @@ -14,6 +14,9 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; +import java.io.IOException; +import java.util.List; +import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.state.MapStateDescriptor; @@ -24,10 +27,6 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import java.io.IOException; -import java.util.List; -import java.util.regex.Pattern; - @Slf4j public abstract class SearchIndexJob { protected static final String PARAMS_SCHEMA_REFRESH_INTERVAL = "schema.refresh"; @@ -47,7 +46,7 @@ protected final StreamExecutionEnvironment createPipeline(ParameterTool params) DataStream messages; // allow a general exclusion of any source that matches a regex pattern upfront to avoid retry loops - if (StringUtils.isNotBlank(params.get(PARAM_EXCLUDE_PATTERN,""))) { + if (StringUtils.isNotBlank(params.get(PARAM_EXCLUDE_PATTERN, ""))) { final Pattern filterExpression = Pattern.compile(params.get(PARAM_EXCLUDE_PATTERN)); messages = source.filter(m -> !filterExpression.matcher(m.getSource()).matches()); } else { @@ -57,7 +56,7 @@ protected final StreamExecutionEnvironment createPipeline(ParameterTool params) DataStream configSource = createConfigSource(env, params); logConfig(configSource, params); BroadcastStream entryStringKeyedStream = configSource - .broadcast(Descriptors.broadcastState); + .broadcast(Descriptors.broadcastState); // TODO - apply any row filtering logic here // or at least you would be able to if that didn't rely on the broadcast state for config @@ -68,18 +67,18 @@ protected final StreamExecutionEnvironment createPipeline(ParameterTool params) * as a flat version of message, which may be more long term useful. */ MapStateDescriptor> broadcastState = new MapStateDescriptor<>( - "fieldsByIndex", - Types.STRING, - Types.LIST(Types.STRING) + "fieldsByIndex", + Types.STRING, + Types.LIST(Types.STRING) ); broadcastState.setQueryable("fieldsByIndex"); // only process messages that we have a collection for, and extract only the fields that are accounted for in target index DataStream output = messages - .keyBy(Message::getSource) - .connect(entryStringKeyedStream) - .process(new FilterStreamFieldsByConfig(broadcastState)) - .name("Index Entry Extractor").uid("index-entry-extract"); + .keyBy(Message::getSource) + .connect(entryStringKeyedStream) + .process(new FilterStreamFieldsByConfig(broadcastState)) + .name("Index Entry Extractor").uid("index-entry-extract"); // now add the correctly formed version for tables @@ -94,14 +93,19 @@ protected final StreamExecutionEnvironment createPipeline(ParameterTool params) protected static class Descriptors { public static final MapStateDescriptor> broadcastState = new MapStateDescriptor<>( - "fieldsByIndex", - Types.STRING, - Types.LIST(Types.STRING) + "fieldsByIndex", + Types.STRING, + Types.LIST(Types.STRING) ); } + protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); - protected abstract DataStream createConfigSource(StreamExecutionEnvironment env, ParameterTool params); + + protected abstract DataStream createConfigSource(StreamExecutionEnvironment env, + ParameterTool params); + protected abstract void writeResults(DataStream results, ParameterTool params) throws IOException; + protected abstract void logConfig(DataStream configSource, ParameterTool params); } diff --git a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrClientBuilder.java b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrClientBuilder.java index 574cb921..5f267395 100644 --- a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrClientBuilder.java +++ b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrClientBuilder.java @@ -12,6 +12,7 @@ package com.cloudera.cyber.indexing; +import java.util.List; import lombok.Builder; import lombok.Data; import org.apache.solr.client.solrj.SolrClient; @@ -20,8 +21,6 @@ import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; -import java.util.List; - @Data @Builder public class SolrClientBuilder { diff --git a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrCollectionFieldsSource.java b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrCollectionFieldsSource.java index f3adfe40..d17d28b0 100644 --- a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrCollectionFieldsSource.java +++ b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrCollectionFieldsSource.java @@ -12,6 +12,14 @@ package com.cloudera.cyber.indexing; +import static java.util.stream.Collectors.toList; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.function.Consumer; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -25,18 +33,10 @@ import org.apache.solr.client.solrj.request.schema.SchemaRequest; import org.apache.solr.client.solrj.response.schema.SchemaResponse; -import java.io.IOException; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.function.Consumer; - -import static java.util.stream.Collectors.toList; - @Slf4j @RequiredArgsConstructor -public class SolrCollectionFieldsSource extends RichParallelSourceFunction implements ResultTypeQueryable { +public class SolrCollectionFieldsSource extends RichParallelSourceFunction + implements ResultTypeQueryable { @NonNull private List solrUrls; @NonNull @@ -78,19 +78,19 @@ private List fieldsForCollection(SolrClient solrClient, String collectio protected Collection loadFieldsFromIndex() throws IOException { SolrClient solrClient = SolrClientBuilder.builder() - .solrUrls(solrUrls) - .build().build(); + .solrUrls(solrUrls) + .build().build(); try { List collections = CollectionAdminRequest.listCollections(solrClient); return collections.stream() - .map( - collection -> - CollectionField.builder() - .key(collection) - .values(fieldsForCollection(solrClient, collection)) - .build() - ) - .collect(toList()); + .map( + collection -> + CollectionField.builder() + .key(collection) + .values(fieldsForCollection(solrClient, collection)) + .build() + ) + .collect(toList()); } catch (SolrServerException e) { log.error("Problem with Solr Schema inspection", e); throw new IOException(e); diff --git a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrIndexer.java b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrIndexer.java index aa61eb75..e4739017 100644 --- a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrIndexer.java +++ b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrIndexer.java @@ -12,7 +12,15 @@ package com.cloudera.cyber.indexing; +import static java.util.stream.Collectors.groupingBy; + import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.StreamSupport; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.configuration.Configuration; @@ -24,15 +32,6 @@ import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrInputDocument; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.StreamSupport; - -import static java.util.stream.Collectors.groupingBy; - @Slf4j public class SolrIndexer extends RichWindowFunction { @@ -54,10 +53,11 @@ public SolrIndexer(ParameterTool params) { } @Override - public void apply(String k, TimeWindow w, Iterable logs, Collector output) throws Exception { + public void apply(String k, TimeWindow w, Iterable logs, Collector output) + throws Exception { Map> collect = StreamSupport - .stream(logs.spliterator(), false) - .collect(groupingBy(IndexEntry::getIndex)); + .stream(logs.spliterator(), false) + .collect(groupingBy(IndexEntry::getIndex)); // TODO - account for all the errors, not just the last AtomicReference lastError = new AtomicReference<>(); collect.forEach((collection, entries) -> { @@ -69,7 +69,7 @@ public void apply(String k, TimeWindow w, Iterable logs, Collector mapToSolrDocuments(Iterable logs) { @Override public void open(Configuration config) { solrClient = SolrClientBuilder.builder() - .solrUrls(solrUrls) - .build().build(); + .solrUrls(solrUrls) + .build().build(); } @Override diff --git a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrJob.java b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrJob.java index fe021666..849c83d3 100644 --- a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrJob.java +++ b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrJob.java @@ -28,11 +28,12 @@ public abstract class SolrJob extends SearchIndexJob { private static final long DEFAULT_MAX_EVENTS = 10000; protected void writeResults(DataStream results, ParameterTool params) { - results.keyBy(IndexEntry::getIndex).timeWindow(Time.milliseconds(params.getLong(PARAMS_INDEX_WINDOW_MAX_MS, DEFAULT_INDEX_WINDOW_MAX_MS))) - .trigger(EventTimeAndCountTrigger.of(DEFAULT_MAX_EVENTS)) - .apply(new SolrIndexer(params)) - .name("Solr Indexer") - .uid("Solr Indexer"); + results.keyBy(IndexEntry::getIndex) + .timeWindow(Time.milliseconds(params.getLong(PARAMS_INDEX_WINDOW_MAX_MS, DEFAULT_INDEX_WINDOW_MAX_MS))) + .trigger(EventTimeAndCountTrigger.of(DEFAULT_MAX_EVENTS)) + .apply(new SolrIndexer(params)) + .name("Solr Indexer") + .uid("Solr Indexer"); } protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); diff --git a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrJobKafka.java b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrJobKafka.java index e241d1b3..cf0c5de0 100644 --- a/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrJobKafka.java +++ b/flink-cyber/flink-indexing/flink-indexing-solr/src/main/java/com/cloudera/cyber/indexing/SolrJobKafka.java @@ -12,9 +12,12 @@ package com.cloudera.cyber.indexing; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; + import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; +import java.util.Arrays; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; @@ -24,10 +27,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.Arrays; - -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; - @Slf4j public class SolrJobKafka extends SolrJob { private static final String DEFAULT_TOPIC_CONFIG_LOG = "solr.config.log"; @@ -41,28 +40,29 @@ public static void main(String[] args) throws Exception { @Override public DataStream createSource(StreamExecutionEnvironment env, ParameterTool params) { return env.fromSource( - FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, "indexer-solr"), - WatermarkStrategy.noWatermarks(), "Message Source").uid("message-source"); + FlinkUtils.createKafkaSource(params.getRequired(PARAMS_TOPIC_INPUT), params, "indexer-solr"), + WatermarkStrategy.noWatermarks(), "Message Source").uid("message-source"); } protected DataStream createConfigSource(StreamExecutionEnvironment env, ParameterTool params) { DataStreamSource dataStreamSource = env.addSource( - new SolrCollectionFieldsSource(Arrays.asList(params.getRequired("solr.urls").split(",")), - params.getLong(PARAMS_SCHEMA_REFRESH_INTERVAL, DEFAULT_SCHEMA_REFRESH_INTERVAL)) + new SolrCollectionFieldsSource(Arrays.asList(params.getRequired("solr.urls").split(",")), + params.getLong(PARAMS_SCHEMA_REFRESH_INTERVAL, DEFAULT_SCHEMA_REFRESH_INTERVAL)) ); return dataStreamSource - .name("Schema Stream").uid("schema-stream") - .setMaxParallelism(1) - .setParallelism(1); + .name("Schema Stream").uid("schema-stream") + .setMaxParallelism(1) + .setParallelism(1); } @Override protected void logConfig(DataStream configSource, ParameterTool params) { String topic = params.get(PARAMS_TOPIC_CONFIG_LOG, DEFAULT_TOPIC_CONFIG_LOG); - KafkaSink sink = new FlinkUtils<>(CollectionField.class).createKafkaSink(topic,"indexer-solr-schema", params); + KafkaSink sink = + new FlinkUtils<>(CollectionField.class).createKafkaSink(topic, "indexer-solr-schema", params); configSource.sinkTo(sink).name("Schema Log").uid("schema-log") - .setParallelism(1); + .setParallelism(1); } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/com/cloudera/cyber/stellar/MetronCompatibilityParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/com/cloudera/cyber/stellar/MetronCompatibilityParser.java index 310b0c6c..e3aff274 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/com/cloudera/cyber/stellar/MetronCompatibilityParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/com/cloudera/cyber/stellar/MetronCompatibilityParser.java @@ -12,9 +12,17 @@ package com.cloudera.cyber.stellar; +import static org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver.Config.STELLAR_SEARCH_INCLUDES_KEY; + import com.cloudera.cyber.parser.MessageToParse; import com.github.benmanes.caffeine.cache.Cache; import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -35,25 +43,16 @@ import org.apache.metron.stellar.dsl.StellarFunctions; import org.json.simple.JSONObject; -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver.Config.STELLAR_SEARCH_INCLUDES_KEY; - @Slf4j @RequiredArgsConstructor public class MetronCompatibilityParser { private static final Map cacheConfig = ImmutableMap.of( - CachingStellarProcessor.MAX_CACHE_SIZE_PARAM, 3000, - CachingStellarProcessor.MAX_TIME_RETAIN_PARAM, 10, - CachingStellarProcessor.RECORD_STATS, true + CachingStellarProcessor.MAX_CACHE_SIZE_PARAM, 3000, + CachingStellarProcessor.MAX_TIME_RETAIN_PARAM, 10, + CachingStellarProcessor.RECORD_STATS, true ); private static final Map stellarConfig = ImmutableMap.of( - STELLAR_SEARCH_INCLUDES_KEY.param(), "org.apache.metron.*" + STELLAR_SEARCH_INCLUDES_KEY.param(), "org.apache.metron.*" ); private final SensorParserConfig parserConfig; private final MessageFilter filter; @@ -68,8 +67,8 @@ public static MetronCompatibilityParser of(String sensorType, InputStream config MessageFilter filter = null; if (!StringUtils.isEmpty(parserConfig.getFilterClassName())) { filter = Filters.get( - parserConfig.getFilterClassName(), - parserConfig.getParserConfig() + parserConfig.getFilterClassName(), + parserConfig.getParserConfig() ); } log.info("Loading parser class {}", parserClassName); @@ -82,16 +81,18 @@ public static MetronCompatibilityParser of(String sensorType, InputStream config Cache cache = CachingStellarProcessor.createCache(cacheConfig); Context stellarContext = new Context.Builder() - .with(Context.Capabilities.CACHE, () -> cache) - .with(Context.Capabilities.STELLAR_CONFIG, () -> stellarConfig) - .build(); + .with(Context.Capabilities.CACHE, () -> cache) + .with(Context.Capabilities.STELLAR_CONFIG, () -> stellarConfig) + .build(); StellarFunctions.FUNCTION_RESOLVER().initialize(stellarContext); return new MetronCompatibilityParser(parserConfig, filter, parser, stellarContext, sensorType); } public Optional> parse(MessageToParse messageToParse) { - RawMessage metronRawMessage = MetronRawDataExtractor.INSTANCE.getRawMessage(parserConfig.getRawMessageStrategy(), messageToParse, parserConfig.getReadMetadata(), - parserConfig.getRawMessageStrategyConfig()); + RawMessage metronRawMessage = + MetronRawDataExtractor.INSTANCE.getRawMessage(parserConfig.getRawMessageStrategy(), messageToParse, + parserConfig.getReadMetadata(), + parserConfig.getRawMessageStrategyConfig()); Optional> result = parser.parseOptionalResult(metronRawMessage.getMessage()); if (result.isPresent()) { @@ -100,15 +101,15 @@ public Optional> parse(MessageToParse messageToP if (CollectionUtils.isNotEmpty(parsedMessages)) { JSONObject parsedMessage = parsedMessages.get(0); parserConfig.getRawMessageStrategy().mergeMetadata( - parsedMessage, - metronRawMessage.getMetadata(), - parserConfig.getMergeMetadata(), - parserConfig.getRawMessageStrategyConfig() + parsedMessage, + metronRawMessage.getMetadata(), + parserConfig.getMergeMetadata(), + parserConfig.getRawMessageStrategyConfig() ); parsedMessage.put(Constants.SENSOR_TYPE, sensorType); parsedMessage.putIfAbsent(Constants.Fields.ORIGINAL.getName(), - new String(metronRawMessage.getMessage(), parser.getReadCharset())); + new String(metronRawMessage.getMessage(), parser.getReadCharset())); applyFieldTransformations(parsedMessage, metronRawMessage); // return empty message - message should be filtered @@ -126,8 +127,8 @@ public Optional> parse(MessageToParse messageToP /** * Applies Stellar field transformations defined in the sensor parser config. * - * @param message Message parsed by the MessageParser - * @param rawMessage Raw message including metadata + * @param message Message parsed by the MessageParser + * @param rawMessage Raw message including metadata */ private void applyFieldTransformations(JSONObject message, RawMessage rawMessage) { for (FieldTransformer handler : parserConfig.getFieldTransformations()) { @@ -135,20 +136,20 @@ private void applyFieldTransformations(JSONObject message, RawMessage rawMessage if (!parserConfig.getMergeMetadata()) { //if we haven't merged metadata, then we need to pass them along as configuration params. handler.transformAndUpdate( - message, - stellarContext, - parserConfig.getParserConfig(), - rawMessage.getMetadata() + message, + stellarContext, + parserConfig.getParserConfig(), + rawMessage.getMetadata() ); } else { handler.transformAndUpdate( - message, - stellarContext, - parserConfig.getParserConfig() + message, + stellarContext, + parserConfig.getParserConfig() ); } } } } - } \ No newline at end of file +} \ No newline at end of file diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/com/cloudera/cyber/stellar/MetronRawDataExtractor.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/com/cloudera/cyber/stellar/MetronRawDataExtractor.java index c9a182e7..071ce6ce 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/com/cloudera/cyber/stellar/MetronRawDataExtractor.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/com/cloudera/cyber/stellar/MetronRawDataExtractor.java @@ -7,17 +7,25 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * http://www.apache.org/licenses/LICENSE-2.0 * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.cloudera.cyber.stellar; import com.cloudera.cyber.parser.MessageToParse; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.metron.common.message.metadata.MetadataUtil; import org.apache.metron.common.message.metadata.RawMessage; @@ -26,12 +34,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - public enum MetronRawDataExtractor { INSTANCE; @@ -41,15 +43,17 @@ public enum MetronRawDataExtractor { /** * Extract the raw message given the strategy, the tuple and the metadata configs. - * @param strategy The {@link RawMessageStrategy} to use for extraction + * + * @param strategy The {@link RawMessageStrategy} to use for extraction * @param messageToParse The original message and its topic metadata - * @param readMetadata True if read metadata, false otherwise - * @param config The config to use during extraction + * @param readMetadata True if read metadata, false otherwise + * @param config The config to use during extraction * @return The resulting {@link RawMessage} */ - public RawMessage getRawMessage(RawMessageStrategy strategy, MessageToParse messageToParse, boolean readMetadata, Map config) { + public RawMessage getRawMessage(RawMessageStrategy strategy, MessageToParse messageToParse, boolean readMetadata, + Map config) { Map metadata = new HashMap<>(); - if(readMetadata) { + if (readMetadata) { String prefix = MetadataUtil.INSTANCE.getMetadataPrefix(config); metadata = extractMetadata(prefix, messageToParse); } @@ -63,8 +67,10 @@ public RawMessage getRawMessage(RawMessageStrategy strategy, MessageToParse mess *

  • The tuple fields outside of the value (e.g. the topic)
    • * * - *

    In addition to extracting the metadata into a map, it applies the appropriate prefix (as configured in the rawMessageStrategyConfig). - * @param prefix The prefix of the metadata keys + *

    In addition to extracting the metadata into a map, + * it applies the appropriate prefix (as configured in the rawMessageStrategyConfig). + * + * @param prefix The prefix of the metadata keys * @param messageToParse The message read from kafka and the topic information * @return A map containing the metadata */ @@ -91,7 +97,8 @@ public Map extractMetadata(String prefix, MessageToParse message return metadata; } - private void addMetaData(String prefix, Map metadata, String envMetadataFieldName, Object envMetadataFieldValue) { + private void addMetaData(String prefix, Map metadata, String envMetadataFieldName, + Object envMetadataFieldValue) { if (!StringUtils.isEmpty(envMetadataFieldName) && envMetadataFieldValue != null) { metadata.put(MetadataUtil.INSTANCE.prefixKey(prefix, envMetadataFieldName), envMetadataFieldValue); } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/common/Constants.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/common/Constants.java index 0dd1ca95..809ede9f 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/common/Constants.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/common/Constants.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common; import java.util.HashMap; @@ -22,114 +25,92 @@ public class Constants { - public static final String ZOOKEEPER_ROOT = "/metron"; - public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology"; - public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; - public static final String SENSOR_TYPE = Fields.SENSOR_TYPE.getName(); - public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field"; - public static final String THREAT_SCORE_FIELD_PROPERTY = "threat.triage.score.field"; - public static final String ENRICHMENT_TOPIC = "enrichments"; - public static final String INDEXING_TOPIC = "indexing"; - public static final String ERROR_STREAM = "error"; - public static final String ERROR_TYPE = "error"; - public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment"; - public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel"; - public static final String STELLAR_CONTEXT_CONF = "stellarContext"; - public static final String GUID = Fields.GUID.getName(); - - /** - * The key in the global configuration that defines the global parser error topic. - * - *

    This value is used only if the error topic is left undefined in a sensor's parser configuration. - */ - public static final String PARSER_ERROR_TOPIC_GLOBALS_KEY = "parser.error.topic"; - - public interface Field { - String getName(); - } - - public enum Fields implements Field { - SRC_ADDR("ip_src_addr") - ,SRC_PORT("ip_src_port") - ,DST_ADDR("ip_dst_addr") - ,DST_PORT("ip_dst_port") - ,PROTOCOL("protocol") - ,TIMESTAMP("timestamp") - ,ORIGINAL("original_string") - ,GUID("guid") - ,SENSOR_TYPE("source.type") - ,INCLUDES_REVERSE_TRAFFIC("includes_reverse_traffic") - ; - private static Map nameToField; - - static { - nameToField = new HashMap<>(); - for (Fields f : Fields.values()) { - nameToField.put(f.getName(), f); - } + public static final String ZOOKEEPER_ROOT = "/metron"; + public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology"; + public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; + public static final String SENSOR_TYPE = Fields.SENSOR_TYPE.getName(); + public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field"; + public static final String THREAT_SCORE_FIELD_PROPERTY = "threat.triage.score.field"; + public static final String ENRICHMENT_TOPIC = "enrichments"; + public static final String INDEXING_TOPIC = "indexing"; + public static final String ERROR_STREAM = "error"; + public static final String ERROR_TYPE = "error"; + public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment"; + public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel"; + public static final String STELLAR_CONTEXT_CONF = "stellarContext"; + public static final String GUID = Fields.GUID.getName(); + + /** + * The key in the global configuration that defines the global parser error topic. + * + *

    This value is used only if the error topic is left undefined in a sensor's parser configuration. + */ + public static final String PARSER_ERROR_TOPIC_GLOBALS_KEY = "parser.error.topic"; + + public interface Field { + String getName(); } - private String name; + public enum Fields implements Field { + SRC_ADDR("ip_src_addr"), SRC_PORT("ip_src_port"), DST_ADDR("ip_dst_addr"), DST_PORT("ip_dst_port"), + PROTOCOL("protocol"), TIMESTAMP("timestamp"), ORIGINAL("original_string"), GUID("guid"), + SENSOR_TYPE("source.type"), INCLUDES_REVERSE_TRAFFIC("includes_reverse_traffic"); + private static final Map nameToField; - Fields(String name) { - this.name = name; - } + static { + nameToField = new HashMap<>(); + for (Fields f : Fields.values()) { + nameToField.put(f.getName(), f); + } + } - public String getName() { - return name; - } + private final String name; - public static Fields fromString(String fieldName) { - return nameToField.get(fieldName); - } - } - - public enum ErrorFields { - MESSAGE("message") - ,FAILED_SENSOR_TYPE("failed_sensor_type") - ,ERROR_TYPE("error_type") - ,EXCEPTION("exception") - ,STACK("stack") - ,TIMESTAMP("timestamp") - ,HOSTNAME("hostname") - ,RAW_MESSAGE("raw_message") - ,RAW_MESSAGE_BYTES("raw_message_bytes") - ,ERROR_FIELDS("error_fields") - ,ERROR_HASH("error_hash") - ,METADATA("metadata") - ; - - private String name; - - ErrorFields(String name) { - this.name = name; - } + Fields(String name) { + this.name = name; + } - public String getName() { - return name; + public String getName() { + return name; + } + + public static Fields fromString(String fieldName) { + return nameToField.get(fieldName); + } } - } - public enum ErrorType { + public enum ErrorFields { + MESSAGE("message"), FAILED_SENSOR_TYPE("failed_sensor_type"), ERROR_TYPE("error_type"), EXCEPTION("exception"), + STACK("stack"), TIMESTAMP("timestamp"), HOSTNAME("hostname"), RAW_MESSAGE("raw_message"), + RAW_MESSAGE_BYTES("raw_message_bytes"), ERROR_FIELDS("error_fields"), ERROR_HASH("error_hash"), + METADATA("metadata"); - PARSER_ERROR("parser_error") - ,PARSER_INVALID("parser_invalid") - ,ENRICHMENT_ERROR("enrichments_error") - ,THREAT_INTEL_ERROR("threatintel_error") - ,INDEXING_ERROR("indexing_error") - ,DEFAULT_ERROR("error") - ; + private final String name; - private String type; + ErrorFields(String name) { + this.name = name; + } - ErrorType(String type) { - this.type = type; + public String getName() { + return name; + } } - public String getType() { - return type; + public enum ErrorType { + + PARSER_ERROR("parser_error"), PARSER_INVALID("parser_invalid"), ENRICHMENT_ERROR("enrichments_error"), + THREAT_INTEL_ERROR("threatintel_error"), INDEXING_ERROR("indexing_error"), DEFAULT_ERROR("error"); + + private final String type; + + ErrorType(String type) { + this.type = type; + } + + public String getType() { + return type; + } } - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/BasicParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/BasicParser.java index 62f7090f..9df0b920 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/BasicParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/BasicParser.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers; import static org.apache.metron.stellar.common.Constants.Fields.DST_ADDR; @@ -26,68 +29,72 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.Charset; import java.util.Map; - import org.apache.metron.parsers.interfaces.MessageParser; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public abstract class BasicParser implements - MessageParser, - Serializable { + MessageParser, + Serializable { - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private Charset readCharset; + private Charset readCharset; - @Override - public boolean validate(JSONObject message) { - JSONObject value = message; - final String invalidMessageTemplate = "[Metron] Message does not have {}: {}"; - if (!(value.containsKey(ORIGINAL.getName()))) { - LOG.trace(invalidMessageTemplate, ORIGINAL.getName(), message); - return false; - } else if (!(value.containsKey(TIMESTAMP.getName()))) { - LOG.trace(invalidMessageTemplate, TIMESTAMP.getName(), message); - return false; - } else { - LOG.trace("[Metron] Message conforms to schema: {}", message); - return true; + @Override + public boolean validate(JSONObject message) { + JSONObject value = message; + final String invalidMessageTemplate = "[Metron] Message does not have {}: {}"; + if (!(value.containsKey(ORIGINAL.getName()))) { + LOG.trace(invalidMessageTemplate, ORIGINAL.getName(), message); + return false; + } else if (!(value.containsKey(TIMESTAMP.getName()))) { + LOG.trace(invalidMessageTemplate, TIMESTAMP.getName(), message); + return false; + } else { + LOG.trace("[Metron] Message conforms to schema: {}", message); + return true; + } } - } - public String getKey(JSONObject value) { - try { - String ipSrcAddr = null; - String ipDstAddr = null; - if (value.containsKey(SRC_ADDR.getName())) - ipSrcAddr = value.get(SRC_ADDR.getName()).toString(); - if (value.containsKey(DST_ADDR.getName())) - ipDstAddr = value.get(DST_ADDR.getName()).toString(); - if (ipSrcAddr == null && ipDstAddr == null) - return "0"; - if (ipSrcAddr == null || ipSrcAddr.length() == 0) - return ipDstAddr; - if (ipDstAddr == null || ipDstAddr.length() == 0) - return ipSrcAddr; - double ip1 = Double.parseDouble(ipSrcAddr.replace(".", "")); - double ip2 = Double.parseDouble(ipDstAddr.replace(".", "")); - return String.valueOf(ip1 + ip2); - } catch (Exception e) { - return "0"; + public String getKey(JSONObject value) { + try { + String ipSrcAddr = null; + String ipDstAddr = null; + if (value.containsKey(SRC_ADDR.getName())) { + ipSrcAddr = value.get(SRC_ADDR.getName()).toString(); + } + if (value.containsKey(DST_ADDR.getName())) { + ipDstAddr = value.get(DST_ADDR.getName()).toString(); + } + if (ipSrcAddr == null && ipDstAddr == null) { + return "0"; + } + if (ipSrcAddr == null || ipSrcAddr.length() == 0) { + return ipDstAddr; + } + if (ipDstAddr == null || ipDstAddr.length() == 0) { + return ipSrcAddr; + } + double ip1 = Double.parseDouble(ipSrcAddr.replace(".", "")); + double ip2 = Double.parseDouble(ipDstAddr.replace(".", "")); + return String.valueOf(ip1 + ip2); + } catch (Exception e) { + return "0"; + } } - } - public void setReadCharset(Map config) { - if (config.containsKey(READ_CHARSET)) { - readCharset = Charset.forName((String) config.get(READ_CHARSET)); - } else { - readCharset = MessageParser.super.getReadCharset(); + public void setReadCharset(Map config) { + if (config.containsKey(READ_CHARSET)) { + readCharset = Charset.forName((String) config.get(READ_CHARSET)); + } else { + readCharset = MessageParser.super.getReadCharset(); + } } - } - @Override - public Charset getReadCharset() { - return null == this.readCharset ? MessageParser.super.getReadCharset() : this.readCharset; - } + @Override + public Charset getReadCharset() { + return null == this.readCharset ? MessageParser.super.getReadCharset() : this.readCharset; + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java index 11d15eb8..01efad46 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java @@ -18,59 +18,58 @@ package org.apache.metron.parsers; -import org.apache.metron.parsers.interfaces.MessageParserResult; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import org.apache.metron.parsers.interfaces.MessageParserResult; public class DefaultMessageParserResult implements MessageParserResult { - private List messages = new ArrayList<>(); - private Map errors = new HashMap<>(); - private Throwable masterThrowable; + private final List messages = new ArrayList<>(); + private final Map errors = new HashMap<>(); + private Throwable masterThrowable; - public DefaultMessageParserResult() { - } + public DefaultMessageParserResult() { + } - public DefaultMessageParserResult(Throwable masterThrowable) { - this.masterThrowable = masterThrowable; - } + public DefaultMessageParserResult(Throwable masterThrowable) { + this.masterThrowable = masterThrowable; + } - public DefaultMessageParserResult(List list) { - messages.addAll(list); - } + public DefaultMessageParserResult(List list) { + messages.addAll(list); + } - public DefaultMessageParserResult(Map map) { - errors.putAll(map); - } + public DefaultMessageParserResult(Map map) { + errors.putAll(map); + } - public DefaultMessageParserResult(List list, Map map) { - messages.addAll(list); - errors.putAll(map); - } + public DefaultMessageParserResult(List list, Map map) { + messages.addAll(list); + errors.putAll(map); + } - public void addMessage(T message) { - messages.add(message); - } + public void addMessage(T message) { + messages.add(message); + } - public void addError(Object message, Throwable throwable) { - errors.put(message, throwable); - } + public void addError(Object message, Throwable throwable) { + errors.put(message, throwable); + } - @Override - public List getMessages() { - return messages; - } + @Override + public List getMessages() { + return messages; + } - @Override - public Map getMessageThrowables() { - return errors; - } + @Override + public Map getMessageThrowables() { + return errors; + } - @Override - public Optional getMasterThrowable() { - return Optional.ofNullable(masterThrowable); - } + @Override + public Optional getMasterThrowable() { + return Optional.ofNullable(masterThrowable); + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/GrokParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/GrokParser.java index 269b0258..48e92427 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/GrokParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/GrokParser.java @@ -23,16 +23,6 @@ import io.krakens.grok.api.Grok; import io.krakens.grok.api.GrokCompiler; import io.krakens.grok.api.Match; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.Path; -import org.apache.metron.common.utils.LazyLogger; -import org.apache.metron.common.utils.LazyLoggerFactory; -import org.apache.metron.parsers.interfaces.MessageParser; -import org.apache.metron.parsers.interfaces.MessageParserResult; -import org.apache.metron.stellar.common.Constants; -import org.json.simple.JSONObject; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -51,267 +41,282 @@ import java.util.Map; import java.util.Optional; import java.util.TimeZone; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.Path; +import org.apache.metron.common.utils.LazyLogger; +import org.apache.metron.common.utils.LazyLoggerFactory; +import org.apache.metron.parsers.interfaces.MessageParser; +import org.apache.metron.parsers.interfaces.MessageParserResult; +import org.apache.metron.stellar.common.Constants; +import org.json.simple.JSONObject; public class GrokParser implements MessageParser, Serializable { - protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - protected GrokCompiler grokCompiler; - protected transient Grok grok; - protected String grokPath; - protected boolean multiLine = false; - protected String patternLabel; - protected List timeFields = new ArrayList<>(); - protected String timestampField; - protected SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z"); - protected String patternsCommonDir = "/patterns/common"; - private Charset readCharset; - - @Override - @SuppressWarnings("unchecked") - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - this.grokPath = (String) parserConfig.get("grokPath"); - String multiLineString = (String) parserConfig.get("multiLine"); - if (!StringUtils.isBlank(multiLineString)) { - multiLine = Boolean.parseBoolean(multiLineString); - } - this.patternLabel = (String) parserConfig.get("patternLabel"); - this.timestampField = (String) parserConfig.get("timestampField"); - List timeFieldsParam = (List) parserConfig.get("timeFields"); - if (timeFieldsParam != null) { - this.timeFields = timeFieldsParam; - } - String dateFormatParam = (String) parserConfig.get("dateFormat"); - if (dateFormatParam != null) { - this.dateFormat = new SimpleDateFormat(dateFormatParam); - } - String timeZoneParam = (String) parserConfig.get("timeZone"); - if (timeZoneParam != null) { - dateFormat.setTimeZone(TimeZone.getTimeZone(timeZoneParam)); - LOG.debug("Grok Parser using provided TimeZone: {}", timeZoneParam); - } else { - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - LOG.debug("Grok Parser using default TimeZone (UTC)"); - } - } - - public InputStream openInputStream(String streamName) throws IOException { - Path path = new Path(streamName); - FileSystem fs = path.getFileSystem(); - if (fs.exists(path)) { - LOG.info("Loading {} from file system.", streamName); - return fs.open(path); - } else { - LOG.info("File not found in HDFS, attempting to load {} from classpath using classloader for {}.", streamName, getClass()); - return getClass().getResourceAsStream(streamName); - } - } - - @Override - public void init() { - grokCompiler = GrokCompiler.newInstance(); - grokCompiler.registerDefaultPatterns(); - try { - InputStream commonInputStream = openInputStream(patternsCommonDir); - LOG.info("Grok parser loading common patterns from: {}", patternsCommonDir); - - if (commonInputStream == null) { - throw new RuntimeException( - "Unable to initialize grok parser: Unable to load " + patternsCommonDir + " from either classpath or HDFS"); - } - - grokCompiler.register(new InputStreamReader(commonInputStream, getReadCharset())); - LOG.info("Loading parser-specific patterns from: {}", grokPath); - - InputStream patterInputStream = openInputStream(grokPath); - if (patterInputStream == null) { - throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + grokPath - + " from either classpath or HDFS"); - } - grokCompiler.register(new InputStreamReader(patterInputStream, getReadCharset())); - - if (!grokCompiler.getPatternDefinitions().containsKey(patternLabel)){ - throw new RuntimeException( - String.format("Grok parser init error: not able to find pattern [%s] in the [%s]", - patternLabel, grokPath)); - } - - LOG.info("Grok parser set the following grok expression for '{}': {}", () ->patternLabel, - () -> grokCompiler.getPatternDefinitions().get(patternLabel)); - - String grokPattern = "%{" + patternLabel + "}"; - - grok = grokCompiler.compile(grokPattern); - LOG.info("Compiled grok pattern {}", grokPattern); - - } catch (Throwable e) { - LOG.error(e.getMessage(), e); - throw new RuntimeException("Grok parser Error: " + e.getMessage(), e); + protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected GrokCompiler grokCompiler; + protected transient Grok grok; + protected String grokPath; + protected boolean multiLine = false; + protected String patternLabel; + protected List timeFields = new ArrayList<>(); + protected String timestampField; + protected SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z"); + protected String patternsCommonDir = "/patterns/common"; + private Charset readCharset; + + @Override + @SuppressWarnings("unchecked") + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + this.grokPath = (String) parserConfig.get("grokPath"); + String multiLineString = (String) parserConfig.get("multiLine"); + if (!StringUtils.isBlank(multiLineString)) { + multiLine = Boolean.parseBoolean(multiLineString); + } + this.patternLabel = (String) parserConfig.get("patternLabel"); + this.timestampField = (String) parserConfig.get("timestampField"); + List timeFieldsParam = (List) parserConfig.get("timeFields"); + if (timeFieldsParam != null) { + this.timeFields = timeFieldsParam; + } + String dateFormatParam = (String) parserConfig.get("dateFormat"); + if (dateFormatParam != null) { + this.dateFormat = new SimpleDateFormat(dateFormatParam); + } + String timeZoneParam = (String) parserConfig.get("timeZone"); + if (timeZoneParam != null) { + dateFormat.setTimeZone(TimeZone.getTimeZone(timeZoneParam)); + LOG.debug("Grok Parser using provided TimeZone: {}", timeZoneParam); + } else { + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + LOG.debug("Grok Parser using default TimeZone (UTC)"); + } } - } - @Override - public Optional> parseOptionalResult(byte[] rawMessage) { - if (grok == null) { - init(); - } - if (multiLine) { - return parseMultiLine(rawMessage); + public InputStream openInputStream(String streamName) throws IOException { + Path path = new Path(streamName); + FileSystem fs = path.getFileSystem(); + if (fs.exists(path)) { + LOG.info("Loading {} from file system.", streamName); + return fs.open(path); + } else { + LOG.info("File not found in HDFS, attempting to load {} from classpath using classloader for {}.", + streamName, getClass()); + return getClass().getResourceAsStream(streamName); + } } - return parseSingleLine(rawMessage); - } - - @SuppressWarnings("unchecked") - private Optional> parseMultiLine(byte[] rawMessage) { - List messages = new ArrayList<>(); - Map errors = new HashMap<>(); - String originalMessage = null; - // read the incoming raw data as if it may have multiple lines of logs - // if there is only only one line, it will just get processed. - try (BufferedReader reader = new BufferedReader(new StringReader(new String(rawMessage, getReadCharset())))) { - while ((originalMessage = reader.readLine()) != null) { - LOG.debug("Grok parser parsing message: {}", originalMessage); + + @Override + public void init() { + grokCompiler = GrokCompiler.newInstance(); + grokCompiler.registerDefaultPatterns(); try { - Match gm = grok.match(originalMessage); - JSONObject message = new JSONObject(); - message.putAll(gm.capture()); - - if (message.size() == 0) { - Throwable rte = new RuntimeException("Grok statement produced a null message. Original message was: " - + originalMessage + " and the parsed message was: " + message + " . Check the pattern at: " - + grokPath); - errors.put(originalMessage, rte); - continue; - } - message.put("original_string", originalMessage); - for (String timeField : timeFields) { - String fieldValue = (String) message.get(timeField); - if (fieldValue != null) { - message.put(timeField, toEpoch(fieldValue)); + InputStream commonInputStream = openInputStream(patternsCommonDir); + LOG.info("Grok parser loading common patterns from: {}", patternsCommonDir); + + if (commonInputStream == null) { + throw new RuntimeException( + "Unable to initialize grok parser: Unable to load " + patternsCommonDir + + " from either classpath or HDFS"); } - } - if (timestampField != null) { - message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField))); - } - message.remove(patternLabel); - postParse(message); - messages.add(message); - LOG.debug("Grok parser parsed message: {}", message); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - errors.put(originalMessage, e); + + grokCompiler.register(new InputStreamReader(commonInputStream, getReadCharset())); + LOG.info("Loading parser-specific patterns from: {}", grokPath); + + InputStream patterInputStream = openInputStream(grokPath); + if (patterInputStream == null) { + throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + grokPath + + " from either classpath or HDFS"); + } + grokCompiler.register(new InputStreamReader(patterInputStream, getReadCharset())); + + if (!grokCompiler.getPatternDefinitions().containsKey(patternLabel)) { + throw new RuntimeException( + String.format("Grok parser init error: not able to find pattern [%s] in the [%s]", + patternLabel, grokPath)); + } + + LOG.info("Grok parser set the following grok expression for '{}': {}", () -> patternLabel, + () -> grokCompiler.getPatternDefinitions().get(patternLabel)); + + String grokPattern = "%{" + patternLabel + "}"; + + grok = grokCompiler.compile(grokPattern); + LOG.info("Compiled grok pattern {}", grokPattern); + + } catch (Throwable e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException("Grok parser Error: " + e.getMessage(), e); } - } - } catch (IOException e) { - LOG.error(e.getMessage(), e); - Exception innerException = new IllegalStateException("Grok parser Error: " - + e.getMessage() - + " on " - + originalMessage, e); - return Optional.of(new DefaultMessageParserResult<>(innerException)); } - return Optional.of(new DefaultMessageParserResult<>(messages, errors)); - } - - @SuppressWarnings("unchecked") - private Optional> parseSingleLine(byte[] rawMessage) { - List messages = new ArrayList<>(); - Map errors = new HashMap<>(); - String originalMessage = null; - try { - originalMessage = new String(rawMessage, StandardCharsets.UTF_8); - LOG.debug("Grok parser parsing message: {}",originalMessage); - Match gm = grok.match(originalMessage); - JSONObject message = new JSONObject(); - message.putAll(gm.capture()); - - if (message.size() == 0) { - Throwable rte = new RuntimeException("Grok statement produced a null message. Original message was: " - + originalMessage + " and the parsed message was: " + message + " . Check the pattern at: " - + grokPath); - errors.put(originalMessage, rte); - } else { - message.put("original_string", originalMessage); - for (String timeField : timeFields) { - String fieldValue = (String) message.get(timeField); - if (fieldValue != null) { - message.put(timeField, toEpoch(fieldValue)); - } + + @Override + public Optional> parseOptionalResult(byte[] rawMessage) { + if (grok == null) { + init(); } - if (timestampField != null) { - message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField))); + if (multiLine) { + return parseMultiLine(rawMessage); } - message.remove(patternLabel); - postParse(message); - messages.add(message); - LOG.debug("Grok parser parsed message: {}", message); - } - } catch (Exception e) { - LOG.error(e.getMessage(), e); - Exception innerException = new IllegalStateException("Grok parser Error: " - + e.getMessage() - + " on " - + originalMessage, e); - return Optional.of(new DefaultMessageParserResult<>(innerException)); + return parseSingleLine(rawMessage); } - return Optional.of(new DefaultMessageParserResult<>(messages, errors)); - } - - @Override - public boolean validate(JSONObject message) { - LOG.debug("Grok parser validating message: {}", message); - - Object timestampObject = message.get(Constants.Fields.TIMESTAMP.getName()); - if (timestampObject instanceof Long) { - Long timestamp = (Long) timestampObject; - if (timestamp > 0) { - LOG.debug("Grok parser validated message: {}", message); - return true; - } + + @SuppressWarnings("unchecked") + private Optional> parseMultiLine(byte[] rawMessage) { + List messages = new ArrayList<>(); + Map errors = new HashMap<>(); + String originalMessage = null; + // read the incoming raw data as if it may have multiple lines of logs + // if there is only only one line, it will just get processed. + try (BufferedReader reader = new BufferedReader(new StringReader(new String(rawMessage, getReadCharset())))) { + while ((originalMessage = reader.readLine()) != null) { + LOG.debug("Grok parser parsing message: {}", originalMessage); + try { + Match gm = grok.match(originalMessage); + JSONObject message = new JSONObject(); + message.putAll(gm.capture()); + + if (message.size() == 0) { + Throwable rte = + new RuntimeException("Grok statement produced a null message. Original message was: " + + originalMessage + " and the parsed message was: " + message + + " . Check the pattern at: " + + grokPath); + errors.put(originalMessage, rte); + continue; + } + message.put("original_string", originalMessage); + for (String timeField : timeFields) { + String fieldValue = (String) message.get(timeField); + if (fieldValue != null) { + message.put(timeField, toEpoch(fieldValue)); + } + } + if (timestampField != null) { + message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField))); + } + message.remove(patternLabel); + postParse(message); + messages.add(message); + LOG.debug("Grok parser parsed message: {}", message); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + errors.put(originalMessage, e); + } + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + Exception innerException = new IllegalStateException("Grok parser Error: " + + e.getMessage() + + " on " + + originalMessage, e); + return Optional.of(new DefaultMessageParserResult<>(innerException)); + } + return Optional.of(new DefaultMessageParserResult<>(messages, errors)); } - LOG.debug("Grok parser did not validate message: {}", message); - return false; - } + @SuppressWarnings("unchecked") + private Optional> parseSingleLine(byte[] rawMessage) { + List messages = new ArrayList<>(); + Map errors = new HashMap<>(); + String originalMessage = null; + try { + originalMessage = new String(rawMessage, StandardCharsets.UTF_8); + LOG.debug("Grok parser parsing message: {}", originalMessage); + Match gm = grok.match(originalMessage); + JSONObject message = new JSONObject(); + message.putAll(gm.capture()); + + if (message.size() == 0) { + Throwable rte = new RuntimeException("Grok statement produced a null message. Original message was: " + + originalMessage + " and the parsed message was: " + message + + " . Check the pattern at: " + + grokPath); + errors.put(originalMessage, rte); + } else { + message.put("original_string", originalMessage); + for (String timeField : timeFields) { + String fieldValue = (String) message.get(timeField); + if (fieldValue != null) { + message.put(timeField, toEpoch(fieldValue)); + } + } + if (timestampField != null) { + message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField))); + } + message.remove(patternLabel); + postParse(message); + messages.add(message); + LOG.debug("Grok parser parsed message: {}", message); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + Exception innerException = new IllegalStateException("Grok parser Error: " + + e.getMessage() + + " on " + + originalMessage, e); + return Optional.of(new DefaultMessageParserResult<>(innerException)); + } + return Optional.of(new DefaultMessageParserResult<>(messages, errors)); + } - protected void postParse(JSONObject message) {} + @Override + public boolean validate(JSONObject message) { + LOG.debug("Grok parser validating message: {}", message); - protected long toEpoch(String datetime) throws ParseException { - LOG.debug("Grok parser converting timestamp to epoch: {}", datetime); - LOG.debug("Grok parser's DateFormat has TimeZone: {}", () -> dateFormat.getTimeZone()); + Object timestampObject = message.get(Constants.Fields.TIMESTAMP.getName()); + if (timestampObject instanceof Long) { + Long timestamp = (Long) timestampObject; + if (timestamp > 0) { + LOG.debug("Grok parser validated message: {}", message); + return true; + } + } - Date date = dateFormat.parse(datetime); - LOG.debug("Grok parser converted timestamp to epoch: {}", date); + LOG.debug("Grok parser did not validate message: {}", message); + return false; + } - return date.getTime(); - } + protected void postParse(JSONObject message) { + } - protected long formatTimestamp(Object value) { - LOG.debug("Grok parser formatting timestamp {}", value); + protected long toEpoch(String datetime) throws ParseException { + LOG.debug("Grok parser converting timestamp to epoch: {}", datetime); + LOG.debug("Grok parser's DateFormat has TimeZone: {}", () -> dateFormat.getTimeZone()); - if (value == null) { - throw new RuntimeException(patternLabel + " pattern does not include field " + timestampField); + Date date = dateFormat.parse(datetime); + LOG.debug("Grok parser converted timestamp to epoch: {}", date); + + return date.getTime(); } - if (value instanceof Number) { - return ((Number) value).longValue(); - } else { - return Long.parseLong(Joiner.on("").join(Splitter.on('.').split(value + ""))); + + protected long formatTimestamp(Object value) { + LOG.debug("Grok parser formatting timestamp {}", value); + + if (value == null) { + throw new RuntimeException(patternLabel + " pattern does not include field " + timestampField); + } + if (value instanceof Number) { + return ((Number) value).longValue(); + } else { + return Long.parseLong(Joiner.on("").join(Splitter.on('.').split(value + ""))); + } } - } - public void setReadCharset(Map config) { - if (config.containsKey(READ_CHARSET)) { - readCharset = Charset.forName((String) config.get(READ_CHARSET)); - } else { - readCharset = MessageParser.super.getReadCharset(); + public void setReadCharset(Map config) { + if (config.containsKey(READ_CHARSET)) { + readCharset = Charset.forName((String) config.get(READ_CHARSET)); + } else { + readCharset = MessageParser.super.getReadCharset(); + } } - } - @Override - public Charset getReadCharset() { - return null == this.readCharset ? MessageParser.super.getReadCharset() : this.readCharset; - } + @Override + public Charset getReadCharset() { + return null == this.readCharset ? MessageParser.super.getReadCharset() : this.readCharset; + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ParseException.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ParseException.java index c657696e..1b311fdc 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ParseException.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ParseException.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers; public class ParseException extends Exception { diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java index bc865e80..cfdec668 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.asa; import com.google.common.collect.ImmutableMap; @@ -22,14 +25,6 @@ import io.krakens.grok.api.GrokCompiler; import io.krakens.grok.api.Match; import io.krakens.grok.api.exception.GrokException; -import org.apache.metron.common.Constants; -import org.apache.metron.common.utils.LazyLogger; -import org.apache.metron.common.utils.LazyLoggerFactory; -import org.apache.metron.parsers.BasicParser; -import org.apache.metron.parsers.ParseException; -import org.apache.metron.parsers.utils.SyslogUtils; -import org.json.simple.JSONObject; - import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -42,200 +37,232 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import org.apache.metron.common.Constants; +import org.apache.metron.common.utils.LazyLogger; +import org.apache.metron.common.utils.LazyLoggerFactory; +import org.apache.metron.parsers.BasicParser; +import org.apache.metron.parsers.ParseException; +import org.apache.metron.parsers.utils.SyslogUtils; +import org.json.simple.JSONObject; public class BasicAsaParser extends BasicParser { - protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - protected Clock deviceClock; - private final String syslogPattern = "%{CISCO_TAGGED_SYSLOG}"; - - private GrokCompiler grokCompiler; - private Grok syslogGrok; - - private static final Map patternMap = ImmutableMap. builder() - .put("ASA-2-106001", "CISCOFW106001") - .put("ASA-2-106006", "CISCOFW106006_106007_106010") - .put("ASA-2-106007", "CISCOFW106006_106007_106010") - .put("ASA-2-106010", "CISCOFW106006_106007_106010") - .put("ASA-3-106014", "CISCOFW106014") - .put("ASA-6-106015", "CISCOFW106015") - .put("ASA-1-106021", "CISCOFW106021") - .put("ASA-4-106023", "CISCOFW106023") - .put("ASA-5-106100", "CISCOFW106100") - .put("ASA-6-110002", "CISCOFW110002") - .put("ASA-6-302010", "CISCOFW302010") - .put("ASA-6-302013", "CISCOFW302013_302014_302015_302016") - .put("ASA-6-302014", "CISCOFW302013_302014_302015_302016") - .put("ASA-6-302015", "CISCOFW302013_302014_302015_302016") - .put("ASA-6-302016", "CISCOFW302013_302014_302015_302016") - .put("ASA-6-302020", "CISCOFW302020_302021") - .put("ASA-6-302021", "CISCOFW302020_302021") - .put("ASA-6-305011", "CISCOFW305011") - .put("ASA-3-313001", "CISCOFW313001_313004_313008") - .put("ASA-3-313004", "CISCOFW313001_313004_313008") - .put("ASA-3-313008", "CISCOFW313001_313004_313008") - .put("ASA-4-313005", "CISCOFW313005") - .put("ASA-4-402117", "CISCOFW402117") - .put("ASA-4-402119", "CISCOFW402119") - .put("ASA-4-419001", "CISCOFW419001") - .put("ASA-4-419002", "CISCOFW419002") - .put("ASA-4-500004", "CISCOFW500004") - .put("ASA-6-602303", "CISCOFW602303_602304") - .put("ASA-6-602304", "CISCOFW602303_602304") - .put("ASA-7-710001", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-7-710002", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-7-710003", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-7-710005", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-7-710006", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-6-713172", "CISCOFW713172") - .put("ASA-4-733100", "CISCOFW733100") - .put("ASA-6-305012", "CISCOFW305012") - .put("ASA-7-609001", "CISCOFW609001") - .put("ASA-7-609002", "CISCOFW609002") - .put("ASA-5-713041", "CISCOFW713041") - .build(); - - private Map grokers = new HashMap(patternMap.size()); - - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - String timeZone = (String) parserConfig.get("deviceTimeZone"); - if (timeZone != null) - deviceClock = Clock.system(ZoneId.of(timeZone)); - else { - deviceClock = Clock.systemUTC(); - LOG.warn("[Metron] No device time zone provided; defaulting to UTC"); - } - } - - private void addGrok(String key, String pattern) throws GrokException, IOException { - InputStream patternStream = this.getClass().getResourceAsStream("/patterns/asa"); - grokCompiler.register(new InputStreamReader(patternStream, StandardCharsets.UTF_8)); - Grok grok = grokCompiler.compile("%{" + pattern + "}"); - grokers.put(key, grok); - } - - @Override - public void init() { - grokCompiler = GrokCompiler.newInstance(); - grokCompiler.registerDefaultPatterns(); - InputStream syslogStream = this.getClass().getResourceAsStream("/patterns/asa"); - try { - grokCompiler.register(new InputStreamReader(syslogStream, StandardCharsets.UTF_8)); - syslogGrok = grokCompiler.compile(syslogPattern); - } catch (IOException | GrokException e) { - LOG.error("[Metron] Failed to load grok patterns from jar", e); - throw new RuntimeException(e.getMessage(), e); - } - - for (Entry pattern : patternMap.entrySet()) { - try { - addGrok(pattern.getKey(), pattern.getValue()); - } catch (IOException | GrokException e) { - LOG.error("[Metron] Failed to load grok pattern {} for ASA tag {}", pattern.getValue(), pattern.getKey()); - } + protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected Clock deviceClock; + private final String syslogPattern = "%{CISCO_TAGGED_SYSLOG}"; + + private GrokCompiler grokCompiler; + private Grok syslogGrok; + + private static final Map patternMap = ImmutableMap.builder() + .put("ASA-2-106001", "CISCOFW106001") + .put("ASA-2-106006", + "CISCOFW106006_106007_106010") + .put("ASA-2-106007", + "CISCOFW106006_106007_106010") + .put("ASA-2-106010", + "CISCOFW106006_106007_106010") + .put("ASA-3-106014", "CISCOFW106014") + .put("ASA-6-106015", "CISCOFW106015") + .put("ASA-1-106021", "CISCOFW106021") + .put("ASA-4-106023", "CISCOFW106023") + .put("ASA-5-106100", "CISCOFW106100") + .put("ASA-6-110002", "CISCOFW110002") + .put("ASA-6-302010", "CISCOFW302010") + .put("ASA-6-302013", + "CISCOFW302013_302014_302015_302016") + .put("ASA-6-302014", + "CISCOFW302013_302014_302015_302016") + .put("ASA-6-302015", + "CISCOFW302013_302014_302015_302016") + .put("ASA-6-302016", + "CISCOFW302013_302014_302015_302016") + .put("ASA-6-302020", "CISCOFW302020_302021") + .put("ASA-6-302021", "CISCOFW302020_302021") + .put("ASA-6-305011", "CISCOFW305011") + .put("ASA-3-313001", + "CISCOFW313001_313004_313008") + .put("ASA-3-313004", + "CISCOFW313001_313004_313008") + .put("ASA-3-313008", + "CISCOFW313001_313004_313008") + .put("ASA-4-313005", "CISCOFW313005") + .put("ASA-4-402117", "CISCOFW402117") + .put("ASA-4-402119", "CISCOFW402119") + .put("ASA-4-419001", "CISCOFW419001") + .put("ASA-4-419002", "CISCOFW419002") + .put("ASA-4-500004", "CISCOFW500004") + .put("ASA-6-602303", "CISCOFW602303_602304") + .put("ASA-6-602304", "CISCOFW602303_602304") + .put("ASA-7-710001", + "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-7-710002", + "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-7-710003", + "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-7-710005", + "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-7-710006", + "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-6-713172", "CISCOFW713172") + .put("ASA-4-733100", "CISCOFW733100") + .put("ASA-6-305012", "CISCOFW305012") + .put("ASA-7-609001", "CISCOFW609001") + .put("ASA-7-609002", "CISCOFW609002") + .put("ASA-5-713041", "CISCOFW713041") + .build(); + + private final Map grokers = new HashMap(patternMap.size()); + + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + String timeZone = (String) parserConfig.get("deviceTimeZone"); + if (timeZone != null) { + deviceClock = Clock.system(ZoneId.of(timeZone)); + } else { + deviceClock = Clock.systemUTC(); + LOG.warn("[Metron] No device time zone provided; defaulting to UTC"); + } } - LOG.info("[Metron] CISCO ASA Parser Initialized"); - } - - @Override - public List parse(byte[] rawMessage) { - String logLine; - String messagePattern; - JSONObject metronJson = new JSONObject(); - List messages = new ArrayList<>(); - Map syslogJson; - - logLine = new String(rawMessage, StandardCharsets.UTF_8); - - try { - LOG.debug("[Metron] Started parsing raw message: {}", logLine); - Match syslogMatch = syslogGrok.match(logLine); - syslogJson = syslogMatch.capture(); - if (!syslogMatch.isNull()) { - LOG.trace("[Metron] Grok CISCO ASA syslog matches: {}", syslogMatch); - - metronJson.put(Constants.Fields.ORIGINAL.getName(), logLine); - metronJson.put(Constants.Fields.TIMESTAMP.getName(), - SyslogUtils.parseTimestampToEpochMillis((String) syslogJson.get("CISCOTIMESTAMP"), deviceClock)); - metronJson.put("ciscotag", syslogJson.get("CISCOTAG")); - metronJson.put("syslog_severity", SyslogUtils.getSeverityFromPriority((int) syslogJson.get("syslog_pri"))); - metronJson.put("syslog_facility", SyslogUtils.getFacilityFromPriority((int) syslogJson.get("syslog_pri"))); - - if (syslogJson.get("syslog_host") != null) { - metronJson.put("syslog_host", syslogJson.get("syslog_host")); - } - if (syslogJson.get("syslog_prog") != null) { - metronJson.put("syslog_prog", syslogJson.get("syslog_prog")); - } - - } else { - throw new RuntimeException( - String.format("[Metron] Message '%s' does not match pattern '%s'", logLine, syslogPattern)); - } - } catch (ParseException e) { - LOG.error("[Metron] Could not parse message timestamp", e); - throw new RuntimeException(e.getMessage(), e); - } catch (RuntimeException e) { - LOG.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); + private void addGrok(String key, String pattern) throws GrokException, IOException { + InputStream patternStream = this.getClass().getResourceAsStream("/patterns/asa"); + grokCompiler.register(new InputStreamReader(patternStream, StandardCharsets.UTF_8)); + Grok grok = grokCompiler.compile("%{" + pattern + "}"); + grokers.put(key, grok); } - try { - messagePattern = (String) syslogJson.get("CISCOTAG"); - Grok asaGrok = grokers.get(messagePattern); - - if (asaGrok == null) { - LOG.info("[Metron] No pattern for ciscotag '{}'", syslogJson.get("CISCOTAG")); - } else { - - String messageContent = (String) syslogJson.get("message"); - Match messageMatch = asaGrok.match(messageContent); - Map messageJson = messageMatch.capture(); - - if (!messageMatch.isNull()) { - LOG.trace("[Metron] Grok CISCO ASA message matches: {}", messageJson); - - String src_ip = (String) messageJson.get("src_ip"); - if (src_ip != null) - metronJson.put(Constants.Fields.SRC_ADDR.getName(), src_ip); - - Integer src_port = (Integer) messageJson.get("src_port"); - if (src_port != null) - metronJson.put(Constants.Fields.SRC_PORT.getName(), src_port); - - String dst_ip = (String) messageJson.get("dst_ip"); - if (dst_ip != null) - metronJson.put(Constants.Fields.DST_ADDR.getName(), dst_ip); + @Override + public void init() { + grokCompiler = GrokCompiler.newInstance(); + grokCompiler.registerDefaultPatterns(); + InputStream syslogStream = this.getClass().getResourceAsStream("/patterns/asa"); + try { + grokCompiler.register(new InputStreamReader(syslogStream, StandardCharsets.UTF_8)); + syslogGrok = grokCompiler.compile(syslogPattern); + } catch (IOException | GrokException e) { + LOG.error("[Metron] Failed to load grok patterns from jar", e); + throw new RuntimeException(e.getMessage(), e); + } - Integer dst_port = (Integer) messageJson.get("dst_port"); - if (dst_port != null) - metronJson.put(Constants.Fields.DST_PORT.getName(), dst_port); + for (Entry pattern : patternMap.entrySet()) { + try { + addGrok(pattern.getKey(), pattern.getValue()); + } catch (IOException | GrokException e) { + LOG.error("[Metron] Failed to load grok pattern {} for ASA tag {}", pattern.getValue(), + pattern.getKey()); + } + } - String protocol = (String) messageJson.get("protocol"); - if (protocol != null) - metronJson.put(Constants.Fields.PROTOCOL.getName(), protocol.toLowerCase()); + LOG.info("[Metron] CISCO ASA Parser Initialized"); + } - String action = (String) messageJson.get("action"); - if (action != null) - metronJson.put("action", action.toLowerCase()); - } else { - LOG.warn("[Metron] Message '{}' did not match pattern for ciscotag '{}'", logLine, - syslogJson.get("CISCOTAG")); + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") + @Override + public List parse(byte[] rawMessage) { + String logLine; + String messagePattern; + JSONObject metronJson = new JSONObject(); + List messages = new ArrayList<>(); + Map syslogJson; + + logLine = new String(rawMessage, StandardCharsets.UTF_8); + + try { + LOG.debug("[Metron] Started parsing raw message: {}", logLine); + Match syslogMatch = syslogGrok.match(logLine); + syslogJson = syslogMatch.capture(); + if (!syslogMatch.isNull()) { + LOG.trace("[Metron] Grok CISCO ASA syslog matches: {}", syslogMatch); + + metronJson.put(Constants.Fields.ORIGINAL.getName(), logLine); + metronJson.put(Constants.Fields.TIMESTAMP.getName(), + SyslogUtils.parseTimestampToEpochMillis((String) syslogJson.get("CISCOTIMESTAMP"), deviceClock)); + metronJson.put("ciscotag", syslogJson.get("CISCOTAG")); + metronJson.put("syslog_severity", + SyslogUtils.getSeverityFromPriority((int) syslogJson.get("syslog_pri"))); + metronJson.put("syslog_facility", + SyslogUtils.getFacilityFromPriority((int) syslogJson.get("syslog_pri"))); + + if (syslogJson.get("syslog_host") != null) { + metronJson.put("syslog_host", syslogJson.get("syslog_host")); + } + if (syslogJson.get("syslog_prog") != null) { + metronJson.put("syslog_prog", syslogJson.get("syslog_prog")); + } + + } else { + throw new RuntimeException( + String.format("[Metron] Message '%s' does not match pattern '%s'", logLine, syslogPattern)); + } + } catch (ParseException e) { + LOG.error("[Metron] Could not parse message timestamp", e); + throw new RuntimeException(e.getMessage(), e); + } catch (RuntimeException e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } - } - LOG.debug("[Metron] Final normalized message: {}", metronJson::toString); + try { + messagePattern = (String) syslogJson.get("CISCOTAG"); + Grok asaGrok = grokers.get(messagePattern); + + if (asaGrok == null) { + LOG.info("[Metron] No pattern for ciscotag '{}'", syslogJson.get("CISCOTAG")); + } else { + + String messageContent = (String) syslogJson.get("message"); + Match messageMatch = asaGrok.match(messageContent); + Map messageJson = messageMatch.capture(); + + if (!messageMatch.isNull()) { + LOG.trace("[Metron] Grok CISCO ASA message matches: {}", messageJson); + + String srcIp = (String) messageJson.get("src_ip"); + if (srcIp != null) { + metronJson.put(Constants.Fields.SRC_ADDR.getName(), srcIp); + } + + Integer srcPort = (Integer) messageJson.get("src_port"); + if (srcPort != null) { + metronJson.put(Constants.Fields.SRC_PORT.getName(), srcPort); + } + + String dstIp = (String) messageJson.get("dst_ip"); + if (dstIp != null) { + metronJson.put(Constants.Fields.DST_ADDR.getName(), dstIp); + } + + Integer dstPort = (Integer) messageJson.get("dst_port"); + if (dstPort != null) { + metronJson.put(Constants.Fields.DST_PORT.getName(), dstPort); + } + + String protocol = (String) messageJson.get("protocol"); + if (protocol != null) { + metronJson.put(Constants.Fields.PROTOCOL.getName(), protocol.toLowerCase()); + } + + String action = (String) messageJson.get("action"); + if (action != null) { + metronJson.put("action", action.toLowerCase()); + } + } else { + LOG.warn("[Metron] Message '{}' did not match pattern for ciscotag '{}'", logLine, + syslogJson.get("CISCOTAG")); + } + } + + LOG.debug("[Metron] Final normalized message: {}", metronJson::toString); + + } catch (RuntimeException e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } - } catch (RuntimeException e) { - LOG.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); + messages.add(metronJson); + return messages; } - - messages.add(metronJson); - return messages; - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java index 33304bd6..76bbe0e3 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,147 +36,150 @@ @SuppressWarnings("serial") public class BasicBroParser extends BasicParser { - protected static final LazyLogger _LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final ThreadLocal DECIMAL_FORMAT = new ThreadLocal() { + protected static final LazyLogger _LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final ThreadLocal DECIMAL_FORMAT = new ThreadLocal() { + @Override + protected NumberFormat initialValue() { + return new DecimalFormat("0.0#####"); + } + }; + private final JSONCleaner cleaner = new JSONCleaner(); + @Override - protected NumberFormat initialValue() { - return new DecimalFormat("0.0#####"); + public void configure(Map parserConfig) { + setReadCharset(parserConfig); } - }; - private JSONCleaner cleaner = new JSONCleaner(); - - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - } - - @Override - public void init() { - - } - @Override - @SuppressWarnings("unchecked") - public List parse(byte[] msg) { - - _LOG.trace("[Metron] Starting to parse incoming message"); - - String rawMessage = null; - List messages = new ArrayList<>(); - try { - rawMessage = new String(msg, getReadCharset()); - _LOG.trace("[Metron] Received message: {}", rawMessage); + @Override + public void init() { - JSONObject cleanedMessage = cleaner.clean(rawMessage); - _LOG.debug("[Metron] Cleaned message: {}", cleanedMessage); + } - if (cleanedMessage == null || cleanedMessage.isEmpty()) { - throw new Exception("Unable to clean message: " + rawMessage); - } + @Override + @SuppressWarnings("unchecked") + public List parse(byte[] msg) { - String key; - JSONObject payload; - if (cleanedMessage.containsKey("type")) { - key = cleanedMessage.get("type").toString(); - payload = cleanedMessage; - } else { - key = cleanedMessage.keySet().iterator().next().toString(); + _LOG.trace("[Metron] Starting to parse incoming message"); - if (key == null) { - throw new Exception("Unable to retrieve key for message: " - + rawMessage); + String rawMessage = null; + List messages = new ArrayList<>(); + try { + rawMessage = new String(msg, getReadCharset()); + _LOG.trace("[Metron] Received message: {}", rawMessage); + + JSONObject cleanedMessage = cleaner.clean(rawMessage); + _LOG.debug("[Metron] Cleaned message: {}", cleanedMessage); + + if (cleanedMessage == null || cleanedMessage.isEmpty()) { + throw new Exception("Unable to clean message: " + rawMessage); + } + + String key; + JSONObject payload; + if (cleanedMessage.containsKey("type")) { + key = cleanedMessage.get("type").toString(); + payload = cleanedMessage; + } else { + key = cleanedMessage.keySet().iterator().next().toString(); + + if (key == null) { + throw new Exception("Unable to retrieve key for message: " + + rawMessage); + } + + payload = (JSONObject) cleanedMessage.get(key); + } + + if (payload == null) { + throw new Exception("Unable to retrieve payload for message: " + + rawMessage); + } + + String originalString = key.toUpperCase() + " |"; + for (Object k : payload.keySet()) { + Object raw = payload.get(k); + String value = raw.toString(); + if (raw instanceof Double) { + value = DECIMAL_FORMAT.get().format(raw); + } + originalString += " " + k.toString() + ":" + value; + } + payload.put("original_string", originalString); + + replaceKey(payload, Constants.Fields.TIMESTAMP.getName(), new String[] {"ts"}); + + long timestamp = 0L; + if (payload.containsKey(Constants.Fields.TIMESTAMP.getName())) { + try { + Double broTimestamp = ((Number) payload.get(Constants.Fields.TIMESTAMP.getName())).doubleValue(); + String broTimestampFormatted = DECIMAL_FORMAT.get().format(broTimestamp); + timestamp = convertToMillis(broTimestamp); + payload.put(Constants.Fields.TIMESTAMP.getName(), timestamp); + payload.put("bro_timestamp", broTimestampFormatted); + _LOG.trace("[Metron] new bro record - timestamp : {}", + () -> payload.get(Constants.Fields.TIMESTAMP.getName())); + } catch (NumberFormatException nfe) { + _LOG.error("[Metron] timestamp is invalid: {}", payload.get("timestamp")); + payload.put(Constants.Fields.TIMESTAMP.getName(), 0); + } + } + + boolean ipSrcReplaced = + replaceKey(payload, Constants.Fields.SRC_ADDR.getName(), new String[] {"source_ip", "id.orig_h"}); + if (!ipSrcReplaced) { + replaceKeyArray(payload, Constants.Fields.SRC_ADDR.getName(), new String[] {"tx_hosts"}); + } + + boolean ipDstReplaced = + replaceKey(payload, Constants.Fields.DST_ADDR.getName(), new String[] {"dest_ip", "id.resp_h"}); + if (!ipDstReplaced) { + replaceKeyArray(payload, Constants.Fields.DST_ADDR.getName(), new String[] {"rx_hosts"}); + } + + replaceKey(payload, Constants.Fields.SRC_PORT.getName(), new String[] {"source_port", "id.orig_p"}); + replaceKey(payload, Constants.Fields.DST_PORT.getName(), new String[] {"dest_port", "id.resp_p"}); + + payload.put(Constants.Fields.PROTOCOL.getName(), key); + _LOG.debug("[Metron] Returning parsed message: {}", payload); + messages.add(payload); + return messages; + + } catch (Exception e) { + String message = "Unable to parse Message: " + rawMessage; + _LOG.error(message, e); + throw new IllegalStateException(message, e); } - payload = (JSONObject) cleanedMessage.get(key); - } - - if (payload == null) { - throw new Exception("Unable to retrieve payload for message: " - + rawMessage); - } - - String originalString = key.toUpperCase() + " |"; - for (Object k : payload.keySet()) { - Object raw = payload.get(k); - String value = raw.toString(); - if (raw instanceof Double) { - value = DECIMAL_FORMAT.get().format(raw); - } - originalString += " " + k.toString() + ":" + value; - } - payload.put("original_string", originalString); + } - replaceKey(payload, Constants.Fields.TIMESTAMP.getName(), new String[]{ "ts" }); + private Long convertToMillis(Double timestampSeconds) { + return ((Double) (timestampSeconds * 1000)).longValue(); + } - long timestamp = 0L; - if (payload.containsKey(Constants.Fields.TIMESTAMP.getName())) { - try { - Double broTimestamp = ((Number) payload.get(Constants.Fields.TIMESTAMP.getName())).doubleValue(); - String broTimestampFormatted = DECIMAL_FORMAT.get().format(broTimestamp); - timestamp = convertToMillis(broTimestamp); - payload.put(Constants.Fields.TIMESTAMP.getName(), timestamp); - payload.put("bro_timestamp", broTimestampFormatted); - _LOG.trace("[Metron] new bro record - timestamp : {}", () -> payload.get(Constants.Fields.TIMESTAMP.getName())); - } catch (NumberFormatException nfe) { - _LOG.error("[Metron] timestamp is invalid: {}", payload.get("timestamp")); - payload.put(Constants.Fields.TIMESTAMP.getName(), 0); + private boolean replaceKey(JSONObject payload, String toKey, String[] fromKeys) { + for (String fromKey : fromKeys) { + if (payload.containsKey(fromKey)) { + Object value = payload.remove(fromKey); + payload.put(toKey, value); + _LOG.trace("[Metron] Added {} to {}", toKey, payload); + return true; + } } - } - - boolean ipSrcReplaced = replaceKey(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{"source_ip", "id.orig_h"}); - if (!ipSrcReplaced) { - replaceKeyArray(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{ "tx_hosts" }); - } - - boolean ipDstReplaced = replaceKey(payload, Constants.Fields.DST_ADDR.getName(), new String[]{"dest_ip", "id.resp_h"}); - if (!ipDstReplaced) { - replaceKeyArray(payload, Constants.Fields.DST_ADDR.getName(), new String[]{ "rx_hosts" }); - } - - replaceKey(payload, Constants.Fields.SRC_PORT.getName(), new String[]{"source_port", "id.orig_p"}); - replaceKey(payload, Constants.Fields.DST_PORT.getName(), new String[]{"dest_port", "id.resp_p"}); - - payload.put(Constants.Fields.PROTOCOL.getName(), key); - _LOG.debug("[Metron] Returning parsed message: {}", payload); - messages.add(payload); - return messages; - - } catch (Exception e) { - String message = "Unable to parse Message: " + rawMessage; - _LOG.error(message, e); - throw new IllegalStateException(message, e); + return false; } - } - - private Long convertToMillis(Double timestampSeconds) { - return ((Double) (timestampSeconds * 1000)).longValue(); - } - - private boolean replaceKey(JSONObject payload, String toKey, String[] fromKeys) { - for (String fromKey : fromKeys) { - if (payload.containsKey(fromKey)) { - Object value = payload.remove(fromKey); - payload.put(toKey, value); - _LOG.trace("[Metron] Added {} to {}", toKey, payload); - return true; - } - } - return false; - } - - private boolean replaceKeyArray(JSONObject payload, String toKey, String[] fromKeys) { - for (String fromKey : fromKeys) { - if (payload.containsKey(fromKey)) { - JSONArray value = (JSONArray) payload.remove(fromKey); - if (value != null && !value.isEmpty()) { - payload.put(toKey, value.get(0)); - _LOG.trace("[Metron] Added {} to {}", toKey, payload); - return true; + private boolean replaceKeyArray(JSONObject payload, String toKey, String[] fromKeys) { + for (String fromKey : fromKeys) { + if (payload.containsKey(fromKey)) { + JSONArray value = (JSONArray) payload.remove(fromKey); + if (value != null && !value.isEmpty()) { + payload.put(toKey, value.get(0)); + _LOG.trace("[Metron] Added {} to {}", toKey, payload); + return true; + } + } } - } + return false; } - return false; - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java index d910d129..0907a845 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java @@ -6,86 +6,81 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.bro; import java.io.Serializable; import java.util.HashMap; import java.util.Iterator; import java.util.Map; - import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; public class JSONCleaner implements Serializable { - /** - * - */ - private static final long serialVersionUID = 1L; - - - /** - * @param jsonString - * @return - * @throws ParseException - * Takes a json String as input and modifies the keys to remove any characters other than . _ a-z A-Z or 0-9 - */ - @SuppressWarnings({"unchecked","rawtypes"}) - public JSONObject clean(String jsonString) throws ParseException - { - JSONParser parser = new JSONParser(); - - - Map json = (Map) parser.parse(jsonString); - JSONObject output = new JSONObject(); - Iterator iter = json.entrySet().iterator(); - - while(iter.hasNext()){ - Map.Entry entry = (Map.Entry)iter.next(); - - String key = ((String)entry.getKey()).replaceAll("[^\\._a-zA-Z0-9]+",""); - output.put(key, entry.getValue()); - } - - return output; - } - - - @SuppressWarnings({ "unchecked", "rawtypes", "unused" }) - public static void main(String args[]) - { - String jsonText = "{\"first_1\": 123, \"second\": [4, 5, 6], \"third\": 789}"; - JSONCleaner cleaner = new JSONCleaner(); - try { - //cleaner.clean(jsonText); - Map obj=new HashMap(); - obj.put("name","foo"); - obj.put("num", 100); - obj.put("balance", 1000.21); - obj.put("is_vip", true); - obj.put("nickname",null); - Map obj1 = new HashMap(); - obj1.put("sourcefile", obj); - - JSONObject json = new JSONObject(obj1); - System.out.println(json); - - - - System.out.print(jsonText); - } catch (Exception e) { - e.printStackTrace(); - } - } - + private static final long serialVersionUID = 1L; + + + /** + * Applies cleaning to the json. + * + * @throws ParseException Takes a json String as input and modifies the keys to remove any characters other than . _ a-z A-Z or 0-9 + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public JSONObject clean(String jsonString) throws ParseException { + JSONParser parser = new JSONParser(); + + + Map json = (Map) parser.parse(jsonString); + JSONObject output = new JSONObject(); + Iterator iter = json.entrySet().iterator(); + + while (iter.hasNext()) { + Map.Entry entry = (Map.Entry) iter.next(); + + String key = ((String) entry.getKey()).replaceAll("[^\\._a-zA-Z0-9]+", ""); + output.put(key, entry.getValue()); + } + + return output; + } + + + @SuppressWarnings({"unchecked", "rawtypes", "unused"}) + public static void main(String[] args) { + String jsonText = "{\"first_1\": 123, \"second\": [4, 5, 6], \"third\": 789}"; + JSONCleaner cleaner = new JSONCleaner(); + try { + //cleaner.clean(jsonText); + Map obj = new HashMap(); + obj.put("name", "foo"); + obj.put("num", 100); + obj.put("balance", 1000.21); + obj.put("is_vip", true); + obj.put("nickname", null); + Map obj1 = new HashMap(); + obj1.put("sourcefile", obj); + + JSONObject json = new JSONObject(obj1); + System.out.println(json); + + + System.out.print(jsonText); + } catch (Exception e) { + e.printStackTrace(); + } + } + } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/cef/CEFParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/cef/CEFParser.java index 147c7a5a..f687a0d6 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/cef/CEFParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/cef/CEFParser.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,238 +37,244 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SuppressWarnings("checkstyle:MemberName") public class CEFParser extends BasicParser { - private static final long serialVersionUID = 1L; - - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*"; - private static final String EXTENSION_CAPTURE_PATTERN = "(?"; - String syslogHost = "[a-z0-9\\.\\\\-_]+"; - - StringBuilder sb = new StringBuilder(""); - sb.append("(?"); - sb.append(syslogPriority); - sb.append(")?"); - sb.append("(?"); - sb.append(syslogTime); - sb.append("|"); - sb.append(syslogTime5424); - sb.append(")?"); - - sb.append("(?"); - sb.append(syslogHost); - sb.append(")?"); - - sb.append(".*"); - - sb.append("CEF: ?0\\|"); - - headerBlock("DeviceVendor", sb); - sb.append("\\|"); - headerBlock("DeviceProduct", sb); - sb.append("\\|"); - headerBlock("DeviceVersion", sb); - sb.append("\\|"); - headerBlock("DeviceEvent", sb); - sb.append("\\|"); - headerBlock("Name", sb); - sb.append("\\|"); - headerBlock("Severity", sb); - sb.append("\\|"); - - // extension capture: - sb.append("(?.*)"); - String pattern = sb.toString(); - - p = Pattern.compile(pattern); - - } - - public static void parseExtensions(String ext, JSONObject obj) { - Matcher m = patternExtensions.matcher(ext); - - int index = 0; - String key = null; - String value = null; - Map labelMap = new HashMap(); - - while (m.find()) { - if (key == null) { - key = ext.substring(index, m.start()); - index = m.end(); - if (!m.find()) { - break; - } - } - value = ext.substring(index, m.start()); - index = m.end(); - int v = value.lastIndexOf(" "); - if (v > 0) { - String temp = value.substring(0, v).trim(); - if (key.endsWith("Label")) { - labelMap.put(key.substring(0, key.length() - 5), temp); - } else { - obj.put(key, temp); - } - key = value.substring(v).trim(); - } - } - value = ext.substring(index); - - // Build a map of Label extensions to apply later - if (key.endsWith("Label")) { - labelMap.put(key.substring(0, key.length() - 5), value); - } else { - obj.put(key, value); - } - - // Apply the labels to custom fields - for (Entry label : labelMap.entrySet()) { - mutate(obj, label.getKey(), label.getValue()); - } - } - - @SuppressWarnings("unchecked") - public List parse(byte[] rawMessage) { - List messages = new ArrayList<>(); - - String cefString = new String(rawMessage, getReadCharset()); - - Matcher matcher = p.matcher(cefString); - - while (matcher.find()) { - JSONObject obj = new JSONObject(); - if (matcher.matches()) { - LOG.debug("Found %d groups", matcher.groupCount()); - obj.put("DeviceVendor", matcher.group("DeviceVendor")); - obj.put("DeviceProduct", matcher.group("DeviceProduct")); - obj.put("DeviceVersion", matcher.group("DeviceVersion")); - obj.put("DeviceEvent", matcher.group("DeviceEvent")); - obj.put("Name", matcher.group("Name")); - obj.put("Severity", standardizeSeverity(matcher.group("Severity"))); - } - - parseExtensions(matcher.group("extensions"), obj); - - // Rename standard CEF fields to comply with Metron standards - obj = mutate(obj, "dst", "ip_dst_addr"); - obj = mutate(obj, "dpt", "ip_dst_port"); - obj = convertToInt(obj, "ip_dst_port"); - - obj = mutate(obj, "src", "ip_src_addr"); - obj = mutate(obj, "spt", "ip_src_port"); - obj = convertToInt(obj, "ip_src_port"); - - obj = mutate(obj, "act", "deviceAction"); - // applicationProtocol - obj = mutate(obj, "app", "protocol"); - - obj.put("original_string", cefString); - - // apply timestamp from message if present, using rt, syslog - // timestamp, - // default to current system time - - if (obj.containsKey("rt")) { - String rt = (String) obj.get("rt"); - try { - obj.put("timestamp", DateUtils.parseMultiformat(rt, DateUtils.DATE_FORMATS_CEF)); - } catch (java.text.ParseException e) { - throw new IllegalStateException("rt field present in CEF but cannot be parsed", e); - } - } else { - String logTimestamp = matcher.group("syslogTime"); - if (!(logTimestamp == null || logTimestamp.isEmpty())) { - try { - obj.put("timestamp", SyslogUtils.parseTimestampToEpochMillis(logTimestamp, Clock.systemUTC())); - } catch (ParseException e) { - throw new IllegalStateException("Cannot parse syslog timestamp", e); - } - } else { - obj.put("timestamp", System.currentTimeMillis()); - } - } - - // add the host - String host = matcher.group("syslogHost"); - if (!(host == null || host.isEmpty())) { - obj.put("host", host); - } - - messages.add(obj); - } - return messages; - } - - @SuppressWarnings("unchecked") - private JSONObject convertToInt(JSONObject obj, String key) { - if (obj.containsKey(key)) { - obj.put(key, Integer.valueOf((String) obj.get(key))); - } - return obj; - } - - private void headerBlock(String name, StringBuilder sb) { - sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")"); - } - - /** - * Maps string based severity in CEF format to integer. - * - * The strings are mapped according to the CEF 23 specification, taking the - * integer value as the value of the range buckets rounded up - * - * The valid string values are: Unknown, Low, Medium, High, and Very-High. - * The valid integer values are: 0-3=Low, 4-6=Medium, 7- 8=High, and - * 9-10=Very-High. - * - * @param severity - * String or Integer - * @return Integer value mapped from the string - */ - private Integer standardizeSeverity(String severity) { - if (severity.length() < 3) { - // should be a number - return Integer.valueOf(severity); - } else { - switch (severity) { - case "Low": - return 2; - case "Medium": - return 5; - case "High": - return 8; - case "Very-High": - return 10; - default: - return 0; - } - } - } - - @Override - public void configure(Map config) { - setReadCharset(config); - } - - @SuppressWarnings("unchecked") - private static JSONObject mutate(JSONObject json, String oldKey, String newKey) { - if (json.containsKey(oldKey)) { - json.put(newKey, json.remove(oldKey)); - } - return json; - } + private static final long serialVersionUID = 1L; + + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*"; + private static final String EXTENSION_CAPTURE_PATTERN = "(?"; + String syslogHost = "[a-z0-9\\.\\\\-_]+"; + + StringBuilder sb = new StringBuilder(); + sb.append("(?"); + sb.append(syslogPriority); + sb.append(")?"); + sb.append("(?"); + sb.append(syslogTime); + sb.append("|"); + sb.append(syslogTime5424); + sb.append(")?"); + + sb.append("(?"); + sb.append(syslogHost); + sb.append(")?"); + + sb.append(".*"); + + sb.append("CEF: ?0\\|"); + + headerBlock("DeviceVendor", sb); + sb.append("\\|"); + headerBlock("DeviceProduct", sb); + sb.append("\\|"); + headerBlock("DeviceVersion", sb); + sb.append("\\|"); + headerBlock("DeviceEvent", sb); + sb.append("\\|"); + headerBlock("Name", sb); + sb.append("\\|"); + headerBlock("Severity", sb); + sb.append("\\|"); + + // extension capture: + sb.append("(?.*)"); + String pattern = sb.toString(); + + p = Pattern.compile(pattern); + + } + + public static void parseExtensions(String ext, JSONObject obj) { + Matcher m = patternExtensions.matcher(ext); + + int index = 0; + String key = null; + String value = null; + Map labelMap = new HashMap(); + + while (m.find()) { + if (key == null) { + key = ext.substring(index, m.start()); + index = m.end(); + if (!m.find()) { + break; + } + } + value = ext.substring(index, m.start()); + index = m.end(); + int v = value.lastIndexOf(" "); + if (v > 0) { + String temp = value.substring(0, v).trim(); + if (key.endsWith("Label")) { + labelMap.put(key.substring(0, key.length() - 5), temp); + } else { + obj.put(key, temp); + } + key = value.substring(v).trim(); + } + } + value = ext.substring(index); + + // Build a map of Label extensions to apply later + if (key.endsWith("Label")) { + labelMap.put(key.substring(0, key.length() - 5), value); + } else { + obj.put(key, value); + } + + // Apply the labels to custom fields + for (Entry label : labelMap.entrySet()) { + mutate(obj, label.getKey(), label.getValue()); + } + } + + @SuppressWarnings("unchecked") + public List parse(byte[] rawMessage) { + List messages = new ArrayList<>(); + + String cefString = new String(rawMessage, getReadCharset()); + + Matcher matcher = p.matcher(cefString); + + while (matcher.find()) { + JSONObject obj = new JSONObject(); + if (matcher.matches()) { + LOG.debug("Found %d groups", matcher.groupCount()); + obj.put("DeviceVendor", matcher.group("DeviceVendor")); + obj.put("DeviceProduct", matcher.group("DeviceProduct")); + obj.put("DeviceVersion", matcher.group("DeviceVersion")); + obj.put("DeviceEvent", matcher.group("DeviceEvent")); + obj.put("Name", matcher.group("Name")); + obj.put("Severity", standardizeSeverity(matcher.group("Severity"))); + } + + parseExtensions(matcher.group("extensions"), obj); + + // Rename standard CEF fields to comply with Metron standards + obj = mutate(obj, "dst", "ip_dst_addr"); + obj = mutate(obj, "dpt", "ip_dst_port"); + obj = convertToInt(obj, "ip_dst_port"); + + obj = mutate(obj, "src", "ip_src_addr"); + obj = mutate(obj, "spt", "ip_src_port"); + obj = convertToInt(obj, "ip_src_port"); + + obj = mutate(obj, "act", "deviceAction"); + // applicationProtocol + obj = mutate(obj, "app", "protocol"); + + obj.put("original_string", cefString); + + // apply timestamp from message if present, using rt, syslog + // timestamp, + // default to current system time + + if (obj.containsKey("rt")) { + String rt = (String) obj.get("rt"); + try { + obj.put("timestamp", DateUtils.parseMultiformat(rt, DateUtils.DATE_FORMATS_CEF)); + } catch (java.text.ParseException e) { + throw new IllegalStateException("rt field present in CEF but cannot be parsed", e); + } + } else { + String logTimestamp = matcher.group("syslogTime"); + if (!(logTimestamp == null || logTimestamp.isEmpty())) { + try { + obj.put("timestamp", SyslogUtils.parseTimestampToEpochMillis(logTimestamp, Clock.systemUTC())); + } catch (ParseException e) { + throw new IllegalStateException("Cannot parse syslog timestamp", e); + } + } else { + obj.put("timestamp", System.currentTimeMillis()); + } + } + + // add the host + String host = matcher.group("syslogHost"); + if (!(host == null || host.isEmpty())) { + obj.put("host", host); + } + + messages.add(obj); + } + return messages; + } + + @SuppressWarnings("unchecked") + private JSONObject convertToInt(JSONObject obj, String key) { + if (obj.containsKey(key)) { + obj.put(key, Integer.valueOf((String) obj.get(key))); + } + return obj; + } + + private void headerBlock(String name, StringBuilder sb) { + sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")"); + } + + /** + * Maps string based severity in CEF format to integer. + * + *

    + * The strings are mapped according to the CEF 23 specification, taking the + * integer value as the value of the range buckets rounded up + * + *

    + * The valid string values are: Unknown, Low, Medium, High, and Very-High. + * The valid integer values are: 0-3=Low, 4-6=Medium, 7- 8=High, and + * 9-10=Very-High. + * + * @param severity String or Integer + * @return Integer value mapped from the string + */ + private Integer standardizeSeverity(String severity) { + if (severity.length() < 3) { + // should be a number + return Integer.valueOf(severity); + } else { + switch (severity) { + case "Low": + return 2; + case "Medium": + return 5; + case "High": + return 8; + case "Very-High": + return 10; + default: + return 0; + } + } + } + + @Override + public void configure(Map config) { + setReadCharset(config); + } + + @SuppressWarnings("unchecked") + private static JSONObject mutate(JSONObject json, String oldKey, String newKey) { + if (json.containsKey(oldKey)) { + json.put(newKey, json.remove(oldKey)); + } + return json; + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/csv/CSVParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/csv/CSVParser.java index 88567d64..fbcbe891 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/csv/CSVParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/csv/CSVParser.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,66 +34,62 @@ import org.slf4j.LoggerFactory; public class CSVParser extends BasicParser { - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String TIMESTAMP_FORMAT_CONF = "timestampFormat"; - private transient CSVConverter converter; - private SimpleDateFormat timestampFormat; + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String TIMESTAMP_FORMAT_CONF = "timestampFormat"; + private transient CSVConverter converter; + private SimpleDateFormat timestampFormat; - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - converter = new CSVConverter(); - converter.initialize(parserConfig); - Object tsFormatObj = parserConfig.get(TIMESTAMP_FORMAT_CONF); - if(tsFormatObj != null) { - timestampFormat = new SimpleDateFormat(tsFormatObj.toString()); + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + converter = new CSVConverter(); + converter.initialize(parserConfig); + Object tsFormatObj = parserConfig.get(TIMESTAMP_FORMAT_CONF); + if (tsFormatObj != null) { + timestampFormat = new SimpleDateFormat(tsFormatObj.toString()); + } } - } - @Override - public void init() { + @Override + public void init() { - } + } - @Override - public List parse(byte[] rawMessage) { - try { - String msg = new String(rawMessage, getReadCharset()); - Map value = converter.toMap(msg); - if(value != null) { - value.put("original_string", msg); - Object timestampObj = value.get("timestamp"); - Long timestamp = null; - if(timestampObj == null) { - timestamp = System.currentTimeMillis(); - } - else { - if(timestampFormat == null) { - timestamp = ConversionUtils.convert(timestampObj, Long.class); - } - else { - try { - timestamp = timestampFormat.parse(timestampObj.toString()).getTime(); + @Override + public List parse(byte[] rawMessage) { + try { + String msg = new String(rawMessage, getReadCharset()); + Map value = converter.toMap(msg); + if (value != null) { + value.put("original_string", msg); + Object timestampObj = value.get("timestamp"); + Long timestamp = null; + if (timestampObj == null) { + timestamp = System.currentTimeMillis(); + } else { + if (timestampFormat == null) { + timestamp = ConversionUtils.convert(timestampObj, Long.class); + } else { + try { + timestamp = timestampFormat.parse(timestampObj.toString()).getTime(); + } catch (Exception e) { + LOG.error("Unable to format {}", timestampObj); + } + } + } + JSONObject jsonVal = new JSONObject(value); + if (timestamp != null) { + jsonVal.put("timestamp", timestamp); + } + return ImmutableList.of(jsonVal); + } else { + return Collections.emptyList(); } - catch(Exception e) { - LOG.error("Unable to format {}", timestampObj.toString()); - } - } - } - JSONObject jsonVal = new JSONObject(value); - if(timestamp != null) { - jsonVal.put("timestamp", timestamp); + } catch (Throwable e) { + String message = "Unable to parse " + new String(rawMessage, getReadCharset()) + ": " + e.getMessage(); + LOG.error(message, e); + throw new IllegalStateException(message, e); } - return ImmutableList.of(jsonVal); - } - else { - return Collections.emptyList(); - } - } catch (Throwable e) { - String message = "Unable to parse " + new String(rawMessage, getReadCharset()) + ": " + e.getMessage(); - LOG.error(message, e); - throw new IllegalStateException(message, e); } - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/AbstractMessageFilter.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/AbstractMessageFilter.java index 8fa2af9a..2c4ef189 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/AbstractMessageFilter.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/AbstractMessageFilter.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,13 +20,12 @@ package org.apache.metron.parsers.filters; -import org.apache.metron.parsers.interfaces.MessageFilter; -import org.json.simple.JSONObject; - import java.io.Serializable; import java.util.Map; +import org.apache.metron.parsers.interfaces.MessageFilter; +import org.json.simple.JSONObject; -public abstract class AbstractMessageFilter implements MessageFilter, Serializable{ - public AbstractMessageFilter(Map config) { - } +public abstract class AbstractMessageFilter implements MessageFilter, Serializable { + public AbstractMessageFilter(Map config) { + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java index 1fa1feb8..f10fbb76 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java @@ -7,68 +7,72 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.parsers.filters; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.parsers.interfaces.MessageFilter; -import org.json.simple.JSONObject; +package org.apache.metron.parsers.filters; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.metron.parsers.interfaces.MessageFilter; +import org.apache.metron.stellar.dsl.Context; +import org.json.simple.JSONObject; -public class BroMessageFilter implements MessageFilter{ - - /** - * Filter protocols based on whitelists and blacklists - */ +@SuppressWarnings("checkstyle:MemberName") +public class BroMessageFilter implements MessageFilter { - private static final long serialVersionUID = -3824683649114625033L; - private String _key; - private final Set _known_protocols; + /** + * Filter protocols based on whitelists and blacklists. + */ - public BroMessageFilter() { - _known_protocols = new HashSet<>(); - } + private static final long serialVersionUID = -3824683649114625033L; + private String _key; + private final Set _known_protocols; - @Override - public void configure(Map config) { - Object protocolsObj = config.get("bro.filter.source.known.protocols"); - Object keyObj = config.get("bro.filter.source.key"); - if(keyObj != null) { - _key = keyObj.toString(); + public BroMessageFilter() { + _known_protocols = new HashSet<>(); } - if(protocolsObj != null) { - if(protocolsObj instanceof String) { - _known_protocols.clear(); - _known_protocols.add(protocolsObj.toString()); - } - else if(protocolsObj instanceof List) { - _known_protocols.clear(); - for(Object o : (List)protocolsObj) { - _known_protocols.add(o.toString()); + + @Override + public void configure(Map config) { + Object protocolsObj = config.get("bro.filter.source.known.protocols"); + Object keyObj = config.get("bro.filter.source.key"); + if (keyObj != null) { + _key = keyObj.toString(); + } + if (protocolsObj != null) { + if (protocolsObj instanceof String) { + _known_protocols.clear(); + _known_protocols.add(protocolsObj.toString()); + } else if (protocolsObj instanceof List) { + _known_protocols.clear(); + for (Object o : (List) protocolsObj) { + _known_protocols.add(o.toString()); + } + } } - } } - } - /** - * @param message JSON representation of a message with a protocol field - * @return False if message if filtered and True if message is not filtered - */ + /** + * Checks if message was filtered or not. + * + * @param message JSON representation of a message with a protocol field + * @return False if message if filtered and True if message is not filtered + */ - @Override - public boolean emit(JSONObject message, Context context) { - String protocol = (String) message.get(_key); - return _known_protocols.contains(protocol); - } + @Override + public boolean emit(JSONObject message, Context context) { + String protocol = (String) message.get(_key); + return _known_protocols.contains(protocol); + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/Filters.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/Filters.java index 03a4098e..d7e3d896 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/Filters.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/Filters.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,42 +20,39 @@ package org.apache.metron.parsers.filters; +import java.util.Map; import org.apache.metron.common.utils.ReflectionUtils; import org.apache.metron.parsers.interfaces.MessageFilter; import org.json.simple.JSONObject; -import java.util.Map; - public enum Filters { - BRO(BroMessageFilter.class) - ,STELLAR(StellarFilter.class) - ,DEFAULT(null) - ; - Class clazz; - Filters(Class clazz) { - this.clazz = clazz; - } - public static MessageFilter get(String filterName, Map config) { - if(filterName == null || filterName.trim().isEmpty()) { - return null; - } - Class filterClass; - try { - Filters f = Filters.valueOf(filterName); - filterClass = f.clazz; - } - catch(Exception ex) { - try { - filterClass = (Class) Class.forName(filterName); - } catch (ClassNotFoundException e) { - throw new IllegalStateException("Unable to find class " + filterName, e); - } + BRO(BroMessageFilter.class), STELLAR(StellarFilter.class), DEFAULT(null); + Class clazz; + + Filters(Class clazz) { + this.clazz = clazz; } - if(filterClass != null) { - MessageFilter filter = ReflectionUtils.createInstance(filterClass); - filter.configure(config); - return filter; + + public static MessageFilter get(String filterName, Map config) { + if (filterName == null || filterName.trim().isEmpty()) { + return null; + } + Class filterClass; + try { + Filters f = Filters.valueOf(filterName); + filterClass = f.clazz; + } catch (Exception ex) { + try { + filterClass = (Class) Class.forName(filterName); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Unable to find class " + filterName, e); + } + } + if (filterClass != null) { + MessageFilter filter = ReflectionUtils.createInstance(filterClass); + filter.configure(config); + return filter; + } + return null; } - return null; - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java index 8300ff40..9795a05d 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,44 +20,42 @@ package org.apache.metron.parsers.filters; +import java.util.Map; +import org.apache.metron.parsers.interfaces.MessageFilter; +import org.apache.metron.stellar.common.StellarPredicateProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.stellar.common.StellarPredicateProcessor; -import org.apache.metron.parsers.interfaces.MessageFilter; import org.json.simple.JSONObject; -import java.util.Map; - public class StellarFilter implements MessageFilter { - public static final String QUERY_STRING_CONF = "filter.query"; - private StellarPredicateProcessor processor = new StellarPredicateProcessor(); - private String query; - private FunctionResolver functionResolver = StellarFunctions.FUNCTION_RESOLVER(); - - public StellarFilter() - { + public static final String QUERY_STRING_CONF = "filter.query"; + private final StellarPredicateProcessor processor = new StellarPredicateProcessor(); + private String query; + private final FunctionResolver functionResolver = StellarFunctions.FUNCTION_RESOLVER(); - } + public StellarFilter() { - @Override - public void configure(Map config) { - Object o = config.get(QUERY_STRING_CONF); - if(o instanceof String) { - query= o.toString(); } - Context stellarContext = (Context) config.get("stellarContext"); - if(stellarContext == null) { - stellarContext = Context.EMPTY_CONTEXT(); + + @Override + public void configure(Map config) { + Object o = config.get(QUERY_STRING_CONF); + if (o instanceof String) { + query = o.toString(); + } + Context stellarContext = (Context) config.get("stellarContext"); + if (stellarContext == null) { + stellarContext = Context.EMPTY_CONTEXT(); + } + processor.validate(query, true, stellarContext); } - processor.validate(query, true, stellarContext); - } - @Override - public boolean emit(JSONObject message, Context context) { - VariableResolver resolver = new MapVariableResolver(message); - return processor.parse(query, resolver, functionResolver, context); - } + @Override + public boolean emit(JSONObject message, Context context) { + VariableResolver resolver = new MapVariableResolver(message); + return processor.parse(query, resolver, functionResolver, context); + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java index 4316836f..5880c9b6 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java @@ -38,156 +38,157 @@ import org.slf4j.LoggerFactory; - public class BasicFireEyeParser extends BasicParser { - private static final long serialVersionUID = 6328907550159134550L; - protected static final Logger LOG = LoggerFactory + private static final long serialVersionUID = 6328907550159134550L; + protected static final Logger LOG = LoggerFactory .getLogger(MethodHandles.lookup().lookupClass()); - private static final String tsRegex = "([a-zA-Z]{3})\\s+(\\d+)\\s+(\\d+\\:\\d+\\:\\d+)" - + "\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+)"; - private static final Pattern tsPattern = Pattern.compile(tsRegex); - private static final String syslogPriorityRegex = "<[1-9][0-9]*>"; - private static final Pattern syslogPriorityPattern = Pattern.compile(syslogPriorityRegex); - private static final String nvRegex = "([\\w\\d]+)=([^=]*)(?=\\s*\\w+=|\\s*$) "; - private static final Pattern nvPattern = Pattern.compile(nvRegex); - - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - } - - @Override - public void init() {} - - - @Override - @SuppressWarnings("unchecked") - public List parse(byte[] rawMessage) { - String toParse; - List messages = new ArrayList<>(); - try { - - toParse = new String(rawMessage, getReadCharset()); + private static final String tsRegex = "([a-zA-Z]{3})\\s+(\\d+)\\s+(\\d+\\:\\d+\\:\\d+)" + + "\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+)"; + private static final Pattern tsPattern = Pattern.compile(tsRegex); + private static final String syslogPriorityRegex = "<[1-9][0-9]*>"; + private static final Pattern syslogPriorityPattern = Pattern.compile(syslogPriorityRegex); + private static final String nvRegex = "([\\w\\d]+)=([^=]*)(?=\\s*\\w+=|\\s*$) "; + private static final Pattern nvPattern = Pattern.compile(nvRegex); - // because we support what is basically a malformed syslog 3164 message having - // some form of text before the PRIORITY, we need to use the priority as - // a delimiter - Matcher m = syslogPriorityPattern.matcher(toParse); + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + } - String delimiter = ""; + @Override + public void init() { + } - while (m.find()) { - delimiter = m.group(); - } - if (!StringUtils.isBlank(delimiter)) { - String[] tokens = toParse.split(delimiter); - if (tokens.length > 1) { - toParse = delimiter + tokens[1]; + @Override + @SuppressWarnings("unchecked") + public List parse(byte[] rawMessage) { + String toParse; + List messages = new ArrayList<>(); + try { + + toParse = new String(rawMessage, getReadCharset()); + + // because we support what is basically a malformed syslog 3164 message having + // some form of text before the PRIORITY, we need to use the priority as + // a delimiter + Matcher m = syslogPriorityPattern.matcher(toParse); + + String delimiter = ""; + + while (m.find()) { + delimiter = m.group(); + } + + if (!StringUtils.isBlank(delimiter)) { + String[] tokens = toParse.split(delimiter); + if (tokens.length > 1) { + toParse = delimiter + tokens[1]; + } + } + + // parse the main message + JSONObject toReturn = parseMessage(toParse); + toReturn.put("timestamp", getTimeStamp(toParse)); + messages.add(toReturn); + return messages; + } catch (Exception e) { + String message = + "Unable to parse " + new String(rawMessage, StandardCharsets.UTF_8) + ": " + e.getMessage(); + LOG.error(message, e); + throw new IllegalStateException(message, e); } - } - - // parse the main message - JSONObject toReturn = parseMessage(toParse); - toReturn.put("timestamp", getTimeStamp(toParse)); - messages.add(toReturn); - return messages; - } catch (Exception e) { - String message = "Unable to parse " + new String(rawMessage, StandardCharsets.UTF_8) + ": " + e.getMessage(); - LOG.error(message, e); - throw new IllegalStateException(message, e); } - } - - private long getTimeStamp(String toParse) throws ParseException { - long timestamp = 0; - String month; - String day; - String time; - Matcher tsMatcher = tsPattern.matcher(toParse); - if (tsMatcher.find()) { - month = tsMatcher.group(1); - day = tsMatcher.group(2); - time = tsMatcher.group(3); - timestamp = ParserUtils.convertToEpoch(month, day, time, true); - } else { - LOG.warn("Unable to find timestamp in message: {}", toParse); + + private long getTimeStamp(String toParse) throws ParseException { + long timestamp = 0; + String month; + String day; + String time; + Matcher tsMatcher = tsPattern.matcher(toParse); + if (tsMatcher.find()) { + month = tsMatcher.group(1); + day = tsMatcher.group(2); + time = tsMatcher.group(3); + timestamp = ParserUtils.convertToEpoch(month, day, time, true); + } else { + LOG.warn("Unable to find timestamp in message: {}", toParse); + } + return timestamp; } - return timestamp; - } - @SuppressWarnings("unchecked") - private JSONObject parseMessage(String toParse) { + @SuppressWarnings("unchecked") + private JSONObject parseMessage(String toParse) { - JSONObject toReturn = new JSONObject(); - String[] messageTokens = toParse.split("\\s+"); - String id = messageTokens[4]; + JSONObject toReturn = new JSONObject(); + String[] messageTokens = toParse.split("\\s+"); + String id = messageTokens[4]; - // We are not parsing the fedata for multi part message as we cannot - // determine how we can split the message and how many multi part - // messages can there be. - // The message itself will be stored in the response. + // We are not parsing the fedata for multi part message as we cannot + // determine how we can split the message and how many multi part + // messages can there be. + // The message itself will be stored in the response. - String[] tokens = id.split("\\."); - if (tokens.length == 2) { + String[] tokens = id.split("\\."); + if (tokens.length == 2) { - String[] array = Arrays.copyOfRange(messageTokens, 1, messageTokens.length - 1); - String syslog = Joiner.on(" ").join(array); + String[] array = Arrays.copyOfRange(messageTokens, 1, messageTokens.length - 1); + String syslog = Joiner.on(" ").join(array); - Multimap multiMap = formatMain(syslog); + Multimap multiMap = formatMain(syslog); - for (String key : multiMap.keySet()) { - String value = Joiner.on(",").join(multiMap.get(key)); - toReturn.put(key, value.trim()); - } - } + for (String key : multiMap.keySet()) { + String value = Joiner.on(",").join(multiMap.get(key)); + toReturn.put(key, value.trim()); + } + } - toReturn.put("original_string", toParse); + toReturn.put("original_string", toParse); - final String ipSrcAddr = (String) toReturn.get("dvc"); - final String ipSrcPort = (String) toReturn.get("src_port"); - final String ipDstDddr = (String) toReturn.get("dst_ip"); - final String ipDstPort = (String) toReturn.get("dst_port"); + final String ipSrcAddr = (String) toReturn.get("dvc"); + final String ipSrcPort = (String) toReturn.get("src_port"); + final String ipDstDddr = (String) toReturn.get("dst_ip"); + final String ipDstPort = (String) toReturn.get("dst_port"); - if (ipSrcAddr != null) { - toReturn.put("ip_src_addr", ipSrcAddr); - } - if (ipSrcPort != null) { - toReturn.put("ip_src_port", ipSrcPort); - } - if (ipDstDddr != null) { - toReturn.put("ip_dst_addr", ipDstDddr); - } - if (ipDstPort != null) { - toReturn.put("ip_dst_port", ipDstPort); + if (ipSrcAddr != null) { + toReturn.put("ip_src_addr", ipSrcAddr); + } + if (ipSrcPort != null) { + toReturn.put("ip_src_port", ipSrcPort); + } + if (ipDstDddr != null) { + toReturn.put("ip_dst_addr", ipDstDddr); + } + if (ipDstPort != null) { + toReturn.put("ip_dst_port", ipDstPort); + } + return toReturn; } - return toReturn; - } - - private Multimap formatMain(String in) { - Multimap multiMap = ArrayListMultimap.create(); - String input = in.replaceAll("cn3", "dst_port") - .replaceAll("cs5", "cncHost").replaceAll("proto", "protocol") - .replaceAll("rt=", "timestamp=").replaceAll("cs1", "malware") - .replaceAll("dst=", "dst_ip=") - .replaceAll("shost", "src_hostname") - .replaceAll("dmac", "dst_mac").replaceAll("smac", "src_mac") - .replaceAll("spt", "src_port") - .replaceAll("\\bsrc\\b", "src_ip"); - String[] tokens = input.split("\\|"); - - if (tokens.length > 0) { - String message = tokens[tokens.length - 1]; - Matcher m = nvPattern.matcher(message); - - while (m.find()) { - String[] str = m.group().split("="); - multiMap.put(str[0], str[1]); - } + + private Multimap formatMain(String in) { + Multimap multiMap = ArrayListMultimap.create(); + String input = in.replaceAll("cn3", "dst_port") + .replaceAll("cs5", "cncHost").replaceAll("proto", "protocol") + .replaceAll("rt=", "timestamp=").replaceAll("cs1", "malware") + .replaceAll("dst=", "dst_ip=") + .replaceAll("shost", "src_hostname") + .replaceAll("dmac", "dst_mac").replaceAll("smac", "src_mac") + .replaceAll("spt", "src_port") + .replaceAll("\\bsrc\\b", "src_ip"); + String[] tokens = input.split("\\|"); + + if (tokens.length > 0) { + String message = tokens[tokens.length - 1]; + Matcher m = nvPattern.matcher(message); + + while (m.find()) { + String[] str = m.group().split("="); + multiMap.put(str[0], str[1]); + } + } + return multiMap; } - return multiMap; - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/Configurable.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/Configurable.java index 4a707e69..88f07a95 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/Configurable.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/Configurable.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,5 +24,5 @@ import java.util.Map; public interface Configurable extends Serializable { - void configure(Map config); + void configure(Map config); } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java index 207c0707..900a6194 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java @@ -7,19 +7,22 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.interfaces; import org.apache.metron.stellar.dsl.Context; -public interface MessageFilter extends Configurable{ +public interface MessageFilter extends Configurable { - boolean emit(T message, Context context); + boolean emit(T message, Context context); } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java index 50e9f15e..0f07a96a 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.interfaces; import java.nio.charset.Charset; @@ -26,66 +29,70 @@ public interface MessageParser extends Configurable { - String READ_CHARSET = "readCharset"; // property to use for getting the read charset from parser config + String READ_CHARSET = "readCharset"; // property to use for getting the read charset from parser config - /** - * Initialize the message parser. This is done once. - */ - void init(); + /** + * Initialize the message parser. This is done once. + */ + void init(); - /** - * Take raw data and convert it to a list of messages. - * - * @param rawMessage the raw bytes of the message - * @return If null is returned, this is treated as an empty list. - */ - @Deprecated - default List parse(byte[] rawMessage) { - throw new NotImplementedException("parse is not implemented"); - } + /** + * Take raw data and convert it to a list of messages. + * + * @param rawMessage the raw bytes of the message + * @return If null is returned, this is treated as an empty list. + */ + @Deprecated + default List parse(byte[] rawMessage) { + throw new NotImplementedException("parse is not implemented"); + } - /** - * Take raw data and convert it to an optional list of messages. - * @param parseMessage the raw bytes of the message - * @return If null is returned, this is treated as an empty list. - */ - @Deprecated - default Optional> parseOptional(byte[] parseMessage) { - return Optional.ofNullable(parse(parseMessage)); - } + /** + * Take raw data and convert it to an optional list of messages. + * + * @param parseMessage the raw bytes of the message + * @return If null is returned, this is treated as an empty list. + */ + @Deprecated + default Optional> parseOptional(byte[] parseMessage) { + return Optional.ofNullable(parse(parseMessage)); + } - /** - * Take raw data and convert it to messages. Each raw message may produce multiple messages and therefore - * multiple errors. A {@link MessageParserResult} is returned, which will have both the messages produced - * and the errors. - * @param parseMessage the raw bytes of the message - * @return Optional of {@link MessageParserResult} - */ - default Optional> parseOptionalResult(byte[] parseMessage) { - Optional> result = Optional.empty(); - try { - Optional> optionalMessages = parseOptional(parseMessage); - if (optionalMessages.isPresent()) { - result = Optional.of(new DefaultMessageParserResult<>(optionalMessages.get())); - } - } catch (Throwable t) { - return Optional.of(new DefaultMessageParserResult<>(t)); + /** + * Take raw data and convert it to messages. Each raw message may produce multiple messages and therefore + * multiple errors. A {@link MessageParserResult} is returned, which will have both the messages produced + * and the errors. + * + * @param parseMessage the raw bytes of the message + * @return Optional of {@link MessageParserResult} + */ + default Optional> parseOptionalResult(byte[] parseMessage) { + Optional> result = Optional.empty(); + try { + Optional> optionalMessages = parseOptional(parseMessage); + if (optionalMessages.isPresent()) { + result = Optional.of(new DefaultMessageParserResult<>(optionalMessages.get())); + } + } catch (Throwable t) { + return Optional.of(new DefaultMessageParserResult<>(t)); + } + return result; } - return result; - } - /** - * Validate the message to ensure that it's correct. - * @param message the message to validate - * @return true if the message is valid, false if not - */ - boolean validate(T message); + /** + * Validate the message to ensure that it's correct. + * + * @param message the message to validate + * @return true if the message is valid, false if not + */ + boolean validate(T message); - /** - * Provides a hook to override the default charset parsers use to read data. - * @return Charset to use for for reading - */ - default Charset getReadCharset() { - return StandardCharsets.UTF_8; - } + /** + * Provides a hook to override the default charset parsers use to read data. + * + * @return Charset to use for for reading + */ + default Charset getReadCharset() { + return StandardCharsets.UTF_8; + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java index 891e94ff..b894c8ef 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java @@ -24,25 +24,28 @@ /** * Result object MessageParser calls. - * @param + * */ public interface MessageParserResult { - /** - * Returns the Message objects of {@code T} - * @return {@code List} - */ - List getMessages(); + /** + * Returns the Message objects of {@code T}. + * + * @return {@code List} + */ + List getMessages(); - /** - * Returns a map of raw message objects to the {@code Throwable} they triggered. - * @return {@code Map} - */ - Map getMessageThrowables(); + /** + * Returns a map of raw message objects to the {@code Throwable} they triggered. + * + * @return {@code Map} + */ + Map getMessageThrowables(); - /** - * Returns a master {@code Throwable} for a parse call. This represents a complete - * call failure, as opposed to one associated with a message. - * @return {@code Optional}{@code Throwable} - */ - Optional getMasterThrowable(); + /** + * Returns a master {@code Throwable} for a parse call. This represents a complete + * call failure, as opposed to one associated with a message. + * + * @return {@code Optional}{@code Throwable} + */ + Optional getMasterThrowable(); } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java index 2fcc795b..e082e4e9 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,72 +31,75 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@SuppressWarnings("serial") public class BasicIseParser extends BasicParser { - private static final Logger _LOG = LoggerFactory - .getLogger(BasicIseParser.class); - static final transient ISEParser _parser = new ISEParser("header="); - - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - } - - @Override - public void init() { - - } - - @SuppressWarnings("unchecked") - @Override - public List parse(byte[] msg) { - - String raw_message = ""; - List messages = new ArrayList<>(); - try { - - raw_message = new String(msg, getReadCharset()); - _LOG.debug("Received message: {}", raw_message); - - /* - * Reinitialize Parser. It has the effect of calling the constructor again. - */ - _parser.ReInit(new StringReader("header=" + raw_message.trim())); - - JSONObject payload = _parser.parseObject(); - - String ip_src_addr = (String) payload.get("Device IP Address"); - String ip_src_port = (String) payload.get("Device Port"); - String ip_dst_addr = (String) payload.get("DestinationIPAddress"); - String ip_dst_port = (String) payload.get("DestinationPort"); - - /* - * Standard Fields for Metron. - */ - - if(ip_src_addr != null) - payload.put("ip_src_addr", ip_src_addr); - if(ip_src_port != null) - payload.put("ip_src_port", ip_src_port); - if(ip_dst_addr != null) - payload.put("ip_dst_addr", ip_dst_addr); - if(ip_dst_port != null) - payload.put("ip_dst_port", ip_dst_port); - messages.add(payload); - return messages; - - } catch (Exception e) { - Log.error(e.toString()); - e.printStackTrace(); - } - return null; - } - - @Override - public boolean validate(JSONObject message) { - return true; - } - - + private static final Logger _LOG = LoggerFactory + .getLogger(BasicIseParser.class); + static final ISEParser _parser = new ISEParser("header="); + + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + } + + @Override + public void init() { + + } + + @SuppressWarnings({"unchecked", "checkstyle:VariableDeclarationUsageDistance"}) + @Override + public List parse(byte[] msg) { + + String rawMessage = ""; + List messages = new ArrayList<>(); + try { + + rawMessage = new String(msg, getReadCharset()); + _LOG.debug("Received message: {}", rawMessage); + + /* + * Reinitialize Parser. It has the effect of calling the constructor again. + */ + _parser.ReInit(new StringReader("header=" + rawMessage.trim())); + + JSONObject payload = _parser.parseObject(); + + String ipSrcAddr = (String) payload.get("Device IP Address"); + String ipSrcPort = (String) payload.get("Device Port"); + String ipDstAddr = (String) payload.get("DestinationIPAddress"); + String ipDstPort = (String) payload.get("DestinationPort"); + + /* + * Standard Fields for Metron. + */ + + if (ipSrcAddr != null) { + payload.put("ip_src_addr", ipSrcAddr); + } + if (ipSrcPort != null) { + payload.put("ip_src_port", ipSrcPort); + } + if (ipDstAddr != null) { + payload.put("ip_dst_addr", ipDstAddr); + } + if (ipDstPort != null) { + payload.put("ip_dst_port", ipDstPort); + } + messages.add(payload); + return messages; + + } catch (Exception e) { + Log.error(e.toString()); + e.printStackTrace(); + } + return null; + } + + @Override + public boolean validate(JSONObject message) { + return true; + } + + } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParser.java index 0f542615..e16a1f21 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,645 +18,841 @@ * limitations under the License. */ /* Generated By:JavaCC: Do not edit this line. ISEParser.java */ + package org.apache.metron.parsers.ise; -import java.io.*; -import java.util.*; -import org.json.simple.*; + +import java.io.Serializable; +import java.io.StringReader; +import java.util.Map; +import org.json.simple.JSONObject; /** -* Basic ISE data parser generated by JavaCC. -*/ + * Basic ISE data parser generated by JavaCC. + */ +@SuppressWarnings({"checkstyle:MethodName", "checkstyle:MemberName"}) public class ISEParser implements Serializable, ISEParserConstants { - // private boolean nativeNumbers = false; - - private static final long serialVersionUID = -2531656825360044979L; - - public ISEParser() - { //do nothing - } - - public ISEParser(String input) - { - this (new StringReader(input)); - } - - /** - * Parses a ISE String into a JSON object {@code Map}. - */ - public JSONObject parseObject() throws ParseException - { - JSONObject toReturn = object(); - if (!ensureEOF()) throw new IllegalStateException("Expected EOF, but still had content to parse"); - return toReturn; - } - - @SuppressWarnings("unused") -final public boolean ensureEOF() throws ParseException { - switch (jj_nt.kind) { - case COMMA: - jj_consume_token(COMMA); - break; - default: - jj_la1[0] = jj_gen; - ; - } - jj_consume_token(0); - {if (true) return true;} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings({ "unchecked", "unused" }) -final public JSONObject innerMap() throws ParseException { - final JSONObject json = new JSONObject(); - String key; - Object value; - key = objectKey(); - jj_consume_token(EQUALS); - value = value(); - json.put(key, value); - key = null; - value = null; - label_1: - while (true) { - switch (jj_nt.kind) { - case SLASH: - ; - break; - default: - jj_la1[1] = jj_gen; - break label_1; - } - jj_consume_token(SLASH); - jj_consume_token(COMMA); - key = objectKey(); - jj_consume_token(EQUALS); - value = value(); - json.put(key, value); - key = null; - value = null; - } - {if (true) return json;} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings({ "unused", "unchecked" }) -final public JSONObject object() throws ParseException { - final JSONObject json = new JSONObject(); - String key; - Object value; - key = objectKey(); - jj_consume_token(EQUALS); - value = value(); - json.put(key, value); - key = null; - value = null; - label_2: - while (true) { - if (jj_2_1(2)) { - ; - } else { - break label_2; - } - jj_consume_token(COMMA); - key = objectKey(); - jj_consume_token(EQUALS); - value = value(); + // private boolean nativeNumbers = false; + + private static final long serialVersionUID = -2531656825360044979L; + + public ISEParser() { //do nothing + } + + public ISEParser(String input) { + this(new StringReader(input)); + } + + /** + * Parses a ISE String into a JSON object {@code Map}. + */ + public JSONObject parseObject() throws ParseException { + JSONObject toReturn = object(); + if (!ensureEOF()) { + throw new IllegalStateException("Expected EOF, but still had content to parse"); + } + return toReturn; + } + + @SuppressWarnings("unused") + public final boolean ensureEOF() throws ParseException { + switch (jj_nt.kind) { + case COMMA: + jj_consume_token(COMMA); + break; + default: + jj_la1[0] = jj_gen; + } + jj_consume_token(0); + { + if (true) { + return true; + } + } + throw new Error("Missing return statement in function"); + } + + @SuppressWarnings({"unchecked", "unused"}) + public final JSONObject innerMap() throws ParseException { + final JSONObject json = new JSONObject(); + String key; + Object value; + key = objectKey(); + jj_consume_token(EQUALS); + value = value(); + json.put(key, value); + key = null; + value = null; + label_1: + while (true) { + switch (jj_nt.kind) { + case SLASH: + break; + default: + jj_la1[1] = jj_gen; + break label_1; + } + jj_consume_token(SLASH); + jj_consume_token(COMMA); + key = objectKey(); + jj_consume_token(EQUALS); + value = value(); + json.put(key, value); + key = null; + value = null; + } + { + if (true) { + return json; + } + } + throw new Error("Missing return statement in function"); + } + + @SuppressWarnings({"unused", "unchecked"}) + public final JSONObject object() throws ParseException { + final JSONObject json = new JSONObject(); + String key; + Object value; + key = objectKey(); + jj_consume_token(EQUALS); + value = value(); json.put(key, value); key = null; value = null; + while (true) { + if (jj_2_1(2)) { + } else { + break; + } + jj_consume_token(COMMA); + key = objectKey(); + jj_consume_token(EQUALS); + value = value(); + json.put(key, value); + key = null; + value = null; + } + { + if (true) { + return json; + } + } + throw new Error("Missing return statement in function"); + } + + @SuppressWarnings("unused") + public final String objectKey() throws ParseException { + String k; + k = string(); + // System.out.println("key == " + k); + { + if (true) { + return k.trim(); + } + } + throw new Error("Missing return statement in function"); } - {if (true) return json;} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings("unused") -final public String objectKey() throws ParseException { - String k; - k = string(); - // System.out.println("key == " + k); - {if (true) return k.trim();} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings({ "unused", "rawtypes" }) -final public Object value() throws ParseException { - Object x; - String eof = "EOF"; - Map m = null; - if (jj_2_2(2147483647)) { - x = nullValue(); - } else if (jj_2_3(2147483647)) { - x = innerMap(); - } else { - switch (jj_nt.kind) { - case TAG: - x = tagString(); - break; - default: - jj_la1[2] = jj_gen; - if (jj_2_4(2147483647)) { - x = blankValue(); - } else if (jj_2_5(2147483647)) { - x = braced_string(); - } else if (jj_2_6(2)) { - x = string(); + + @SuppressWarnings({"unused", "rawtypes"}) + public final Object value() throws ParseException { + Object x; + String eof = "EOF"; + Map m = null; + if (jj_2_2(2147483647)) { + x = nullValue(); + } else if (jj_2_3(2147483647)) { + x = innerMap(); } else { - jj_consume_token(-1); - throw new ParseException(); - } - } - } - // System.out.println("val == " + x); - //if (x instanceof Map) return "Map"; - //return (String) x; - {if (true) return x;} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings("unused") -final public String nullValue() throws ParseException { - {if (true) return null;} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings("unused") -final public String tagString() throws ParseException { - String output = "(tag=0)"; - jj_consume_token(TAG); - jj_consume_token(STRING_BODY); - {if (true) return output + token.image;} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings("unused") -final public String blankValue() throws ParseException { - {if (true) return null;} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings("unused") -final public String string() throws ParseException { - String s; - jj_consume_token(STRING_BODY); - {if (true) return token.image.trim();} - throw new Error("Missing return statement in function"); - } - - @SuppressWarnings("unused") -final public String braced_string() throws ParseException { - String s; - jj_consume_token(BRACED_STRING); - // System.out.println("braced == " + token.image); - s = token.image; - jj_consume_token(COMMA); - {if (true) return s.trim();} - throw new Error("Missing return statement in function"); - } - - private boolean jj_2_1(int xla) { - jj_la = xla; jj_lastpos = jj_scanpos = token; - try { return !jj_3_1(); } - catch(LookaheadSuccess ls) { return true; } - finally { jj_save(0, xla); } - } - - private boolean jj_2_2(int xla) { - jj_la = xla; jj_lastpos = jj_scanpos = token; - try { return !jj_3_2(); } - catch(LookaheadSuccess ls) { return true; } - finally { jj_save(1, xla); } - } - - private boolean jj_2_3(int xla) { - jj_la = xla; jj_lastpos = jj_scanpos = token; - try { return !jj_3_3(); } - catch(LookaheadSuccess ls) { return true; } - finally { jj_save(2, xla); } - } - - private boolean jj_2_4(int xla) { - jj_la = xla; jj_lastpos = jj_scanpos = token; - try { return !jj_3_4(); } - catch(LookaheadSuccess ls) { return true; } - finally { jj_save(3, xla); } - } - - private boolean jj_2_5(int xla) { - jj_la = xla; jj_lastpos = jj_scanpos = token; - try { return !jj_3_5(); } - catch(LookaheadSuccess ls) { return true; } - finally { jj_save(4, xla); } - } - - private boolean jj_2_6(int xla) { - jj_la = xla; jj_lastpos = jj_scanpos = token; - try { return !jj_3_6(); } - catch(LookaheadSuccess ls) { return true; } - finally { jj_save(5, xla); } - } - - private boolean jj_3_5() { - if (jj_3R_5()) return true; - return false; - } - - private boolean jj_3_4() { - if (jj_scan_token(0)) return true; - return false; - } - - private boolean jj_3R_5() { - if (jj_scan_token(BRACED_STRING)) return true; - if (jj_scan_token(COMMA)) return true; - return false; - } - - private boolean jj_3_3() { - if (jj_3R_4()) return true; - return false; - } - - private boolean jj_3R_4() { - if (jj_3R_3()) return true; - if (jj_scan_token(EQUALS)) return true; - if (jj_3R_7()) return true; - Token xsp; - while (true) { - xsp = jj_scanpos; - if (jj_3R_8()) { jj_scanpos = xsp; break; } - } - return false; - } - - private boolean jj_3_2() { - if (jj_scan_token(COMMA)) return true; - return false; - } - - private boolean jj_3_6() { - if (jj_3R_6()) return true; - return false; - } - - private boolean jj_3_1() { - if (jj_scan_token(COMMA)) return true; - if (jj_3R_3()) return true; - return false; - } - - private boolean jj_3R_13() { - if (jj_3R_5()) return true; - return false; - } - - private boolean jj_3R_12() { - if (jj_3R_16()) return true; - return false; - } - - private boolean jj_3R_11() { - if (jj_3R_15()) return true; - return false; - } - - private boolean jj_3R_6() { - if (jj_scan_token(STRING_BODY)) return true; - return false; - } - - private boolean jj_3R_10() { - if (jj_3R_4()) return true; - return false; - } - - private boolean jj_3R_9() { - if (jj_3R_14()) return true; - return false; - } - - private boolean jj_3R_7() { - Token xsp; - xsp = jj_scanpos; - if (jj_3R_9()) { - jj_scanpos = xsp; - if (jj_3R_10()) { - jj_scanpos = xsp; - if (jj_3R_11()) { - jj_scanpos = xsp; - if (jj_3R_12()) { - jj_scanpos = xsp; - if (jj_3R_13()) { - jj_scanpos = xsp; - if (jj_3_6()) return true; - } - } - } - } - } - return false; - } - - private boolean jj_3R_16() { - return false; - } - - private boolean jj_3R_15() { - if (jj_scan_token(TAG)) return true; - if (jj_scan_token(STRING_BODY)) return true; - return false; - } - - private boolean jj_3R_3() { - if (jj_3R_6()) return true; - return false; - } - - private boolean jj_3R_8() { - if (jj_scan_token(SLASH)) return true; - if (jj_scan_token(COMMA)) return true; - if (jj_3R_3()) return true; - if (jj_scan_token(EQUALS)) return true; - if (jj_3R_7()) return true; - return false; - } - - private boolean jj_3R_14() { - return false; - } - - /** Generated Token Manager. */ - public ISEParserTokenManager token_source; - JavaCharStream jj_input_stream; - /** Current token. */ - public Token token; - /** Next token. */ - public Token jj_nt; - private Token jj_scanpos, jj_lastpos; - private int jj_la; - private int jj_gen; - final private int[] jj_la1 = new int[3]; - static private int[] jj_la1_0; - static { - jj_la1_init_0(); - } - private static void jj_la1_init_0() { - jj_la1_0 = new int[] {0x20,0x80,0x100,}; - } - final private JJCalls[] jj_2_rtns = new JJCalls[6]; - private boolean jj_rescan = false; - private int jj_gc = 0; - - /** Constructor with InputStream. */ - public ISEParser(java.io.InputStream stream) { - this(stream, null); - } - /** Constructor with InputStream and supplied encoding */ - public ISEParser(java.io.InputStream stream, String encoding) { - try { jj_input_stream = new JavaCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); } - token_source = new ISEParserTokenManager(jj_input_stream); - token = new Token(); - token.next = jj_nt = token_source.getNextToken(); - jj_gen = 0; - for (int i = 0; i < 3; i++) jj_la1[i] = -1; - for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream stream) { - ReInit(stream, null); - } - /** Reinitialise. */ - public void ReInit(java.io.InputStream stream, String encoding) { - try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); } - token_source.ReInit(jj_input_stream); - token = new Token(); - token.next = jj_nt = token_source.getNextToken(); - jj_gen = 0; - for (int i = 0; i < 3; i++) jj_la1[i] = -1; - for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); - } - - /** Constructor. */ - public ISEParser(java.io.Reader stream) { - jj_input_stream = new JavaCharStream(stream, 1, 1); - token_source = new ISEParserTokenManager(jj_input_stream); - token = new Token(); - token.next = jj_nt = token_source.getNextToken(); - jj_gen = 0; - for (int i = 0; i < 3; i++) jj_la1[i] = -1; - for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); - } - - /** Reinitialise. */ - public void ReInit(java.io.Reader stream) { - jj_input_stream.ReInit(stream, 1, 1); - token_source.ReInit(jj_input_stream); - token = new Token(); - token.next = jj_nt = token_source.getNextToken(); - jj_gen = 0; - for (int i = 0; i < 3; i++) jj_la1[i] = -1; - for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); - } - - /** Constructor with generated Token Manager. */ - public ISEParser(ISEParserTokenManager tm) { - token_source = tm; - token = new Token(); - token.next = jj_nt = token_source.getNextToken(); - jj_gen = 0; - for (int i = 0; i < 3; i++) jj_la1[i] = -1; - for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); - } - - /** Reinitialise. */ - public void ReInit(ISEParserTokenManager tm) { - token_source = tm; - token = new Token(); - token.next = jj_nt = token_source.getNextToken(); - jj_gen = 0; - for (int i = 0; i < 3; i++) jj_la1[i] = -1; - for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); - } - - private Token jj_consume_token(int kind) throws ParseException { - Token oldToken = token; - if ((token = jj_nt).next != null) jj_nt = jj_nt.next; - else jj_nt = jj_nt.next = token_source.getNextToken(); - if (token.kind == kind) { - jj_gen++; - if (++jj_gc > 100) { - jj_gc = 0; + switch (jj_nt.kind) { + case TAG: + x = tagString(); + break; + default: + jj_la1[2] = jj_gen; + if (jj_2_4(2147483647)) { + x = blankValue(); + } else if (jj_2_5(2147483647)) { + x = braced_string(); + } else if (jj_2_6(2)) { + x = string(); + } else { + jj_consume_token(-1); + throw new ParseException(); + } + } + } + // System.out.println("val == " + x); + //if (x instanceof Map) return "Map"; + //return (String) x; + { + if (true) { + return x; + } + } + throw new Error("Missing return statement in function"); + } + + @SuppressWarnings("unused") + public final String nullValue() throws ParseException { + { + if (true) { + return null; + } + } + throw new Error("Missing return statement in function"); + } + + @SuppressWarnings("unused") + public final String tagString() throws ParseException { + String output = "(tag=0)"; + jj_consume_token(TAG); + jj_consume_token(STRING_BODY); + { + if (true) { + return output + token.image; + } + } + throw new Error("Missing return statement in function"); + } + + @SuppressWarnings("unused") + public final String blankValue() throws ParseException { + { + if (true) { + return null; + } + } + throw new Error("Missing return statement in function"); + } + + @SuppressWarnings("unused") + public final String string() throws ParseException { + String s; + jj_consume_token(STRING_BODY); + { + if (true) { + return token.image.trim(); + } + } + throw new Error("Missing return statement in function"); + } + + @SuppressWarnings("unused") + public final String braced_string() throws ParseException { + String s; + jj_consume_token(BRACED_STRING); + // System.out.println("braced == " + token.image); + s = token.image; + jj_consume_token(COMMA); + { + if (true) { + return s.trim(); + } + } + throw new Error("Missing return statement in function"); + } + + private boolean jj_2_1(int xla) { + jj_la = xla; + jj_lastpos = jj_scanpos = token; + try { + return !jj_3_1(); + } catch (LookaheadSuccess ls) { + return true; + } finally { + jj_save(0, xla); + } + } + + private boolean jj_2_2(int xla) { + jj_la = xla; + jj_lastpos = jj_scanpos = token; + try { + return !jj_3_2(); + } catch (LookaheadSuccess ls) { + return true; + } finally { + jj_save(1, xla); + } + } + + private boolean jj_2_3(int xla) { + jj_la = xla; + jj_lastpos = jj_scanpos = token; + try { + return !jj_3_3(); + } catch (LookaheadSuccess ls) { + return true; + } finally { + jj_save(2, xla); + } + } + + private boolean jj_2_4(int xla) { + jj_la = xla; + jj_lastpos = jj_scanpos = token; + try { + return !jj_3_4(); + } catch (LookaheadSuccess ls) { + return true; + } finally { + jj_save(3, xla); + } + } + + private boolean jj_2_5(int xla) { + jj_la = xla; + jj_lastpos = jj_scanpos = token; + try { + return !jj_3_5(); + } catch (LookaheadSuccess ls) { + return true; + } finally { + jj_save(4, xla); + } + } + + private boolean jj_2_6(int xla) { + jj_la = xla; + jj_lastpos = jj_scanpos = token; + try { + return !jj_3_6(); + } catch (LookaheadSuccess ls) { + return true; + } finally { + jj_save(5, xla); + } + } + + private boolean jj_3_5() { + return jj_3R_5(); + } + + private boolean jj_3_4() { + return jj_scan_token(0); + } + + private boolean jj_3R_5() { + if (jj_scan_token(BRACED_STRING)) { + return true; + } + return jj_scan_token(COMMA); + } + + private boolean jj_3_3() { + return jj_3R_4(); + } + + private boolean jj_3R_4() { + if (jj_3R_3()) { + return true; + } + if (jj_scan_token(EQUALS)) { + return true; + } + if (jj_3R_7()) { + return true; + } + Token xsp; + while (true) { + xsp = jj_scanpos; + if (jj_3R_8()) { + jj_scanpos = xsp; + break; + } + } + return false; + } + + private boolean jj_3_2() { + return jj_scan_token(COMMA); + } + + private boolean jj_3_6() { + return jj_3R_6(); + } + + private boolean jj_3_1() { + if (jj_scan_token(COMMA)) { + return true; + } + return jj_3R_3(); + } + + private boolean jj_3R_13() { + return jj_3R_5(); + } + + private boolean jj_3R_12() { + return jj_3R_16(); + } + + private boolean jj_3R_11() { + return jj_3R_15(); + } + + private boolean jj_3R_6() { + return jj_scan_token(STRING_BODY); + } + + private boolean jj_3R_10() { + return jj_3R_4(); + } + + private boolean jj_3R_9() { + return jj_3R_14(); + } + + private boolean jj_3R_7() { + Token xsp; + xsp = jj_scanpos; + if (jj_3R_9()) { + jj_scanpos = xsp; + if (jj_3R_10()) { + jj_scanpos = xsp; + if (jj_3R_11()) { + jj_scanpos = xsp; + if (jj_3R_12()) { + jj_scanpos = xsp; + if (jj_3R_13()) { + jj_scanpos = xsp; + return jj_3_6(); + } + } + } + } + } + return false; + } + + private boolean jj_3R_16() { + return false; + } + + private boolean jj_3R_15() { + if (jj_scan_token(TAG)) { + return true; + } + return jj_scan_token(STRING_BODY); + } + + private boolean jj_3R_3() { + return jj_3R_6(); + } + + private boolean jj_3R_8() { + if (jj_scan_token(SLASH)) { + return true; + } + if (jj_scan_token(COMMA)) { + return true; + } + if (jj_3R_3()) { + return true; + } + if (jj_scan_token(EQUALS)) { + return true; + } + return jj_3R_7(); + } + + private boolean jj_3R_14() { + return false; + } + + /** + * Generated Token Manager. + */ + public ISEParserTokenManager token_source; + JavaCharStream jj_input_stream; + /** + * Current token. + */ + public Token token; + /** + * Next token. + */ + public Token jj_nt; + private Token jj_scanpos; + private Token jj_lastpos; + private int jj_la; + private int jj_gen; + private final int[] jj_la1 = new int[3]; + private static int[] jj_la1_0; + + static { + jj_la1_init_0(); + } + + private static void jj_la1_init_0() { + jj_la1_0 = new int[] {0x20, 0x80, 0x100, }; + } + + private final JJCalls[] jj_2_rtns = new JJCalls[6]; + private boolean jj_rescan = false; + private int jj_gc = 0; + + /** + * Constructor with InputStream. + */ + public ISEParser(java.io.InputStream stream) { + this(stream, null); + } + + /** + * Constructor with InputStream and supplied encoding. + */ + public ISEParser(java.io.InputStream stream, String encoding) { + try { + jj_input_stream = new JavaCharStream(stream, encoding, 1, 1); + } catch (java.io.UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + token_source = new ISEParserTokenManager(jj_input_stream); + token = new Token(); + token.next = jj_nt = token_source.getNextToken(); + jj_gen = 0; + for (int i = 0; i < 3; i++) { + jj_la1[i] = -1; + } + for (int i = 0; i < jj_2_rtns.length; i++) { + jj_2_rtns[i] = new JJCalls(); + } + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.InputStream stream) { + ReInit(stream, null); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.InputStream stream, String encoding) { + try { + jj_input_stream.ReInit(stream, encoding, 1, 1); + } catch (java.io.UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + token_source.ReInit(jj_input_stream); + token = new Token(); + token.next = jj_nt = token_source.getNextToken(); + jj_gen = 0; + for (int i = 0; i < 3; i++) { + jj_la1[i] = -1; + } for (int i = 0; i < jj_2_rtns.length; i++) { - JJCalls c = jj_2_rtns[i]; - while (c != null) { - if (c.gen < jj_gen) c.first = null; - c = c.next; - } - } - } - return token; - } - jj_nt = token; - token = oldToken; - jj_kind = kind; - throw generateParseException(); - } - - static private final class LookaheadSuccess extends java.lang.Error { - - private static final long serialVersionUID = -5724812746511794505L; } - final private LookaheadSuccess jj_ls = new LookaheadSuccess(); - private boolean jj_scan_token(int kind) { - if (jj_scanpos == jj_lastpos) { - jj_la--; - if (jj_scanpos.next == null) { - jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken(); - } else { - jj_lastpos = jj_scanpos = jj_scanpos.next; - } - } else { - jj_scanpos = jj_scanpos.next; - } - if (jj_rescan) { - int i = 0; Token tok = token; - while (tok != null && tok != jj_scanpos) { i++; tok = tok.next; } - if (tok != null) jj_add_error_token(kind, i); - } - if (jj_scanpos.kind != kind) return true; - if (jj_la == 0 && jj_scanpos == jj_lastpos) throw jj_ls; - return false; - } - - -/** Get the next Token. */ - final public Token getNextToken() { - if ((token = jj_nt).next != null) jj_nt = jj_nt.next; - else jj_nt = jj_nt.next = token_source.getNextToken(); - jj_gen++; - return token; - } - -/** Get the specific Token. */ - final public Token getToken(int index) { - Token t = token; - for (int i = 0; i < index; i++) { - if (t.next != null) t = t.next; - else t = t.next = token_source.getNextToken(); - } - return t; - } - - private java.util.List jj_expentries = new java.util.ArrayList(); - private int[] jj_expentry; - private int jj_kind = -1; - private int[] jj_lasttokens = new int[100]; - private int jj_endpos; - - private void jj_add_error_token(int kind, int pos) { - if (pos >= 100) return; - if (pos == jj_endpos + 1) { - jj_lasttokens[jj_endpos++] = kind; - } else if (jj_endpos != 0) { - jj_expentry = new int[jj_endpos]; - for (int i = 0; i < jj_endpos; i++) { - jj_expentry[i] = jj_lasttokens[i]; - } - jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) { - int[] oldentry = (int[])(it.next()); - if (oldentry.length == jj_expentry.length) { - for (int i = 0; i < jj_expentry.length; i++) { - if (oldentry[i] != jj_expentry[i]) { - continue jj_entries_loop; + jj_2_rtns[i] = new JJCalls(); + } + } + + /** + * Constructor. + */ + public ISEParser(java.io.Reader stream) { + jj_input_stream = new JavaCharStream(stream, 1, 1); + token_source = new ISEParserTokenManager(jj_input_stream); + token = new Token(); + token.next = jj_nt = token_source.getNextToken(); + jj_gen = 0; + for (int i = 0; i < 3; i++) { + jj_la1[i] = -1; + } + for (int i = 0; i < jj_2_rtns.length; i++) { + jj_2_rtns[i] = new JJCalls(); + } + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.Reader stream) { + jj_input_stream.ReInit(stream, 1, 1); + token_source.ReInit(jj_input_stream); + token = new Token(); + token.next = jj_nt = token_source.getNextToken(); + jj_gen = 0; + for (int i = 0; i < 3; i++) { + jj_la1[i] = -1; + } + for (int i = 0; i < jj_2_rtns.length; i++) { + jj_2_rtns[i] = new JJCalls(); + } + } + + /** + * Constructor with generated Token Manager. + */ + public ISEParser(ISEParserTokenManager tm) { + token_source = tm; + token = new Token(); + token.next = jj_nt = token_source.getNextToken(); + jj_gen = 0; + for (int i = 0; i < 3; i++) { + jj_la1[i] = -1; + } + for (int i = 0; i < jj_2_rtns.length; i++) { + jj_2_rtns[i] = new JJCalls(); + } + } + + /** + * Reinitialise. + */ + public void ReInit(ISEParserTokenManager tm) { + token_source = tm; + token = new Token(); + token.next = jj_nt = token_source.getNextToken(); + jj_gen = 0; + for (int i = 0; i < 3; i++) { + jj_la1[i] = -1; + } + for (int i = 0; i < jj_2_rtns.length; i++) { + jj_2_rtns[i] = new JJCalls(); + } + } + + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") + private Token jj_consume_token(int kind) throws ParseException { + Token oldToken = token; + if ((token = jj_nt).next != null) { + jj_nt = jj_nt.next; + } else { + jj_nt = jj_nt.next = token_source.getNextToken(); + } + if (token.kind == kind) { + jj_gen++; + if (++jj_gc > 100) { + jj_gc = 0; + for (int i = 0; i < jj_2_rtns.length; i++) { + JJCalls c = jj_2_rtns[i]; + while (c != null) { + if (c.gen < jj_gen) { + c.first = null; + } + c = c.next; + } + } + } + return token; + } + jj_nt = token; + token = oldToken; + jj_kind = kind; + throw generateParseException(); + } + + private static final class LookaheadSuccess extends java.lang.Error { + + private static final long serialVersionUID = -5724812746511794505L; + } + + private final LookaheadSuccess jj_ls = new LookaheadSuccess(); + + private boolean jj_scan_token(int kind) { + if (jj_scanpos == jj_lastpos) { + jj_la--; + if (jj_scanpos.next == null) { + jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken(); + } else { + jj_lastpos = jj_scanpos = jj_scanpos.next; + } + } else { + jj_scanpos = jj_scanpos.next; + } + if (jj_rescan) { + int i = 0; + Token tok = token; + while (tok != null && tok != jj_scanpos) { + i++; + tok = tok.next; + } + if (tok != null) { + jj_add_error_token(kind, i); } - } - jj_expentries.add(jj_expentry); - break jj_entries_loop; - } - } - if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind; - } - } - - /** Generate ParseException. */ - public ParseException generateParseException() { - jj_expentries.clear(); - boolean[] la1tokens = new boolean[11]; - if (jj_kind >= 0) { - la1tokens[jj_kind] = true; - jj_kind = -1; - } - for (int i = 0; i < 3; i++) { - if (jj_la1[i] == jj_gen) { - for (int j = 0; j < 32; j++) { - if ((jj_la1_0[i] & (1< jj_gen) { - jj_la = p.arg; jj_lastpos = jj_scanpos = p.first; - switch (i) { - case 0: jj_3_1(); break; - case 1: jj_3_2(); break; - case 2: jj_3_3(); break; - case 3: jj_3_4(); break; - case 4: jj_3_5(); break; - case 5: jj_3_6(); break; - } - } - p = p.next; - } while (p != null); - } catch(LookaheadSuccess ls) { } - } - jj_rescan = false; - } - - private void jj_save(int index, int xla) { - JJCalls p = jj_2_rtns[index]; - while (p.gen > jj_gen) { - if (p.next == null) { p = p.next = new JJCalls(); break; } - p = p.next; - } - p.gen = jj_gen + xla - jj_la; p.first = token; p.arg = xla; - } - - static final class JJCalls { - int gen; - Token first; - int arg; - JJCalls next; - } + } + if (jj_scanpos.kind != kind) { + return true; + } + if (jj_la == 0 && jj_scanpos == jj_lastpos) { + throw jj_ls; + } + return false; + } + + + /** + * Get the next Token. + */ + public final Token getNextToken() { + if ((token = jj_nt).next != null) { + jj_nt = jj_nt.next; + } else { + jj_nt = jj_nt.next = token_source.getNextToken(); + } + jj_gen++; + return token; + } + + /** + * Get the specific Token. + */ + public final Token getToken(int index) { + Token t = token; + for (int i = 0; i < index; i++) { + if (t.next != null) { + t = t.next; + } else { + t = t.next = token_source.getNextToken(); + } + } + return t; + } + + private final java.util.List jj_expentries = new java.util.ArrayList(); + private int[] jj_expentry; + private int jj_kind = -1; + private final int[] jj_lasttokens = new int[100]; + private int jj_endpos; + + private void jj_add_error_token(int kind, int pos) { + if (pos >= 100) { + return; + } + if (pos == jj_endpos + 1) { + jj_lasttokens[jj_endpos++] = kind; + } else if (jj_endpos != 0) { + jj_expentry = new int[jj_endpos]; + System.arraycopy(jj_lasttokens, 0, jj_expentry, 0, jj_endpos); + jj_entries_loop: + for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext(); ) { + int[] oldentry = (int[]) (it.next()); + if (oldentry.length == jj_expentry.length) { + for (int i = 0; i < jj_expentry.length; i++) { + if (oldentry[i] != jj_expentry[i]) { + continue jj_entries_loop; + } + } + jj_expentries.add(jj_expentry); + break; + } + } + if (pos != 0) { + jj_lasttokens[(jj_endpos = pos) - 1] = kind; + } + } + } + + /** + * Generate ParseException. + */ + public ParseException generateParseException() { + jj_expentries.clear(); + boolean[] la1tokens = new boolean[11]; + if (jj_kind >= 0) { + la1tokens[jj_kind] = true; + jj_kind = -1; + } + for (int i = 0; i < 3; i++) { + if (jj_la1[i] == jj_gen) { + for (int j = 0; j < 32; j++) { + if ((jj_la1_0[i] & (1 << j)) != 0) { + la1tokens[j] = true; + } + } + } + } + for (int i = 0; i < 11; i++) { + if (la1tokens[i]) { + jj_expentry = new int[1]; + jj_expentry[0] = i; + jj_expentries.add(jj_expentry); + } + } + jj_endpos = 0; + jj_rescan_token(); + jj_add_error_token(0, 0); + int[][] exptokseq = new int[jj_expentries.size()][]; + for (int i = 0; i < jj_expentries.size(); i++) { + exptokseq[i] = jj_expentries.get(i); + } + return new ParseException(token, exptokseq, tokenImage); + } + + /** + * Enable tracing. + */ + public final void enable_tracing() { + } + + /** + * Disable tracing. + */ + public final void disable_tracing() { + } + + @SuppressWarnings("checkstyle:EmptyCatchBlock") + private void jj_rescan_token() { + jj_rescan = true; + for (int i = 0; i < 6; i++) { + try { + JJCalls p = jj_2_rtns[i]; + do { + if (p.gen > jj_gen) { + jj_la = p.arg; + jj_lastpos = jj_scanpos = p.first; + switch (i) { + case 0: + jj_3_1(); + break; + case 1: + jj_3_2(); + break; + case 2: + jj_3_3(); + break; + case 3: + jj_3_4(); + break; + case 4: + jj_3_5(); + break; + case 5: + jj_3_6(); + break; + default: + break; + } + } + p = p.next; + } while (p != null); + } catch (LookaheadSuccess ls) { + } + } + jj_rescan = false; + } + + private void jj_save(int index, int xla) { + JJCalls p = jj_2_rtns[index]; + while (p.gen > jj_gen) { + if (p.next == null) { + p = p.next = new JJCalls(); + break; + } + p = p.next; + } + p.gen = jj_gen + xla - jj_la; + p.first = token; + p.arg = xla; + } + + static final class JJCalls { + int gen; + Token first; + int arg; + JJCalls next; + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParserConstants.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParserConstants.java index 126d120f..d51fcac2 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParserConstants.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParserConstants.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,6 +18,7 @@ * limitations under the License. */ /* Generated By:JavaCC: Do not edit this line. ISEParserConstants.java */ + package org.apache.metron.parsers.ise; @@ -25,45 +28,71 @@ */ interface ISEParserConstants { - /** End of File. */ - int EOF = 0; - /** RegularExpression Id. */ - int C_SINGLE_COMMENT = 1; - /** RegularExpression Id. */ - int C_MULTILINE_COMMENT = 2; - /** RegularExpression Id. */ - int SH_SINGLE_COMMENT = 3; - /** RegularExpression Id. */ - int EOL = 4; - /** RegularExpression Id. */ - int COMMA = 5; - /** RegularExpression Id. */ - int EQUALS = 6; - /** RegularExpression Id. */ - int SLASH = 7; - /** RegularExpression Id. */ - int TAG = 8; - /** RegularExpression Id. */ - int STRING_BODY = 9; - /** RegularExpression Id. */ - int BRACED_STRING = 10; + /** + * End of File. + */ + int EOF = 0; + /** + * RegularExpression Id. + */ + int C_SINGLE_COMMENT = 1; + /** + * RegularExpression Id. + */ + int C_MULTILINE_COMMENT = 2; + /** + * RegularExpression Id. + */ + int SH_SINGLE_COMMENT = 3; + /** + * RegularExpression Id. + */ + int EOL = 4; + /** + * RegularExpression Id. + */ + int COMMA = 5; + /** + * RegularExpression Id. + */ + int EQUALS = 6; + /** + * RegularExpression Id. + */ + int SLASH = 7; + /** + * RegularExpression Id. + */ + int TAG = 8; + /** + * RegularExpression Id. + */ + int STRING_BODY = 9; + /** + * RegularExpression Id. + */ + int BRACED_STRING = 10; - /** Lexical state. */ - int DEFAULT = 0; + /** + * Lexical state. + */ + int DEFAULT = 0; - /** Literal token values. */ - String[] tokenImage = { - "", - "", - "", - "", - "", - "\",\"", - "\"=\"", - "\"\\\\\"", - "\"(tag=0)\"", - "", - "", - }; + /** + * Literal token values. + */ + String[] tokenImage = { + "", + "", + "", + "", + "", + "\",\"", + "\"=\"", + "\"\\\\\"", + "\"(tag=0)\"", + "", + "", + }; } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParserTokenManager.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParserTokenManager.java index 9bd53471..f3ad18b9 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParserTokenManager.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ISEParserTokenManager.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,661 +18,709 @@ * limitations under the License. */ /* Generated By:JavaCC: Do not edit this line. ISEParserTokenManager.java */ + package org.apache.metron.parsers.ise; -/** Token Manager. */ -class ISEParserTokenManager implements ISEParserConstants -{ - - /** Debug output. */ - public java.io.PrintStream debugStream = System.out; - /** Set debug output. */ - public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; } -private final int jjStopStringLiteralDfa_0(int pos, long active0) -{ - switch (pos) - { - case 0: - if ((active0 & 0x100L) != 0L) - { - jjmatchedKind = 9; - return 18; - } - if ((active0 & 0x80L) != 0L) +/** + * Token Manager. + */ +@SuppressWarnings({"checkstyle:MethodName", "checkstyle:MemberName"}) +class ISEParserTokenManager implements ISEParserConstants { + + /** + * Debug output. + */ + public java.io.PrintStream debugStream = System.out; + + /** + * Set debug output. + */ + public void setDebugStream(java.io.PrintStream ds) { + debugStream = ds; + } + + private final int jjStopStringLiteralDfa_0(int pos, long active0) { + switch (pos) { + case 0: + if ((active0 & 0x100L) != 0L) { + jjmatchedKind = 9; + return 18; + } + if ((active0 & 0x80L) != 0L) { + return 6; + } + return -1; + case 1: + if ((active0 & 0x100L) != 0L) { + jjmatchedKind = 9; + jjmatchedPos = 1; + return 18; + } + return -1; + case 2: + if ((active0 & 0x100L) != 0L) { + jjmatchedKind = 9; + jjmatchedPos = 2; + return 18; + } + return -1; + case 3: + if ((active0 & 0x100L) != 0L) { + jjmatchedKind = 9; + jjmatchedPos = 3; + return 18; + } + return -1; + case 4: + if ((active0 & 0x100L) != 0L) { + if (jjmatchedPos < 3) { + jjmatchedKind = 9; + jjmatchedPos = 3; + } + return -1; + } + return -1; + case 5: + if ((active0 & 0x100L) != 0L) { + if (jjmatchedPos < 3) { + jjmatchedKind = 9; + jjmatchedPos = 3; + } + return -1; + } + return -1; + default: + return -1; + } + } + + private final int jjStartNfa_0(int pos, long active0) { + return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0), pos + 1); + } + + private int jjStopAtPos(int pos, int kind) { + jjmatchedKind = kind; + jjmatchedPos = pos; + return pos + 1; + } + + private int jjMoveStringLiteralDfa0_0() { + switch (curChar) { + case 40: + return jjMoveStringLiteralDfa1_0(0x100L); + case 44: + return jjStopAtPos(0, 5); + case 61: + return jjStopAtPos(0, 6); + case 92: + return jjStartNfaWithStates_0(0, 7, 6); + default: + return jjMoveNfa_0(0, 0); + } + } + + private int jjMoveStringLiteralDfa1_0(long active0) { + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + jjStopStringLiteralDfa_0(0, active0); + return 1; + } + switch (curChar) { + case 84: + case 116: + return jjMoveStringLiteralDfa2_0(active0, 0x100L); + default: + break; + } + return jjStartNfa_0(0, active0); + } + + private int jjMoveStringLiteralDfa2_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { + return jjStartNfa_0(0, old0); + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + jjStopStringLiteralDfa_0(1, active0); + return 2; + } + switch (curChar) { + case 65: + case 97: + return jjMoveStringLiteralDfa3_0(active0, 0x100L); + default: + break; + } + return jjStartNfa_0(1, active0); + } + + private int jjMoveStringLiteralDfa3_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { + return jjStartNfa_0(1, old0); + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + jjStopStringLiteralDfa_0(2, active0); + return 3; + } + switch (curChar) { + case 71: + case 103: + return jjMoveStringLiteralDfa4_0(active0, 0x100L); + default: + break; + } + return jjStartNfa_0(2, active0); + } + + private int jjMoveStringLiteralDfa4_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { + return jjStartNfa_0(2, old0); + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + jjStopStringLiteralDfa_0(3, active0); + return 4; + } + switch (curChar) { + case 61: + return jjMoveStringLiteralDfa5_0(active0, 0x100L); + default: + break; + } + return jjStartNfa_0(3, active0); + } + + private int jjMoveStringLiteralDfa5_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { + return jjStartNfa_0(3, old0); + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + jjStopStringLiteralDfa_0(4, active0); + return 5; + } + switch (curChar) { + case 48: + return jjMoveStringLiteralDfa6_0(active0, 0x100L); + default: + break; + } + return jjStartNfa_0(4, active0); + } + + private int jjMoveStringLiteralDfa6_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { + return jjStartNfa_0(4, old0); + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + jjStopStringLiteralDfa_0(5, active0); return 6; - return -1; - case 1: - if ((active0 & 0x100L) != 0L) - { - jjmatchedKind = 9; - jjmatchedPos = 1; - return 18; - } - return -1; - case 2: - if ((active0 & 0x100L) != 0L) - { - jjmatchedKind = 9; - jjmatchedPos = 2; - return 18; - } - return -1; - case 3: - if ((active0 & 0x100L) != 0L) - { - jjmatchedKind = 9; - jjmatchedPos = 3; - return 18; - } - return -1; - case 4: - if ((active0 & 0x100L) != 0L) - { - if (jjmatchedPos < 3) - { - jjmatchedKind = 9; - jjmatchedPos = 3; + } + switch (curChar) { + case 41: + if ((active0 & 0x100L) != 0L) { + return jjStopAtPos(6, 8); + } + break; + default: + break; + } + return jjStartNfa_0(5, active0); + } + + private int jjStartNfaWithStates_0(int pos, int kind, int state) { + jjmatchedKind = kind; + jjmatchedPos = pos; + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + return pos + 1; + } + return jjMoveNfa_0(state, pos + 1); + } + + static final long[] jjbitVec0 = { + 0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL + }; + static final long[] jjbitVec2 = { + 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL + }; + + private int jjMoveNfa_0(int startState, int curPos) { + int startsAt = 0; + jjnewStateCnt = 18; + int i = 1; + jjstateSet[0] = startState; + int kind = 0x7fffffff; + for (; ; ) { + if (++jjround == 0x7fffffff) { + ReInitRounds(); } - return -1; - } - return -1; - case 5: - if ((active0 & 0x100L) != 0L) - { - if (jjmatchedPos < 3) - { - jjmatchedKind = 9; - jjmatchedPos = 3; + if (curChar < 64) { + long l = 1L << curChar; + do { + switch (jjstateSet[--i]) { + case 18: + case 4: + if ((0xdfffeffbffffc9ffL & l) == 0L) { + break; + } + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + break; + case 0: + if ((0xdfffeffbffffc9ffL & l) != 0L) { + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + } else if ((0x3400L & l) != 0L) { + if (kind > 4) { + kind = 4; + } + } + if (curChar == 47) { + jjAddStates(0, 1); + } else if (curChar == 35) { + jjCheckNAddTwoStates(1, 2); + } + break; + case 6: + if ((0xdfffeffbffffc9ffL & l) != 0L) { + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + } + if ((0x900400000000L & l) != 0L) { + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + } + break; + case 1: + if ((0xffffffffffffcbffL & l) != 0L) { + jjCheckNAddTwoStates(1, 2); + } + break; + case 2: + if ((0x3400L & l) != 0L && kind > 3) { + kind = 3; + } + break; + case 3: + if ((0x3400L & l) != 0L && kind > 4) { + kind = 4; + } + break; + case 8: + jjAddStates(2, 3); + break; + case 10: + if (curChar == 47) { + jjAddStates(0, 1); + } + break; + case 11: + if (curChar == 47) { + jjCheckNAddTwoStates(12, 13); + } + break; + case 12: + if ((0xffffffffffffcbffL & l) != 0L) { + jjCheckNAddTwoStates(12, 13); + } + break; + case 13: + if ((0x3400L & l) != 0L && kind > 1) { + kind = 1; + } + break; + case 14: + if (curChar == 42) { + jjCheckNAddTwoStates(15, 17); + } + break; + case 15: + jjCheckNAddTwoStates(15, 17); + break; + case 16: + if (curChar == 47 && kind > 2) { + kind = 2; + } + break; + case 17: + if (curChar == 42) { + jjstateSet[jjnewStateCnt++] = 16; + } + break; + default: + break; + } + } while (i != startsAt); + } else if (curChar < 128) { + long l = 1L << (curChar & 077); + do { + switch (jjstateSet[--i]) { + case 18: + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + if (curChar == 92) { + jjstateSet[jjnewStateCnt++] = 6; + } + break; + case 0: + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + if (curChar == 123) { + jjCheckNAdd(8); + } else if (curChar == 92) { + jjstateSet[jjnewStateCnt++] = 6; + } + break; + case 6: + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + if ((0x14404410144044L & l) != 0L) { + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + } + if (curChar == 92) { + jjstateSet[jjnewStateCnt++] = 6; + } + break; + case 1: + jjAddStates(4, 5); + break; + case 4: + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + break; + case 5: + if (curChar == 92) { + jjstateSet[jjnewStateCnt++] = 6; + } + break; + case 7: + if (curChar == 123) { + jjCheckNAdd(8); + } + break; + case 8: + if ((0xd7ffffffffffffffL & l) != 0L) { + jjCheckNAddTwoStates(8, 9); + } + break; + case 9: + if (curChar == 125 && kind > 10) { + kind = 10; + } + break; + case 12: + jjAddStates(6, 7); + break; + case 15: + jjAddStates(8, 9); + break; + default: + break; + } + } while (i != startsAt); + } else { + int hiByte = curChar >> 8; + int i1 = hiByte >> 6; + long l1 = 1L << (hiByte & 077); + int i2 = (curChar & 0xff) >> 6; + long l2 = 1L << (curChar & 077); + do { + switch (jjstateSet[--i]) { + case 18: + case 4: + if (!jjCanMove_0(hiByte, i1, i2, l1, l2)) { + break; + } + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + break; + case 0: + if (!jjCanMove_0(hiByte, i1, i2, l1, l2)) { + break; + } + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + break; + case 6: + if (!jjCanMove_0(hiByte, i1, i2, l1, l2)) { + break; + } + if (kind > 9) { + kind = 9; + } + jjCheckNAddTwoStates(4, 5); + break; + case 1: + if (jjCanMove_0(hiByte, i1, i2, l1, l2)) { + jjAddStates(4, 5); + } + break; + case 8: + if (jjCanMove_0(hiByte, i1, i2, l1, l2)) { + jjAddStates(2, 3); + } + break; + case 12: + if (jjCanMove_0(hiByte, i1, i2, l1, l2)) { + jjAddStates(6, 7); + } + break; + case 15: + if (jjCanMove_0(hiByte, i1, i2, l1, l2)) { + jjAddStates(8, 9); + } + break; + default: + break; + } + } while (i != startsAt); } - return -1; - } - return -1; - default : - return -1; - } -} -private final int jjStartNfa_0(int pos, long active0) -{ - return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0), pos + 1); -} -private int jjStopAtPos(int pos, int kind) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - return pos + 1; -} -private int jjMoveStringLiteralDfa0_0() -{ - switch(curChar) - { - case 40: - return jjMoveStringLiteralDfa1_0(0x100L); - case 44: - return jjStopAtPos(0, 5); - case 61: - return jjStopAtPos(0, 6); - case 92: - return jjStartNfaWithStates_0(0, 7, 6); - default : - return jjMoveNfa_0(0, 0); - } -} -private int jjMoveStringLiteralDfa1_0(long active0) -{ - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { - jjStopStringLiteralDfa_0(0, active0); - return 1; - } - switch(curChar) - { - case 84: - case 116: - return jjMoveStringLiteralDfa2_0(active0, 0x100L); - default : - break; - } - return jjStartNfa_0(0, active0); -} -private int jjMoveStringLiteralDfa2_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) - return jjStartNfa_0(0, old0); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { - jjStopStringLiteralDfa_0(1, active0); - return 2; - } - switch(curChar) - { - case 65: - case 97: - return jjMoveStringLiteralDfa3_0(active0, 0x100L); - default : - break; - } - return jjStartNfa_0(1, active0); -} -private int jjMoveStringLiteralDfa3_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) - return jjStartNfa_0(1, old0); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { - jjStopStringLiteralDfa_0(2, active0); - return 3; - } - switch(curChar) - { - case 71: - case 103: - return jjMoveStringLiteralDfa4_0(active0, 0x100L); - default : - break; - } - return jjStartNfa_0(2, active0); -} -private int jjMoveStringLiteralDfa4_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) - return jjStartNfa_0(2, old0); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { - jjStopStringLiteralDfa_0(3, active0); - return 4; - } - switch(curChar) - { - case 61: - return jjMoveStringLiteralDfa5_0(active0, 0x100L); - default : - break; - } - return jjStartNfa_0(3, active0); -} -private int jjMoveStringLiteralDfa5_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) - return jjStartNfa_0(3, old0); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { - jjStopStringLiteralDfa_0(4, active0); - return 5; - } - switch(curChar) - { - case 48: - return jjMoveStringLiteralDfa6_0(active0, 0x100L); - default : - break; - } - return jjStartNfa_0(4, active0); -} -private int jjMoveStringLiteralDfa6_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) - return jjStartNfa_0(4, old0); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { - jjStopStringLiteralDfa_0(5, active0); - return 6; - } - switch(curChar) - { - case 41: - if ((active0 & 0x100L) != 0L) - return jjStopAtPos(6, 8); - break; - default : - break; - } - return jjStartNfa_0(5, active0); -} -private int jjStartNfaWithStates_0(int pos, int kind, int state) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return pos + 1; } - return jjMoveNfa_0(state, pos + 1); -} -static final long[] jjbitVec0 = { - 0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL -}; -static final long[] jjbitVec2 = { - 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL -}; -private int jjMoveNfa_0(int startState, int curPos) -{ - int startsAt = 0; - jjnewStateCnt = 18; - int i = 1; - jjstateSet[0] = startState; - int kind = 0x7fffffff; - for (;;) - { - if (++jjround == 0x7fffffff) - ReInitRounds(); - if (curChar < 64) - { - long l = 1L << curChar; - do - { - switch(jjstateSet[--i]) - { - case 18: - case 4: - if ((0xdfffeffbffffc9ffL & l) == 0L) - break; - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - break; - case 0: - if ((0xdfffeffbffffc9ffL & l) != 0L) - { - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - } - else if ((0x3400L & l) != 0L) - { - if (kind > 4) - kind = 4; - } - if (curChar == 47) - jjAddStates(0, 1); - else if (curChar == 35) - jjCheckNAddTwoStates(1, 2); - break; - case 6: - if ((0xdfffeffbffffc9ffL & l) != 0L) - { - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - } - if ((0x900400000000L & l) != 0L) - { - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - } - break; - case 1: - if ((0xffffffffffffcbffL & l) != 0L) - jjCheckNAddTwoStates(1, 2); - break; - case 2: - if ((0x3400L & l) != 0L && kind > 3) - kind = 3; - break; - case 3: - if ((0x3400L & l) != 0L && kind > 4) - kind = 4; - break; - case 8: - jjAddStates(2, 3); - break; - case 10: - if (curChar == 47) - jjAddStates(0, 1); - break; - case 11: - if (curChar == 47) - jjCheckNAddTwoStates(12, 13); - break; - case 12: - if ((0xffffffffffffcbffL & l) != 0L) - jjCheckNAddTwoStates(12, 13); - break; - case 13: - if ((0x3400L & l) != 0L && kind > 1) - kind = 1; - break; - case 14: - if (curChar == 42) - jjCheckNAddTwoStates(15, 17); - break; - case 15: - jjCheckNAddTwoStates(15, 17); - break; - case 16: - if (curChar == 47 && kind > 2) - kind = 2; - break; - case 17: - if (curChar == 42) - jjstateSet[jjnewStateCnt++] = 16; - break; - default : break; + if (kind != 0x7fffffff) { + jjmatchedKind = kind; + jjmatchedPos = curPos; + kind = 0x7fffffff; } - } while(i != startsAt); - } - else if (curChar < 128) - { - long l = 1L << (curChar & 077); - do - { - switch(jjstateSet[--i]) - { - case 18: - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - if (curChar == 92) - jjstateSet[jjnewStateCnt++] = 6; - break; - case 0: - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - if (curChar == 123) - jjCheckNAdd(8); - else if (curChar == 92) - jjstateSet[jjnewStateCnt++] = 6; - break; - case 6: - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - if ((0x14404410144044L & l) != 0L) - { - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - } - if (curChar == 92) - jjstateSet[jjnewStateCnt++] = 6; - break; - case 1: - jjAddStates(4, 5); - break; - case 4: - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - break; - case 5: - if (curChar == 92) - jjstateSet[jjnewStateCnt++] = 6; - break; - case 7: - if (curChar == 123) - jjCheckNAdd(8); - break; - case 8: - if ((0xd7ffffffffffffffL & l) != 0L) - jjCheckNAddTwoStates(8, 9); - break; - case 9: - if (curChar == 125 && kind > 10) - kind = 10; - break; - case 12: - jjAddStates(6, 7); - break; - case 15: - jjAddStates(8, 9); - break; - default : break; + ++curPos; + if ((i = jjnewStateCnt) == (startsAt = 18 - (jjnewStateCnt = startsAt))) { + return curPos; } - } while(i != startsAt); - } - else - { - int hiByte = (int)(curChar >> 8); - int i1 = hiByte >> 6; - long l1 = 1L << (hiByte & 077); - int i2 = (curChar & 0xff) >> 6; - long l2 = 1L << (curChar & 077); - do - { - switch(jjstateSet[--i]) - { - case 18: - case 4: - if (!jjCanMove_0(hiByte, i1, i2, l1, l2)) - break; - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - break; - case 0: - if (!jjCanMove_0(hiByte, i1, i2, l1, l2)) - break; - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - break; - case 6: - if (!jjCanMove_0(hiByte, i1, i2, l1, l2)) - break; - if (kind > 9) - kind = 9; - jjCheckNAddTwoStates(4, 5); - break; - case 1: - if (jjCanMove_0(hiByte, i1, i2, l1, l2)) - jjAddStates(4, 5); - break; - case 8: - if (jjCanMove_0(hiByte, i1, i2, l1, l2)) - jjAddStates(2, 3); - break; - case 12: - if (jjCanMove_0(hiByte, i1, i2, l1, l2)) - jjAddStates(6, 7); - break; - case 15: - if (jjCanMove_0(hiByte, i1, i2, l1, l2)) - jjAddStates(8, 9); - break; - default : break; + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + return curPos; } - } while(i != startsAt); - } - if (kind != 0x7fffffff) - { - jjmatchedKind = kind; - jjmatchedPos = curPos; - kind = 0x7fffffff; - } - ++curPos; - if ((i = jjnewStateCnt) == (startsAt = 18 - (jjnewStateCnt = startsAt))) - return curPos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return curPos; } - } -} -static final int[] jjnextStates = { - 11, 14, 8, 9, 1, 2, 12, 13, 15, 17, -}; -private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2) -{ - switch(hiByte) - { - case 0: - return ((jjbitVec2[i2] & l2) != 0L); - default : - if ((jjbitVec0[i1] & l1) != 0L) - return true; - return false; - } -} + } + } -/** Token literal values. */ -public static final String[] jjstrLiteralImages = { -"", null, null, null, null, "\54", "\75", "\134", null, null, null, }; - -/** Lexer state names. */ -public static final String[] lexStateNames = { - "DEFAULT", -}; -static final long[] jjtoToken = { - 0x7e1L, -}; -static final long[] jjtoSkip = { - 0x1eL, -}; -protected JavaCharStream input_stream; -private final int[] jjrounds = new int[18]; -private final int[] jjstateSet = new int[36]; -protected char curChar; -/** Constructor. */ -public ISEParserTokenManager(JavaCharStream stream){ - if (JavaCharStream.staticFlag) - throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer."); - input_stream = stream; -} + static final int[] jjnextStates = { + 11, 14, 8, 9, 1, 2, 12, 13, 15, 17, + }; -/** Constructor. */ -public ISEParserTokenManager(JavaCharStream stream, int lexState){ - this(stream); - SwitchTo(lexState); -} + private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2) { + switch (hiByte) { + case 0: + return ((jjbitVec2[i2] & l2) != 0L); + default: + return (jjbitVec0[i1] & l1) != 0L; + } + } -/** Reinitialise parser. */ -public void ReInit(JavaCharStream stream) -{ - jjmatchedPos = jjnewStateCnt = 0; - curLexState = defaultLexState; - input_stream = stream; - ReInitRounds(); -} -private void ReInitRounds() -{ - int i; - jjround = 0x80000001; - for (i = 18; i-- > 0;) - jjrounds[i] = 0x80000000; -} + /** + * Token literal values. + */ + @SuppressWarnings("checkstyle:IllegalTokenText") + public static final String[] jjstrLiteralImages = { + "", null, null, null, null, "\54", "\75", "\134", null, null, null, }; -/** Reinitialise parser. */ -public void ReInit(JavaCharStream stream, int lexState) -{ - ReInit(stream); - SwitchTo(lexState); -} + /** + * Lexer state names. + */ + public static final String[] lexStateNames = { + "DEFAULT", + }; + static final long[] jjtoToken = { + 0x7e1L, + }; + static final long[] jjtoSkip = { + 0x1eL, + }; + protected JavaCharStream input_stream; + private final int[] jjrounds = new int[18]; + private final int[] jjstateSet = new int[36]; + protected char curChar; -/** Switch to specified lex state. */ -public void SwitchTo(int lexState) -{ - if (lexState >= 1 || lexState < 0) - throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); - else - curLexState = lexState; -} + /** + * Constructor. + */ + public ISEParserTokenManager(JavaCharStream stream) { + if (JavaCharStream.staticFlag) { + throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer."); + } + input_stream = stream; + } -protected Token jjFillToken() -{ - final Token t; - final String curTokenImage; - final int beginLine; - final int endLine; - final int beginColumn; - final int endColumn; - String im = jjstrLiteralImages[jjmatchedKind]; - curTokenImage = (im == null) ? input_stream.GetImage() : im; - beginLine = input_stream.getBeginLine(); - beginColumn = input_stream.getBeginColumn(); - endLine = input_stream.getEndLine(); - endColumn = input_stream.getEndColumn(); - t = Token.newToken(jjmatchedKind, curTokenImage); - - t.beginLine = beginLine; - t.endLine = endLine; - t.beginColumn = beginColumn; - t.endColumn = endColumn; - - return t; -} + /** + * Constructor. + */ + public ISEParserTokenManager(JavaCharStream stream, int lexState) { + this(stream); + SwitchTo(lexState); + } -int curLexState = 0; -int defaultLexState = 0; -int jjnewStateCnt; -int jjround; -int jjmatchedPos; -int jjmatchedKind; - -/** Get the next Token. */ -public Token getNextToken() -{ - Token matchedToken; - int curPos = 0; - - EOFLoop : - for (;;) - { - try - { - curChar = input_stream.BeginToken(); - } - catch(java.io.IOException e) - { - jjmatchedKind = 0; - matchedToken = jjFillToken(); - return matchedToken; - } - - jjmatchedKind = 0x7fffffff; - jjmatchedPos = 0; - curPos = jjMoveStringLiteralDfa0_0(); - if (jjmatchedKind != 0x7fffffff) - { - if (jjmatchedPos + 1 < curPos) - input_stream.backup(curPos - jjmatchedPos - 1); - if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) - { - matchedToken = jjFillToken(); - return matchedToken; - } - else - { - continue EOFLoop; - } - } - int error_line = input_stream.getEndLine(); - int error_column = input_stream.getEndColumn(); - String error_after = null; - boolean EOFSeen = false; - try { input_stream.readChar(); input_stream.backup(1); } - catch (java.io.IOException e1) { - EOFSeen = true; - error_after = curPos <= 1 ? "" : input_stream.GetImage(); - if (curChar == '\n' || curChar == '\r') { - error_line++; - error_column = 0; - } - else - error_column++; - } - if (!EOFSeen) { - input_stream.backup(1); - error_after = curPos <= 1 ? "" : input_stream.GetImage(); - } - throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR); - } -} + /** + * Reinitialise parser. + */ + public void ReInit(JavaCharStream stream) { + jjmatchedPos = jjnewStateCnt = 0; + curLexState = defaultLexState; + input_stream = stream; + ReInitRounds(); + } -private void jjCheckNAdd(int state) -{ - if (jjrounds[state] != jjround) - { - jjstateSet[jjnewStateCnt++] = state; - jjrounds[state] = jjround; - } -} -private void jjAddStates(int start, int end) -{ - do { - jjstateSet[jjnewStateCnt++] = jjnextStates[start]; - } while (start++ != end); -} -private void jjCheckNAddTwoStates(int state1, int state2) -{ - jjCheckNAdd(state1); - jjCheckNAdd(state2); -} + /** + * Reinitialise parser. + */ + public void ReInit(JavaCharStream stream, int lexState) { + ReInit(stream); + SwitchTo(lexState); + } + + private void ReInitRounds() { + int i; + jjround = 0x80000001; + for (i = 18; i-- > 0; ) { + jjrounds[i] = 0x80000000; + } + } + + /** + * Switch to specified lex state. + */ + public void SwitchTo(int lexState) { + if (lexState >= 1 || lexState < 0) { + throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", + TokenMgrError.INVALID_LEXICAL_STATE); + } else { + curLexState = lexState; + } + } + + protected Token jjFillToken() { + final Token t; + final String curTokenImage; + final int beginLine; + final int endLine; + final int beginColumn; + final int endColumn; + String im = jjstrLiteralImages[jjmatchedKind]; + curTokenImage = (im == null) ? input_stream.GetImage() : im; + beginLine = input_stream.getBeginLine(); + beginColumn = input_stream.getBeginColumn(); + endLine = input_stream.getEndLine(); + endColumn = input_stream.getEndColumn(); + t = Token.newToken(jjmatchedKind, curTokenImage); + + t.beginLine = beginLine; + t.endLine = endLine; + t.beginColumn = beginColumn; + t.endColumn = endColumn; + + return t; + } + + int curLexState = 0; + int defaultLexState = 0; + int jjnewStateCnt; + int jjround; + int jjmatchedPos; + int jjmatchedKind; + + /** + * Get the next Token. + */ + public Token getNextToken() { + Token matchedToken; + int curPos = 0; + + EOFLoop: + for (; ; ) { + try { + curChar = input_stream.BeginToken(); + } catch (java.io.IOException e) { + jjmatchedKind = 0; + matchedToken = jjFillToken(); + return matchedToken; + } + + jjmatchedKind = 0x7fffffff; + jjmatchedPos = 0; + curPos = jjMoveStringLiteralDfa0_0(); + if (jjmatchedKind != 0x7fffffff) { + if (jjmatchedPos + 1 < curPos) { + input_stream.backup(curPos - jjmatchedPos - 1); + } + if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) { + matchedToken = jjFillToken(); + return matchedToken; + } else { + continue EOFLoop; + } + } + int errorLine = input_stream.getEndLine(); + int errorColumn = input_stream.getEndColumn(); + String errorAfter = null; + boolean eofSeen = false; + try { + input_stream.readChar(); + input_stream.backup(1); + } catch (java.io.IOException e1) { + eofSeen = true; + errorAfter = curPos <= 1 ? "" : input_stream.GetImage(); + if (curChar == '\n' || curChar == '\r') { + errorLine++; + errorColumn = 0; + } else { + errorColumn++; + } + } + if (!eofSeen) { + input_stream.backup(1); + errorAfter = curPos <= 1 ? "" : input_stream.GetImage(); + } + throw new TokenMgrError(eofSeen, curLexState, errorLine, errorColumn, errorAfter, curChar, + TokenMgrError.LEXICAL_ERROR); + } + } + + private void jjCheckNAdd(int state) { + if (jjrounds[state] != jjround) { + jjstateSet[jjnewStateCnt++] = state; + jjrounds[state] = jjround; + } + } + + private void jjAddStates(int start, int end) { + do { + jjstateSet[jjnewStateCnt++] = jjnextStates[start]; + } while (start++ != end); + } + + private void jjCheckNAddTwoStates(int state1, int state2) { + jjCheckNAdd(state1); + jjCheckNAdd(state2); + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java index f23f5bcc..588d1578 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +19,7 @@ */ /* Generated By:JavaCC: Do not edit this line. JavaCharStream.java Version 5.0 */ /* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=false */ + package org.apache.metron.parsers.ise; import java.nio.charset.StandardCharsets; @@ -25,611 +28,630 @@ * An implementation of interface CharStream, where the stream is assumed to * contain only ASCII characters (with java-like unicode escape processing). */ +@SuppressWarnings({"checkstyle:MethodName", "checkstyle:SummaryJavadoc", "checkstyle:NonEmptyAtclauseDescription"}) +class JavaCharStream { + /** + * Whether parser is static. + */ + public static final boolean staticFlag = false; + + static final int hexval(char c) throws java.io.IOException { + switch (c) { + case '0': + return 0; + case '1': + return 1; + case '2': + return 2; + case '3': + return 3; + case '4': + return 4; + case '5': + return 5; + case '6': + return 6; + case '7': + return 7; + case '8': + return 8; + case '9': + return 9; + + case 'a': + case 'A': + return 10; + case 'b': + case 'B': + return 11; + case 'c': + case 'C': + return 12; + case 'd': + case 'D': + return 13; + case 'e': + case 'E': + return 14; + case 'f': + case 'F': + return 15; + default: + break; + } + + throw new java.io.IOException(); // Should never come here + } + + /** + * Position in buffer. + */ + public int bufpos = -1; + int bufsize; + int available; + int tokenBegin; + protected int[] bufline; + protected int[] bufcolumn; + + protected int column = 0; + protected int line = 1; + + protected boolean prevCharIsCR = false; + protected boolean prevCharIsLF = false; + + protected java.io.Reader inputStream; + + protected char[] nextCharBuf; + protected char[] buffer; + protected int maxNextCharInd = 0; + protected int nextCharInd = -1; + protected int inBuf = 0; + protected int tabSize = 8; + + protected void setTabSize(int i) { + tabSize = i; + } + + protected int getTabSize(int i) { + return tabSize; + } + + protected void ExpandBuff(boolean wrapAround) { + char[] newbuffer = new char[bufsize + 2048]; + int[] newbufline = new int[bufsize + 2048]; + int[] newbufcolumn = new int[bufsize + 2048]; + + try { + if (wrapAround) { + System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); + System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos); + buffer = newbuffer; + + System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); + System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); + bufline = newbufline; + + System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); + System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); + bufcolumn = newbufcolumn; + + bufpos += (bufsize - tokenBegin); + } else { + System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); + buffer = newbuffer; + + System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); + bufline = newbufline; + + System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); + bufcolumn = newbufcolumn; + + bufpos -= tokenBegin; + } + } catch (Throwable t) { + throw new Error(t.getMessage()); + } + + available = (bufsize += 2048); + tokenBegin = 0; + } + + protected void FillBuff() throws java.io.IOException { + int i; + if (maxNextCharInd == 4096) { + maxNextCharInd = nextCharInd = 0; + } + + try { + if ((i = inputStream.read(nextCharBuf, maxNextCharInd, + 4096 - maxNextCharInd)) == -1) { + inputStream.close(); + throw new java.io.IOException(); + } else { + maxNextCharInd += i; + } + } catch (java.io.IOException e) { + if (bufpos != 0) { + --bufpos; + backup(0); + } else { + bufline[bufpos] = line; + bufcolumn[bufpos] = column; + } + throw e; + } + } + + protected char ReadByte() throws java.io.IOException { + if (++nextCharInd >= maxNextCharInd) { + FillBuff(); + } + + return nextCharBuf[nextCharInd]; + } + + /** + * Reads the begin token. + * + * @return starting character for token. + */ + public char BeginToken() throws java.io.IOException { + if (inBuf > 0) { + --inBuf; + + if (++bufpos == bufsize) { + bufpos = 0; + } + + tokenBegin = bufpos; + return buffer[bufpos]; + } + + tokenBegin = 0; + bufpos = -1; + + return readChar(); + } + + protected void AdjustBuffSize() { + if (available == bufsize) { + if (tokenBegin > 2048) { + bufpos = 0; + available = tokenBegin; + } else { + ExpandBuff(false); + } + } else if (available > tokenBegin) { + available = bufsize; + } else if ((tokenBegin - available) < 2048) { + ExpandBuff(true); + } else { + available = tokenBegin; + } + } + + protected void UpdateLineColumn(char c) { + column++; + + if (prevCharIsLF) { + prevCharIsLF = false; + line += (column = 1); + } else if (prevCharIsCR) { + prevCharIsCR = false; + if (c == '\n') { + prevCharIsLF = true; + } else { + line += (column = 1); + } + } + + switch (c) { + case '\r': + prevCharIsCR = true; + break; + case '\n': + prevCharIsLF = true; + break; + case '\t': + column--; + column += (tabSize - (column % tabSize)); + break; + default: + break; + } -class JavaCharStream -{ - /** Whether parser is static. */ - public static final boolean staticFlag = false; - - static final int hexval(char c) throws java.io.IOException { - switch(c) - { - case '0' : - return 0; - case '1' : - return 1; - case '2' : - return 2; - case '3' : - return 3; - case '4' : - return 4; - case '5' : - return 5; - case '6' : - return 6; - case '7' : - return 7; - case '8' : - return 8; - case '9' : - return 9; - - case 'a' : - case 'A' : - return 10; - case 'b' : - case 'B' : - return 11; - case 'c' : - case 'C' : - return 12; - case 'd' : - case 'D' : - return 13; - case 'e' : - case 'E' : - return 14; - case 'f' : - case 'F' : - return 15; - } - - throw new java.io.IOException(); // Should never come here - } - -/** Position in buffer. */ - public int bufpos = -1; - int bufsize; - int available; - int tokenBegin; - protected int bufline[]; - protected int bufcolumn[]; - - protected int column = 0; - protected int line = 1; - - protected boolean prevCharIsCR = false; - protected boolean prevCharIsLF = false; - - protected java.io.Reader inputStream; - - protected char[] nextCharBuf; - protected char[] buffer; - protected int maxNextCharInd = 0; - protected int nextCharInd = -1; - protected int inBuf = 0; - protected int tabSize = 8; - - protected void setTabSize(int i) { tabSize = i; } - protected int getTabSize(int i) { return tabSize; } - - protected void ExpandBuff(boolean wrapAround) - { - char[] newbuffer = new char[bufsize + 2048]; - int newbufline[] = new int[bufsize + 2048]; - int newbufcolumn[] = new int[bufsize + 2048]; - - try - { - if (wrapAround) - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos); - buffer = newbuffer; - - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); - bufline = newbufline; - - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); - bufcolumn = newbufcolumn; - - bufpos += (bufsize - tokenBegin); - } - else - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - buffer = newbuffer; - - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - bufline = newbufline; - - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - bufcolumn = newbufcolumn; - - bufpos -= tokenBegin; - } - } - catch (Throwable t) - { - throw new Error(t.getMessage()); - } - - available = (bufsize += 2048); - tokenBegin = 0; - } - - protected void FillBuff() throws java.io.IOException - { - int i; - if (maxNextCharInd == 4096) - maxNextCharInd = nextCharInd = 0; - - try { - if ((i = inputStream.read(nextCharBuf, maxNextCharInd, - 4096 - maxNextCharInd)) == -1) - { - inputStream.close(); - throw new java.io.IOException(); - } - else - maxNextCharInd += i; - return; - } - catch(java.io.IOException e) { - if (bufpos != 0) - { - --bufpos; - backup(0); - } - else - { bufline[bufpos] = line; bufcolumn[bufpos] = column; - } - throw e; - } - } - - protected char ReadByte() throws java.io.IOException - { - if (++nextCharInd >= maxNextCharInd) - FillBuff(); - - return nextCharBuf[nextCharInd]; - } - -/** @return starting character for token. */ - public char BeginToken() throws java.io.IOException - { - if (inBuf > 0) - { - --inBuf; - - if (++bufpos == bufsize) - bufpos = 0; - - tokenBegin = bufpos; - return buffer[bufpos]; - } - - tokenBegin = 0; - bufpos = -1; - - return readChar(); - } - - protected void AdjustBuffSize() - { - if (available == bufsize) - { - if (tokenBegin > 2048) - { - bufpos = 0; - available = tokenBegin; - } - else - ExpandBuff(false); - } - else if (available > tokenBegin) - available = bufsize; - else if ((tokenBegin - available) < 2048) - ExpandBuff(true); - else - available = tokenBegin; - } - - protected void UpdateLineColumn(char c) - { - column++; - - if (prevCharIsLF) - { - prevCharIsLF = false; - line += (column = 1); - } - else if (prevCharIsCR) - { - prevCharIsCR = false; - if (c == '\n') - { - prevCharIsLF = true; - } - else - line += (column = 1); - } - - switch (c) - { - case '\r' : - prevCharIsCR = true; - break; - case '\n' : - prevCharIsLF = true; - break; - case '\t' : - column--; - column += (tabSize - (column % tabSize)); - break; - default : - break; - } - - bufline[bufpos] = line; - bufcolumn[bufpos] = column; - } - -/** Read a character. */ - public char readChar() throws java.io.IOException - { - if (inBuf > 0) - { - --inBuf; - - if (++bufpos == bufsize) - bufpos = 0; - - return buffer[bufpos]; - } - - char c; - - if (++bufpos == available) - AdjustBuffSize(); - - if ((buffer[bufpos] = c = ReadByte()) == '\\') - { - UpdateLineColumn(c); - - int backSlashCnt = 1; - - for (;;) // Read all the backslashes - { - if (++bufpos == available) - AdjustBuffSize(); - - try - { - if ((buffer[bufpos] = c = ReadByte()) != '\\') - { + } + + /** + * Read a character. + */ + public char readChar() throws java.io.IOException { + if (inBuf > 0) { + --inBuf; + + if (++bufpos == bufsize) { + bufpos = 0; + } + + return buffer[bufpos]; + } + + char c; + + if (++bufpos == available) { + AdjustBuffSize(); + } + + if ((buffer[bufpos] = c = ReadByte()) == '\\') { UpdateLineColumn(c); - // found a non-backslash char. - if ((c == 'u') && ((backSlashCnt & 1) == 1)) - { - if (--bufpos < 0) - bufpos = bufsize - 1; - break; + int backSlashCnt = 1; + + // Read all the backslashes + for (; ; ) { + if (++bufpos == available) { + AdjustBuffSize(); + } + + try { + if ((buffer[bufpos] = c = ReadByte()) != '\\') { + UpdateLineColumn(c); + // found a non-backslash char. + if ((c == 'u') && ((backSlashCnt & 1) == 1)) { + if (--bufpos < 0) { + bufpos = bufsize - 1; + } + + break; + } + + backup(backSlashCnt); + return '\\'; + } + } catch (java.io.IOException e) { + // We are returning one backslash so we should only backup (count-1) + if (backSlashCnt > 1) { + backup(backSlashCnt - 1); + } + + return '\\'; + } + + UpdateLineColumn(c); + backSlashCnt++; } - backup(backSlashCnt); - return '\\'; - } + // Here, we have seen an odd number of backslash's followed by a 'u' + try { + while ((c = ReadByte()) == 'u') { + ++column; + } + + buffer[bufpos] = c = (char) (hexval(c) << 12 + | hexval(ReadByte()) << 8 + | hexval(ReadByte()) << 4 + | hexval(ReadByte())); + + column += 4; + } catch (java.io.IOException e) { + throw new Error("Invalid escape character at line " + line + + " column " + column + "."); + } + + if (backSlashCnt == 1) { + return c; + } else { + backup(backSlashCnt - 1); + return '\\'; + } + } else { + UpdateLineColumn(c); + return c; + } + } + + /** + * @see #getEndColumn + * @deprecated + */ + @Deprecated + public int getColumn() { + return bufcolumn[bufpos]; + } + + /** + * @see #getEndLine + * @deprecated + */ + @Deprecated + public int getLine() { + return bufline[bufpos]; + } + + /** + * Get end column. + */ + public int getEndColumn() { + return bufcolumn[bufpos]; + } + + /** + * Get end line. + */ + public int getEndLine() { + return bufline[bufpos]; + } + + /** + * @return column of token start + */ + public int getBeginColumn() { + return bufcolumn[tokenBegin]; + } + + /** + * @return line number of token start + */ + public int getBeginLine() { + return bufline[tokenBegin]; + } + + /** + * Retreat. + */ + public void backup(int amount) { + + inBuf += amount; + if ((bufpos -= amount) < 0) { + bufpos += bufsize; + } + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.Reader dstream, + int startline, int startcolumn, int buffersize) { + inputStream = dstream; + line = startline; + column = startcolumn - 1; + + available = bufsize = buffersize; + buffer = new char[buffersize]; + bufline = new int[buffersize]; + bufcolumn = new int[buffersize]; + nextCharBuf = new char[4096]; + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.Reader dstream, + int startline, int startcolumn) { + this(dstream, startline, startcolumn, 4096); + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.Reader dstream) { + this(dstream, 1, 1, 4096); + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.InputStream dstream, String encoding, int startline, + int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException { + this(encoding == null ? new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8) + : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.InputStream dstream, int startline, + int startcolumn, int buffersize) { + this(new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8), startline, startcolumn, buffersize); + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.InputStream dstream, String encoding, int startline, + int startcolumn) throws java.io.UnsupportedEncodingException { + this(dstream, encoding, startline, startcolumn, 4096); + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.InputStream dstream, int startline, + int startcolumn) { + this(dstream, startline, startcolumn, 4096); + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException { + this(dstream, encoding, 1, 1, 4096); + } + + /** + * Constructor. + */ + public JavaCharStream(java.io.InputStream dstream) { + this(dstream, 1, 1, 4096); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.Reader dstream, + int startline, int startcolumn, int buffersize) { + inputStream = dstream; + line = startline; + column = startcolumn - 1; + + if (buffer == null || buffersize != buffer.length) { + available = bufsize = buffersize; + buffer = new char[buffersize]; + bufline = new int[buffersize]; + bufcolumn = new int[buffersize]; + nextCharBuf = new char[4096]; } - catch(java.io.IOException e) - { - // We are returning one backslash so we should only backup (count-1) - if (backSlashCnt > 1) - backup(backSlashCnt-1); + prevCharIsLF = prevCharIsCR = false; + tokenBegin = inBuf = maxNextCharInd = 0; + nextCharInd = bufpos = -1; + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.Reader dstream, + int startline, int startcolumn) { + ReInit(dstream, startline, startcolumn, 4096); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.Reader dstream) { + ReInit(dstream, 1, 1, 4096); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.InputStream dstream, String encoding, int startline, + int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException { + ReInit(encoding == null ? new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8) + : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.InputStream dstream, int startline, + int startcolumn, int buffersize) { + ReInit(new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8), startline, startcolumn, buffersize); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.InputStream dstream, String encoding, int startline, + int startcolumn) throws java.io.UnsupportedEncodingException { + ReInit(dstream, encoding, startline, startcolumn, 4096); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.InputStream dstream, int startline, + int startcolumn) { + ReInit(dstream, startline, startcolumn, 4096); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException { + ReInit(dstream, encoding, 1, 1, 4096); + } + + /** + * Reinitialise. + */ + public void ReInit(java.io.InputStream dstream) { + ReInit(dstream, 1, 1, 4096); + } - return '\\'; + /** + * @return token image as String + */ + public String GetImage() { + if (bufpos >= tokenBegin) { + return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); + } else { + return new String(buffer, tokenBegin, bufsize - tokenBegin) + + new String(buffer, 0, bufpos + 1); } + } + + /** + * @return suffix + */ + public char[] GetSuffix(int len) { + char[] ret = new char[len]; + + if ((bufpos + 1) >= len) { + System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); + } else { + System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, + len - bufpos - 1); + System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); + } + + return ret; + } + + /** + * Set buffers back to null when finished. + */ + public void Done() { + nextCharBuf = null; + buffer = null; + bufline = null; + bufcolumn = null; + } + + /** + * Method to adjust line and column numbers for the start of a token. + */ + public void adjustBeginLineColumn(int newLine, int newCol) { + int start = tokenBegin; + int len; + + if (bufpos >= tokenBegin) { + len = bufpos - tokenBegin + inBuf + 1; + } else { + len = bufsize - tokenBegin + bufpos + 1 + inBuf; + } + + int i = 0; + int j = 0; + int k = 0; + int nextColDiff = 0; + int columnDiff = 0; + + while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) { + bufline[j] = newLine; + nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j]; + bufcolumn[j] = newCol + columnDiff; + columnDiff = nextColDiff; + i++; + } + + if (i < len) { + bufline[j] = newLine++; + bufcolumn[j] = newCol + columnDiff; - UpdateLineColumn(c); - backSlashCnt++; - } - - // Here, we have seen an odd number of backslash's followed by a 'u' - try - { - while ((c = ReadByte()) == 'u') - ++column; - - buffer[bufpos] = c = (char)(hexval(c) << 12 | - hexval(ReadByte()) << 8 | - hexval(ReadByte()) << 4 | - hexval(ReadByte())); - - column += 4; - } - catch(java.io.IOException e) - { - throw new Error("Invalid escape character at line " + line + - " column " + column + "."); - } - - if (backSlashCnt == 1) - return c; - else - { - backup(backSlashCnt - 1); - return '\\'; - } - } - else - { - UpdateLineColumn(c); - return c; - } - } - - @Deprecated - /** - * @deprecated - * @see #getEndColumn - */ - public int getColumn() { - return bufcolumn[bufpos]; - } - - @Deprecated - /** - * @deprecated - * @see #getEndLine - */ - public int getLine() { - return bufline[bufpos]; - } - -/** Get end column. */ - public int getEndColumn() { - return bufcolumn[bufpos]; - } - -/** Get end line. */ - public int getEndLine() { - return bufline[bufpos]; - } - -/** @return column of token start */ - public int getBeginColumn() { - return bufcolumn[tokenBegin]; - } - -/** @return line number of token start */ - public int getBeginLine() { - return bufline[tokenBegin]; - } - -/** Retreat. */ - public void backup(int amount) { - - inBuf += amount; - if ((bufpos -= amount) < 0) - bufpos += bufsize; - } - -/** Constructor. */ - public JavaCharStream(java.io.Reader dstream, - int startline, int startcolumn, int buffersize) - { - inputStream = dstream; - line = startline; - column = startcolumn - 1; - - available = bufsize = buffersize; - buffer = new char[buffersize]; - bufline = new int[buffersize]; - bufcolumn = new int[buffersize]; - nextCharBuf = new char[4096]; - } - -/** Constructor. */ - public JavaCharStream(java.io.Reader dstream, - int startline, int startcolumn) - { - this(dstream, startline, startcolumn, 4096); - } - -/** Constructor. */ - public JavaCharStream(java.io.Reader dstream) - { - this(dstream, 1, 1, 4096); - } -/** Reinitialise. */ - public void ReInit(java.io.Reader dstream, - int startline, int startcolumn, int buffersize) - { - inputStream = dstream; - line = startline; - column = startcolumn - 1; - - if (buffer == null || buffersize != buffer.length) - { - available = bufsize = buffersize; - buffer = new char[buffersize]; - bufline = new int[buffersize]; - bufcolumn = new int[buffersize]; - nextCharBuf = new char[4096]; - } - prevCharIsLF = prevCharIsCR = false; - tokenBegin = inBuf = maxNextCharInd = 0; - nextCharInd = bufpos = -1; - } - -/** Reinitialise. */ - public void ReInit(java.io.Reader dstream, - int startline, int startcolumn) - { - ReInit(dstream, startline, startcolumn, 4096); - } - -/** Reinitialise. */ - public void ReInit(java.io.Reader dstream) - { - ReInit(dstream, 1, 1, 4096); - } -/** Constructor. */ - public JavaCharStream(java.io.InputStream dstream, String encoding, int startline, - int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException - { - this(encoding == null ? new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); - } - -/** Constructor. */ - public JavaCharStream(java.io.InputStream dstream, int startline, - int startcolumn, int buffersize) - { - this(new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8), startline, startcolumn, buffersize); - } - -/** Constructor. */ - public JavaCharStream(java.io.InputStream dstream, String encoding, int startline, - int startcolumn) throws java.io.UnsupportedEncodingException - { - this(dstream, encoding, startline, startcolumn, 4096); - } - -/** Constructor. */ - public JavaCharStream(java.io.InputStream dstream, int startline, - int startcolumn) - { - this(dstream, startline, startcolumn, 4096); - } - -/** Constructor. */ - public JavaCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException - { - this(dstream, encoding, 1, 1, 4096); - } - -/** Constructor. */ - public JavaCharStream(java.io.InputStream dstream) - { - this(dstream, 1, 1, 4096); - } - -/** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding, int startline, - int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException - { - ReInit(encoding == null ? new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); - } - -/** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, int startline, - int startcolumn, int buffersize) - { - ReInit(new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8), startline, startcolumn, buffersize); - } -/** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding, int startline, - int startcolumn) throws java.io.UnsupportedEncodingException - { - ReInit(dstream, encoding, startline, startcolumn, 4096); - } -/** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, int startline, - int startcolumn) - { - ReInit(dstream, startline, startcolumn, 4096); - } -/** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException - { - ReInit(dstream, encoding, 1, 1, 4096); - } - -/** Reinitialise. */ - public void ReInit(java.io.InputStream dstream) - { - ReInit(dstream, 1, 1, 4096); - } - - /** @return token image as String */ - public String GetImage() - { - if (bufpos >= tokenBegin) - return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); - else - return new String(buffer, tokenBegin, bufsize - tokenBegin) + - new String(buffer, 0, bufpos + 1); - } - - /** @return suffix */ - public char[] GetSuffix(int len) - { - char[] ret = new char[len]; - - if ((bufpos + 1) >= len) - System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); - else - { - System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, - len - bufpos - 1); - System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); - } - - return ret; - } - - /** Set buffers back to null when finished. */ - public void Done() - { - nextCharBuf = null; - buffer = null; - bufline = null; - bufcolumn = null; - } - - /** - * Method to adjust line and column numbers for the start of a token. - */ - public void adjustBeginLineColumn(int newLine, int newCol) - { - int start = tokenBegin; - int len; - - if (bufpos >= tokenBegin) - { - len = bufpos - tokenBegin + inBuf + 1; - } - else - { - len = bufsize - tokenBegin + bufpos + 1 + inBuf; - } - - int i = 0, j = 0, k = 0; - int nextColDiff = 0, columnDiff = 0; - - while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) - { - bufline[j] = newLine; - nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j]; - bufcolumn[j] = newCol + columnDiff; - columnDiff = nextColDiff; - i++; - } - - if (i < len) - { - bufline[j] = newLine++; - bufcolumn[j] = newCol + columnDiff; - - while (i++ < len) - { - if (bufline[j = start % bufsize] != bufline[++start % bufsize]) - bufline[j] = newLine++; - else - bufline[j] = newLine; - } - } - - line = bufline[j]; - column = bufcolumn[j]; - } + while (i++ < len) { + if (bufline[j = start % bufsize] != bufline[++start % bufsize]) { + bufline[j] = newLine++; + } else { + bufline[j] = newLine; + } + } + } + + line = bufline[j]; + column = bufcolumn[j]; + } } /* JavaCC - OriginalChecksum=96a5b0b0fa09286690f250998f047719 (do not edit this line) */ diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ParseException.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ParseException.java index fc21aa1d..ec3638c1 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ParseException.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/ParseException.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +19,7 @@ */ /* Generated By:JavaCC: Do not edit this line. ParseException.java Version 5.0 */ /* JavaCCOptions:KEEP_LINE_COL=null */ + package org.apache.metron.parsers.ise; /** @@ -25,180 +28,184 @@ * calling the method generateParseException in the generated * parser. * + *

    * You can modify this class to customize your error reporting * mechanisms so long as you retain the public fields. */ +@SuppressWarnings("checkstyle:MethodName") public class ParseException extends Exception { - /** - * The version identifier for this Serializable class. - * Increment only if the serialized form of the - * class changes. - */ - private static final long serialVersionUID = 1L; + /** + * The version identifier for this Serializable class. + * Increment only if the serialized form of the + * class changes. + */ + private static final long serialVersionUID = 1L; - /** - * This constructor is used by the method "generateParseException" - * in the generated parser. Calling this constructor generates - * a new object of this type with the fields "currentToken", - * "expectedTokenSequences", and "tokenImage" set. - */ - public ParseException(Token currentTokenVal, - int[][] expectedTokenSequencesVal, - String[] tokenImageVal - ) - { - super(initialise(currentTokenVal, expectedTokenSequencesVal, tokenImageVal)); - currentToken = currentTokenVal; - expectedTokenSequences = expectedTokenSequencesVal; - tokenImage = tokenImageVal; - } + /** + * This constructor is used by the method "generateParseException" + * in the generated parser. Calling this constructor generates + * a new object of this type with the fields "currentToken", + * "expectedTokenSequences", and "tokenImage" set. + */ + public ParseException(Token currentTokenVal, + int[][] expectedTokenSequencesVal, + String[] tokenImageVal + ) { + super(initialise(currentTokenVal, expectedTokenSequencesVal, tokenImageVal)); + currentToken = currentTokenVal; + expectedTokenSequences = expectedTokenSequencesVal; + tokenImage = tokenImageVal; + } - /** - * The following constructors are for use by you for whatever - * purpose you can think of. Constructing the exception in this - * manner makes the exception behave in the normal way - i.e., as - * documented in the class "Throwable". The fields "errorToken", - * "expectedTokenSequences", and "tokenImage" do not contain - * relevant information. The JavaCC generated code does not use - * these constructors. - */ + /** + * The following constructors are for use by you for whatever + * purpose you can think of. Constructing the exception in this + * manner makes the exception behave in the normal way - i.e., as + * documented in the class "Throwable". The fields "errorToken", + * "expectedTokenSequences", and "tokenImage" do not contain + * relevant information. The JavaCC generated code does not use + * these constructors. + */ - public ParseException() { - super(); - } + public ParseException() { + super(); + } - /** Constructor with message. */ - public ParseException(String message) { - super(message); - } + /** + * Constructor with message. + */ + public ParseException(String message) { + super(message); + } - /** - * This is the last token that has been consumed successfully. If - * this object has been created due to a parse error, the token - * followng this token will (therefore) be the first error token. - */ - public Token currentToken; + /** + * This is the last token that has been consumed successfully. If + * this object has been created due to a parse error, the token + * followng this token will (therefore) be the first error token. + */ + public Token currentToken; - /** - * Each entry in this array is an array of integers. Each array - * of integers represents a sequence of tokens (by their ordinal - * values) that is expected at this point of the parse. - */ - public int[][] expectedTokenSequences; + /** + * Each entry in this array is an array of integers. Each array + * of integers represents a sequence of tokens (by their ordinal + * values) that is expected at this point of the parse. + */ + public int[][] expectedTokenSequences; - /** - * This is a reference to the "tokenImage" array of the generated - * parser within which the parse error occurred. This array is - * defined in the generated ...Constants interface. - */ - public String[] tokenImage; + /** + * This is a reference to the "tokenImage" array of the generated + * parser within which the parse error occurred. This array is + * defined in the generated ...Constants interface. + */ + public String[] tokenImage; - /** - * It uses "currentToken" and "expectedTokenSequences" to generate a parse - * error message and returns it. If this object has been created - * due to a parse error, and you do not catch it (it gets thrown - * from the parser) the correct error message - * gets displayed. - */ - private static String initialise(Token currentToken, - int[][] expectedTokenSequences, - String[] tokenImage) { - String eol = System.getProperty("line.separator", "\n"); - StringBuffer expected = new StringBuffer(); - int maxSize = 0; - for (int i = 0; i < expectedTokenSequences.length; i++) { - if (maxSize < expectedTokenSequences[i].length) { - maxSize = expectedTokenSequences[i].length; - } - for (int j = 0; j < expectedTokenSequences[i].length; j++) { - expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' '); - } - if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) { - expected.append("..."); - } - expected.append(eol).append(" "); - } - String retval = "Encountered \""; - Token tok = currentToken.next; - for (int i = 0; i < maxSize; i++) { - if (i != 0) retval += " "; - if (tok.kind == 0) { - retval += tokenImage[0]; - break; - } - retval += " " + tokenImage[tok.kind]; - retval += " \""; - retval += add_escapes(tok.image); - retval += " \""; - tok = tok.next; - } - retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn; - retval += "." + eol; - if (expectedTokenSequences.length == 1) { - retval += "Was expecting:" + eol + " "; - } else { - retval += "Was expecting one of:" + eol + " "; + /** + * It uses "currentToken" and "expectedTokenSequences" to generate a parse + * error message and returns it. If this object has been created + * due to a parse error, and you do not catch it (it gets thrown + * from the parser) the correct error message + * gets displayed. + */ + private static String initialise(Token currentToken, + int[][] expectedTokenSequences, + String[] tokenImage) { + String eol = System.getProperty("line.separator", "\n"); + StringBuffer expected = new StringBuffer(); + int maxSize = 0; + for (int i = 0; i < expectedTokenSequences.length; i++) { + if (maxSize < expectedTokenSequences[i].length) { + maxSize = expectedTokenSequences[i].length; + } + for (int j = 0; j < expectedTokenSequences[i].length; j++) { + expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' '); + } + if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) { + expected.append("..."); + } + expected.append(eol).append(" "); + } + String retval = "Encountered \""; + Token tok = currentToken.next; + for (int i = 0; i < maxSize; i++) { + if (i != 0) { + retval += " "; + } + if (tok.kind == 0) { + retval += tokenImage[0]; + break; + } + retval += " " + tokenImage[tok.kind]; + retval += " \""; + retval += add_escapes(tok.image); + retval += " \""; + tok = tok.next; + } + retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn; + retval += "." + eol; + if (expectedTokenSequences.length == 1) { + retval += "Was expecting:" + eol + " "; + } else { + retval += "Was expecting one of:" + eol + " "; + } + retval += expected.toString(); + return retval; } - retval += expected.toString(); - return retval; - } - /** - * The end of line string for this machine. - */ - protected String eol = System.getProperty("line.separator", "\n"); + /** + * The end of line string for this machine. + */ + protected String eol = System.getProperty("line.separator", "\n"); - /** - * Used to convert raw characters to their escaped version - * when these raw version cannot be used as part of an ASCII - * string literal. - */ - static String add_escapes(String str) { - StringBuffer retval = new StringBuffer(); - char ch; - for (int i = 0; i < str.length(); i++) { - switch (str.charAt(i)) - { - case 0 : - continue; - case '\b': - retval.append("\\b"); - continue; - case '\t': - retval.append("\\t"); - continue; - case '\n': - retval.append("\\n"); - continue; - case '\f': - retval.append("\\f"); - continue; - case '\r': - retval.append("\\r"); - continue; - case '\"': - retval.append("\\\""); - continue; - case '\'': - retval.append("\\\'"); - continue; - case '\\': - retval.append("\\\\"); - continue; - default: - if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { - String s = "0000" + Integer.toString(ch, 16); - retval.append("\\u" + s.substring(s.length() - 4, s.length())); - } else { - retval.append(ch); - } - continue; + /** + * Used to convert raw characters to their escaped version + * when these raw version cannot be used as part of an ASCII + * string literal. + */ + static String add_escapes(String str) { + StringBuffer retval = new StringBuffer(); + char ch; + for (int i = 0; i < str.length(); i++) { + switch (str.charAt(i)) { + case 0: + continue; + case '\b': + retval.append("\\b"); + continue; + case '\t': + retval.append("\\t"); + continue; + case '\n': + retval.append("\\n"); + continue; + case '\f': + retval.append("\\f"); + continue; + case '\r': + retval.append("\\r"); + continue; + case '\"': + retval.append("\\\""); + continue; + case '\'': + retval.append("\\'"); + continue; + case '\\': + retval.append("\\\\"); + continue; + default: + if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { + String s = "0000" + Integer.toString(ch, 16); + retval.append("\\u" + s.substring(s.length() - 4)); + } else { + retval.append(ch); + } + continue; + } } - } - return retval.toString(); - } + return retval.toString(); + } } /* JavaCC - OriginalChecksum=f9f7217056f99de5708d01ebd497dede (do not edit this line) */ diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/Token.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/Token.java index 3545ec47..2bc1f5f4 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/Token.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/Token.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +19,7 @@ */ /* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */ /* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=false */ + package org.apache.metron.parsers.ise; /** @@ -25,124 +28,130 @@ class Token implements java.io.Serializable { - /** - * The version identifier for this Serializable class. - * Increment only if the serialized form of the - * class changes. - */ - private static final long serialVersionUID = 1L; - - /** - * An integer that describes the kind of this token. This numbering - * system is determined by JavaCCParser, and a table of these numbers is - * stored in the file ...Constants.java. - */ - public int kind; - - /** The line number of the first character of this Token. */ - public int beginLine; - /** The column number of the first character of this Token. */ - public int beginColumn; - /** The line number of the last character of this Token. */ - public int endLine; - /** The column number of the last character of this Token. */ - public int endColumn; - - /** - * The string image of the token. - */ - public String image; - - /** - * A reference to the next regular (non-special) token from the input - * stream. If this is the last token from the input stream, or if the - * token manager has not read tokens beyond this one, this field is - * set to null. This is true only if this token is also a regular - * token. Otherwise, see below for a description of the contents of - * this field. - */ - public Token next; - - /** - * This field is used to access special tokens that occur prior to this - * token, but after the immediately preceding regular (non-special) token. - * If there are no such special tokens, this field is set to null. - * When there are more than one such special token, this field refers - * to the last of these special tokens, which in turn refers to the next - * previous special token through its specialToken field, and so on - * until the first special token (whose specialToken field is null). - * The next fields of special tokens refer to other special tokens that - * immediately follow it (without an intervening regular token). If there - * is no such token, this field is null. - */ - public Token specialToken; - - /** - * An optional attribute value of the Token. - * Tokens which are not used as syntactic sugar will often contain - * meaningful values that will be used later on by the compiler or - * interpreter. This attribute value is often different from the image. - * Any subclass of Token that actually wants to return a non-null value can - * override this method as appropriate. - */ - public Object getValue() { - return null; - } - - /** - * No-argument constructor - */ - public Token() {} - - /** - * Constructs a new token for the specified Image. - */ - public Token(int kind) - { - this(kind, null); - } - - /** - * Constructs a new token for the specified Image and Kind. - */ - public Token(int kind, String image) - { - this.kind = kind; - this.image = image; - } - - /** - * Returns the image. - */ - public String toString() - { - return image; - } - - /** - * Returns a new Token object, by default. However, if you want, you - * can create and return subclass objects based on the value of ofKind. - * Simply add the cases to the switch for all those special cases. - * For example, if you have a subclass of Token called IDToken that - * you want to create if ofKind is ID, simply add something like : - * - * case MyParserConstants.ID : return new IDToken(ofKind, image); - * - * to the following switch statement. Then you can cast matchedToken - * variable to the appropriate type and use sit in your lexical actions. - */ - public static Token newToken(int ofKind, String image) - { - switch(ofKind) - { - default : return new Token(ofKind, image); + /** + * The version identifier for this Serializable class. + * Increment only if the serialized form of the + * class changes. + */ + private static final long serialVersionUID = 1L; + + /** + * An integer that describes the kind of this token. This numbering + * system is determined by JavaCCParser, and a table of these numbers is + * stored in the file ...Constants.java. + */ + public int kind; + + /** + * The line number of the first character of this Token. + */ + public int beginLine; + /** + * The column number of the first character of this Token. + */ + public int beginColumn; + /** + * The line number of the last character of this Token. + */ + public int endLine; + /** + * The column number of the last character of this Token. + */ + public int endColumn; + + /** + * The string image of the token. + */ + public String image; + + /** + * A reference to the next regular (non-special) token from the input + * stream. If this is the last token from the input stream, or if the + * token manager has not read tokens beyond this one, this field is + * set to null. This is true only if this token is also a regular + * token. Otherwise, see below for a description of the contents of + * this field. + */ + public Token next; + + /** + * This field is used to access special tokens that occur prior to this + * token, but after the immediately preceding regular (non-special) token. + * If there are no such special tokens, this field is set to null. + * When there are more than one such special token, this field refers + * to the last of these special tokens, which in turn refers to the next + * previous special token through its specialToken field, and so on + * until the first special token (whose specialToken field is null). + * The next fields of special tokens refer to other special tokens that + * immediately follow it (without an intervening regular token). If there + * is no such token, this field is null. + */ + public Token specialToken; + + /** + * An optional attribute value of the Token. + * Tokens which are not used as syntactic sugar will often contain + * meaningful values that will be used later on by the compiler or + * interpreter. This attribute value is often different from the image. + * Any subclass of Token that actually wants to return a non-null value can + * override this method as appropriate. + */ + public Object getValue() { + return null; + } + + /** + * No-argument constructor. + */ + public Token() { } - } - public static Token newToken(int ofKind) - { - return newToken(ofKind, null); - } + /** + * Constructs a new token for the specified Image. + */ + public Token(int kind) { + this(kind, null); + } + + /** + * Constructs a new token for the specified Image and Kind. + */ + public Token(int kind, String image) { + this.kind = kind; + this.image = image; + } + + /** + * Returns the image. + */ + public String toString() { + return image; + } + + /** + * Returns a new Token object, by default. However, if you want, you + * can create and return subclass objects based on the value of ofKind. + * Simply add the cases to the switch for all those special cases. + * For example, if you have a subclass of Token called IDToken that + * you want to create if ofKind is ID, simply add something like : + * + *

    + * case MyParserConstants.ID : return new IDToken(ofKind, image); + * + *

    + * to the following switch statement. Then you can cast matchedToken + * variable to the appropriate type and use sit in your lexical actions. + */ + public static Token newToken(int ofKind, String image) { + switch (ofKind) { + default: + return new Token(ofKind, image); + } + } + + public static Token newToken(int ofKind) { + return newToken(ofKind, null); + } } /* JavaCC - OriginalChecksum=99daf0baa94b6c270eea5be0575db6aa (do not edit this line) */ diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java index 2ccc23af..05528bbb 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,148 +19,161 @@ */ /* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */ /* JavaCCOptions: */ + package org.apache.metron.parsers.ise; -/** Token Manager Error. */ -class TokenMgrError extends Error -{ - - /** - * The version identifier for this Serializable class. - * Increment only if the serialized form of the - * class changes. - */ - private static final long serialVersionUID = 1L; - - /* - * Ordinals for various reasons why an Error of this type can be thrown. - */ - - /** - * Lexical error occurred. - */ - static final int LEXICAL_ERROR = 0; - - /** - * An attempt was made to create a second instance of a static token manager. - */ - static final int STATIC_LEXER_ERROR = 1; - - /** - * Tried to change to an invalid lexical state. - */ - static final int INVALID_LEXICAL_STATE = 2; - - /** - * Detected (and bailed out of) an infinite loop in the token manager. - */ - static final int LOOP_DETECTED = 3; - - /** - * Indicates the reason why the exception is thrown. It will have - * one of the above 4 values. - */ - int errorCode; - - /** - * Replaces unprintable characters by their escaped (or unicode escaped) - * equivalents in the given string - */ - protected static final String addEscapes(String str) { - StringBuffer retval = new StringBuffer(); - char ch; - for (int i = 0; i < str.length(); i++) { - switch (str.charAt(i)) - { - case 0 : - continue; - case '\b': - retval.append("\\b"); - continue; - case '\t': - retval.append("\\t"); - continue; - case '\n': - retval.append("\\n"); - continue; - case '\f': - retval.append("\\f"); - continue; - case '\r': - retval.append("\\r"); - continue; - case '\"': - retval.append("\\\""); - continue; - case '\'': - retval.append("\\\'"); - continue; - case '\\': - retval.append("\\\\"); - continue; - default: - if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { - String s = "0000" + Integer.toString(ch, 16); - retval.append("\\u" + s.substring(s.length() - 4, s.length())); - } else { - retval.append(ch); - } - continue; - } +/** + * Token Manager Error. + */ +class TokenMgrError extends Error { + + /** + * The version identifier for this Serializable class. + * Increment only if the serialized form of the + * class changes. + */ + private static final long serialVersionUID = 1L; + + /* + * Ordinals for various reasons why an Error of this type can be thrown. + */ + + /** + * Lexical error occurred. + */ + static final int LEXICAL_ERROR = 0; + + /** + * An attempt was made to create a second instance of a static token manager. + */ + static final int STATIC_LEXER_ERROR = 1; + + /** + * Tried to change to an invalid lexical state. + */ + static final int INVALID_LEXICAL_STATE = 2; + + /** + * Detected (and bailed out of) an infinite loop in the token manager. + */ + static final int LOOP_DETECTED = 3; + + /** + * Indicates the reason why the exception is thrown. It will have + * one of the above 4 values. + */ + int errorCode; + + /** + * Replaces unprintable characters by their escaped (or unicode escaped) + * equivalents in the given string. + */ + protected static final String addEscapes(String str) { + StringBuffer retval = new StringBuffer(); + char ch; + for (int i = 0; i < str.length(); i++) { + switch (str.charAt(i)) { + case 0: + continue; + case '\b': + retval.append("\\b"); + continue; + case '\t': + retval.append("\\t"); + continue; + case '\n': + retval.append("\\n"); + continue; + case '\f': + retval.append("\\f"); + continue; + case '\r': + retval.append("\\r"); + continue; + case '\"': + retval.append("\\\""); + continue; + case '\'': + retval.append("\\'"); + continue; + case '\\': + retval.append("\\\\"); + continue; + default: + if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { + String s = "0000" + Integer.toString(ch, 16); + retval.append("\\u" + s.substring(s.length() - 4)); + } else { + retval.append(ch); + } + continue; + } + } + return retval.toString(); + } + + /** + * Returns a detailed message for the Error when it is thrown by the + * token manager to indicate a lexical error. + * Parameters : + * eofSeen : indicates if EOF caused the lexical error + * curLexState : lexical state in which this error occurred + * errorLine : line number when the error occurred + * errorColumn : column number when the error occurred + * errorAfter : prefix that was seen before this error occurred + * curchar : the offending character + * Note: You can customize the lexical error message by modifying this method. + */ + @SuppressWarnings("checkstyle:MethodName") + protected static String LexicalError(boolean eofSeen, int lexState, int errorLine, int errorColumn, + String errorAfter, char curChar) { + return ("Lexical error at line " + + errorLine + ", column " + + errorColumn + ". Encountered: " + + (eofSeen ? " " + : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int) curChar + "), ") + + "after : \"" + addEscapes(errorAfter) + "\""); + } + + /** + * You can also modify the body of this method to customize your error messages. + * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not + * of end-users concern, so you can return something like : + * + *

    + * "Internal Error : Please file a bug report .... " + * + *

    + * from this method for such cases in the release version of your parser. + */ + public String getMessage() { + return super.getMessage(); + } + + /* + * Constructors of various flavors follow. + */ + + /** + * No arg constructor. + */ + public TokenMgrError() { + } + + /** + * Constructor with message and reason. + */ + public TokenMgrError(String message, int reason) { + super(message); + errorCode = reason; + } + + /** + * Full Constructor. + */ + public TokenMgrError(boolean eofSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, + int reason) { + this(LexicalError(eofSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); } - return retval.toString(); - } - - /** - * Returns a detailed message for the Error when it is thrown by the - * token manager to indicate a lexical error. - * Parameters : - * EOFSeen : indicates if EOF caused the lexical error - * curLexState : lexical state in which this error occurred - * errorLine : line number when the error occurred - * errorColumn : column number when the error occurred - * errorAfter : prefix that was seen before this error occurred - * curchar : the offending character - * Note: You can customize the lexical error message by modifying this method. - */ - protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { - return("Lexical error at line " + - errorLine + ", column " + - errorColumn + ". Encountered: " + - (EOFSeen ? " " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") + - "after : \"" + addEscapes(errorAfter) + "\""); - } - - /** - * You can also modify the body of this method to customize your error messages. - * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not - * of end-users concern, so you can return something like : - * - * "Internal Error : Please file a bug report .... " - * - * from this method for such cases in the release version of your parser. - */ - public String getMessage() { - return super.getMessage(); - } - - /* - * Constructors of various flavors follow. - */ - - /** No arg constructor. */ - public TokenMgrError() { - } - - /** Constructor with message and reason. */ - public TokenMgrError(String message, int reason) { - super(message); - errorCode = reason; - } - - /** Full Constructor. */ - public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) { - this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); - } } /* JavaCC - OriginalChecksum=5fbf6813c9d6a1d713f1d4a002af1322 (do not edit this line) */ diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java index 6f8d1de3..6f9cd5ee 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java @@ -42,196 +42,197 @@ public class JSONMapParser extends BasicParser { - private interface Handler { - - JSONObject handle(String key, Map value, JSONObject obj); - } - - @SuppressWarnings("unchecked") - public enum MapStrategy implements Handler { - DROP((key, value, obj) -> obj), UNFOLD((key, value, obj) -> { - return recursiveUnfold(key, value, obj); - }), ALLOW((key, value, obj) -> { - obj.put(key, value); - return obj; - }), ERROR((key, value, obj) -> { - throw new IllegalStateException( - "Unable to process " + key + " => " + value + " because value is a map."); - }); - Handler handler; - - MapStrategy(Handler handler) { - this.handler = handler; + private interface Handler { + + JSONObject handle(String key, Map value, JSONObject obj); } @SuppressWarnings("unchecked") - private static JSONObject recursiveUnfold(String key, Map value, JSONObject obj) { - Set> entrySet = value.entrySet(); - for (Map.Entry kv : entrySet) { - String newKey = Joiner.on(".").join(key, kv.getKey().toString()); - if (kv.getValue() instanceof Map) { - recursiveUnfold(newKey, (Map) kv.getValue(), obj); - } else { - obj.put(newKey, kv.getValue()); + public enum MapStrategy implements Handler { + DROP((key, value, obj) -> obj), UNFOLD((key, value, obj) -> { + return recursiveUnfold(key, value, obj); + }), ALLOW((key, value, obj) -> { + obj.put(key, value); + return obj; + }), ERROR((key, value, obj) -> { + throw new IllegalStateException( + "Unable to process " + key + " => " + value + " because value is a map."); + }); + Handler handler; + + MapStrategy(Handler handler) { + this.handler = handler; } - } - return obj; - } - @Override - public JSONObject handle(String key, Map value, JSONObject obj) { - return handler.handle(key, value, obj); - } - - } - - public static final String MAP_STRATEGY_CONFIG = "mapStrategy"; - public static final String JSONP_QUERY = "jsonpQuery"; - public static final String WRAP_JSON = "wrapInEntityArray"; - public static final String WRAP_ENTITY_NAME = "wrapEntityName"; - public static final String DEFAULT_WRAP_ENTITY_NAME = "messages"; - public static final String OVERRIDE_ORIGINAL_STRING = "overrideOriginalString"; - - private static final String WRAP_START_FMT = "{ \"%s\" : ["; - private static final String WRAP_END = "]}"; - - private MapStrategy mapStrategy = MapStrategy.DROP; - private transient TypeRef>> typeRef = null; - private String jsonpQuery = null; - private String wrapEntityName = DEFAULT_WRAP_ENTITY_NAME; - private boolean wrapJson = false; - private boolean overrideOriginalString = false; // adds original string values per sub-map - - - @Override - public void configure(Map config) { - setReadCharset(config); - String strategyStr = (String) config.getOrDefault(MAP_STRATEGY_CONFIG, MapStrategy.DROP.name()); - mapStrategy = MapStrategy.valueOf(strategyStr); - overrideOriginalString = (Boolean) config.getOrDefault(OVERRIDE_ORIGINAL_STRING, false); - if (config.containsKey(JSONP_QUERY)) { - typeRef = new TypeRef>>() { }; - jsonpQuery = (String) config.get(JSONP_QUERY); - - if (!StringUtils.isBlank(jsonpQuery) && config.containsKey(WRAP_JSON)) { - Object wrapObject = config.get(WRAP_JSON); - if (wrapObject instanceof String) { - wrapJson = Boolean.valueOf((String)wrapObject); - } else if (wrapObject instanceof Boolean) { - wrapJson = (Boolean) config.get(WRAP_JSON); + @SuppressWarnings("unchecked") + private static JSONObject recursiveUnfold(String key, Map value, JSONObject obj) { + Set> entrySet = value.entrySet(); + for (Map.Entry kv : entrySet) { + String newKey = Joiner.on(".").join(key, kv.getKey().toString()); + if (kv.getValue() instanceof Map) { + recursiveUnfold(newKey, (Map) kv.getValue(), obj); + } else { + obj.put(newKey, kv.getValue()); + } + } + return obj; } - String entityName = (String)config.get(WRAP_ENTITY_NAME); - if (!StringUtils.isBlank(entityName)) { - wrapEntityName = entityName; + + @Override + public JSONObject handle(String key, Map value, JSONObject obj) { + return handler.handle(key, value, obj); } - } - Configuration.setDefaults(new Configuration.Defaults() { + } - private final JsonProvider jsonProvider = new JacksonJsonProvider(); - private final MappingProvider mappingProvider = new JacksonMappingProvider(); + public static final String MAP_STRATEGY_CONFIG = "mapStrategy"; + public static final String JSONP_QUERY = "jsonpQuery"; + public static final String WRAP_JSON = "wrapInEntityArray"; + public static final String WRAP_ENTITY_NAME = "wrapEntityName"; + public static final String DEFAULT_WRAP_ENTITY_NAME = "messages"; + public static final String OVERRIDE_ORIGINAL_STRING = "overrideOriginalString"; - @Override - public JsonProvider jsonProvider() { - return jsonProvider; - } + private static final String WRAP_START_FMT = "{ \"%s\" : ["; + private static final String WRAP_END = "]}"; - @Override - public MappingProvider mappingProvider() { - return mappingProvider; - } + private MapStrategy mapStrategy = MapStrategy.DROP; + private transient TypeRef>> typeRef = null; + private String jsonpQuery = null; + private String wrapEntityName = DEFAULT_WRAP_ENTITY_NAME; + private boolean wrapJson = false; + private boolean overrideOriginalString = false; // adds original string values per sub-map - @Override - public Set

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,66 +32,68 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@SuppressWarnings("serial") public class BasicLancopeParser extends BasicParser { - // Sample Lancope Message - // {"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.40.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.55M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.992Z","type":"syslog","host":"10.122.196.201"} + // Sample Lancope Message + // {"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: + // 2014-07-17T15:58:30Z 10.40.10.254 0.0.0.0 Minor High Concern Index + // The host's concern index has either exceeded the CI threshold or rapidly increased. + // Observed 36.55M points. Policy maximum allows up to 20M points.","@version":"1", + // "@timestamp":"2014-07-17T15:56:05.992Z","type":"syslog","host":"10.122.196.201"} - private static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - } + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + } - @Override - public void init() { + @Override + public void init() { - } + } - //@SuppressWarnings("unchecked") - @Override - public List parse(byte[] msg) { + //@SuppressWarnings("unchecked") + @Override + public List parse(byte[] msg) { - JSONObject payload = null; - List messages = new ArrayList<>(); - try { - - String raw_message = new String(msg, getReadCharset()); - - payload = (JSONObject) JSONValue.parse(raw_message); - - + JSONObject payload = null; + List messages = new ArrayList<>(); + try { - String message = payload.get("message").toString(); - String[] parts = message.split(" "); - payload.put("ip_src_addr", parts[6]); - payload.put("ip_dst_addr", parts[7]); + String rawMessage = new String(msg, getReadCharset()); - String fixed_date = parts[5].replace('T', ' '); - fixed_date = fixed_date.replace('Z', ' ').trim(); + payload = (JSONObject) JSONValue.parse(rawMessage); - SimpleDateFormat formatter = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - Date date; + String message = payload.get("message").toString(); + String[] parts = message.split(" "); + payload.put("ip_src_addr", parts[6]); + payload.put("ip_dst_addr", parts[7]); - date = formatter.parse(fixed_date); - long timestamp = date.getTime(); - payload.put("timestamp", timestamp); + String fixedDate = parts[5].replace('T', ' '); + fixedDate = fixedDate.replace('Z', ' ').trim(); - payload.remove("@timestamp"); - payload.remove("message"); - payload.put("original_string", message); + SimpleDateFormat formatter = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); - messages.add(payload); - return messages; - } catch (Exception e) { + Date date; + + date = formatter.parse(fixedDate); + long timestamp = date.getTime(); + payload.put("timestamp", timestamp); + + payload.remove("@timestamp"); + payload.remove("message"); + payload.put("original_string", message); + + messages.add(payload); + return messages; + } catch (Exception e) { + + _LOG.error("Unable to parse message: {}", payload.toJSONString()); + return null; + } + } - _LOG.error("Unable to parse message: {}", payload.toJSONString()); - return null; - } - } - } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java index 86ec0439..ca8c3215 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.leef; import java.io.BufferedReader; @@ -45,240 +48,246 @@ /** * LEEF Parser * + *

    * Based primarily on the IBM LEEF guide at https://www.ibm.com/support/knowledgecenter/SS42VS_DSM/c_LEEF_Format_Guide_intro.html * with support for other found in the wild examples shown in tests. - * */ public class LEEFParser extends BasicParser { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1L; - public enum HeaderFields { - DEVICE_VENDOR("DeviceVendor"), - DEVICE_PRODUCT("DeviceProduct"), - DEVICE_VERSION("DeviceVersion"), - DEVICE_EVENT("DeviceEvent"), - DELIMITER("Delimiter"), - VERSION("Version") - ; + public enum HeaderFields { + DEVICE_VENDOR("DeviceVendor"), + DEVICE_PRODUCT("DeviceProduct"), + DEVICE_VERSION("DeviceVersion"), + DEVICE_EVENT("DeviceEvent"), + DELIMITER("Delimiter"), + VERSION("Version"); - private String name; + private final String name; + + HeaderFields(String name) { + this.name = name; + } - HeaderFields(String name) { - this.name = name; + public String getName() { + return name; + } } - public String getName() { - return name; + // Field name for custom device time in LEEF + private static final String DEV_TIME = "devTime"; + private static final String DEV_TIME_FORMAT = "devTimeFormat"; + + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*"; + + private Pattern pattern; + + public void init() { + + // LEEF Headers: Version|Device Vendor|Device Product|Device Version|Device Event|Delimiter + String syslogTime = + "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?" + + "|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b +(?:(?:0[1-9])" + + + "|(?:[12][0-9])|(?:3[01])|[1-9]) (?!<[0-9])(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])(?::(?:(?:[0-5]?[0-9]|60)" + + "(?:[:.,][0-9]+)?))(?![0-9])?"; + String syslogTime5424 = "(?:\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2}))"; + String syslogPriority = "<(?:[0-9]+)>"; + String syslogHost = "[a-z0-9\\.\\\\-_]+"; + + StringBuilder sb = new StringBuilder(); + sb.append("(?"); + sb.append(syslogPriority); + sb.append(")?"); + sb.append("(?"); + sb.append(syslogTime); + sb.append("|"); + sb.append(syslogTime5424); + sb.append(")?"); + + sb.append("(?"); + sb.append(syslogHost); + sb.append(")?"); + + sb.append(".*"); + + sb.append("LEEF:(?<"); + sb.append(HeaderFields.VERSION.getName()); + sb.append(">1.0|2.0|0)?\\|"); + + headerBlock(HeaderFields.DEVICE_VENDOR.getName(), sb); + sb.append("\\|"); + headerBlock(HeaderFields.DEVICE_PRODUCT.getName(), sb); + sb.append("\\|"); + headerBlock(HeaderFields.DEVICE_VERSION.getName(), sb); + sb.append("\\|"); + headerBlock(HeaderFields.DEVICE_EVENT.getName(), sb); + sb.append("\\|"); + + // add optional delimiter header (only applicable for LEEF 2.0) + sb.append("("); + headerBlock(HeaderFields.DELIMITER.getName(), sb); + sb.append("\\|"); + sb.append(")?"); + + // extension capture: + sb.append(" ?(?.*)"); + pattern = Pattern.compile(sb.toString()); } - } - - // Field name for custom device time in LEEF - private static final String DEV_TIME = "devTime"; - private static final String DEV_TIME_FORMAT = "devTimeFormat"; - - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*"; - - private Pattern pattern; - - public void init() { - - // LEEF Headers: Version|Device Vendor|Device Product|Device Version|Device Event|Delimiter - String syslogTime = "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b +(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?!<[0-9])(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])(?::(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?))(?![0-9])?"; - String syslogTime5424 = "(?:\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2}))"; - String syslogPriority = "<(?:[0-9]+)>"; - String syslogHost = "[a-z0-9\\.\\\\-_]+"; - - StringBuilder sb = new StringBuilder(""); - sb.append("(?"); - sb.append(syslogPriority); - sb.append(")?"); - sb.append("(?"); - sb.append(syslogTime); - sb.append("|"); - sb.append(syslogTime5424); - sb.append(")?"); - - sb.append("(?"); - sb.append(syslogHost); - sb.append(")?"); - - sb.append(".*"); - - sb.append("LEEF:(?<"); - sb.append(HeaderFields.VERSION.getName()); - sb.append(">1.0|2.0|0)?\\|"); - - headerBlock(HeaderFields.DEVICE_VENDOR.getName(), sb); - sb.append("\\|"); - headerBlock(HeaderFields.DEVICE_PRODUCT.getName(), sb); - sb.append("\\|"); - headerBlock(HeaderFields.DEVICE_VERSION.getName(), sb); - sb.append("\\|"); - headerBlock(HeaderFields.DEVICE_EVENT.getName(), sb); - sb.append("\\|"); - - // add optional delimiter header (only applicable for LEEF 2.0) - sb.append("("); - headerBlock(HeaderFields.DELIMITER.getName(), sb); - sb.append("\\|"); - sb.append(")?"); - - // extension capture: - sb.append(" ?(?.*)"); - pattern = Pattern.compile(sb.toString()); - } - - public Optional> parseOptionalResult(byte[] rawMessage) { - List messages = new ArrayList<>(); - Map errors = new HashMap<>(); - String originalMessage = null; - - try (BufferedReader reader = new BufferedReader(new StringReader(new String(rawMessage, getReadCharset())))) { - while ((originalMessage = reader.readLine()) != null) { - Matcher matcher = pattern.matcher(originalMessage); - while (matcher.find()) { - JSONObject obj = new JSONObject(); - if (!matcher.matches()) { - break; - } - LOG.debug("Found %d groups", matcher.groupCount()); - obj.put(HeaderFields.DEVICE_VENDOR.getName(), - matcher.group(HeaderFields.DEVICE_VENDOR.getName())); - obj.put(HeaderFields.DEVICE_PRODUCT.getName(), - matcher.group(HeaderFields.DEVICE_PRODUCT.getName())); - obj.put(HeaderFields.DEVICE_VERSION.getName(), - matcher.group(HeaderFields.DEVICE_VERSION.getName())); - obj.put(HeaderFields.DEVICE_EVENT.getName(), - matcher.group(HeaderFields.DEVICE_EVENT.getName())); - - String ext = matcher.group("extensions"); - - // In LEEF 2.0 the delimiter can be specified - String version = matcher.group(HeaderFields.VERSION.getName()); - if (version.equals("2.0")) { - String delimiter = matcher.group(HeaderFields.DELIMITER.getName()); - if (delimiter == null || delimiter.length() == 0) { - delimiter = "\\t"; - } - delimiter = "(? formats = (obj.containsKey(DEV_TIME_FORMAT)) ? - new ArrayList() {{ - add(new SimpleDateFormat(devTimeFormat)); - }} : - DateUtils.DATE_FORMATS_LEEF; - obj.put(Fields.TIMESTAMP.getName(), DateUtils.parseMultiformat(devTime, formats)); - } catch (java.text.ParseException e) { - errors.put(originalMessage, - new IllegalStateException("devTime field present in LEEF but cannot be parsed", - e)); - continue; + public Optional> parseOptionalResult(byte[] rawMessage) { + List messages = new ArrayList<>(); + Map errors = new HashMap<>(); + String originalMessage = null; + + try (BufferedReader reader = new BufferedReader(new StringReader(new String(rawMessage, getReadCharset())))) { + while ((originalMessage = reader.readLine()) != null) { + Matcher matcher = pattern.matcher(originalMessage); + while (matcher.find()) { + JSONObject obj = new JSONObject(); + if (!matcher.matches()) { + break; + } + LOG.debug("Found %d groups", matcher.groupCount()); + obj.put(HeaderFields.DEVICE_VENDOR.getName(), + matcher.group(HeaderFields.DEVICE_VENDOR.getName())); + obj.put(HeaderFields.DEVICE_PRODUCT.getName(), + matcher.group(HeaderFields.DEVICE_PRODUCT.getName())); + obj.put(HeaderFields.DEVICE_VERSION.getName(), + matcher.group(HeaderFields.DEVICE_VERSION.getName())); + obj.put(HeaderFields.DEVICE_EVENT.getName(), + matcher.group(HeaderFields.DEVICE_EVENT.getName())); + + String ext = matcher.group("extensions"); + + // In LEEF 2.0 the delimiter can be specified + String version = matcher.group(HeaderFields.VERSION.getName()); + if (version.equals("2.0")) { + String delimiter = matcher.group(HeaderFields.DELIMITER.getName()); + if (delimiter == null || delimiter.length() == 0) { + delimiter = "\\t"; + } + delimiter = "(? formats = (obj.containsKey(DEV_TIME_FORMAT)) + ? new ArrayList() { + { + add(new SimpleDateFormat(devTimeFormat)); + } + } + : DateUtils.DATE_FORMATS_LEEF; + obj.put(Fields.TIMESTAMP.getName(), DateUtils.parseMultiformat(devTime, formats)); + } catch (java.text.ParseException e) { + errors.put(originalMessage, + new IllegalStateException("devTime field present in LEEF but cannot be parsed", + e)); + continue; + } + } else { + String logTimestamp = matcher.group("syslogTime"); + if (!(logTimestamp == null || logTimestamp.isEmpty())) { + try { + obj.put(Fields.TIMESTAMP.getName(), + SyslogUtils.parseTimestampToEpochMillis(logTimestamp, Clock.systemUTC())); + } catch (ParseException e) { + errors.put(originalMessage, + new IllegalStateException("Cannot parse syslog timestamp", e)); + continue; + } + } else { + obj.put(Fields.TIMESTAMP.getName(), System.currentTimeMillis()); + } + } + messages.add(obj); + } } - } else { - String logTimestamp = matcher.group("syslogTime"); - if (!(logTimestamp == null || logTimestamp.isEmpty())) { - try { - obj.put(Fields.TIMESTAMP.getName(), - SyslogUtils.parseTimestampToEpochMillis(logTimestamp, Clock.systemUTC())); - } catch (ParseException e) { - errors.put(originalMessage, - new IllegalStateException("Cannot parse syslog timestamp", e)); - continue; - } - } else { - obj.put(Fields.TIMESTAMP.getName(), System.currentTimeMillis()); - } - } - messages.add(obj); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + Exception innerException = new IllegalStateException("LEEF parser Error: " + + e.getMessage() + + " on " + + originalMessage, e); + return Optional.of(new DefaultMessageParserResult<>(innerException)); + } + return Optional.of(new DefaultMessageParserResult<>(messages, errors)); + } + + @SuppressWarnings("unchecked") + private JSONObject convertToInt(JSONObject obj, String key) { + if (obj.containsKey(key)) { + obj.put(key, Integer.valueOf((String) obj.get(key))); } - } - } catch (IOException e) { - LOG.error(e.getMessage(), e); - Exception innerException = new IllegalStateException("LEEF parser Error: " - + e.getMessage() - + " on " - + originalMessage, e); - return Optional.of(new DefaultMessageParserResult<>(innerException)); + return obj; } - return Optional.of(new DefaultMessageParserResult<>(messages, errors)); - } - @SuppressWarnings("unchecked") - private JSONObject convertToInt(JSONObject obj, String key) { - if (obj.containsKey(key)) { - obj.put(key, Integer.valueOf((String) obj.get(key))); + private void headerBlock(String name, StringBuilder sb) { + sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")"); } - return obj; - } - - private void headerBlock(String name, StringBuilder sb) { - sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")"); - } - - @Override - public void configure(Map config) { - setReadCharset(config); - } - - @SuppressWarnings("unchecked") - private JSONObject mutate(JSONObject json, String oldKey, String newKey) { - if (json.containsKey(oldKey)) { - json.put(newKey, json.remove(oldKey)); + + @Override + public void configure(Map config) { + setReadCharset(config); + } + + @SuppressWarnings("unchecked") + private JSONObject mutate(JSONObject json, String oldKey, String newKey) { + if (json.containsKey(oldKey)) { + json.put(newKey, json.remove(oldKey)); + } + return json; } - return json; - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java index 783fac42..f3d1dff5 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.logstash; import java.nio.charset.StandardCharsets; @@ -28,68 +31,68 @@ public class BasicLogstashParser extends BasicParser { - @Override - public void configure(Map parserConfig) { - - } - - @Override - public void init() { - - } - - @Override - public List parse(byte[] raw_message) { - List messages = new ArrayList<>(); - try { - - /* - * We need to create a new JSONParser each time because its - * not serializable and the parser is created on the storm nimbus - * node, then transfered to the workers. - */ - JSONParser jsonParser = new JSONParser(); - String rawString = new String(raw_message, StandardCharsets.UTF_8); - JSONObject rawJson = (JSONObject) jsonParser.parse(rawString); - - // remove logstash meta fields - rawJson.remove("@version"); - rawJson.remove("type"); - rawJson.remove("host"); - rawJson.remove("tags"); - - // rename other keys - rawJson = mutate(rawJson, "message", "original_string"); - rawJson = mutate(rawJson, "src_ip", "ip_src_addr"); - rawJson = mutate(rawJson, "dst_ip", "ip_dst_addr"); - rawJson = mutate(rawJson, "src_port", "ip_src_port"); - rawJson = mutate(rawJson, "dst_port", "ip_dst_port"); - rawJson = mutate(rawJson, "src_ip", "ip_src_addr"); - - // convert timestamp to milli since epoch - long timestamp = LogstashToEpoch((String) rawJson.remove("@timestamp")); - rawJson.put("timestamp", timestamp); - messages.add(rawJson); - return messages; - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - private JSONObject mutate(JSONObject json, String oldKey, String newKey) { - if (json.containsKey(oldKey)) { - json.put(newKey, json.remove(oldKey)); - } - return json; - } - - private long LogstashToEpoch(String timestamp) throws java.text.ParseException { - SimpleDateFormat logstashDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - return logstashDateFormat.parse(timestamp).getTime(); - - } - - + @Override + public void configure(Map parserConfig) { + + } + + @Override + public void init() { + + } + + @Override + public List parse(byte[] rawMessage) { + List messages = new ArrayList<>(); + try { + + /* + * We need to create a new JSONParser each time because its + * not serializable and the parser is created on the storm nimbus + * node, then transfered to the workers. + */ + JSONParser jsonParser = new JSONParser(); + String rawString = new String(rawMessage, StandardCharsets.UTF_8); + JSONObject rawJson = (JSONObject) jsonParser.parse(rawString); + + // remove logstash meta fields + rawJson.remove("@version"); + rawJson.remove("type"); + rawJson.remove("host"); + rawJson.remove("tags"); + + // rename other keys + rawJson = mutate(rawJson, "message", "original_string"); + rawJson = mutate(rawJson, "src_ip", "ip_src_addr"); + rawJson = mutate(rawJson, "dst_ip", "ip_dst_addr"); + rawJson = mutate(rawJson, "src_port", "ip_src_port"); + rawJson = mutate(rawJson, "dst_port", "ip_dst_port"); + rawJson = mutate(rawJson, "src_ip", "ip_src_addr"); + + // convert timestamp to milli since epoch + long timestamp = LogstashToEpoch((String) rawJson.remove("@timestamp")); + rawJson.put("timestamp", timestamp); + messages.add(rawJson); + return messages; + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + private JSONObject mutate(JSONObject json, String oldKey, String newKey) { + if (json.containsKey(oldKey)) { + json.put(newKey, json.remove(oldKey)); + } + return json; + } + + @SuppressWarnings("checkstyle:MethodName") + private long LogstashToEpoch(String timestamp) throws java.text.ParseException { + SimpleDateFormat logstashDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + return logstashDateFormat.parse(timestamp).getTime(); + + } + } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java index 238eb322..e50c408b 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.paloalto; @@ -31,422 +34,669 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SuppressWarnings("checkstyle:MethodName") public class BasicPaloAltoFirewallParser extends BasicParser { - private static boolean empty_attribute(final String s) { - return s == null || s.trim().isEmpty() || s.equals("\"\""); - } - - private static String unquoted_attribute(String s) { - s = s.trim(); - if (s.startsWith("\"") && s.endsWith("\"")) - return s.substring(1, s.length() - 1); - return s; - } - - private static final Logger _LOG = LoggerFactory.getLogger(BasicPaloAltoFirewallParser.class); - - private static final long serialVersionUID = 3147090149725343999L; - - private static final String LogTypeConfig = "CONFIG"; - private static final String LogTypeSystem = "SYSTEM"; - private static final String LogTypeThreat = "THREAT"; - private static final String LogTypeTraffic = "TRAFFIC"; - - public static final String PaloAltoDomain = "palo_alto_domain"; - public static final String ReceiveTime = "receive_time"; - public static final String SerialNum = "serial"; - public static final String Type = "type"; - public static final String ThreatContentType = "subtype"; - public static final String ConfigVersion = "config_version"; - public static final String GenerateTime = "time_generated"; - public static final String SourceAddress = "ip_src_addr"; // Palo Alto name: "src" - public static final String DestinationAddress = "ip_dst_addr"; // Palo Alto name: "dst" - public static final String NATSourceIP = "natsrc"; - public static final String NATDestinationIP = "natdst"; - public static final String Rule = "rule"; - public static final String SourceUser = "srcuser"; - public static final String DestinationUser = "dstuser"; - public static final String Application = "app"; - public static final String VirtualSystem = "vsys"; - public static final String SourceZone = "from"; - public static final String DestinationZone = "to"; - public static final String InboundInterface = "inbound_if"; - public static final String OutboundInterface = "outbound_if"; - public static final String LogAction = "log_action"; - public static final String TimeLogged = "start"; - public static final String SessionID = "sessionid"; - public static final String RepeatCount = "repeatcnt"; - public static final String SourcePort = "ip_src_port"; // Palo Alto name: "sport" - public static final String DestinationPort = "ip_dst_port"; // Palo Alto name: "dport" - public static final String NATSourcePort = "natsport"; - public static final String NATDestinationPort = "natdport"; - public static final String Flags = "flags"; - public static final String IPProtocol = "protocol"; // Palo Alto name: "proto" - public static final String Action = "action"; - public static final String Seqno = "seqno"; - public static final String ActionFlags = "actionflags"; - public static final String Category = "category"; - public static final String DGH1 = "dg_hier_level_1"; - public static final String DGH2 = "dg_hier_level_2"; - public static final String DGH3 = "dg_hier_level_3"; - public static final String DGH4 = "dg_hier_level_4"; - public static final String VSYSName = "vsys_name"; - public static final String DeviceName = "device_name"; - public static final String ActionSource = "action_source"; - public static final String ParserVersion = "parser_version"; - public static final String Tokens = "tokens_seen"; - - public static final String SourceVmUuid = "source_vm_uuid"; - public static final String DestinationVmUuid = "destination_vm_uuid"; - public static final String TunnelId = "tunnel_id"; - public static final String MonitorTag = "monitor_tag"; - public static final String ParentSessionId = "parent_session_id"; - public static final String ParentSessionStartTime = "parent_session_start_time"; - public static final String TunnelType = "tunnel_type"; - - //System - public static final String EventId = "event_id"; - public static final String Object = "object"; - public static final String Module = "module"; - public static final String Description = "description"; - - //Config - public static final String Command = "command"; - public static final String Admin = "admin"; - public static final String Client = "client"; - public static final String Result = "result"; - public static final String ConfigurationPath = "configuration_path"; - public static final String BeforeChangeDetail = "before_change_detail"; - public static final String AfterChangeDetail = "after_change_detail"; - - //Threat - public static final String URL = "url"; - public static final String HOST = "host"; - public static final String ThreatID = "threatid"; - public static final String Severity = "severity"; - public static final String Direction = "direction"; - public static final String SourceLocation = "srcloc"; - public static final String DestinationLocation = "dstloc"; - public static final String ContentType = "contenttype"; - public static final String PCAPID = "pcap_id"; - public static final String WFFileDigest = "filedigest"; - public static final String WFCloud = "cloud"; - public static final String UserAgent = "user_agent"; - public static final String WFFileType = "filetype"; - public static final String XForwardedFor = "xff"; - public static final String Referer = "referer"; - public static final String WFSender = "sender"; - public static final String WFSubject = "subject"; - public static final String WFRecipient = "recipient"; - public static final String WFReportID = "reportid"; - public static final String URLIndex = "url_idx"; - public static final String HTTPMethod = "http_method"; - public static final String ThreatCategory = "threat_category"; - public static final String ContentVersion = "content_version"; - - - //Traffic - public static final String Bytes = "bytes"; - public static final String BytesSent = "bytes_sent"; - public static final String BytesReceived = "bytes_received"; - public static final String Packets = "packets"; - public static final String StartTime = "start"; - public static final String ElapsedTimeInSec = "elapsed"; - public static final String PktsSent = "pkts_sent"; - public static final String PktsReceived = "pkts_received"; - public static final String EndReason = "session_end_reason"; - - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - } - - @Override - public void init() { - - } - - @Override - @SuppressWarnings({"unchecked", "unused"}) - public List parse(byte[] msg) { - - JSONObject outputMessage = new JSONObject(); - String toParse = ""; - List messages = new ArrayList<>(); - try { - - toParse = new String(msg, getReadCharset()); - _LOG.debug("Received message: {}", toParse); - parseMessage(toParse, outputMessage); - long timestamp = System.currentTimeMillis(); - outputMessage.put("timestamp", System.currentTimeMillis()); - outputMessage.put("original_string", toParse); - messages.add(outputMessage); - return messages; - } catch (Exception e) { - e.printStackTrace(); - _LOG.error("Failed to parse: {}", toParse); - return null; + private static boolean empty_attribute(final String s) { + return s == null || s.trim().isEmpty() || s.equals("\"\""); } - } - - @SuppressWarnings("unchecked") - private void parseMessage(String message, JSONObject outputMessage) { - String[] tokens = Iterables.toArray(Splitter.on(Pattern.compile(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)")).split(message), String.class); + private static String unquoted_attribute(String s) { + s = s.trim(); + if (s.startsWith("\"") && s.endsWith("\"")) { + return s.substring(1, s.length() - 1); + } + return s; + } - int parser_version = 0; + private static final Logger _LOG = LoggerFactory.getLogger(BasicPaloAltoFirewallParser.class); + + private static final long serialVersionUID = 3147090149725343999L; + + private static final String LogTypeConfig = "CONFIG"; + private static final String LogTypeSystem = "SYSTEM"; + private static final String LogTypeThreat = "THREAT"; + private static final String LogTypeTraffic = "TRAFFIC"; + + public static final String PaloAltoDomain = "palo_alto_domain"; + public static final String ReceiveTime = "receive_time"; + public static final String SerialNum = "serial"; + public static final String Type = "type"; + public static final String ThreatContentType = "subtype"; + public static final String ConfigVersion = "config_version"; + public static final String GenerateTime = "time_generated"; + public static final String SourceAddress = "ip_src_addr"; // Palo Alto name: "src" + public static final String DestinationAddress = "ip_dst_addr"; // Palo Alto name: "dst" + public static final String NATSourceIP = "natsrc"; + public static final String NATDestinationIP = "natdst"; + public static final String Rule = "rule"; + public static final String SourceUser = "srcuser"; + public static final String DestinationUser = "dstuser"; + public static final String Application = "app"; + public static final String VirtualSystem = "vsys"; + public static final String SourceZone = "from"; + public static final String DestinationZone = "to"; + public static final String InboundInterface = "inbound_if"; + public static final String OutboundInterface = "outbound_if"; + public static final String LogAction = "log_action"; + public static final String TimeLogged = "start"; + public static final String SessionID = "sessionid"; + public static final String RepeatCount = "repeatcnt"; + public static final String SourcePort = "ip_src_port"; // Palo Alto name: "sport" + public static final String DestinationPort = "ip_dst_port"; // Palo Alto name: "dport" + public static final String NATSourcePort = "natsport"; + public static final String NATDestinationPort = "natdport"; + public static final String Flags = "flags"; + public static final String IPProtocol = "protocol"; // Palo Alto name: "proto" + public static final String Action = "action"; + public static final String Seqno = "seqno"; + public static final String ActionFlags = "actionflags"; + public static final String Category = "category"; + public static final String DGH1 = "dg_hier_level_1"; + public static final String DGH2 = "dg_hier_level_2"; + public static final String DGH3 = "dg_hier_level_3"; + public static final String DGH4 = "dg_hier_level_4"; + public static final String VSYSName = "vsys_name"; + public static final String DeviceName = "device_name"; + public static final String ActionSource = "action_source"; + public static final String ParserVersion = "parser_version"; + public static final String Tokens = "tokens_seen"; + + public static final String SourceVmUuid = "source_vm_uuid"; + public static final String DestinationVmUuid = "destination_vm_uuid"; + public static final String TunnelId = "tunnel_id"; + public static final String MonitorTag = "monitor_tag"; + public static final String ParentSessionId = "parent_session_id"; + public static final String ParentSessionStartTime = "parent_session_start_time"; + public static final String TunnelType = "tunnel_type"; + + //System + public static final String EventId = "event_id"; + public static final String Object = "object"; + public static final String Module = "module"; + public static final String Description = "description"; + + //Config + public static final String Command = "command"; + public static final String Admin = "admin"; + public static final String Client = "client"; + public static final String Result = "result"; + public static final String ConfigurationPath = "configuration_path"; + public static final String BeforeChangeDetail = "before_change_detail"; + public static final String AfterChangeDetail = "after_change_detail"; + + //Threat + public static final String URL = "url"; + public static final String HOST = "host"; + public static final String ThreatID = "threatid"; + public static final String Severity = "severity"; + public static final String Direction = "direction"; + public static final String SourceLocation = "srcloc"; + public static final String DestinationLocation = "dstloc"; + public static final String ContentType = "contenttype"; + public static final String PCAPID = "pcap_id"; + public static final String WFFileDigest = "filedigest"; + public static final String WFCloud = "cloud"; + public static final String UserAgent = "user_agent"; + public static final String WFFileType = "filetype"; + public static final String XForwardedFor = "xff"; + public static final String Referer = "referer"; + public static final String WFSender = "sender"; + public static final String WFSubject = "subject"; + public static final String WFRecipient = "recipient"; + public static final String WFReportID = "reportid"; + public static final String URLIndex = "url_idx"; + public static final String HTTPMethod = "http_method"; + public static final String ThreatCategory = "threat_category"; + public static final String ContentVersion = "content_version"; + + + //Traffic + public static final String Bytes = "bytes"; + public static final String BytesSent = "bytes_sent"; + public static final String BytesReceived = "bytes_received"; + public static final String Packets = "packets"; + public static final String StartTime = "start"; + public static final String ElapsedTimeInSec = "elapsed"; + public static final String PktsSent = "pkts_sent"; + public static final String PktsReceived = "pkts_received"; + public static final String EndReason = "session_end_reason"; + + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + } - String type = tokens[3].trim(); + @Override + public void init() { - //validate log types - if (!type.equals(LogTypeConfig) && - !type.equals(LogTypeThreat) && - !type.equals(LogTypeTraffic) && - !type.equals(LogTypeSystem)) { - throw new UnsupportedOperationException("Unsupported log type."); } - //populate common objects - if (!empty_attribute(tokens[0])) outputMessage.put(PaloAltoDomain, tokens[0].trim()); - if (!empty_attribute(tokens[1])) outputMessage.put(ReceiveTime, tokens[1].trim()); - if (!empty_attribute(tokens[2])) outputMessage.put(SerialNum, tokens[2].trim()); - outputMessage.put(Type, type); - if (!empty_attribute(tokens[4])) outputMessage.put(ThreatContentType, unquoted_attribute(tokens[4])); - if (!empty_attribute(tokens[5])) outputMessage.put(ConfigVersion, tokens[5].trim()); - if (!empty_attribute(tokens[6])) outputMessage.put(GenerateTime, tokens[6].trim()); - - if (LogTypeConfig.equals(type.toUpperCase())) { - // There are two fields in custom logs only and they are not in the default format. - // But we need to parse them if they exist - if (tokens.length == 16 || tokens.length == 18) parser_version = 61; - else if (tokens.length == 22 || tokens.length == 24) parser_version = 80; - - if (parser_version >= 61) { - if (!empty_attribute(tokens[7])) outputMessage.put(HOST, tokens[7].trim()); - if (!empty_attribute(tokens[8])) outputMessage.put(VirtualSystem, tokens[8].trim()); - if (!empty_attribute(tokens[9])) outputMessage.put(Command, tokens[9].trim()); - if (!empty_attribute(tokens[10])) outputMessage.put(Admin, tokens[10].trim()); - if (!empty_attribute(tokens[11])) outputMessage.put(Client, unquoted_attribute(tokens[11])); - if (!empty_attribute(tokens[12])) outputMessage.put(Result, unquoted_attribute(tokens[12])); - if (!empty_attribute(tokens[13])) outputMessage.put(ConfigurationPath, unquoted_attribute(tokens[13])); - } - - if (parser_version == 61) { - if (!empty_attribute(tokens[14])) outputMessage.put(Seqno, unquoted_attribute(tokens[14])); - if (!empty_attribute(tokens[15])) outputMessage.put(ActionFlags, unquoted_attribute(tokens[15])); - if (tokens.length == 18) { - if (!empty_attribute(tokens[16])) - outputMessage.put(BeforeChangeDetail, unquoted_attribute(tokens[16])); - if (!empty_attribute(tokens[17])) - outputMessage.put(AfterChangeDetail, unquoted_attribute(tokens[17])); - } - } - - if (parser_version >= 70) { - int custom_fields_offset = 0; - if (tokens.length == 24) { - if (!empty_attribute(tokens[14])) { - outputMessage.put(BeforeChangeDetail, unquoted_attribute(tokens[14 + custom_fields_offset])); - } - if (!empty_attribute(tokens[15])) { - outputMessage.put(AfterChangeDetail, unquoted_attribute(tokens[15 + custom_fields_offset])); - } - custom_fields_offset = 2; - } - if (!empty_attribute(tokens[14 + custom_fields_offset])) { - outputMessage.put(Seqno, unquoted_attribute(tokens[14 + custom_fields_offset])); - } - if (!empty_attribute(tokens[15 + custom_fields_offset])) { - outputMessage.put(ActionFlags, unquoted_attribute(tokens[15 + custom_fields_offset])); - } - if (!empty_attribute(tokens[16 + custom_fields_offset])) { - outputMessage.put(DGH1, unquoted_attribute(tokens[16 + custom_fields_offset])); - } - if (!empty_attribute(tokens[17 + custom_fields_offset])) { - outputMessage.put(DGH2, unquoted_attribute(tokens[17 + custom_fields_offset])); - } - if (!empty_attribute(tokens[18 + custom_fields_offset])) { - outputMessage.put(DGH3, unquoted_attribute(tokens[18 + custom_fields_offset])); + @Override + @SuppressWarnings({"unchecked", "unused"}) + public List parse(byte[] msg) { + + JSONObject outputMessage = new JSONObject(); + String toParse = ""; + List messages = new ArrayList<>(); + try { + + toParse = new String(msg, getReadCharset()); + _LOG.debug("Received message: {}", toParse); + parseMessage(toParse, outputMessage); + long timestamp = System.currentTimeMillis(); + outputMessage.put("timestamp", System.currentTimeMillis()); + outputMessage.put("original_string", toParse); + messages.add(outputMessage); + return messages; + } catch (Exception e) { + e.printStackTrace(); + _LOG.error("Failed to parse: {}", toParse); + return null; } - if (!empty_attribute(tokens[19 + custom_fields_offset])) { - outputMessage.put(DGH4, unquoted_attribute(tokens[19 + custom_fields_offset])); - } - if (!empty_attribute(tokens[20 + custom_fields_offset])) { - outputMessage.put(VSYSName, unquoted_attribute(tokens[20 + custom_fields_offset])); - } - if (!empty_attribute(tokens[21 + custom_fields_offset])) { - outputMessage.put(DeviceName, unquoted_attribute(tokens[21 + custom_fields_offset])); + } + + @SuppressWarnings({"unchecked", "checkstyle:VariableDeclarationUsageDistance"}) + private void parseMessage(String message, JSONObject outputMessage) { + + String[] tokens = + Iterables.toArray(Splitter.on(Pattern.compile(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)")).split(message), + String.class); + + int parserVersion = 0; + + String type = tokens[3].trim(); + + //validate log types + if (!type.equals(LogTypeConfig) + && !type.equals(LogTypeThreat) + && !type.equals(LogTypeTraffic) + && !type.equals(LogTypeSystem)) { + throw new UnsupportedOperationException("Unsupported log type."); } - } - } else if (LogTypeSystem.equals(type.toUpperCase())) { - if (tokens.length == 17) parser_version = 61; - else if (tokens.length == 23) parser_version = 80; - - if (parser_version >= 61) { - if (!empty_attribute(tokens[7])) outputMessage.put(VirtualSystem, tokens[7].trim()); - if (!empty_attribute(tokens[8])) outputMessage.put(EventId, tokens[8].trim()); - if (!empty_attribute(tokens[9])) outputMessage.put(Object, tokens[9].trim()); - - if (!empty_attribute(tokens[12])) outputMessage.put(Module, tokens[12].trim()); - if (!empty_attribute(tokens[13])) outputMessage.put(Severity, unquoted_attribute(tokens[13])); - if (!empty_attribute(tokens[14])) outputMessage.put(Description, unquoted_attribute(tokens[14])); - if (!empty_attribute(tokens[15])) outputMessage.put(Seqno, unquoted_attribute(tokens[15])); - if (!empty_attribute(tokens[16])) outputMessage.put(ActionFlags, unquoted_attribute(tokens[16])); - } - - if (parser_version == 80) { - if (!empty_attribute(tokens[17])) outputMessage.put(DGH1, tokens[17].trim()); - if (!empty_attribute(tokens[18])) outputMessage.put(DGH2, tokens[18].trim()); - if (!empty_attribute(tokens[19])) outputMessage.put(DGH3, tokens[19].trim()); - if (!empty_attribute(tokens[20])) outputMessage.put(DGH4, tokens[20].trim()); - if (!empty_attribute(tokens[21])) outputMessage.put(VSYSName, unquoted_attribute(tokens[21])); - if (!empty_attribute(tokens[22])) outputMessage.put(DeviceName, unquoted_attribute(tokens[22])); - } - } else if (LogTypeThreat.equals(type.toUpperCase()) || - LogTypeTraffic.equals(type.toUpperCase())) { - if (!empty_attribute(tokens[7])) outputMessage.put(SourceAddress, tokens[7].trim()); - if (!empty_attribute(tokens[8])) outputMessage.put(DestinationAddress, tokens[8].trim()); - if (!empty_attribute(tokens[9])) outputMessage.put(NATSourceIP, tokens[9].trim()); - if (!empty_attribute(tokens[10])) outputMessage.put(NATDestinationIP, tokens[10].trim()); - if (!empty_attribute(tokens[11])) outputMessage.put(Rule, unquoted_attribute(tokens[11])); - if (!empty_attribute(tokens[12])) outputMessage.put(SourceUser, unquoted_attribute(tokens[12])); - if (!empty_attribute(tokens[13])) outputMessage.put(DestinationUser, unquoted_attribute(tokens[13])); - if (!empty_attribute(tokens[14])) outputMessage.put(Application, unquoted_attribute(tokens[14])); - if (!empty_attribute(tokens[15])) outputMessage.put(VirtualSystem, unquoted_attribute(tokens[15])); - if (!empty_attribute(tokens[16])) outputMessage.put(SourceZone, unquoted_attribute(tokens[16])); - if (!empty_attribute(tokens[17])) outputMessage.put(DestinationZone, unquoted_attribute(tokens[17])); - if (!empty_attribute(tokens[18])) outputMessage.put(InboundInterface, unquoted_attribute(tokens[18])); - if (!empty_attribute(tokens[19])) outputMessage.put(OutboundInterface, unquoted_attribute(tokens[19])); - if (!empty_attribute(tokens[20])) outputMessage.put(LogAction, unquoted_attribute(tokens[20])); - if (!empty_attribute(tokens[21])) outputMessage.put(TimeLogged, tokens[21].trim()); - if (!empty_attribute(tokens[22])) outputMessage.put(SessionID, tokens[22].trim()); - if (!empty_attribute(tokens[23])) outputMessage.put(RepeatCount, tokens[23].trim()); - if (!empty_attribute(tokens[24])) outputMessage.put(SourcePort, tokens[24].trim()); - if (!empty_attribute(tokens[25])) outputMessage.put(DestinationPort, tokens[25].trim()); - if (!empty_attribute(tokens[26])) outputMessage.put(NATSourcePort, tokens[26].trim()); - if (!empty_attribute(tokens[27])) outputMessage.put(NATDestinationPort, tokens[27].trim()); - if (!empty_attribute(tokens[28])) outputMessage.put(Flags, tokens[28].trim()); - if (!empty_attribute(tokens[29])) outputMessage.put(IPProtocol, unquoted_attribute(tokens[29])); - if (!empty_attribute(tokens[30])) outputMessage.put(Action, unquoted_attribute(tokens[30])); - - if (LogTypeThreat.equals(type.toUpperCase())) { - int p1_offset = 0; - if (tokens.length == 45) parser_version = 60; - else if (tokens.length == 53) parser_version = 61; - else if (tokens.length == 61) { - parser_version = 70; - p1_offset = 1; - } else if (tokens.length == 72) { - parser_version = 80; - p1_offset = 1; + + //populate common objects + if (!empty_attribute(tokens[0])) { + outputMessage.put(PaloAltoDomain, tokens[0].trim()); } - if (!empty_attribute(tokens[31])) { - outputMessage.put(URL, unquoted_attribute(tokens[31])); - try { - URL url = new URL(unquoted_attribute(tokens[31])); - outputMessage.put(HOST, url.getHost()); - } catch (MalformedURLException e) { - } + if (!empty_attribute(tokens[1])) { + outputMessage.put(ReceiveTime, tokens[1].trim()); } - if (!empty_attribute(tokens[32])) outputMessage.put(ThreatID, tokens[32].trim()); - if (!empty_attribute(tokens[33])) outputMessage.put(Category, unquoted_attribute(tokens[33])); - if (!empty_attribute(tokens[34])) outputMessage.put(Severity, unquoted_attribute(tokens[34])); - if (!empty_attribute(tokens[35])) outputMessage.put(Direction, unquoted_attribute(tokens[35])); - if (!empty_attribute(tokens[36])) outputMessage.put(Seqno, tokens[36].trim()); - if (!empty_attribute(tokens[37])) outputMessage.put(ActionFlags, unquoted_attribute(tokens[37])); - if (!empty_attribute(tokens[38])) outputMessage.put(SourceLocation, unquoted_attribute(tokens[38])); - if (!empty_attribute(tokens[39])) - outputMessage.put(DestinationLocation, unquoted_attribute(tokens[39])); - if (!empty_attribute(tokens[41])) outputMessage.put(ContentType, unquoted_attribute(tokens[41])); - if (!empty_attribute(tokens[42])) outputMessage.put(PCAPID, tokens[42].trim()); - if (!empty_attribute(tokens[43])) outputMessage.put(WFFileDigest, unquoted_attribute(tokens[43])); - if (!empty_attribute(tokens[44])) outputMessage.put(WFCloud, unquoted_attribute(tokens[44])); - if (parser_version >= 61) { - if (!empty_attribute(tokens[(45 + p1_offset)])) - outputMessage.put(UserAgent, unquoted_attribute(tokens[(45 + p1_offset)])); - if (!empty_attribute(tokens[(46 + p1_offset)])) - outputMessage.put(WFFileType, unquoted_attribute(tokens[(46 + p1_offset)])); - if (!empty_attribute(tokens[(47 + p1_offset)])) - outputMessage.put(XForwardedFor, unquoted_attribute(tokens[(47 + p1_offset)])); - if (!empty_attribute(tokens[(48 + p1_offset)])) - outputMessage.put(Referer, unquoted_attribute(tokens[(48 + p1_offset)])); - if (!empty_attribute(tokens[(49 + p1_offset)])) - outputMessage.put(WFSender, unquoted_attribute(tokens[(49 + p1_offset)])); - if (!empty_attribute(tokens[(50 + p1_offset)])) - outputMessage.put(WFSubject, unquoted_attribute(tokens[(50 + p1_offset)])); - if (!empty_attribute(tokens[(51 + p1_offset)])) - outputMessage.put(WFRecipient, unquoted_attribute(tokens[(51 + p1_offset)])); - if (!empty_attribute(tokens[(52 + p1_offset)])) - outputMessage.put(WFReportID, unquoted_attribute(tokens[(52 + p1_offset)])); + if (!empty_attribute(tokens[2])) { + outputMessage.put(SerialNum, tokens[2].trim()); } - if (parser_version >= 70) { - if (!empty_attribute(tokens[45])) outputMessage.put(URLIndex, tokens[45].trim()); - if (!empty_attribute(tokens[54])) outputMessage.put(DGH1, tokens[54].trim()); - if (!empty_attribute(tokens[55])) outputMessage.put(DGH2, tokens[55].trim()); - if (!empty_attribute(tokens[56])) outputMessage.put(DGH3, tokens[56].trim()); - if (!empty_attribute(tokens[57])) outputMessage.put(DGH4, tokens[57].trim()); - if (!empty_attribute(tokens[58])) outputMessage.put(VSYSName, unquoted_attribute(tokens[58])); - if (!empty_attribute(tokens[59])) outputMessage.put(DeviceName, unquoted_attribute(tokens[59])); + outputMessage.put(Type, type); + if (!empty_attribute(tokens[4])) { + outputMessage.put(ThreatContentType, unquoted_attribute(tokens[4])); } - if (parser_version >= 80) { - if (!empty_attribute(tokens[61])) outputMessage.put(SourceVmUuid, tokens[61].trim()); - if (!empty_attribute(tokens[62])) outputMessage.put(DestinationVmUuid, tokens[62].trim()); - if (!empty_attribute(tokens[63])) outputMessage.put(HTTPMethod, tokens[63].trim()); - if (!empty_attribute(tokens[64])) outputMessage.put(TunnelId, tokens[64].trim()); - if (!empty_attribute(tokens[65])) outputMessage.put(MonitorTag, tokens[65].trim()); - if (!empty_attribute(tokens[66])) outputMessage.put(ParentSessionId, tokens[66].trim()); - if (!empty_attribute(tokens[67])) outputMessage.put(ParentSessionStartTime, tokens[67].trim()); - if (!empty_attribute(tokens[68])) outputMessage.put(TunnelType, tokens[68].trim()); - if (!empty_attribute(tokens[69])) outputMessage.put(ThreatCategory, tokens[69].trim()); - if (!empty_attribute(tokens[70])) outputMessage.put(ContentVersion, tokens[70].trim()); + if (!empty_attribute(tokens[5])) { + outputMessage.put(ConfigVersion, tokens[5].trim()); } - } else if (LogTypeTraffic.equals(type.toUpperCase())) { - if (tokens.length == 46) parser_version = 60; - else if (tokens.length == 47) parser_version = 61; - else if (tokens.length == 54) parser_version = 70; - else if (tokens.length == 61) parser_version = 80; - if (!empty_attribute(tokens[31])) outputMessage.put(Bytes, tokens[31].trim()); - if (!empty_attribute(tokens[32])) outputMessage.put(BytesSent, tokens[32].trim()); - if (!empty_attribute(tokens[33])) outputMessage.put(BytesReceived, tokens[33].trim()); - if (!empty_attribute(tokens[34])) outputMessage.put(Packets, tokens[34].trim()); - if (!empty_attribute(tokens[35])) outputMessage.put(StartTime, tokens[35].trim()); - if (!empty_attribute(tokens[36])) outputMessage.put(ElapsedTimeInSec, tokens[36].trim()); - if (!empty_attribute(tokens[37])) outputMessage.put(Category, unquoted_attribute(tokens[37])); - if (!empty_attribute(tokens[39])) outputMessage.put(Seqno, tokens[39].trim()); - if (!empty_attribute(tokens[40])) outputMessage.put(ActionFlags, unquoted_attribute(tokens[40])); - if (!empty_attribute(tokens[41])) outputMessage.put(SourceLocation, unquoted_attribute(tokens[41])); - if (!empty_attribute(tokens[42])) - outputMessage.put(DestinationLocation, unquoted_attribute(tokens[42])); - if (!empty_attribute(tokens[44])) outputMessage.put(PktsSent, tokens[44].trim()); - if (!empty_attribute(tokens[45])) outputMessage.put(PktsReceived, tokens[45].trim()); - if (parser_version >= 61) { - if (!empty_attribute(tokens[46])) outputMessage.put(EndReason, unquoted_attribute(tokens[46])); + if (!empty_attribute(tokens[6])) { + outputMessage.put(GenerateTime, tokens[6].trim()); } - if (parser_version >= 70) { - if (!empty_attribute(tokens[47])) outputMessage.put(DGH1, tokens[47].trim()); - if (!empty_attribute(tokens[48])) outputMessage.put(DGH2, tokens[48].trim()); - if (!empty_attribute(tokens[49])) outputMessage.put(DGH3, tokens[49].trim()); - if (!empty_attribute(tokens[50])) outputMessage.put(DGH4, tokens[50].trim()); - if (!empty_attribute(tokens[51])) outputMessage.put(VSYSName, unquoted_attribute(tokens[51])); - if (!empty_attribute(tokens[52])) outputMessage.put(DeviceName, unquoted_attribute(tokens[52])); - if (!empty_attribute(tokens[53])) outputMessage.put(ActionSource, unquoted_attribute(tokens[53])); + + if (LogTypeConfig.equalsIgnoreCase(type)) { + // There are two fields in custom logs only and they are not in the default format. + // But we need to parse them if they exist + if (tokens.length == 16 || tokens.length == 18) { + parserVersion = 61; + } else if (tokens.length == 22 || tokens.length == 24) { + parserVersion = 80; + } + + if (parserVersion >= 61) { + if (!empty_attribute(tokens[7])) { + outputMessage.put(HOST, tokens[7].trim()); + } + if (!empty_attribute(tokens[8])) { + outputMessage.put(VirtualSystem, tokens[8].trim()); + } + if (!empty_attribute(tokens[9])) { + outputMessage.put(Command, tokens[9].trim()); + } + if (!empty_attribute(tokens[10])) { + outputMessage.put(Admin, tokens[10].trim()); + } + if (!empty_attribute(tokens[11])) { + outputMessage.put(Client, unquoted_attribute(tokens[11])); + } + if (!empty_attribute(tokens[12])) { + outputMessage.put(Result, unquoted_attribute(tokens[12])); + } + if (!empty_attribute(tokens[13])) { + outputMessage.put(ConfigurationPath, unquoted_attribute(tokens[13])); + } + } + + if (parserVersion == 61) { + if (!empty_attribute(tokens[14])) { + outputMessage.put(Seqno, unquoted_attribute(tokens[14])); + } + if (!empty_attribute(tokens[15])) { + outputMessage.put(ActionFlags, unquoted_attribute(tokens[15])); + } + if (tokens.length == 18) { + if (!empty_attribute(tokens[16])) { + outputMessage.put(BeforeChangeDetail, unquoted_attribute(tokens[16])); + } + if (!empty_attribute(tokens[17])) { + outputMessage.put(AfterChangeDetail, unquoted_attribute(tokens[17])); + } + } + } + + if (parserVersion >= 70) { + int customFieldsOffset = 0; + if (tokens.length == 24) { + if (!empty_attribute(tokens[14])) { + outputMessage.put(BeforeChangeDetail, unquoted_attribute(tokens[14 + customFieldsOffset])); + } + if (!empty_attribute(tokens[15])) { + outputMessage.put(AfterChangeDetail, unquoted_attribute(tokens[15 + customFieldsOffset])); + } + customFieldsOffset = 2; + } + if (!empty_attribute(tokens[14 + customFieldsOffset])) { + outputMessage.put(Seqno, unquoted_attribute(tokens[14 + customFieldsOffset])); + } + if (!empty_attribute(tokens[15 + customFieldsOffset])) { + outputMessage.put(ActionFlags, unquoted_attribute(tokens[15 + customFieldsOffset])); + } + if (!empty_attribute(tokens[16 + customFieldsOffset])) { + outputMessage.put(DGH1, unquoted_attribute(tokens[16 + customFieldsOffset])); + } + if (!empty_attribute(tokens[17 + customFieldsOffset])) { + outputMessage.put(DGH2, unquoted_attribute(tokens[17 + customFieldsOffset])); + } + if (!empty_attribute(tokens[18 + customFieldsOffset])) { + outputMessage.put(DGH3, unquoted_attribute(tokens[18 + customFieldsOffset])); + } + if (!empty_attribute(tokens[19 + customFieldsOffset])) { + outputMessage.put(DGH4, unquoted_attribute(tokens[19 + customFieldsOffset])); + } + if (!empty_attribute(tokens[20 + customFieldsOffset])) { + outputMessage.put(VSYSName, unquoted_attribute(tokens[20 + customFieldsOffset])); + } + if (!empty_attribute(tokens[21 + customFieldsOffset])) { + outputMessage.put(DeviceName, unquoted_attribute(tokens[21 + customFieldsOffset])); + } + } + } else if (LogTypeSystem.equalsIgnoreCase(type)) { + if (tokens.length == 17) { + parserVersion = 61; + } else if (tokens.length == 23) { + parserVersion = 80; + } + + if (parserVersion >= 61) { + if (!empty_attribute(tokens[7])) { + outputMessage.put(VirtualSystem, tokens[7].trim()); + } + if (!empty_attribute(tokens[8])) { + outputMessage.put(EventId, tokens[8].trim()); + } + if (!empty_attribute(tokens[9])) { + outputMessage.put(Object, tokens[9].trim()); + } + + if (!empty_attribute(tokens[12])) { + outputMessage.put(Module, tokens[12].trim()); + } + if (!empty_attribute(tokens[13])) { + outputMessage.put(Severity, unquoted_attribute(tokens[13])); + } + if (!empty_attribute(tokens[14])) { + outputMessage.put(Description, unquoted_attribute(tokens[14])); + } + if (!empty_attribute(tokens[15])) { + outputMessage.put(Seqno, unquoted_attribute(tokens[15])); + } + if (!empty_attribute(tokens[16])) { + outputMessage.put(ActionFlags, unquoted_attribute(tokens[16])); + } + } + + if (parserVersion == 80) { + if (!empty_attribute(tokens[17])) { + outputMessage.put(DGH1, tokens[17].trim()); + } + if (!empty_attribute(tokens[18])) { + outputMessage.put(DGH2, tokens[18].trim()); + } + if (!empty_attribute(tokens[19])) { + outputMessage.put(DGH3, tokens[19].trim()); + } + if (!empty_attribute(tokens[20])) { + outputMessage.put(DGH4, tokens[20].trim()); + } + if (!empty_attribute(tokens[21])) { + outputMessage.put(VSYSName, unquoted_attribute(tokens[21])); + } + if (!empty_attribute(tokens[22])) { + outputMessage.put(DeviceName, unquoted_attribute(tokens[22])); + } + } + } else if (LogTypeThreat.equalsIgnoreCase(type) + || LogTypeTraffic.equalsIgnoreCase(type)) { + if (!empty_attribute(tokens[7])) { + outputMessage.put(SourceAddress, tokens[7].trim()); + } + if (!empty_attribute(tokens[8])) { + outputMessage.put(DestinationAddress, tokens[8].trim()); + } + if (!empty_attribute(tokens[9])) { + outputMessage.put(NATSourceIP, tokens[9].trim()); + } + if (!empty_attribute(tokens[10])) { + outputMessage.put(NATDestinationIP, tokens[10].trim()); + } + if (!empty_attribute(tokens[11])) { + outputMessage.put(Rule, unquoted_attribute(tokens[11])); + } + if (!empty_attribute(tokens[12])) { + outputMessage.put(SourceUser, unquoted_attribute(tokens[12])); + } + if (!empty_attribute(tokens[13])) { + outputMessage.put(DestinationUser, unquoted_attribute(tokens[13])); + } + if (!empty_attribute(tokens[14])) { + outputMessage.put(Application, unquoted_attribute(tokens[14])); + } + if (!empty_attribute(tokens[15])) { + outputMessage.put(VirtualSystem, unquoted_attribute(tokens[15])); + } + if (!empty_attribute(tokens[16])) { + outputMessage.put(SourceZone, unquoted_attribute(tokens[16])); + } + if (!empty_attribute(tokens[17])) { + outputMessage.put(DestinationZone, unquoted_attribute(tokens[17])); + } + if (!empty_attribute(tokens[18])) { + outputMessage.put(InboundInterface, unquoted_attribute(tokens[18])); + } + if (!empty_attribute(tokens[19])) { + outputMessage.put(OutboundInterface, unquoted_attribute(tokens[19])); + } + if (!empty_attribute(tokens[20])) { + outputMessage.put(LogAction, unquoted_attribute(tokens[20])); + } + if (!empty_attribute(tokens[21])) { + outputMessage.put(TimeLogged, tokens[21].trim()); + } + if (!empty_attribute(tokens[22])) { + outputMessage.put(SessionID, tokens[22].trim()); + } + if (!empty_attribute(tokens[23])) { + outputMessage.put(RepeatCount, tokens[23].trim()); + } + if (!empty_attribute(tokens[24])) { + outputMessage.put(SourcePort, tokens[24].trim()); + } + if (!empty_attribute(tokens[25])) { + outputMessage.put(DestinationPort, tokens[25].trim()); + } + if (!empty_attribute(tokens[26])) { + outputMessage.put(NATSourcePort, tokens[26].trim()); + } + if (!empty_attribute(tokens[27])) { + outputMessage.put(NATDestinationPort, tokens[27].trim()); + } + if (!empty_attribute(tokens[28])) { + outputMessage.put(Flags, tokens[28].trim()); + } + if (!empty_attribute(tokens[29])) { + outputMessage.put(IPProtocol, unquoted_attribute(tokens[29])); + } + if (!empty_attribute(tokens[30])) { + outputMessage.put(Action, unquoted_attribute(tokens[30])); + } + + if (LogTypeThreat.equalsIgnoreCase(type)) { + int p1Offset = 0; + if (tokens.length == 45) { + parserVersion = 60; + } else if (tokens.length == 53) { + parserVersion = 61; + } else if (tokens.length == 61) { + parserVersion = 70; + p1Offset = 1; + } else if (tokens.length == 72) { + parserVersion = 80; + p1Offset = 1; + } + if (!empty_attribute(tokens[31])) { + String rawUrl = unquoted_attribute(tokens[31]); + outputMessage.put(URL, rawUrl); + try { + URL url = new URL(rawUrl); + outputMessage.put(HOST, url.getHost()); + } catch (MalformedURLException e) { + _LOG.debug("Malformed URL: " + rawUrl, e); + } + } + if (!empty_attribute(tokens[32])) { + outputMessage.put(ThreatID, tokens[32].trim()); + } + if (!empty_attribute(tokens[33])) { + outputMessage.put(Category, unquoted_attribute(tokens[33])); + } + if (!empty_attribute(tokens[34])) { + outputMessage.put(Severity, unquoted_attribute(tokens[34])); + } + if (!empty_attribute(tokens[35])) { + outputMessage.put(Direction, unquoted_attribute(tokens[35])); + } + if (!empty_attribute(tokens[36])) { + outputMessage.put(Seqno, tokens[36].trim()); + } + if (!empty_attribute(tokens[37])) { + outputMessage.put(ActionFlags, unquoted_attribute(tokens[37])); + } + if (!empty_attribute(tokens[38])) { + outputMessage.put(SourceLocation, unquoted_attribute(tokens[38])); + } + if (!empty_attribute(tokens[39])) { + outputMessage.put(DestinationLocation, unquoted_attribute(tokens[39])); + } + if (!empty_attribute(tokens[41])) { + outputMessage.put(ContentType, unquoted_attribute(tokens[41])); + } + if (!empty_attribute(tokens[42])) { + outputMessage.put(PCAPID, tokens[42].trim()); + } + if (!empty_attribute(tokens[43])) { + outputMessage.put(WFFileDigest, unquoted_attribute(tokens[43])); + } + if (!empty_attribute(tokens[44])) { + outputMessage.put(WFCloud, unquoted_attribute(tokens[44])); + } + if (parserVersion >= 61) { + if (!empty_attribute(tokens[(45 + p1Offset)])) { + outputMessage.put(UserAgent, unquoted_attribute(tokens[(45 + p1Offset)])); + } + if (!empty_attribute(tokens[(46 + p1Offset)])) { + outputMessage.put(WFFileType, unquoted_attribute(tokens[(46 + p1Offset)])); + } + if (!empty_attribute(tokens[(47 + p1Offset)])) { + outputMessage.put(XForwardedFor, unquoted_attribute(tokens[(47 + p1Offset)])); + } + if (!empty_attribute(tokens[(48 + p1Offset)])) { + outputMessage.put(Referer, unquoted_attribute(tokens[(48 + p1Offset)])); + } + if (!empty_attribute(tokens[(49 + p1Offset)])) { + outputMessage.put(WFSender, unquoted_attribute(tokens[(49 + p1Offset)])); + } + if (!empty_attribute(tokens[(50 + p1Offset)])) { + outputMessage.put(WFSubject, unquoted_attribute(tokens[(50 + p1Offset)])); + } + if (!empty_attribute(tokens[(51 + p1Offset)])) { + outputMessage.put(WFRecipient, unquoted_attribute(tokens[(51 + p1Offset)])); + } + if (!empty_attribute(tokens[(52 + p1Offset)])) { + outputMessage.put(WFReportID, unquoted_attribute(tokens[(52 + p1Offset)])); + } + } + if (parserVersion >= 70) { + if (!empty_attribute(tokens[45])) { + outputMessage.put(URLIndex, tokens[45].trim()); + } + if (!empty_attribute(tokens[54])) { + outputMessage.put(DGH1, tokens[54].trim()); + } + if (!empty_attribute(tokens[55])) { + outputMessage.put(DGH2, tokens[55].trim()); + } + if (!empty_attribute(tokens[56])) { + outputMessage.put(DGH3, tokens[56].trim()); + } + if (!empty_attribute(tokens[57])) { + outputMessage.put(DGH4, tokens[57].trim()); + } + if (!empty_attribute(tokens[58])) { + outputMessage.put(VSYSName, unquoted_attribute(tokens[58])); + } + if (!empty_attribute(tokens[59])) { + outputMessage.put(DeviceName, unquoted_attribute(tokens[59])); + } + } + if (parserVersion >= 80) { + if (!empty_attribute(tokens[61])) { + outputMessage.put(SourceVmUuid, tokens[61].trim()); + } + if (!empty_attribute(tokens[62])) { + outputMessage.put(DestinationVmUuid, tokens[62].trim()); + } + if (!empty_attribute(tokens[63])) { + outputMessage.put(HTTPMethod, tokens[63].trim()); + } + if (!empty_attribute(tokens[64])) { + outputMessage.put(TunnelId, tokens[64].trim()); + } + if (!empty_attribute(tokens[65])) { + outputMessage.put(MonitorTag, tokens[65].trim()); + } + if (!empty_attribute(tokens[66])) { + outputMessage.put(ParentSessionId, tokens[66].trim()); + } + if (!empty_attribute(tokens[67])) { + outputMessage.put(ParentSessionStartTime, tokens[67].trim()); + } + if (!empty_attribute(tokens[68])) { + outputMessage.put(TunnelType, tokens[68].trim()); + } + if (!empty_attribute(tokens[69])) { + outputMessage.put(ThreatCategory, tokens[69].trim()); + } + if (!empty_attribute(tokens[70])) { + outputMessage.put(ContentVersion, tokens[70].trim()); + } + } + } else if (LogTypeTraffic.equalsIgnoreCase(type)) { + if (tokens.length == 46) { + parserVersion = 60; + } else if (tokens.length == 47) { + parserVersion = 61; + } else if (tokens.length == 54) { + parserVersion = 70; + } else if (tokens.length == 61) { + parserVersion = 80; + } + if (!empty_attribute(tokens[31])) { + outputMessage.put(Bytes, tokens[31].trim()); + } + if (!empty_attribute(tokens[32])) { + outputMessage.put(BytesSent, tokens[32].trim()); + } + if (!empty_attribute(tokens[33])) { + outputMessage.put(BytesReceived, tokens[33].trim()); + } + if (!empty_attribute(tokens[34])) { + outputMessage.put(Packets, tokens[34].trim()); + } + if (!empty_attribute(tokens[35])) { + outputMessage.put(StartTime, tokens[35].trim()); + } + if (!empty_attribute(tokens[36])) { + outputMessage.put(ElapsedTimeInSec, tokens[36].trim()); + } + if (!empty_attribute(tokens[37])) { + outputMessage.put(Category, unquoted_attribute(tokens[37])); + } + if (!empty_attribute(tokens[39])) { + outputMessage.put(Seqno, tokens[39].trim()); + } + if (!empty_attribute(tokens[40])) { + outputMessage.put(ActionFlags, unquoted_attribute(tokens[40])); + } + if (!empty_attribute(tokens[41])) { + outputMessage.put(SourceLocation, unquoted_attribute(tokens[41])); + } + if (!empty_attribute(tokens[42])) { + outputMessage.put(DestinationLocation, unquoted_attribute(tokens[42])); + } + if (!empty_attribute(tokens[44])) { + outputMessage.put(PktsSent, tokens[44].trim()); + } + if (!empty_attribute(tokens[45])) { + outputMessage.put(PktsReceived, tokens[45].trim()); + } + if (parserVersion >= 61) { + if (!empty_attribute(tokens[46])) { + outputMessage.put(EndReason, unquoted_attribute(tokens[46])); + } + } + if (parserVersion >= 70) { + if (!empty_attribute(tokens[47])) { + outputMessage.put(DGH1, tokens[47].trim()); + } + if (!empty_attribute(tokens[48])) { + outputMessage.put(DGH2, tokens[48].trim()); + } + if (!empty_attribute(tokens[49])) { + outputMessage.put(DGH3, tokens[49].trim()); + } + if (!empty_attribute(tokens[50])) { + outputMessage.put(DGH4, tokens[50].trim()); + } + if (!empty_attribute(tokens[51])) { + outputMessage.put(VSYSName, unquoted_attribute(tokens[51])); + } + if (!empty_attribute(tokens[52])) { + outputMessage.put(DeviceName, unquoted_attribute(tokens[52])); + } + if (!empty_attribute(tokens[53])) { + outputMessage.put(ActionSource, unquoted_attribute(tokens[53])); + } + } + if (parserVersion >= 80) { + if (!empty_attribute(tokens[54])) { + outputMessage.put(SourceVmUuid, tokens[54].trim()); + } + if (!empty_attribute(tokens[55])) { + outputMessage.put(DestinationVmUuid, tokens[55].trim()); + } + if (!empty_attribute(tokens[56])) { + outputMessage.put(TunnelId, tokens[56].trim()); + } + if (!empty_attribute(tokens[57])) { + outputMessage.put(MonitorTag, tokens[57].trim()); + } + if (!empty_attribute(tokens[58])) { + outputMessage.put(ParentSessionId, tokens[58].trim()); + } + if (!empty_attribute(tokens[59])) { + outputMessage.put(ParentSessionStartTime, tokens[59].trim()); + } + if (!empty_attribute(tokens[60])) { + outputMessage.put(TunnelType, tokens[60].trim()); + } + } + } } - if (parser_version >= 80) { - if (!empty_attribute(tokens[54])) outputMessage.put(SourceVmUuid, tokens[54].trim()); - if (!empty_attribute(tokens[55])) outputMessage.put(DestinationVmUuid, tokens[55].trim()); - if (!empty_attribute(tokens[56])) outputMessage.put(TunnelId, tokens[56].trim()); - if (!empty_attribute(tokens[57])) outputMessage.put(MonitorTag, tokens[57].trim()); - if (!empty_attribute(tokens[58])) outputMessage.put(ParentSessionId, tokens[58].trim()); - if (!empty_attribute(tokens[59])) outputMessage.put(ParentSessionStartTime, tokens[59].trim()); - if (!empty_attribute(tokens[60])) outputMessage.put(TunnelType, tokens[60].trim()); + outputMessage.put(ParserVersion, parserVersion); + if (parserVersion == 0) { + outputMessage.put(Tokens, tokens.length); } - } - } - outputMessage.put(ParserVersion, parser_version); - if (parser_version == 0) { - outputMessage.put(Tokens, tokens.length); } - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/paloalto/README.md b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/paloalto/README.md index b6a12306..599b19a3 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/paloalto/README.md +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/paloalto/README.md @@ -15,18 +15,22 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> + # BasicPaloAltoFirewallParser + ## Introduction -The parser is able to parse logs generated by Palo Alto firewall devices powered by Pan OS. The supported log format is CSV. + +The parser is able to parse logs generated by Palo Alto firewall devices powered by Pan OS. The supported log format is +CSV. The supported log types and versions -| Log type | Pan OS version | -|----------|----------------| -| Traffic | 6.0, 6.1, 7.0, 8.0 | -| Threat | 6.0, 6.1, 7.0, 8.0 | -| Config | 6.1, 7.0, 8.0 | -| System | 6.1, 7.0, 8.0 | +| Log type | Pan OS version | +|----------|--------------------| +| Traffic | 6.0, 6.1, 7.0, 8.0 | +| Threat | 6.0, 6.1, 7.0, 8.0 | +| Config | 6.1, 7.0, 8.0 | +| System | 6.1, 7.0, 8.0 | diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java index 3899c690..e431fdad 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java @@ -4,8 +4,10 @@ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at + * *

    * http://www.apache.org/licenses/LICENSE-2.0 + * *

    * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express @@ -17,7 +19,7 @@ import com.google.common.base.CaseFormat; import java.lang.invoke.MethodHandles; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -30,8 +32,8 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import org.apache.commons.lang3.StringUtils; -import org.apache.metron.stellar.common.Constants; import org.apache.metron.parsers.BasicParser; +import org.apache.metron.stellar.common.Constants; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +47,9 @@ * * "convertCamelCaseToUnderScore": true, * "recordTypeRegex": "(?<process>(?<=\\s)\\b(kernel|syslog)\\b(?=\\[|:))", - * "messageHeaderRegex": "(?<syslogpriority>(?<=^<)\\d{1,4}(?=>)).*?(?<timestamp>(?<=>)[A-Za-z]{3}\\s{1,2}\\d{1,2}\\s\\d{1,2}:\\d{1,2}:\\d{1,2}(?=\\s)).*?(?<syslogHost>(?<=\\s).*?(?=\\s))", + * "messageHeaderRegex": "(?<syslogpriority>(?<=^<)\\d{1,4}(?=>)).*? + * (?<timestamp>(?<=>)[A-Za-z]{3}\\s{1,2}\\d{1,2}\\s\\d{1,2}:\\d{1,2}:\\d{1,2}(?=\\s)).*? + * (?<syslogHost>(?<=\\s).*?(?=\\s))", * "fields": [ * { * "recordType": "kernel", @@ -53,12 +57,15 @@ * }, * { * "recordType": "syslog", - * "regex": ".*(?<processid>(?<=PID\\s=\\s).*?(?=\\sLine)).*(?<filePath>(?<=64\\s)\/([A-Za-z0-9_-]+\/)+(?=\\w))(?<fileName>.*?(?=\")).*(?<eventInfo>(?<=\").*?(?=$))" + * "regex": ".*(?<processid>(?<=PID\\s=\\s).*?(?=\\sLine)).* + * (?<filePath>(?<=64\\s)\/([A-Za-z0-9_-]+\/)+(?=\\w))(?<fileName>.*?(?=\")).* + * (?<eventInfo>(?<=\").*?(?=$))" * } * ] * * * + *

    * Note: messageHeaderRegex could be specified as lists also e.g. * * 

    @@ -70,6 +77,7 @@
  * 
  *
    * + *

    * Where regular expression 1 are valid regular expressions and may have named * groups, which would be extracted into fields. This list will be evaluated in order until a * matching regular expression is found.
    @@ -87,9 +95,11 @@ *
    * * + *

    * Here message structure (<7>Jun 26 16:18:01 hostName kernel) is common across all messages. * Hence messageHeaderRegex could be used to extract fields from this part. * + *

    * fields : json list of objects containing recordType and regex. regex could be a further list e.g. * * 

    @@ -114,29 +124,34 @@
  * 
  *
    * + *

    * This means that an _ (underscore), cannot be used as part of a named group name. E.g. this is an * invalid regular expression .*(?<event_info>(?<=\\]|\\w\\:).*?(?=$)) * + *

    * However, this limitation can be easily overcome by adding a parser configuration setting. * + *

    * * "convertCamelCaseToUnderScore": true, - * - * If above property is added to the sensor parser configuration, in parserConfig object, this parser will automatically convert all the camel case property names to underscore seperated. + * + * If above property is added to the sensor parser configuration, + * in parserConfig object, this parser will automatically convert all the camel case property names to underscore seperated. * For example, following conversions will automatically happen: * + *

    * * ipSrcAddr -> ip_src_addr * ipDstAddr -> ip_dst_addr * ipSrcPort -> ip_src_port - * + * * etc. */ //@formatter:on public class RegularExpressionsParser extends BasicParser { protected static final Logger LOG = - LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private List> fields; private Map parserConfig; @@ -145,7 +160,7 @@ public class RegularExpressionsParser extends BasicParser { private Pattern recordTypePattern; private final Set recordTypePatternNamedGroups = new HashSet<>(); private final Map>> recordTypePatternMap = - new LinkedHashMap<>(); + new LinkedHashMap<>(); private final Map> messageHeaderPatternsMap = new LinkedHashMap<>(); /** @@ -163,7 +178,7 @@ public List parse(byte[] rawMessage) { LOG.debug(" raw message. {}", originalMessage); if (originalMessage.isEmpty()) { LOG.warn("Message is empty."); - return Arrays.asList(new JSONObject()); + return Collections.singletonList(new JSONObject()); } } catch (Exception e) { LOG.error("[Metron] Could not read raw message. {} " + originalMessage, e); @@ -176,82 +191,97 @@ public List parse(byte[] rawMessage) { } parsedJson.putAll(parse(originalMessage)); parsedJson.put(Constants.Fields.ORIGINAL.getName(), originalMessage); - /** + /* * Populate the output json with default timestamp. */ parsedJson.put(Constants.Fields.TIMESTAMP.getName(), System.currentTimeMillis()); applyFieldTransformations(parsedJson); - return Arrays.asList(parsedJson); + return Collections.singletonList(parsedJson); + } + + private JSONObject parse(String originalMessage) { + JSONObject parsedJson = new JSONObject(); + Optional recordIdentifier = getField(recordTypePattern, originalMessage); + if (recordIdentifier.isPresent()) { + extractNamedGroups(parsedJson, recordIdentifier.get(), originalMessage); + } + /* + * Extract fields(named groups) from record type regular expression + */ + Matcher matcher = recordTypePattern.matcher(originalMessage); + if (matcher.find()) { + for (String namedGroup : recordTypePatternNamedGroups) { + if (matcher.group(namedGroup) != null) { + parsedJson.put(namedGroup, matcher.group(namedGroup).trim()); + } + } + } + return parsedJson; } private void applyFieldTransformations(JSONObject parsedJson) { - if (getParserConfig().get(ParserConfigConstants.CONVERT_CAMELCASE_TO_UNDERSCORE.getName()) - != null && (Boolean) getParserConfig() - .get(ParserConfigConstants.CONVERT_CAMELCASE_TO_UNDERSCORE.getName())) { + if (getParserConfig().get(ParserConfigConstants.CONVERT_CAMELCASE_TO_UNDERSCORE.getName()) != null + && (Boolean) getParserConfig() + .get(ParserConfigConstants.CONVERT_CAMELCASE_TO_UNDERSCORE.getName())) { convertCamelCaseToUnderScore(parsedJson); } } // @formatter:off - /** - * This method is called during the parser initialization. It parses the parser - * configuration and configures the parser accordingly. It then initializes - * instance variables. - * - * @param parserConfig ParserConfig(Map) supplied to the sensor. - * @see org.apache.metron.parsers.interfaces.Configurable#configure(java.util.Map)
    - *
    - */ - // @formatter:on - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - setParserConfig(parserConfig); - setFields((List>) getParserConfig() - .get(ParserConfigConstants.FIELDS.getName())); - String recordTypeRegex = - (String) getParserConfig().get(ParserConfigConstants.RECORD_TYPE_REGEX.getName()); - - if (StringUtils.isBlank(recordTypeRegex)) { - LOG.error("Invalid config :recordTypeRegex is missing in parserConfig"); - throw new IllegalStateException( - "Invalid config :recordTypeRegex is missing in parserConfig"); - } - - setRecordTypePattern(recordTypeRegex); - recordTypePatternNamedGroups.addAll(getNamedGroups(recordTypeRegex)); - List> fields = - (List>) getParserConfig().get(ParserConfigConstants.FIELDS.getName()); - - try { - configureRecordTypePatterns(fields); - configureMessageHeaderPattern(); - } catch (PatternSyntaxException e) { - LOG.error("Invalid config : {} ", e.getMessage()); - throw new IllegalStateException("Invalid config : " + e.getMessage()); - } - - validateConfig(); - } + /** + * This method is called during the parser initialization. It parses the parser + * configuration and configures the parser accordingly. It then initializes + * instance variables. + * + * @param parserConfig {@code ParserConfig(Map)} supplied to the sensor. + * @see org.apache.metron.parsers.interfaces.Configurable#configure(java.util.Map) + */ + // @formatter:on + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + setParserConfig(parserConfig); + setFields((List>) getParserConfig() + .get(ParserConfigConstants.FIELDS.getName())); + String recordTypeRegex = + (String) getParserConfig().get(ParserConfigConstants.RECORD_TYPE_REGEX.getName()); + if (StringUtils.isBlank(recordTypeRegex)) { + LOG.error("Invalid config :recordTypeRegex is missing in parserConfig"); + throw new IllegalStateException( + "Invalid config :recordTypeRegex is missing in parserConfig"); + } + setRecordTypePattern(recordTypeRegex); + recordTypePatternNamedGroups.addAll(getNamedGroups(recordTypeRegex)); + List> fields = + (List>) getParserConfig().get(ParserConfigConstants.FIELDS.getName()); + try { + configureRecordTypePatterns(fields); + configureMessageHeaderPattern(); + } catch (PatternSyntaxException e) { + LOG.error("Invalid config : {} ", e.getMessage()); + throw new IllegalStateException("Invalid config : " + e.getMessage()); + } + validateConfig(); + } private void configureMessageHeaderPattern() { if (getParserConfig().get(ParserConfigConstants.MESSAGE_HEADER.getName()) != null) { if (getParserConfig() - .get(ParserConfigConstants.MESSAGE_HEADER.getName()) instanceof List) { + .get(ParserConfigConstants.MESSAGE_HEADER.getName()) instanceof List) { List messageHeaderPatternList = (List) getParserConfig() - .get(ParserConfigConstants.MESSAGE_HEADER.getName()); + .get(ParserConfigConstants.MESSAGE_HEADER.getName()); for (String messageHeaderPatternStr : messageHeaderPatternList) { messageHeaderPatternsMap.put(Pattern.compile(messageHeaderPatternStr), - getNamedGroups(messageHeaderPatternStr)); + getNamedGroups(messageHeaderPatternStr)); } } else if (getParserConfig() - .get(ParserConfigConstants.MESSAGE_HEADER.getName()) instanceof String) { + .get(ParserConfigConstants.MESSAGE_HEADER.getName()) instanceof String) { String messageHeaderPatternStr = - (String) getParserConfig().get(ParserConfigConstants.MESSAGE_HEADER.getName()); + (String) getParserConfig().get(ParserConfigConstants.MESSAGE_HEADER.getName()); if (StringUtils.isNotBlank(messageHeaderPatternStr)) { messageHeaderPatternsMap.put(Pattern.compile(messageHeaderPatternStr), - getNamedGroups(messageHeaderPatternStr)); + getNamedGroups(messageHeaderPatternStr)); } } } @@ -261,21 +291,21 @@ private void configureRecordTypePatterns(List> fields) { for (Map field : fields) { if (field.get(ParserConfigConstants.RECORD_TYPE.getName()) != null - && field.get(ParserConfigConstants.REGEX.getName()) != null) { + && field.get(ParserConfigConstants.REGEX.getName()) != null) { String recordType = - ((String) field.get(ParserConfigConstants.RECORD_TYPE.getName())).toLowerCase(); + ((String) field.get(ParserConfigConstants.RECORD_TYPE.getName())).toLowerCase(); recordTypePatternMap.put(recordType, new LinkedHashMap<>()); if (field.get(ParserConfigConstants.REGEX.getName()) instanceof List) { List regexList = - (List) field.get(ParserConfigConstants.REGEX.getName()); + (List) field.get(ParserConfigConstants.REGEX.getName()); regexList.forEach(s -> { recordTypePatternMap.get(recordType) - .put(Pattern.compile(s), getNamedGroups(s)); + .put(Pattern.compile(s), getNamedGroups(s)); }); } else if (field.get(ParserConfigConstants.REGEX.getName()) instanceof String) { recordTypePatternMap.get(recordType).put( - Pattern.compile((String) field.get(ParserConfigConstants.REGEX.getName())), - getNamedGroups((String) field.get(ParserConfigConstants.REGEX.getName()))); + Pattern.compile((String) field.get(ParserConfigConstants.REGEX.getName())), + getNamedGroups((String) field.get(ParserConfigConstants.REGEX.getName()))); } } } @@ -287,28 +317,8 @@ private void setRecordTypePattern(String recordTypeRegex) { } } - private JSONObject parse(String originalMessage) { - JSONObject parsedJson = new JSONObject(); - Optional recordIdentifier = getField(recordTypePattern, originalMessage); - if (recordIdentifier.isPresent()) { - extractNamedGroups(parsedJson, recordIdentifier.get(), originalMessage); - } - /* - * Extract fields(named groups) from record type regular expression - */ - Matcher matcher = recordTypePattern.matcher(originalMessage); - if (matcher.find()) { - for (String namedGroup : recordTypePatternNamedGroups) { - if (matcher.group(namedGroup) != null) { - parsedJson.put(namedGroup, matcher.group(namedGroup).trim()); - } - } - } - return parsedJson; - } - private void extractNamedGroups(Map json, String recordType, - String originalMessage) { + String originalMessage) { Map> patternMap = recordTypePatternMap.get(recordType.toLowerCase()); if (patternMap != null) { for (Map.Entry> entry : patternMap.entrySet()) { @@ -318,7 +328,7 @@ private void extractNamedGroups(Map json, String recordType, Matcher m = pattern.matcher(originalMessage); if (m.matches()) { LOG.debug("RecordType : {} Trying regex : {} for message : {} ", recordType, - pattern.toString(), originalMessage); + pattern, originalMessage); for (String namedGroup : namedGroups) { if (m.group(namedGroup) != null) { json.put(namedGroup, m.group(namedGroup).trim()); @@ -353,7 +363,7 @@ private Set getNamedGroups(String regex) { private Map extractHeaderFields(String originalMessage) { Map messageHeaderJson = new JSONObject(); for (Map.Entry> syslogPatternEntry : messageHeaderPatternsMap - .entrySet()) { + .entrySet()) { Matcher m = syslogPatternEntry.getKey().matcher(originalMessage); if (m.find()) { for (String namedGroup : syslogPatternEntry.getValue()) { @@ -384,7 +394,7 @@ private void convertCamelCaseToUnderScore(Map json) { for (Map.Entry entry : json.entrySet()) { if (capitalLettersPattern.matcher(entry.getKey()).matches()) { oldKeyNewKeyMap.put(entry.getKey(), - CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, entry.getKey())); + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, entry.getKey())); } } oldKeyNewKeyMap.forEach((oldKey, newKey) -> json.put(newKey, json.remove(oldKey))); @@ -408,15 +418,15 @@ public void setParserConfig(Map parserConfig) { enum ParserConfigConstants { //@formatter:off - RECORD_TYPE("recordType"), - RECORD_TYPE_REGEX("recordTypeRegex"), - REGEX("regex"), - FIELDS("fields"), - MESSAGE_HEADER("messageHeaderRegex"), - CONVERT_CAMELCASE_TO_UNDERSCORE("convertCamelCaseToUnderScore"); - //@formatter:on - private final String name; - private static Map nameToField; + RECORD_TYPE("recordType"), + RECORD_TYPE_REGEX("recordTypeRegex"), + REGEX("regex"), + FIELDS("fields"), + MESSAGE_HEADER("messageHeaderRegex"), + CONVERT_CAMELCASE_TO_UNDERSCORE("convertCamelCaseToUnderScore"); + //@formatter:on + private final String name; + private static final Map nameToField; ParserConfigConstants(String name) { this.name = name; diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java index d4c5170b..9c0f08b7 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.snort; import com.google.common.collect.Lists; @@ -36,15 +39,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@SuppressWarnings("serial") public class BasicSnortParser extends BasicParser { - private static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - /** - * The default field names for Snort Alerts. - */ - private String[] fieldNames = new String[] { + /** + * The default field names for Snort Alerts. + */ + private String[] fieldNames = new String[] { Constants.Fields.TIMESTAMP.getName(), "sig_generator", "sig_id", @@ -72,146 +74,146 @@ public class BasicSnortParser extends BasicParser { "icmpcode", "icmpid", "icmpseq" - }; + }; - /** - * Snort alerts are received as CSV records - */ - private String recordDelimiter = ","; + /** + * Snort alerts are received as CSV records. + */ + private String recordDelimiter = ","; - private transient CSVConverter converter; + private transient CSVConverter converter; - private static String defaultDateFormat = "MM/dd/yy-HH:mm:ss.SSSSSS"; - private transient DateTimeFormatter dateTimeFormatter; + private static final String defaultDateFormat = "MM/dd/yy-HH:mm:ss.SSSSSS"; + private transient DateTimeFormatter dateTimeFormatter; - public BasicSnortParser() { + public BasicSnortParser() { - } + } - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - dateTimeFormatter = getDateFormatter(parserConfig); - dateTimeFormatter = getDateFormatterWithZone(dateTimeFormatter, parserConfig); - init(); - } + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + dateTimeFormatter = getDateFormatter(parserConfig); + dateTimeFormatter = getDateFormatterWithZone(dateTimeFormatter, parserConfig); + init(); + } - private DateTimeFormatter getDateFormatter(Map parserConfig) { - String format = (String) parserConfig.get("dateFormat"); - if (StringUtils.isNotEmpty(format)) { - _LOG.info("Using date format '{}'", format); - return DateTimeFormatter.ofPattern(format); - } else { - _LOG.info("Using default date format '{}'", defaultDateFormat); - return DateTimeFormatter.ofPattern(defaultDateFormat); + private DateTimeFormatter getDateFormatter(Map parserConfig) { + String format = (String) parserConfig.get("dateFormat"); + if (StringUtils.isNotEmpty(format)) { + _LOG.info("Using date format '{}'", format); + return DateTimeFormatter.ofPattern(format); + } else { + _LOG.info("Using default date format '{}'", defaultDateFormat); + return DateTimeFormatter.ofPattern(defaultDateFormat); + } } - } - - private DateTimeFormatter getDateFormatterWithZone(DateTimeFormatter formatter, Map parserConfig) { - String timezone = (String) parserConfig.get("timeZone"); - if (StringUtils.isNotEmpty(timezone)) { - if(ZoneId.getAvailableZoneIds().contains(timezone)) { - _LOG.info("Using timezone '{}'", timezone); - return formatter.withZone(ZoneId.of(timezone)); - } else { - throw new IllegalArgumentException("Unable to find ZoneId '" + timezone + "'"); - } - } else { - _LOG.info("Using default timezone '{}'", ZoneId.systemDefault()); - return formatter.withZone(ZoneId.systemDefault()); + + private DateTimeFormatter getDateFormatterWithZone(DateTimeFormatter formatter, Map parserConfig) { + String timezone = (String) parserConfig.get("timeZone"); + if (StringUtils.isNotEmpty(timezone)) { + if (ZoneId.getAvailableZoneIds().contains(timezone)) { + _LOG.info("Using timezone '{}'", timezone); + return formatter.withZone(ZoneId.of(timezone)); + } else { + throw new IllegalArgumentException("Unable to find ZoneId '" + timezone + "'"); + } + } else { + _LOG.info("Using default timezone '{}'", ZoneId.systemDefault()); + return formatter.withZone(ZoneId.systemDefault()); + } } - } - - @Override - public void init() { - if(converter == null) { - converter = new CSVConverter(); - Map config = new HashMap<>(); - config.put(CSVConverter.SEPARATOR_KEY, recordDelimiter); - config.put(CSVConverter.COLUMNS_KEY, Lists.newArrayList(fieldNames)); - converter.initialize(config); + + @Override + public void init() { + if (converter == null) { + converter = new CSVConverter(); + Map config = new HashMap<>(); + config.put(CSVConverter.SEPARATOR_KEY, recordDelimiter); + config.put(CSVConverter.COLUMNS_KEY, Lists.newArrayList(fieldNames)); + converter.initialize(config); + } } - } - - @Override - public List parse(byte[] rawMessage) { - - JSONObject jsonMessage = new JSONObject(); - List messages = new ArrayList<>(); - try { - // snort alerts expected as csv records - String csvMessage = new String(rawMessage, getReadCharset()); - Map records = null; - try { - records = converter.toMap(csvMessage); - } - catch(ArrayIndexOutOfBoundsException aioob) { - throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " in " + csvMessage); - } - - // validate the number of fields - if (records.size() != fieldNames.length) { - throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.size()); - } - long timestamp = 0L; - // build the json record from each field - for (Map.Entry kv : records.entrySet()) { - - String field = kv.getKey(); - String record = kv.getValue(); - - if("timestamp".equals(field)) { - - // convert the timestamp to epoch - timestamp = toEpoch(record); - jsonMessage.put("timestamp", timestamp); - } else { - jsonMessage.put(field, record); + @Override + public List parse(byte[] rawMessage) { + + JSONObject jsonMessage = new JSONObject(); + List messages = new ArrayList<>(); + try { + // snort alerts expected as csv records + String csvMessage = new String(rawMessage, getReadCharset()); + Map records = null; + try { + records = converter.toMap(csvMessage); + } catch (ArrayIndexOutOfBoundsException aioob) { + throw new IllegalArgumentException( + "Unexpected number of fields, expected: " + fieldNames.length + " in " + csvMessage); + } + + // validate the number of fields + if (records.size() != fieldNames.length) { + throw new IllegalArgumentException( + "Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.size()); + } + long timestamp = 0L; + // build the json record from each field + for (Map.Entry kv : records.entrySet()) { + + String field = kv.getKey(); + String record = kv.getValue(); + + if ("timestamp".equals(field)) { + + // convert the timestamp to epoch + timestamp = toEpoch(record); + jsonMessage.put("timestamp", timestamp); + + } else { + jsonMessage.put(field, record); + } + } + + // add original msg; required by 'checkForSchemaCorrectness' + jsonMessage.put("original_string", csvMessage); + jsonMessage.put("is_alert", "true"); + messages.add(jsonMessage); + } catch (Exception e) { + String message = "Unable to parse message: " + (rawMessage == null ? "null" : new String(rawMessage, + StandardCharsets.UTF_8)); + _LOG.error(message, e); + throw new IllegalStateException(message, e); } - } - - // add original msg; required by 'checkForSchemaCorrectness' - jsonMessage.put("original_string", csvMessage); - jsonMessage.put("is_alert", "true"); - messages.add(jsonMessage); - } catch (Exception e) { - String message = "Unable to parse message: " + (rawMessage == null?"null" : new String(rawMessage, - StandardCharsets.UTF_8)); - _LOG.error(message, e); - throw new IllegalStateException(message, e); + + return messages; + } + + /** + * Parses Snort's default date-time representation and + * converts to epoch. + * + * @param snortDatetime Snort's default date-time as String '01/27-16:01:04.877970' + */ + private long toEpoch(String snortDatetime) throws ParseException { + ZonedDateTime zonedDateTime = ZonedDateTime.parse(snortDatetime.trim(), dateTimeFormatter); + return zonedDateTime.toInstant().toEpochMilli(); + } + + public String getRecordDelimiter() { + return this.recordDelimiter; } - return messages; - } - - /** - * Parses Snort's default date-time representation and - * converts to epoch. - * @param snortDatetime Snort's default date-time as String '01/27-16:01:04.877970' - * @return epoch time - * @throws java.text.ParseException - */ - private long toEpoch(String snortDatetime) throws ParseException { - ZonedDateTime zonedDateTime = ZonedDateTime.parse(snortDatetime.trim(), dateTimeFormatter); - return zonedDateTime.toInstant().toEpochMilli(); - } - - public String getRecordDelimiter() { - return this.recordDelimiter; - } - - public void setRecordDelimiter(String recordDelimiter) { - this.recordDelimiter = recordDelimiter; - } - - public String[] getFieldNames() { - return this.fieldNames; - } - - public void setFieldNames(String[] fieldNames) { - this.fieldNames = fieldNames; - } + public void setRecordDelimiter(String recordDelimiter) { + this.recordDelimiter = recordDelimiter; + } + + public String[] getFieldNames() { + return this.fieldNames; + } + + public void setFieldNames(String[] fieldNames) { + this.fieldNames = fieldNames; + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java index 9781e506..0f5e8cb8 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,101 +30,99 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@SuppressWarnings("serial") +@SuppressWarnings("checkstyle:MemberName") public class BasicSourcefireParser extends BasicParser { - private static final Logger _LOG = LoggerFactory - .getLogger(BasicSourcefireParser.class); - - public static final String hostkey = "host"; - String domain_name_regex = "([^\\.]+)\\.([a-z]{2}|[a-z]{3}|([a-z]{2}\\.[a-z]{2}))$"; - String sidRegex = "(.*)(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; - //String sidRegex = "(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; - Pattern sidPattern = Pattern.compile(sidRegex); - Pattern pattern = Pattern.compile(domain_name_regex); - - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - } - - @Override - public void init() { - - } - - @Override - @SuppressWarnings({ "unchecked", "unused" }) - public List parse(byte[] msg) { - - JSONObject payload = new JSONObject(); - String toParse = ""; - List messages = new ArrayList<>(); - try { - - toParse = new String(msg, getReadCharset()); - _LOG.debug("Received message: {}", toParse); - - String tmp = toParse.substring(toParse.lastIndexOf("{")); - payload.put("key", tmp); - - String protocol = tmp.substring(tmp.indexOf("{") + 1, - tmp.indexOf("}")).toLowerCase(); - String source = tmp.substring(tmp.indexOf("}") + 1, - tmp.indexOf("->")).trim(); - String dest = tmp.substring(tmp.indexOf("->") + 2, tmp.length()) - .trim(); - - payload.put("protocol", protocol); - - String source_ip = ""; - String dest_ip = ""; - - if (source.contains(":")) { - String parts[] = source.split(":"); - payload.put("ip_src_addr", parts[0]); - payload.put("ip_src_port", parts[1]); - source_ip = parts[0]; - } else { - payload.put("ip_src_addr", source); - source_ip = source; - - } - - if (dest.contains(":")) { - String parts[] = dest.split(":"); - payload.put("ip_dst_addr", parts[0]); - payload.put("ip_dst_port", parts[1]); - dest_ip = parts[0]; - } else { - payload.put("ip_dst_addr", dest); - dest_ip = dest; - } - long timestamp = System.currentTimeMillis(); - payload.put("timestamp", timestamp); - - Matcher sidMatcher = sidPattern.matcher(toParse); - String originalString = null; - String signatureId = ""; - if (sidMatcher.find()) { - signatureId = sidMatcher.group(2); - originalString = sidMatcher.group(1) +" "+ sidMatcher.group(2) + " " + sidMatcher.group(3); - } else { - _LOG.warn("Unable to find SID in message: {}", toParse); - originalString = toParse; - } - payload.put("original_string", originalString); - payload.put("signature_id", signatureId); - messages.add(payload); - return messages; - } catch (Exception e) { - e.printStackTrace(); - _LOG.error("Failed to parse: {}", toParse); - return null; - } - } - - + private static final Logger _LOG = LoggerFactory + .getLogger(BasicSourcefireParser.class); + + public static final String hostkey = "host"; + String domain_name_regex = "([^\\.]+)\\.([a-z]{2}|[a-z]{3}|([a-z]{2}\\.[a-z]{2}))$"; + String sidRegex = "(.*)(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; + //String sidRegex = "(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; + Pattern sidPattern = Pattern.compile(sidRegex); + Pattern pattern = Pattern.compile(domain_name_regex); + + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + } + + @Override + public void init() { + + } + + @Override + @SuppressWarnings({"unchecked", "unused"}) + public List parse(byte[] msg) { + + JSONObject payload = new JSONObject(); + String toParse = ""; + List messages = new ArrayList<>(); + try { + + toParse = new String(msg, getReadCharset()); + _LOG.debug("Received message: {}", toParse); + + String tmp = toParse.substring(toParse.lastIndexOf("{")); + payload.put("key", tmp); + + String protocol = tmp.substring(tmp.indexOf("{") + 1, + tmp.indexOf("}")).toLowerCase(); + String source = tmp.substring(tmp.indexOf("}") + 1, + tmp.indexOf("->")).trim(); + String dest = tmp.substring(tmp.indexOf("->") + 2) + .trim(); + + payload.put("protocol", protocol); + + String sourceIp = ""; + String destIp = ""; + + if (source.contains(":")) { + String[] parts = source.split(":"); + payload.put("ip_src_addr", parts[0]); + payload.put("ip_src_port", parts[1]); + sourceIp = parts[0]; + } else { + payload.put("ip_src_addr", source); + sourceIp = source; + + } + + if (dest.contains(":")) { + String[] parts = dest.split(":"); + payload.put("ip_dst_addr", parts[0]); + payload.put("ip_dst_port", parts[1]); + destIp = parts[0]; + } else { + payload.put("ip_dst_addr", dest); + destIp = dest; + } + long timestamp = System.currentTimeMillis(); + payload.put("timestamp", timestamp); + + Matcher sidMatcher = sidPattern.matcher(toParse); + String originalString = null; + String signatureId = ""; + if (sidMatcher.find()) { + signatureId = sidMatcher.group(2); + originalString = sidMatcher.group(1) + " " + sidMatcher.group(2) + " " + sidMatcher.group(3); + } else { + _LOG.warn("Unable to find SID in message: {}", toParse); + originalString = toParse; + } + payload.put("original_string", originalString); + payload.put("signature_id", signatureId); + messages.add(payload); + return messages; + } catch (Exception e) { + e.printStackTrace(); + _LOG.error("Failed to parse: {}", toParse); + return null; + } + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/BaseSyslogParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/BaseSyslogParser.java index c154c7c6..7fc2ae6d 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/BaseSyslogParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/BaseSyslogParser.java @@ -54,113 +54,115 @@ */ public abstract class BaseSyslogParser implements MessageParser, Serializable { - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private Charset readCharset; - private Optional> messageProcessorOptional = Optional.empty(); - private transient SyslogParser syslogParser; - protected Clock deviceClock; - - protected void setSyslogParser(SyslogParser syslogParser) { - this.syslogParser = syslogParser; - } - - protected void setMessageProcessor(Consumer function) { - this.messageProcessorOptional = Optional.of(function); - } - - protected abstract SyslogParser buildSyslogParser( Map config); - - @Override - public void configure(Map parserConfig) { - setReadCharset(parserConfig); - // we'll pull out the clock stuff ourselves - String timeZone = (String) parserConfig.get("deviceTimeZone"); - if (timeZone != null) - deviceClock = Clock.system(ZoneId.of(timeZone)); - else { - deviceClock = Clock.systemUTC(); - LOG.warn("[Metron] No device time zone provided; defaulting to UTC"); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private Charset readCharset; + private Optional> messageProcessorOptional = Optional.empty(); + private transient SyslogParser syslogParser; + protected Clock deviceClock; + + protected void setSyslogParser(SyslogParser syslogParser) { + this.syslogParser = syslogParser; + } + + protected void setMessageProcessor(Consumer function) { + this.messageProcessorOptional = Optional.of(function); + } + + protected abstract SyslogParser buildSyslogParser(Map config); + + @Override + public void configure(Map parserConfig) { + setReadCharset(parserConfig); + // we'll pull out the clock stuff ourselves + String timeZone = (String) parserConfig.get("deviceTimeZone"); + if (timeZone != null) { + deviceClock = Clock.system(ZoneId.of(timeZone)); + } else { + deviceClock = Clock.systemUTC(); + LOG.warn("[Metron] No device time zone provided; defaulting to UTC"); + } + syslogParser = buildSyslogParser(parserConfig); } - syslogParser = buildSyslogParser(parserConfig); - } - - @Override - public void init(){} - - @Override - public boolean validate(JSONObject message) { - if (!(message.containsKey("original_string"))) { - LOG.trace("[Metron] Message does not have original_string: {}", message); - return false; - } else if (!(message.containsKey("timestamp"))) { - LOG.trace("[Metron] Message does not have timestamp: {}", message); - return false; - } else { - LOG.trace("[Metron] Message conforms to schema: {}", message); - return true; + + @Override + public void init() { + } + + @Override + public boolean validate(JSONObject message) { + if (!(message.containsKey("original_string"))) { + LOG.trace("[Metron] Message does not have original_string: {}", message); + return false; + } else if (!(message.containsKey("timestamp"))) { + LOG.trace("[Metron] Message does not have timestamp: {}", message); + return false; + } else { + LOG.trace("[Metron] Message conforms to schema: {}", message); + return true; + } } - } - - @Override - @SuppressWarnings("unchecked") - public Optional> parseOptionalResult(byte[] rawMessage) { - try { - if (rawMessage == null || rawMessage.length == 0) { - return Optional.empty(); - } - - String originalString = new String(rawMessage, getReadCharset()); - final List returnList = new ArrayList<>(); - Map errorMap = new HashMap<>(); - try (Reader reader = new BufferedReader(new StringReader(originalString))) { - syslogParser.parseLines(reader, (m) -> { - JSONObject jsonObject = new JSONObject(m); - // be sure to put in the original string, and the timestamp. - // we wil just copy over the timestamp from the syslog - jsonObject.put("original_string", originalString); - try { - setTimestamp(jsonObject); - } catch (ParseException pe) { - errorMap.put(originalString,pe); - return; - } - messageProcessorOptional.ifPresent((c) -> c.accept(jsonObject)); - returnList.add(jsonObject); - },errorMap::put); - - return Optional.of(new DefaultMessageParserResult(returnList,errorMap)); - } - } catch (IOException e) { - String message = "Unable to read buffer " + new String(rawMessage, StandardCharsets.UTF_8) + ": " + e.getMessage(); - LOG.error(message, e); - return Optional.of(new DefaultMessageParserResult( new IllegalStateException(message, e))); + + @Override + @SuppressWarnings("unchecked") + public Optional> parseOptionalResult(byte[] rawMessage) { + try { + if (rawMessage == null || rawMessage.length == 0) { + return Optional.empty(); + } + + String originalString = new String(rawMessage, getReadCharset()); + final List returnList = new ArrayList<>(); + Map errorMap = new HashMap<>(); + try (Reader reader = new BufferedReader(new StringReader(originalString))) { + syslogParser.parseLines(reader, (m) -> { + JSONObject jsonObject = new JSONObject(m); + // be sure to put in the original string, and the timestamp. + // we wil just copy over the timestamp from the syslog + jsonObject.put("original_string", originalString); + try { + setTimestamp(jsonObject); + } catch (ParseException pe) { + errorMap.put(originalString, pe); + return; + } + messageProcessorOptional.ifPresent((c) -> c.accept(jsonObject)); + returnList.add(jsonObject); + }, errorMap::put); + + return Optional.of(new DefaultMessageParserResult(returnList, errorMap)); + } + } catch (IOException e) { + String message = + "Unable to read buffer " + new String(rawMessage, StandardCharsets.UTF_8) + ": " + e.getMessage(); + LOG.error(message, e); + return Optional.of(new DefaultMessageParserResult(new IllegalStateException(message, e))); + } } - } - - @SuppressWarnings("unchecked") - private void setTimestamp(JSONObject message) throws ParseException { - String timeStampString = (String) message.get(SyslogFieldKeys.HEADER_TIMESTAMP.getField()); - if (!StringUtils.isBlank(timeStampString) && !timeStampString.equals("-")) { - message.put("timestamp", SyslogUtils.parseTimestampToEpochMillis(timeStampString, deviceClock)); - } else { - message.put( - "timestamp", - LocalDateTime.now() - .toEpochSecond(ZoneOffset.UTC)); + + @SuppressWarnings("unchecked") + private void setTimestamp(JSONObject message) throws ParseException { + String timeStampString = (String) message.get(SyslogFieldKeys.HEADER_TIMESTAMP.getField()); + if (!StringUtils.isBlank(timeStampString) && !timeStampString.equals("-")) { + message.put("timestamp", SyslogUtils.parseTimestampToEpochMillis(timeStampString, deviceClock)); + } else { + message.put( + "timestamp", + LocalDateTime.now() + .toEpochSecond(ZoneOffset.UTC)); + } } - } - public void setReadCharset(Map config) { - if (config.containsKey(READ_CHARSET)) { - readCharset = Charset.forName((String) config.get(READ_CHARSET)); - } else { - readCharset = MessageParser.super.getReadCharset(); + public void setReadCharset(Map config) { + if (config.containsKey(READ_CHARSET)) { + readCharset = Charset.forName((String) config.get(READ_CHARSET)); + } else { + readCharset = MessageParser.super.getReadCharset(); + } } - } - @Override - public Charset getReadCharset() { - return null == this.readCharset ? MessageParser.super.getReadCharset() : this.readCharset; - } + @Override + public Charset getReadCharset() { + return null == this.readCharset ? MessageParser.super.getReadCharset() : this.readCharset; + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/Syslog3164Parser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/Syslog3164Parser.java index 632bcfd1..51fd480e 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/Syslog3164Parser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/Syslog3164Parser.java @@ -22,7 +22,6 @@ import com.github.palindromicity.syslog.SyslogParser; import com.github.palindromicity.syslog.SyslogParserBuilder; import com.github.palindromicity.syslog.SyslogSpecification; - import java.io.Serializable; import java.util.EnumSet; import java.util.Map; @@ -33,11 +32,11 @@ */ public class Syslog3164Parser extends BaseSyslogParser implements Serializable { - @Override - public SyslogParser buildSyslogParser(Map config) { - return new SyslogParserBuilder() - .forSpecification(SyslogSpecification.RFC_3164) - .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION)) - .build(); - } + @Override + public SyslogParser buildSyslogParser(Map config) { + return new SyslogParserBuilder() + .forSpecification(SyslogSpecification.RFC_3164) + .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION)) + .build(); + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java index cacb0e40..9eab2bef 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java @@ -23,7 +23,6 @@ import com.github.palindromicity.syslog.SyslogParser; import com.github.palindromicity.syslog.SyslogParserBuilder; import com.github.palindromicity.syslog.SyslogSpecification; - import java.io.Serializable; import java.util.EnumSet; import java.util.Map; @@ -33,19 +32,19 @@ * Parser for well structured RFC 5424 messages. */ public class Syslog5424Parser extends BaseSyslogParser implements Serializable { - public static final String NIL_POLICY_CONFIG = "nilPolicy"; + public static final String NIL_POLICY_CONFIG = "nilPolicy"; - @Override - public SyslogParser buildSyslogParser(Map config) { - // Default to OMIT policy for nil fields - // this means they will not be in the returned field set - String nilPolicyStr = (String) config.getOrDefault(NIL_POLICY_CONFIG, NilPolicy.OMIT.name()); - NilPolicy nilPolicy = NilPolicy.valueOf(nilPolicyStr); - return new SyslogParserBuilder() - .forSpecification(SyslogSpecification.RFC_5424) - .withNilPolicy(nilPolicy) - .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION)) - .build(); - } + @Override + public SyslogParser buildSyslogParser(Map config) { + // Default to OMIT policy for nil fields + // this means they will not be in the returned field set + String nilPolicyStr = (String) config.getOrDefault(NIL_POLICY_CONFIG, NilPolicy.OMIT.name()); + NilPolicy nilPolicy = NilPolicy.valueOf(nilPolicyStr); + return new SyslogParserBuilder() + .forSpecification(SyslogSpecification.RFC_5424) + .withNilPolicy(nilPolicy) + .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION)) + .build(); + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/DateUtils.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/DateUtils.java index 4317b627..0f9cc9a7 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/DateUtils.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/DateUtils.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,125 +22,122 @@ import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.*; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; import java.util.List; -import java.util.TimeZone; import java.util.regex.Pattern; - import org.apache.commons.lang.StringUtils; /** - * Various utilities for parsing and extracting dates - * + * Various utilities for parsing and extracting dates. */ public class DateUtils { - // Per IBM LEEF guide at https://www.ibm.com/support/knowledgecenter/SS42VS_DSM/c_LEEF_Format_Guide_intro.html - public static List DATE_FORMATS_LEEF = new ArrayList() { - { - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS Z")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS z")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss")); - } - }; + // Per IBM LEEF guide at https://www.ibm.com/support/knowledgecenter/SS42VS_DSM/c_LEEF_Format_Guide_intro.html + public static List DATE_FORMATS_LEEF = new ArrayList() { + { + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS Z")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS z")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss")); + } + }; - public static List DATE_FORMATS_CEF = new ArrayList() { - { - // as per CEF Spec - add(new SimpleDateFormat("MMM d HH:mm:ss.SSS Z")); - add(new SimpleDateFormat("MMM d HH:mm:ss.SSS z")); - add(new SimpleDateFormat("MMM d HH:mm:ss.SSS")); - add(new SimpleDateFormat("MMM d HH:mm:ss zzz")); - add(new SimpleDateFormat("MMM d HH:mm:ss")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS Z")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS z")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss Z")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss z")); - add(new SimpleDateFormat("MMM d yyyy HH:mm:ss")); - // found in the wild - add(new SimpleDateFormat("d MMMM yyyy HH:mm:ss")); - } - }; + public static List DATE_FORMATS_CEF = new ArrayList() { + { + // as per CEF Spec + add(new SimpleDateFormat("MMM d HH:mm:ss.SSS Z")); + add(new SimpleDateFormat("MMM d HH:mm:ss.SSS z")); + add(new SimpleDateFormat("MMM d HH:mm:ss.SSS")); + add(new SimpleDateFormat("MMM d HH:mm:ss zzz")); + add(new SimpleDateFormat("MMM d HH:mm:ss")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS Z")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS z")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss.SSS")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss Z")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss z")); + add(new SimpleDateFormat("MMM d yyyy HH:mm:ss")); + // found in the wild + add(new SimpleDateFormat("d MMMM yyyy HH:mm:ss")); + } + }; - public static List DATE_FORMATS_SYSLOG = new ArrayList() { - { - // As specified in https://tools.ietf.org/html/rfc5424 - add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ")); + public static List DATE_FORMATS_SYSLOG = new ArrayList() { + { + // As specified in https://tools.ietf.org/html/rfc5424 + add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ")); - // common format per rsyslog defaults e.g. Mar 21 14:05:02 - add(new SimpleDateFormat("MMM dd HH:mm:ss")); - add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss")); + // common format per rsyslog defaults e.g. Mar 21 14:05:02 + add(new SimpleDateFormat("MMM dd HH:mm:ss")); + add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss")); - // additional formats found in the wild - add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")); - add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")); - add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")); + // additional formats found in the wild + add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")); + add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")); + add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")); - } - }; + } + }; - Pattern NUMERIC = Pattern.compile("\\b\\d+\\b"); + @SuppressWarnings("checkstyle:MemberName") + Pattern NUMERIC = Pattern.compile("\\b\\d+\\b"); - /** - * Parse the data according to a sequence of possible parse patterns. - * - * If the given date is entirely numeric, it is assumed to be a unix - * timestamp. - * - * If the year is not specified in the date string, use the current year. - * Assume that any date more than 4 days in the future is in the past as per - * SyslogUtils - * - * @param candidate - * The possible date. - * @param validPatterns - * A list of SimpleDateFormat instances to try parsing with. - * @return A java.util.Date based on the parse result - * @throws ParseException - */ - public static long parseMultiformat(String candidate, List validPatterns) throws ParseException { - if (StringUtils.isNumeric(candidate)) { - return Long.valueOf(candidate); - } else { - for (SimpleDateFormat pattern : validPatterns) { - try { - DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder() - .appendPattern(pattern.toPattern()); - if (!pattern.toPattern().contains("y")) { - formatterBuilder - .parseDefaulting(ChronoField.YEAR, LocalDate.now().getYear()); - } - DateTimeFormatter formatter = formatterBuilder.toFormatter(); - ZonedDateTime parsedValue = parseDateTimeWithDefaultTimezone(candidate, formatter); - ZonedDateTime current = ZonedDateTime.now(parsedValue.getZone()); + /** + * Parse the data according to a sequence of possible parse patterns. + * + *

    + * If the given date is entirely numeric, it is assumed to be a unix timestamp. + * + *

    + * If the year is not specified in the date string, use the current year. + * Assume that any date more than 4 days in the future is in the past as per + * SyslogUtils + * + * @param candidate The possible date. + * @param validPatterns A list of SimpleDateFormat instances to try parsing with. + * @return A java.util.Date based on the parse result + */ + public static long parseMultiformat(String candidate, List validPatterns) throws ParseException { + if (StringUtils.isNumeric(candidate)) { + return Long.valueOf(candidate); + } else { + for (SimpleDateFormat pattern : validPatterns) { + try { + DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder() + .appendPattern(pattern.toPattern()); + if (!pattern.toPattern().contains("y")) { + formatterBuilder + .parseDefaulting(ChronoField.YEAR, LocalDate.now().getYear()); + } + DateTimeFormatter formatter = formatterBuilder.toFormatter(); + ZonedDateTime parsedValue = parseDateTimeWithDefaultTimezone(candidate, formatter); + ZonedDateTime current = ZonedDateTime.now(parsedValue.getZone()); - current = current.plusDays(4); - if (parsedValue.isAfter(current)) { - parsedValue = parsedValue.minusYears(1); - } - return parsedValue.toInstant().toEpochMilli(); - } catch (DateTimeParseException e) { - continue; - } - } - throw new ParseException("Failed to parse any of the given date formats", 0); - } - } + current = current.plusDays(4); + if (parsedValue.isAfter(current)) { + parsedValue = parsedValue.minusYears(1); + } + return parsedValue.toInstant().toEpochMilli(); + } catch (DateTimeParseException e) { + continue; + } + } + throw new ParseException("Failed to parse any of the given date formats", 0); + } + } - private static ZonedDateTime parseDateTimeWithDefaultTimezone(String candidate, DateTimeFormatter formatter) { - TemporalAccessor temporalAccessor = formatter.parseBest(candidate, ZonedDateTime::from, LocalDateTime::from); - return temporalAccessor instanceof ZonedDateTime - ? ((ZonedDateTime) temporalAccessor) - : ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault()); - } + private static ZonedDateTime parseDateTimeWithDefaultTimezone(String candidate, DateTimeFormatter formatter) { + TemporalAccessor temporalAccessor = formatter.parseBest(candidate, ZonedDateTime::from, LocalDateTime::from); + return temporalAccessor instanceof ZonedDateTime + ? ((ZonedDateTime) temporalAccessor) + : ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault()); + } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java index 224839b3..9b76c73f 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/GrokUtils.java @@ -6,38 +6,42 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.utils; + import java.io.Serializable; import java.util.regex.Pattern; public class GrokUtils implements Serializable { - private static final long serialVersionUID = 7465176887422419286L; - /** - * Extract Grok patter like %{FOO} to FOO, Also Grok pattern with semantic. - */ - public static final Pattern GROK_PATTERN = Pattern.compile( - "%\\{" + - "(?" + - "(?[A-z0-9]+)" + - "(?::(?[A-z0-9_:;\\/\\s\\.]+))?" + - ")" + - "(?:=(?" + - "(?:" + - "(?:[^{}]+|\\.+)+" + - ")+" + - ")" + - ")?" + - "\\}"); + private static final long serialVersionUID = 7465176887422419286L; + /** + * Extract Grok patter like %{FOO} to FOO, Also Grok pattern with semantic. + */ + public static final Pattern GROK_PATTERN = Pattern.compile( + "%\\{" + + "(?" + + "(?[A-z0-9]+)" + + "(?::(?[A-z0-9_:;\\/\\s\\.]+))?" + + ")" + + "(?:=(?" + + "(?:" + + "(?:[^{}]+|\\.+)+" + + ")+" + + ")" + + ")?" + + "\\}"); - } +} diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java index 09fe0889..4dc10c6e 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/ParserUtils.java @@ -6,78 +6,78 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.utils; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; - -import org.apache.commons.io.IOUtils; -import org.json.simple.JSONObject; - import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.Locale; import java.util.TimeZone; +import org.apache.commons.io.IOUtils; public class ParserUtils { - public static final String PREFIX = "stream2file"; - public static final String SUFFIX = ".tmp"; + public static final String PREFIX = "stream2file"; + public static final String SUFFIX = ".tmp"; - public static File stream2file(InputStream in) throws IOException { - final File tempFile = File.createTempFile(PREFIX, SUFFIX); - tempFile.deleteOnExit(); - try (FileOutputStream out = new FileOutputStream(tempFile)) { - IOUtils.copy(in, out); + public static File stream2file(InputStream in) throws IOException { + final File tempFile = File.createTempFile(PREFIX, SUFFIX); + tempFile.deleteOnExit(); + try (FileOutputStream out = new FileOutputStream(tempFile)) { + IOUtils.copy(in, out); + } + return tempFile; } - return tempFile; - } - /** - * Converts passed month (human short form), day, time and current year - * to milliseconds since epoch - * - * @param m Month in human short form (3 letters) (MMM) - * @param d Day (dd) - * @param ts Time (HH:mm:ss) - * @param adjust_timezone If True set GMT timezone for input time - * @return Number of milliseconds since epoch - * @exception ParseException If a date parsing error occured - */ - public static Long convertToEpoch(String m, String d, String ts, - boolean adjust_timezone) throws ParseException { - d = d.trim(); - if (d.length() <= 2) { - d = "0" + d; - } - Date date = new SimpleDateFormat("MMM", Locale.ENGLISH).parse(m); - Calendar cal = Calendar.getInstance(); - cal.setTime(date); - String month = String.valueOf(cal.get(Calendar.MONTH) + 1); - int year = Calendar.getInstance().get(Calendar.YEAR); - if (month.length() <= 2) { - month = "0" + month; - } - String coglomerated_ts = year + "-" + month + "-" + d + " " + ts; - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - if (adjust_timezone) { - sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + /** + * Converts passed month (human short form), day, time and current year + * to milliseconds since epoch. + * + * @param m Month in human short form (3 letters) (MMM) + * @param d Day (dd) + * @param ts Time (HH:mm:ss) + * @param adjustTimezone If True set GMT timezone for input time + * @return Number of milliseconds since epoch + * @throws ParseException If a date parsing error occured + */ + public static Long convertToEpoch(String m, String d, String ts, + boolean adjustTimezone) throws ParseException { + d = d.trim(); + if (d.length() <= 2) { + d = "0" + d; + } + Date date = new SimpleDateFormat("MMM", Locale.ENGLISH).parse(m); + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + String month = String.valueOf(cal.get(Calendar.MONTH) + 1); + int year = Calendar.getInstance().get(Calendar.YEAR); + if (month.length() <= 2) { + month = "0" + month; + } + String coglomeratedTs = year + "-" + month + "-" + d + " " + ts; + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + if (adjustTimezone) { + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + } + date = sdf.parse(coglomeratedTs); + long timeInMillisSinceEpoch = date.getTime(); + return timeInMillisSinceEpoch; } - date = sdf.parse(coglomerated_ts); - long timeInMillisSinceEpoch = date.getTime(); - return timeInMillisSinceEpoch; - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/SyslogUtils.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/SyslogUtils.java index 8b513009..a0b5efe6 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/SyslogUtils.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/utils/SyslogUtils.java @@ -7,17 +7,24 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.utils; -import org.apache.metron.parsers.ParseException; +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoField.HOUR_OF_DAY; +import static java.time.temporal.ChronoField.MINUTE_OF_HOUR; +import static java.time.temporal.ChronoField.MONTH_OF_YEAR; +import static java.time.temporal.ChronoField.SECOND_OF_MINUTE; import java.time.Clock; import java.time.ZoneId; @@ -25,8 +32,7 @@ import java.time.format.DateTimeFormatter; import java.time.temporal.TemporalAccessor; import java.util.regex.Pattern; - -import static java.time.temporal.ChronoField.*; +import org.apache.metron.parsers.ParseException; public class SyslogUtils { @@ -50,31 +56,34 @@ public static long parseTimestampToEpochMillis(String logTimestamp, Clock device int currentYear = currentDate.getYear(); ZonedDateTime inputDateWithCurrentYear = - ZonedDateTime.of(currentYear, inputMonth, inputDay, inputHour, inputMinute, inputSecond, 0, deviceTimeZone); + ZonedDateTime.of(currentYear, inputMonth, inputDay, inputHour, inputMinute, inputSecond, 0, + deviceTimeZone); // Since no year is provided, one must be derived. Assume that any date more than 4 days in the future is in the past. - if(inputDateWithCurrentYear.isAfter(currentDate.plusDays(4L))) { + if (inputDateWithCurrentYear.isAfter(currentDate.plusDays(4L))) { ZonedDateTime inputDateWithPreviousYear = - ZonedDateTime.of(currentYear-1, inputMonth, inputDay, inputHour, inputMinute, inputSecond, 0, deviceTimeZone); + ZonedDateTime.of(currentYear - 1, inputMonth, inputDay, inputHour, inputMinute, inputSecond, 0, + deviceTimeZone); return inputDateWithPreviousYear.toInstant().toEpochMilli(); - } - else + } else { return inputDateWithCurrentYear.toInstant().toEpochMilli(); - } - - // CISCO timestamp (standard syslog + year) - // MMM dd yyyy HH:mm:ss - // Oct 09 2015 13:42:11 - else if (Pattern.matches("[A-Z][a-z]{2}\\s\\d{2}\\s\\d{4}\\s\\d{2}:\\d{2}:\\d{2}", logTimestamp)) - return convertToEpochMillis(logTimestamp, DateTimeFormatter.ofPattern("MMM dd yyyy HH:mm:ss").withZone(deviceTimeZone)); + } - // RFC5424 (ISO timestamp) - // 2015-10-09T13:42:11.52Z or 2015-10-09T13:42:11.52-04:00 - else if (Pattern.matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2})", logTimestamp)) + // CISCO timestamp (standard syslog + year) + // MMM dd yyyy HH:mm:ss + // Oct 09 2015 13:42:11 + } else if (Pattern.matches("[A-Z][a-z]{2}\\s\\d{2}\\s\\d{4}\\s\\d{2}:\\d{2}:\\d{2}", logTimestamp)) { + return convertToEpochMillis(logTimestamp, + DateTimeFormatter.ofPattern("MMM dd yyyy HH:mm:ss").withZone(deviceTimeZone)); + + // RFC5424 (ISO timestamp) + // 2015-10-09T13:42:11.52Z or 2015-10-09T13:42:11.52-04:00 + } else if (Pattern.matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2})", + logTimestamp)) { return convertToEpochMillis(logTimestamp, DateTimeFormatter.ISO_OFFSET_DATE_TIME); - - else + } else { throw new ParseException(String.format("Unsupported date format: '%s'", logTimestamp)); + } } private static long convertToEpochMillis(String logTimestamp, DateTimeFormatter logTimeFormat) { @@ -85,46 +94,76 @@ private static long convertToEpochMillis(String logTimestamp, DateTimeFormatter public static String getSeverityFromPriority(int priority) { int severity = priority & 0x07; switch (severity) { - case 0: return "emerg"; - case 1: return "alert"; - case 2: return "crit"; - case 3: return "err"; - case 4: return "warn"; - case 5: return "notice"; - case 6: return "info"; - case 7: return "debug"; - default: return "unknown"; + case 0: + return "emerg"; + case 1: + return "alert"; + case 2: + return "crit"; + case 3: + return "err"; + case 4: + return "warn"; + case 5: + return "notice"; + case 6: + return "info"; + case 7: + return "debug"; + default: + return "unknown"; } } public static String getFacilityFromPriority(int priority) { int facility = priority >> 3; switch (facility) { - case 0: return "kern"; - case 1: return "user"; - case 2: return "mail"; - case 3: return "daemon"; - case 4: return "auth"; - case 5: return "syslog"; - case 6: return "lpr"; - case 7: return "news"; - case 8: return "uucp"; + case 0: + return "kern"; + case 1: + return "user"; + case 2: + return "mail"; + case 3: + return "daemon"; + case 4: + return "auth"; + case 5: + return "syslog"; + case 6: + return "lpr"; + case 7: + return "news"; + case 8: + return "uucp"; //case 9 - case 10: return "authpriv"; - case 11: return "ftp"; + case 10: + return "authpriv"; + case 11: + return "ftp"; //case 12 //case 13 //case 14 - case 15: return "cron"; - case 16: return "local0"; - case 17: return "local1"; - case 18: return "local2"; - case 19: return "local3"; - case 20: return "local4"; - case 21: return "local5"; - case 22: return "local6"; - case 23: return "local7"; - default: return "unknown"; + case 15: + return "cron"; + case 16: + return "local0"; + case 17: + return "local1"; + case 18: + return "local2"; + case 19: + return "local3"; + case 20: + return "local4"; + case 21: + return "local5"; + case 22: + return "local6"; + case 23: + return "local7"; + default: + return "unknown"; } } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java index 44a98aa8..b35230fb 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

    * http://www.apache.org/licenses/LICENSE-2.0 + * *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,114 +28,114 @@ public class GrokWebSphereParser extends GrokParser { - private static final long serialVersionUID = 4860439408055777358L; + private static final long serialVersionUID = 4860439408055777358L; - @Override - protected long formatTimestamp(Object value) { - long epochTimestamp = System.currentTimeMillis(); - if (value != null) { - try { - epochTimestamp = toEpoch(Calendar.getInstance().get(Calendar.YEAR) + " " + value); - } catch (ParseException e) { - //default to current time - } + @Override + protected long formatTimestamp(Object value) { + long epochTimestamp = System.currentTimeMillis(); + if (value != null) { + try { + epochTimestamp = toEpoch(Calendar.getInstance().get(Calendar.YEAR) + " " + value); + } catch (ParseException e) { + //default to current time + } + } + return epochTimestamp; } - return epochTimestamp; - } - @Override - protected void postParse(JSONObject message) { - removeEmptyFields(message); - message.remove("timestamp_string"); - if (message.containsKey("message")) { - String messageValue = (String) message.get("message"); - if (messageValue.contains("logged into")) { - parseLoginMessage(message); - } else if (messageValue.contains("logged out")) { - parseLogoutMessage(message); - } else if (messageValue.contains("rbm(")) { - parseRBMMessage(message); - } else { - parseOtherMessage(message); - } + @Override + protected void postParse(JSONObject message) { + removeEmptyFields(message); + message.remove("timestamp_string"); + if (message.containsKey("message")) { + String messageValue = (String) message.get("message"); + if (messageValue.contains("logged into")) { + parseLoginMessage(message); + } else if (messageValue.contains("logged out")) { + parseLogoutMessage(message); + } else if (messageValue.contains("rbm(")) { + parseRBMMessage(message); + } else { + parseOtherMessage(message); + } + } } - } - @SuppressWarnings("unchecked") - private void removeEmptyFields(JSONObject json) { - Iterator keyIter = json.keySet().iterator(); - while (keyIter.hasNext()) { - Object key = keyIter.next(); - Object value = json.get(key); - if (null == value || "".equals(value.toString())) { - keyIter.remove(); - } + @SuppressWarnings("unchecked") + private void removeEmptyFields(JSONObject json) { + Iterator keyIter = json.keySet().iterator(); + while (keyIter.hasNext()) { + Object key = keyIter.next(); + Object value = json.get(key); + if (null == value || "".equals(value.toString())) { + keyIter.remove(); + } + } } - } - //Extracts the appropriate fields from login messages - @SuppressWarnings("unchecked") - private void parseLoginMessage(JSONObject json) { - json.put("event_subtype", "login"); - String message = (String) json.get("message"); - if (message.contains(":")) { - String[] parts = message.split(":"); - String user = parts[0]; - String ip_src_addr = parts[1]; - if (user.contains("user(") && user.contains(")")) { - user = user.substring(user.indexOf("user(") + "user(".length()); - user = user.substring(0, user.indexOf(")")); - json.put("username", user); - } - if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) { - ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1); - ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]")); - json.put("ip_src_addr", ip_src_addr); - } - json.remove("message"); + //Extracts the appropriate fields from login messages + @SuppressWarnings("unchecked") + private void parseLoginMessage(JSONObject json) { + json.put("event_subtype", "login"); + String message = (String) json.get("message"); + if (message.contains(":")) { + String[] parts = message.split(":"); + String user = parts[0]; + String ipSrcAddr = parts[1]; + if (user.contains("user(") && user.contains(")")) { + user = user.substring(user.indexOf("user(") + "user(".length()); + user = user.substring(0, user.indexOf(")")); + json.put("username", user); + } + if (ipSrcAddr.contains("[") && ipSrcAddr.contains("]")) { + ipSrcAddr = ipSrcAddr.substring(ipSrcAddr.indexOf("[") + 1); + ipSrcAddr = ipSrcAddr.substring(0, ipSrcAddr.indexOf("]")); + json.put("ip_src_addr", ipSrcAddr); + } + json.remove("message"); + } } - } - //Extracts the appropriate fields from logout messages - @SuppressWarnings("unchecked") - private void parseLogoutMessage(JSONObject json) { - json.put("event_subtype", "logout"); - String message = (String) json.get("message"); - if (message.matches(".*'.*'.*'.*'.*")) { - String parts[] = message.split("'"); - String ip_src_addr = parts[0]; - if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) { - ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1); - ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]")); - json.put("ip_src_addr", ip_src_addr); - } - json.put("username", parts[1]); - json.put("security_domain", parts[3]); - json.remove("message"); + //Extracts the appropriate fields from logout messages + @SuppressWarnings("unchecked") + private void parseLogoutMessage(JSONObject json) { + json.put("event_subtype", "logout"); + String message = (String) json.get("message"); + if (message.matches(".*'.*'.*'.*'.*")) { + String[] parts = message.split("'"); + String ipSrcAddr = parts[0]; + if (ipSrcAddr.contains("[") && ipSrcAddr.contains("]")) { + ipSrcAddr = ipSrcAddr.substring(ipSrcAddr.indexOf("[") + 1); + ipSrcAddr = ipSrcAddr.substring(0, ipSrcAddr.indexOf("]")); + json.put("ip_src_addr", ipSrcAddr); + } + json.put("username", parts[1]); + json.put("security_domain", parts[3]); + json.remove("message"); + } } - } - //Extracts the appropriate fields from RBM messages - @SuppressWarnings("unchecked") - private void parseRBMMessage(JSONObject json) { - String message = (String) json.get("message"); - if (message.contains("(")) { - json.put("process", message.substring(0, message.indexOf("("))); - if (message.contains(":")) { - json.put("message", message.substring(message.indexOf(":") + 2)); - } + //Extracts the appropriate fields from RBM messages + @SuppressWarnings("unchecked") + private void parseRBMMessage(JSONObject json) { + String message = (String) json.get("message"); + if (message.contains("(")) { + json.put("process", message.substring(0, message.indexOf("("))); + if (message.contains(":")) { + json.put("message", message.substring(message.indexOf(":") + 2)); + } + } } - } - //Extracts the appropriate fields from other messages - @SuppressWarnings("unchecked") - private void parseOtherMessage(JSONObject json) { - String message = (String) json.get("message"); - if (message.contains("(")) { - json.put("process", message.substring(0, message.indexOf("("))); - if (message.contains(":")) { - json.put("message", message.substring(message.indexOf(":") + 2)); - } + //Extracts the appropriate fields from other messages + @SuppressWarnings("unchecked") + private void parseOtherMessage(JSONObject json) { + String message = (String) json.get("message"); + if (message.contains("(")) { + json.put("process", message.substring(0, message.indexOf("("))); + if (message.contains(":")) { + json.put("message", message.substring(message.indexOf(":") + 2)); + } + } } - } } diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java index 971e80fb..a3e154e5 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

    * http://www.apache.org/licenses/LICENSE-2.0 + * *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,19 +19,25 @@ */ package org.apache.metron.parsers; -import org.apache.commons.io.IOUtils; -import org.apache.metron.parsers.interfaces.MessageParserResult; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.commons.io.IOUtils; +import org.apache.metron.parsers.interfaces.MessageParserResult; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.jupiter.api.Test; public class MultiLineGrokParserTest { diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java index 8922b78f..ef6a3769 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

    * http://www.apache.org/licenses/LICENSE-2.0 + * *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,19 +19,25 @@ */ package org.apache.metron.parsers; -import org.apache.commons.io.IOUtils; -import org.apache.metron.parsers.interfaces.MessageParserResult; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.commons.io.IOUtils; +import org.apache.metron.parsers.interfaces.MessageParserResult; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.jupiter.api.Test; public class MultiLineWithErrorsGrokParserTest { diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java index 79842542..9cb82f16 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java @@ -17,6 +17,15 @@ */ package org.apache.metron.parsers.bro; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.lang3.tuple.Pair; import org.apache.log4j.Level; @@ -30,14 +39,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.*; - public class BasicBroParserTest { private BasicBroParser broParser = new BasicBroParser(); private JSONParser jsonParser = new JSONParser(); @@ -183,8 +184,8 @@ public void testHttpBroMessage() throws ParseException { @Test public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException { { - String rawMessage = "{\"http\": {\"ts\":1467657279,\"uid\":\"CMYLzP3PKiwZAgBa51\",\"id.orig_h\":\"192.168.138.158\",\"id.orig_p\":49206,\"id.resp_h\":\"95.163.121.204\"," + - "\"id.resp_p\":80,\"trans_depth\":2,\"method\":\"GET\",\"host\":\"7oqnsnzwwnm6zb7y.gigapaysun.com\",\"uri\":\"/img/flags/it.png\",\"referrer\":\"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg\",\"user_agent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)\",\"request_body_len\":0,\"response_body_len\":552,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F3m7vB2RjUe4n01aqj\"],\"resp_mime_types\":[\"image/png\"]}}"; + String rawMessage = "{\"http\": {\"ts\":1467657279,\"uid\":\"CMYLzP3PKiwZAgBa51\",\"id.orig_h\":\"192.168.138.158\",\"id.orig_p\":49206,\"id.resp_h\":\"95.163.121.204\"," + + "\"id.resp_p\":80,\"trans_depth\":2,\"method\":\"GET\",\"host\":\"7oqnsnzwwnm6zb7y.gigapaysun.com\",\"uri\":\"/img/flags/it.png\",\"referrer\":\"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg\",\"user_agent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)\",\"request_body_len\":0,\"response_body_len\":552,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F3m7vB2RjUe4n01aqj\"],\"resp_mime_types\":[\"image/png\"]}}"; Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); @@ -196,8 +197,8 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException assertEquals(broJson.get("bro_timestamp").toString(), expectedBroTimestamp); } { - String rawMessage = "{\"http\": {\"ts\":1467657279.0,\"uid\":\"CMYLzP3PKiwZAgBa51\",\"id.orig_h\":\"192.168.138.158\",\"id.orig_p\":49206,\"id.resp_h\":\"95.163.121.204\"," + - "\"id.resp_p\":80,\"trans_depth\":2,\"method\":\"GET\",\"host\":\"7oqnsnzwwnm6zb7y.gigapaysun.com\",\"uri\":\"/img/flags/it.png\",\"referrer\":\"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg\",\"user_agent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)\",\"request_body_len\":0,\"response_body_len\":552,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F3m7vB2RjUe4n01aqj\"],\"resp_mime_types\":[\"image/png\"]}}"; + String rawMessage = "{\"http\": {\"ts\":1467657279.0,\"uid\":\"CMYLzP3PKiwZAgBa51\",\"id.orig_h\":\"192.168.138.158\",\"id.orig_p\":49206,\"id.resp_h\":\"95.163.121.204\"," + + "\"id.resp_p\":80,\"trans_depth\":2,\"method\":\"GET\",\"host\":\"7oqnsnzwwnm6zb7y.gigapaysun.com\",\"uri\":\"/img/flags/it.png\",\"referrer\":\"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg\",\"user_agent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)\",\"request_body_len\":0,\"response_body_len\":552,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F3m7vB2RjUe4n01aqj\"],\"resp_mime_types\":[\"image/png\"]}}"; Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); @@ -209,8 +210,8 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException assertEquals(broJson.get("bro_timestamp").toString(), expectedBroTimestamp); } { - String rawMessage = "{\"http\": {\"ts\":1467657279.1,\"uid\":\"CMYLzP3PKiwZAgBa51\",\"id.orig_h\":\"192.168.138.158\",\"id.orig_p\":49206,\"id.resp_h\":\"95.163.121.204\"," + - "\"id.resp_p\":80,\"trans_depth\":2,\"method\":\"GET\",\"host\":\"7oqnsnzwwnm6zb7y.gigapaysun.com\",\"uri\":\"/img/flags/it.png\",\"referrer\":\"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg\",\"user_agent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)\",\"request_body_len\":0,\"response_body_len\":552,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F3m7vB2RjUe4n01aqj\"],\"resp_mime_types\":[\"image/png\"]}}"; + String rawMessage = "{\"http\": {\"ts\":1467657279.1,\"uid\":\"CMYLzP3PKiwZAgBa51\",\"id.orig_h\":\"192.168.138.158\",\"id.orig_p\":49206,\"id.resp_h\":\"95.163.121.204\"," + + "\"id.resp_p\":80,\"trans_depth\":2,\"method\":\"GET\",\"host\":\"7oqnsnzwwnm6zb7y.gigapaysun.com\",\"uri\":\"/img/flags/it.png\",\"referrer\":\"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg\",\"user_agent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)\",\"request_body_len\":0,\"response_body_len\":552,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F3m7vB2RjUe4n01aqj\"],\"resp_mime_types\":[\"image/png\"]}}"; Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); @@ -222,8 +223,8 @@ public void testHttpBroMessageWithZeroDecimalTruncation() throws ParseException assertEquals(broJson.get("bro_timestamp").toString(), expectedBroTimestamp); } { - String rawMessage = "{\"http\": {\"ts\":1467657279.11,\"uid\":\"CMYLzP3PKiwZAgBa51\",\"id.orig_h\":\"192.168.138.158\",\"id.orig_p\":49206,\"id.resp_h\":\"95.163.121.204\"," + - "\"id.resp_p\":80,\"trans_depth\":2,\"method\":\"GET\",\"host\":\"7oqnsnzwwnm6zb7y.gigapaysun.com\",\"uri\":\"/img/flags/it.png\",\"referrer\":\"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg\",\"user_agent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)\",\"request_body_len\":0,\"response_body_len\":552,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F3m7vB2RjUe4n01aqj\"],\"resp_mime_types\":[\"image/png\"]}}"; + String rawMessage = "{\"http\": {\"ts\":1467657279.11,\"uid\":\"CMYLzP3PKiwZAgBa51\",\"id.orig_h\":\"192.168.138.158\",\"id.orig_p\":49206,\"id.resp_h\":\"95.163.121.204\"," + + "\"id.resp_p\":80,\"trans_depth\":2,\"method\":\"GET\",\"host\":\"7oqnsnzwwnm6zb7y.gigapaysun.com\",\"uri\":\"/img/flags/it.png\",\"referrer\":\"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg\",\"user_agent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)\",\"request_body_len\":0,\"response_body_len\":552,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F3m7vB2RjUe4n01aqj\"],\"resp_mime_types\":[\"image/png\"]}}"; Map rawMessageMap = (Map) jsonParser.parse(rawMessage); JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next()); diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java index db1559e4..42515ec6 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java @@ -18,6 +18,12 @@ package org.apache.metron.parsers.cef; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -26,6 +32,16 @@ import com.github.fge.jsonschema.main.JsonSchemaFactory; import com.github.fge.jsonschema.main.JsonValidator; import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.metron.common.Constants.Fields; import org.apache.metron.parsers.interfaces.MessageParser; import org.json.simple.JSONObject; @@ -35,20 +51,6 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; -import java.io.IOException; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.text.SimpleDateFormat; -import java.time.LocalDate; -import java.time.ZoneId; -import java.time.temporal.TemporalAccessor; -import java.util.*; - -import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.when; - public class CEFParserTest { private CEFParser parser; @@ -251,20 +253,20 @@ private void runTest(String name, List lines, String schema, String targ * -standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/ * processors/standard/TestParseCEF.java) */ - private final static String sample = "CEF:0|TestVendor|TestProduct|TestVersion|TestEventClassID|TestName|Low|" + - // TimeStamp, String and Long - "rt=Feb 09 2015 00:27:43 UTC cn3Label=Test Long cn3=9223372036854775807 " + - // FloatPoint and MacAddress - "cfp1=1.234 cfp1Label=Test FP Number smac=00:00:0c:07:ac:00 " + - // IPv6 and String - "c6a3=2001:cdba::3257:9652 c6a3Label=Test IPv6 cs1Label=Test String cs1=test test test chocolate " + - // IPv4 - "destinationTranslatedAddress=123.123.123.123 " + - // Date without TZ - "deviceCustomDate1=Feb 06 2015 13:27:43 " + - // Integer and IP Address (from v4) - "dpt=1234 agt=123.123.0.124 dlat=40.366633 " + - // A JSON object inside one of CEF's custom Strings + private static final String sample = "CEF:0|TestVendor|TestProduct|TestVersion|TestEventClassID|TestName|Low|" + + // TimeStamp, String and Long + "rt=Feb 09 2015 00:27:43 UTC cn3Label=Test Long cn3=9223372036854775807 " + + // FloatPoint and MacAddress + "cfp1=1.234 cfp1Label=Test FP Number smac=00:00:0c:07:ac:00 " + + // IPv6 and String + "c6a3=2001:cdba::3257:9652 c6a3Label=Test IPv6 cs1Label=Test String cs1=test test test chocolate " + + // IPv4 + "destinationTranslatedAddress=123.123.123.123 " + + // Date without TZ + "deviceCustomDate1=Feb 06 2015 13:27:43 " + + // Integer and IP Address (from v4) + "dpt=1234 agt=123.123.0.124 dlat=40.366633 " + + // A JSON object inside one of CEF's custom Strings "cs2Label=JSON payload " + "cs2={\"test_test_test\": \"chocolate!\", \"what?!?\": \"Simple! test test test chocolate!\"}"; diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java index 573acaa3..a5751e39 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java @@ -17,13 +17,11 @@ */ package org.apache.metron.parsers.fireeye; -import org.apache.metron.parsers.AbstractParserConfigTest; -import org.apache.metron.parsers.interfaces.MessageParser; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.nio.charset.StandardCharsets; import java.time.Year; @@ -32,10 +30,13 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.*; +import org.apache.metron.parsers.AbstractParserConfigTest; +import org.apache.metron.parsers.interfaces.MessageParser; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class BasicFireEyeParserTest extends AbstractParserConfigTest { @@ -68,7 +69,7 @@ public void testParse() throws ParseException { } } - private final static String fireeyeMessage = "<164>Mar 19 05:24:39 10.220.15.15 fenotify-851983.alert: CEF:0|FireEye|CMS|7.2.1.244420|DM|domain-match|1|rt=Feb 09 2015 12:28:26 UTC dvc=10.201.78.57 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 shost=dev001srv02.example.com proto=udp cs5Label=cncHost cs5=mfdclk001.org dvchost=DEVFEYE1 spt=54527 dvc=10.100.25.16 smac=00:00:0c:07:ac:00 cn1Label=vlan cn1=0 externalId=851983 cs4Label=link cs4=https://DEVCMS01.example.com/event_stream/events_for_bot?ev_id\\=851983 dmac=00:1d:a2:af:32:a1 cs1Label=sname cs1=Trojan.Generic.DNS"; + private static final String fireeyeMessage = "<164>Mar 19 05:24:39 10.220.15.15 fenotify-851983.alert: CEF:0|FireEye|CMS|7.2.1.244420|DM|domain-match|1|rt=Feb 09 2015 12:28:26 UTC dvc=10.201.78.57 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 shost=dev001srv02.example.com proto=udp cs5Label=cncHost cs5=mfdclk001.org dvchost=DEVFEYE1 spt=54527 dvc=10.100.25.16 smac=00:00:0c:07:ac:00 cn1Label=vlan cn1=0 externalId=851983 cs4Label=link cs4=https://DEVCMS01.example.com/event_stream/events_for_bot?ev_id\\=851983 dmac=00:1d:a2:af:32:a1 cs1Label=sname cs1=Trojan.Generic.DNS"; @SuppressWarnings("rawtypes") @Test diff --git a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java index 0731b797..a83387b5 100644 --- a/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java +++ b/flink-cyber/flink-parse/flink-parse-metron/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java @@ -4,8 +4,10 @@ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at + * *

    * http://www.apache.org/licenses/LICENSE-2.0 + * *

    * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express @@ -14,6 +16,16 @@ */ package org.apache.metron.parsers.regex; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.parsers.interfaces.MessageParser; import org.json.simple.JSONObject; @@ -22,15 +34,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.*; - public class RegularExpressionsParserTest { private RegularExpressionsParser regularExpressionsParser; diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcConnectionOptions.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcConnectionOptions.java index f23b2330..2bf38c7b 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcConnectionOptions.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcConnectionOptions.java @@ -12,11 +12,10 @@ package com.cloudera.cyber.jdbc.connector.jdbc; -import org.apache.flink.util.Preconditions; - -import javax.annotation.Nullable; import java.io.Serializable; import java.util.Optional; +import javax.annotation.Nullable; +import org.apache.flink.util.Preconditions; public class JdbcConnectionOptions implements Serializable { @@ -24,17 +23,20 @@ public class JdbcConnectionOptions implements Serializable { private static final long serialVersionUID = 1L; protected final String url; - @Nullable protected final String driverName; + @Nullable + protected final String driverName; protected final int connectionCheckTimeoutSeconds; - @Nullable protected final String username; - @Nullable protected final String password; + @Nullable + protected final String username; + @Nullable + protected final String password; protected JdbcConnectionOptions( - String url, - @Nullable String driverName, - @Nullable String username, - @Nullable String password, - int connectionCheckTimeoutSeconds) { + String url, + @Nullable String driverName, + @Nullable String username, + @Nullable String password, + int connectionCheckTimeoutSeconds) { Preconditions.checkArgument(connectionCheckTimeoutSeconds > 0); this.url = Preconditions.checkNotNull(url, "jdbc url is empty"); this.driverName = driverName; @@ -92,14 +94,14 @@ public JdbcConnectionOptionsBuilder withPassword(String password) { } public JdbcConnectionOptionsBuilder withConnectionCheckTimeoutSeconds( - int connectionCheckTimeoutSeconds) { + int connectionCheckTimeoutSeconds) { this.connectionCheckTimeoutSeconds = connectionCheckTimeoutSeconds; return this; } public JdbcConnectionOptions build() { return new JdbcConnectionOptions( - url, driverName, username, password, connectionCheckTimeoutSeconds); + url, driverName, username, password, connectionCheckTimeoutSeconds); } } } diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcExecutionOptions.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcExecutionOptions.java index ff20fceb..bf063c96 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcExecutionOptions.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcExecutionOptions.java @@ -12,10 +12,9 @@ package com.cloudera.cyber.jdbc.connector.jdbc; -import org.apache.flink.util.Preconditions; - import java.io.Serializable; import java.util.Objects; +import org.apache.flink.util.Preconditions; public class JdbcExecutionOptions implements Serializable { public static final int DEFAULT_MAX_RETRY_TIMES = 3; @@ -55,8 +54,8 @@ public boolean equals(Object o) { } JdbcExecutionOptions that = (JdbcExecutionOptions) o; return batchIntervalMs == that.batchIntervalMs - && batchSize == that.batchSize - && maxRetries == that.maxRetries; + && batchSize == that.batchSize + && maxRetries == that.maxRetries; } @Override diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcSink.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcSink.java index 3aeb4882..df3db912 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcSink.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/JdbcSink.java @@ -17,34 +17,33 @@ import com.cloudera.cyber.jdbc.connector.jdbc.internal.JdbcStatementBuilder; import com.cloudera.cyber.jdbc.connector.jdbc.internal.connection.PhoenixJdbcConnectionProvider; import com.cloudera.cyber.jdbc.connector.jdbc.internal.executor.JdbcBatchStatementExecutor; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; - import java.util.function.Function; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; public class JdbcSink { private JdbcSink() { } public static SinkFunction sink( - String sql, - JdbcStatementBuilder statementBuilder, - JdbcConnectionOptions connectionOptions) { + String sql, + JdbcStatementBuilder statementBuilder, + JdbcConnectionOptions connectionOptions) { return sink(sql, statementBuilder, JdbcExecutionOptions.defaults(), connectionOptions); } public static SinkFunction sink( - String sql, - JdbcStatementBuilder statementBuilder, - JdbcExecutionOptions executionOptions, - JdbcConnectionOptions connectionOptions) { + String sql, + JdbcStatementBuilder statementBuilder, + JdbcExecutionOptions executionOptions, + JdbcConnectionOptions connectionOptions) { return new GenericJdbcSinkFunction<>( - new JdbcOutputFormat<>( - new PhoenixJdbcConnectionProvider(connectionOptions), - executionOptions, - context -> - JdbcBatchStatementExecutor.simple( - sql, statementBuilder, Function.identity()), - JdbcOutputFormat.RecordExtractor.identity())); + new JdbcOutputFormat<>( + new PhoenixJdbcConnectionProvider(connectionOptions), + executionOptions, + context -> + JdbcBatchStatementExecutor.simple( + sql, statementBuilder, Function.identity()), + JdbcOutputFormat.RecordExtractor.identity())); } } diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/GenericJdbcSinkFunction.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/GenericJdbcSinkFunction.java index 37cbb7e2..80297517 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/GenericJdbcSinkFunction.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/GenericJdbcSinkFunction.java @@ -18,6 +18,8 @@ package com.cloudera.cyber.jdbc.connector.jdbc.internal; +import java.io.IOException; +import javax.annotation.Nonnull; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -29,11 +31,8 @@ import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.util.Preconditions; -import javax.annotation.Nonnull; -import java.io.IOException; - public class GenericJdbcSinkFunction extends RichSinkFunction - implements CheckpointedFunction, InputTypeConfigurable { + implements CheckpointedFunction, InputTypeConfigurable { private final JdbcOutputFormat outputFormat; public GenericJdbcSinkFunction(@Nonnull JdbcOutputFormat outputFormat) { diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/JdbcOutputFormat.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/JdbcOutputFormat.java index 2617b8a1..3084e71e 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/JdbcOutputFormat.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/JdbcOutputFormat.java @@ -12,9 +12,23 @@ package com.cloudera.cyber.jdbc.connector.jdbc.internal; +import static org.apache.flink.util.Preconditions.checkNotNull; + import com.cloudera.cyber.jdbc.connector.jdbc.JdbcExecutionOptions; import com.cloudera.cyber.jdbc.connector.jdbc.internal.connection.JdbcConnectionProvider; import com.cloudera.cyber.jdbc.connector.jdbc.internal.executor.JdbcBatchStatementExecutor; +import java.io.Flushable; +import java.io.IOException; +import java.io.Serializable; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.RuntimeContext; @@ -27,23 +41,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.io.Flushable; -import java.io.IOException; -import java.io.Serializable; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; - -import static org.apache.flink.util.Preconditions.checkNotNull; - public class JdbcOutputFormat> - extends RichOutputFormat implements Flushable, InputTypeConfigurable { + extends RichOutputFormat implements Flushable, InputTypeConfigurable { protected final JdbcConnectionProvider connectionProvider; @Nullable @@ -64,7 +63,7 @@ static RecordExtractor identity() { } public interface StatementExecutorFactory> - extends Function, Serializable { + extends Function, Serializable { } private static final long serialVersionUID = 1L; @@ -84,10 +83,10 @@ public interface StatementExecutorFactory statementExecutorFactory, - @Nonnull RecordExtractor recordExtractor) { + @Nonnull JdbcConnectionProvider connectionProvider, + @Nonnull JdbcExecutionOptions executionOptions, + @Nonnull StatementExecutorFactory statementExecutorFactory, + @Nonnull RecordExtractor recordExtractor) { this.connectionProvider = checkNotNull(connectionProvider); this.executionOptions = checkNotNull(executionOptions); this.statementExecutorFactory = checkNotNull(statementExecutorFactory); @@ -103,29 +102,29 @@ public void open(int taskNumber, int numTasks) throws IOException { jdbcStatementExecutor = createAndOpenStatementExecutor(statementExecutorFactory); if (executionOptions.getBatchIntervalMs() != 0 && executionOptions.getBatchSize() != 1) { this.scheduler = - Executors.newScheduledThreadPool( - 1, new ExecutorThreadFactory("jdbc-upsert-output-format")); + Executors.newScheduledThreadPool( + 1, new ExecutorThreadFactory("jdbc-upsert-output-format")); this.scheduledFuture = - this.scheduler.scheduleWithFixedDelay( - () -> { - synchronized (JdbcOutputFormat.this) { - if (!closed) { - try { - flush(); - } catch (Exception e) { - flushException = e; - } + this.scheduler.scheduleWithFixedDelay( + () -> { + synchronized (JdbcOutputFormat.this) { + if (!closed) { + try { + flush(); + } catch (Exception e) { + flushException = e; } } - }, - executionOptions.getBatchIntervalMs(), - executionOptions.getBatchIntervalMs(), - TimeUnit.MILLISECONDS); + } + }, + executionOptions.getBatchIntervalMs(), + executionOptions.getBatchIntervalMs(), + TimeUnit.MILLISECONDS); } } private JdbcExec createAndOpenStatementExecutor( - StatementExecutorFactory statementExecutorFactory) throws IOException { + StatementExecutorFactory statementExecutorFactory) throws IOException { JdbcExec exec = statementExecutorFactory.apply(getRuntimeContext()); try { exec.prepareStatements(connectionProvider.getConnection()); @@ -150,8 +149,7 @@ public final synchronized void writeRecord(In rec) throws IOException { JdbcIn extractedRecord = jdbcRecordExtractor.apply(recordCopy); jdbcStatementExecutor.addToBatch(extractedRecord); batchCount++; - if (executionOptions.getBatchSize() > 0 - && batchCount >= executionOptions.getBatchSize()) { + if (executionOptions.getBatchSize() > 0 && batchCount >= executionOptions.getBatchSize()) { flush(); } } catch (Exception e) { @@ -186,8 +184,8 @@ public synchronized void flush() throws IOException { } } catch (Exception exception) { LOG.error( - "JDBC connection is not valid, and reestablish connection failed.", - exception); + "JDBC connection is not valid, and reestablish connection failed.", + exception); throw new IOException("Reestablish JDBC connection failed", exception); } try { @@ -195,7 +193,7 @@ public synchronized void flush() throws IOException { } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new IOException( - "unable to flush; interrupted while doing another attempt", e); + "unable to flush; interrupted while doing another attempt", e); } } } diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/JdbcStatementBuilder.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/JdbcStatementBuilder.java index 1183bde9..6ff6b681 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/JdbcStatementBuilder.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/JdbcStatementBuilder.java @@ -12,11 +12,11 @@ package com.cloudera.cyber.jdbc.connector.jdbc.internal; -import org.apache.flink.util.function.BiConsumerWithException; - import java.io.Serializable; import java.sql.PreparedStatement; import java.sql.SQLException; +import org.apache.flink.util.function.BiConsumerWithException; public interface JdbcStatementBuilder - extends BiConsumerWithException, Serializable {} + extends BiConsumerWithException, Serializable { +} diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/connection/JdbcConnectionProvider.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/connection/JdbcConnectionProvider.java index 58172f36..00747289 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/connection/JdbcConnectionProvider.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/connection/JdbcConnectionProvider.java @@ -17,9 +17,9 @@ package com.cloudera.cyber.jdbc.connector.jdbc.internal.connection; -import javax.annotation.Nullable; import java.sql.Connection; import java.sql.SQLException; +import javax.annotation.Nullable; public interface JdbcConnectionProvider { diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/connection/PhoenixJdbcConnectionProvider.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/connection/PhoenixJdbcConnectionProvider.java index d573c6f3..4c92b4a7 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/connection/PhoenixJdbcConnectionProvider.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/connection/PhoenixJdbcConnectionProvider.java @@ -18,11 +18,6 @@ package com.cloudera.cyber.jdbc.connector.jdbc.internal.connection; import com.cloudera.cyber.jdbc.connector.jdbc.JdbcConnectionOptions; -import org.apache.flink.util.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.concurrent.NotThreadSafe; import java.io.Serializable; import java.sql.Connection; import java.sql.Driver; @@ -30,6 +25,10 @@ import java.sql.SQLException; import java.util.Enumeration; import java.util.Properties; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.flink.util.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @NotThreadSafe public class PhoenixJdbcConnectionProvider implements JdbcConnectionProvider, Serializable { @@ -74,7 +73,7 @@ public boolean isConnectionValid() { } private static Driver loadDriver(String driverName) - throws SQLException, ClassNotFoundException { + throws SQLException, ClassNotFoundException { Preconditions.checkNotNull(driverName); Enumeration drivers = DriverManager.getDrivers(); while (drivers.hasMoreElements()) { @@ -87,7 +86,7 @@ private static Driver loadDriver(String driverName) // * Class loader hell of DriverManager(see JDK-8146872). // * driver is not installed as a service provider. Class clazz = - Class.forName(driverName, true, Thread.currentThread().getContextClassLoader()); + Class.forName(driverName, true, Thread.currentThread().getContextClassLoader()); try { return (Driver) clazz.newInstance(); } catch (Exception ex) { diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/executor/JdbcBatchStatementExecutor.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/executor/JdbcBatchStatementExecutor.java index c8071abc..ef5c67c5 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/executor/JdbcBatchStatementExecutor.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/executor/JdbcBatchStatementExecutor.java @@ -14,7 +14,6 @@ import com.cloudera.cyber.jdbc.connector.jdbc.internal.JdbcStatementBuilder; - import java.sql.Connection; import java.sql.SQLException; import java.util.function.Function; @@ -31,7 +30,7 @@ public interface JdbcBatchStatementExecutor { void closeStatements() throws SQLException; static JdbcBatchStatementExecutor simple( - String sql, JdbcStatementBuilder paramSetter, Function valueTransformer) { + String sql, JdbcStatementBuilder paramSetter, Function valueTransformer) { return new SimpleBatchStatementExecutor<>(sql, paramSetter, valueTransformer); } } diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/executor/SimpleBatchStatementExecutor.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/executor/SimpleBatchStatementExecutor.java index cfaa1905..b2af4de0 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/executor/SimpleBatchStatementExecutor.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/executor/SimpleBatchStatementExecutor.java @@ -19,15 +19,14 @@ package com.cloudera.cyber.jdbc.connector.jdbc.internal.executor; import com.cloudera.cyber.jdbc.connector.jdbc.internal.JdbcStatementBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class SimpleBatchStatementExecutor implements JdbcBatchStatementExecutor { @@ -42,7 +41,7 @@ class SimpleBatchStatementExecutor implements JdbcBatchStatementExecutor statementBuilder, Function valueTransformer) { + String sql, JdbcStatementBuilder statementBuilder, Function valueTransformer) { this.sql = sql; this.parameterSetter = statementBuilder; this.valueTransformer = valueTransformer; diff --git a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/options/JdbcConnectorOptions.java b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/options/JdbcConnectorOptions.java index d606b8f9..12f3e8c1 100644 --- a/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/options/JdbcConnectorOptions.java +++ b/flink-cyber/flink-phoenix-jdbc/src/main/java/com/cloudera/cyber/jdbc/connector/jdbc/internal/options/JdbcConnectorOptions.java @@ -18,12 +18,11 @@ package com.cloudera.cyber.jdbc.connector.jdbc.internal.options; -import com.cloudera.cyber.jdbc.connector.jdbc.JdbcConnectionOptions; +import static org.apache.flink.util.Preconditions.checkNotNull; -import javax.annotation.Nullable; +import com.cloudera.cyber.jdbc.connector.jdbc.JdbcConnectionOptions; import java.util.Objects; - -import static org.apache.flink.util.Preconditions.checkNotNull; +import javax.annotation.Nullable; public class JdbcConnectorOptions extends JdbcConnectionOptions { @@ -34,13 +33,13 @@ public class JdbcConnectorOptions extends JdbcConnectionOptions { private final @Nullable Integer parallelism; private JdbcConnectorOptions( - String dbURL, - String tableName, - @Nullable String driverName, - @Nullable String username, - @Nullable String password, - @Nullable Integer parallelism, - int connectionCheckTimeoutSeconds) { + String dbURL, + String tableName, + @Nullable String driverName, + @Nullable String username, + @Nullable String password, + @Nullable Integer parallelism, + int connectionCheckTimeoutSeconds) { super(dbURL, driverName, username, password, connectionCheckTimeoutSeconds); this.tableName = tableName; this.parallelism = parallelism; @@ -64,13 +63,13 @@ public boolean equals(Object o) { if (o instanceof JdbcConnectorOptions) { JdbcConnectorOptions options = (JdbcConnectorOptions) o; return Objects.equals(url, options.url) - && Objects.equals(tableName, options.tableName) - && Objects.equals(driverName, options.driverName) - && Objects.equals(username, options.username) - && Objects.equals(password, options.password) - && Objects.equals(parallelism, options.parallelism) - && Objects.equals( - connectionCheckTimeoutSeconds, options.connectionCheckTimeoutSeconds); + && Objects.equals(tableName, options.tableName) + && Objects.equals(driverName, options.driverName) + && Objects.equals(username, options.username) + && Objects.equals(password, options.password) + && Objects.equals(parallelism, options.parallelism) + && Objects.equals( + connectionCheckTimeoutSeconds, options.connectionCheckTimeoutSeconds); } else { return false; } @@ -79,13 +78,13 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( - url, - tableName, - driverName, - username, - password, - parallelism, - connectionCheckTimeoutSeconds); + url, + tableName, + driverName, + username, + password, + parallelism, + connectionCheckTimeoutSeconds); } public static class Builder { @@ -146,13 +145,13 @@ public JdbcConnectorOptions build() { } return new JdbcConnectorOptions( - dbURL, - tableName, - driverName, - username, - password, - parallelism, - connectionCheckTimeoutSeconds); + dbURL, + tableName, + driverName, + username, + password, + parallelism, + connectionCheckTimeoutSeconds); } } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FieldValueProfileAggregateFunction.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FieldValueProfileAggregateFunction.java index 7758f81b..c98aefe8 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FieldValueProfileAggregateFunction.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FieldValueProfileAggregateFunction.java @@ -12,9 +12,8 @@ package com.cloudera.cyber.profiler; -import com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc; import com.cloudera.cyber.profiler.accumulator.FieldValueProfileGroupAcc; - +import com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc; import java.text.DecimalFormat; import java.util.Map; import java.util.stream.Collectors; @@ -32,8 +31,8 @@ public ProfileGroupAcc createAccumulator() { @Override protected Map getMeasurementFormats() { - return profileGroupConfig.getMeasurements().stream(). - collect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName, - ProfileMeasurementConfig::getDecimalFormat)); + return profileGroupConfig.getMeasurements().stream() + .llect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName, + ProfileMeasurementConfig::getDecimalFormat)); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FirstSeenHBase.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FirstSeenHBase.java index ba5fa0a2..17b8d4d6 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FirstSeenHBase.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FirstSeenHBase.java @@ -15,14 +15,12 @@ import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; import com.cloudera.cyber.hbase.LookupKey; import com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc; -import lombok.Data; -import org.apache.hadoop.hbase.util.Bytes; - import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.Data; @Data public class FirstSeenHBase implements Serializable { @@ -37,15 +35,19 @@ public FirstSeenHBase(EnrichmentStorageConfig enrichmentStorageConfig, ProfileGr this.enrichmentStorageConfig = enrichmentStorageConfig; this.profileName = profileGroupConfig.getProfileGroupName(); this.keyFieldNames = profileGroupConfig.getKeyFieldNames(); - ProfileMeasurementConfig measurementConfig = profileGroupConfig.getMeasurements().stream().filter(m -> m.getAggregationMethod().equals(ProfileAggregationMethod.FIRST_SEEN)). - findFirst().orElseThrow(() -> new NullPointerException("Expected at least one first seen measurement but none was found.")); + ProfileMeasurementConfig measurementConfig = + profileGroupConfig.getMeasurements().stream() + .filter(m -> m.getAggregationMethod().equals(ProfileAggregationMethod.FIRST_SEEN)) + .findFirst() + .orElseThrow(() -> new NullPointerException( + "Expected at least one first seen measurement but none was found.")); this.firstSeenResultName = measurementConfig.getResultExtensionName(); } public LookupKey getKey(ProfileMessage message) { Map extensions = message.getExtensions(); String key = Stream.concat(Stream.of(profileName), - keyFieldNames.stream().map(extensions::get)).collect(Collectors.joining(":")); + keyFieldNames.stream().map(extensions::get)).collect(Collectors.joining(":")); return enrichmentStorageConfig.getFormat().getLookupBuilder().build(enrichmentStorageConfig, "first_seen", key); } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FirstSeenHbaseLookup.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FirstSeenHbaseLookup.java index a7acfac4..5f94ea98 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FirstSeenHbaseLookup.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FirstSeenHbaseLookup.java @@ -18,18 +18,17 @@ import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; import com.cloudera.cyber.hbase.AbstractHbaseMapFunction; import com.cloudera.cyber.hbase.LookupKey; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.util.Collector; -import org.apache.flink.util.OutputTag; - import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.util.Collector; +import org.apache.flink.util.OutputTag; @EqualsAndHashCode(callSuper = true) @Data @@ -42,16 +41,23 @@ public class FirstSeenHbaseLookup extends AbstractHbaseMapFunction firstSeenUpdateOutput = new OutputTag(ProfileJob.FIRST_SEEN_ENRICHMENT_UPDATE){}; + protected final OutputTag firstSeenUpdateOutput = + new OutputTag(ProfileJob.FIRST_SEEN_ENRICHMENT_UPDATE) { + }; - public FirstSeenHbaseLookup(EnrichmentStorageConfig enrichmentStorageConfig, ProfileGroupConfig profileGroupConfig) throws IllegalStateException { + public FirstSeenHbaseLookup(EnrichmentStorageConfig enrichmentStorageConfig, ProfileGroupConfig profileGroupConfig) + throws IllegalStateException { this.firstSeenHBaseInfo = new FirstSeenHBase(enrichmentStorageConfig, profileGroupConfig); - ProfileMeasurementConfig measurementConfig = profileGroupConfig.getMeasurements().stream().filter(m -> m.getAggregationMethod().equals(ProfileAggregationMethod.FIRST_SEEN)). - findFirst().orElseThrow(() -> new NullPointerException("Expected at least one first seen measurement but none was found.")); + ProfileMeasurementConfig measurementConfig = + profileGroupConfig.getMeasurements().stream() + .filter(m -> m.getAggregationMethod().equals(ProfileAggregationMethod.FIRST_SEEN)) + .findFirst().orElseThrow( + () -> new NullPointerException("Expected at least one first seen measurement but none was found.")); - this.firstSeenExpireDuration = Duration.of(measurementConfig.getFirstSeenExpirationDuration(), ChronoUnit.valueOf(measurementConfig.getFirstSeenExpirationDurationUnit())); + this.firstSeenExpireDuration = Duration.of(measurementConfig.getFirstSeenExpirationDuration(), + ChronoUnit.valueOf(measurementConfig.getFirstSeenExpirationDurationUnit())); this.enrichmentStorageConfig = enrichmentStorageConfig; } @@ -64,11 +70,14 @@ public void processElement(ProfileMessage message, Context context, Collector firstSeenExtensions = new HashMap<>(); String firstSeen = firstSeenHBaseInfo.getFirstSeen(message); if (!first) { - String previousFirstTimestamp = mergeFirstLastSeen((String)previousFirstLastSeen.get(FIRST_SEEN_PROPERTY_NAME), - (String)previousFirstLastSeen.get(LAST_SEEN_PROPERTY_NAME), firstSeenHBaseInfo.getFirstSeen(message)); + String previousFirstTimestamp = + mergeFirstLastSeen((String) previousFirstLastSeen.get(FIRST_SEEN_PROPERTY_NAME), + (String) previousFirstLastSeen.get(LAST_SEEN_PROPERTY_NAME), + firstSeenHBaseInfo.getFirstSeen(message)); if (previousFirstTimestamp != null) { // there was a previous observation - send the first timestamp - firstSeenExtensions.put(firstSeenHBaseInfo.getFirstSeenResultName().concat(FIRST_SEEN_TIME_SUFFIX), previousFirstTimestamp); + firstSeenExtensions.put(firstSeenHBaseInfo.getFirstSeenResultName().concat(FIRST_SEEN_TIME_SUFFIX), + previousFirstTimestamp); firstSeen = previousFirstTimestamp; } else { // previous observation was too old, treat this like a new observation @@ -80,7 +89,8 @@ public void processElement(ProfileMessage message, Context context, Collector{ +public class MessageKeySelector implements KeySelector { private List fieldNames; public MessageKeySelector(List fieldNames) { Preconditions.checkNotNull(fieldNames, "profile key field list is null"); - Preconditions.checkArgument(!fieldNames.isEmpty(),"profiled key field list must contain at least one field name"); + Preconditions.checkArgument(!fieldNames.isEmpty(), + "profiled key field list must contain at least one field name"); this.fieldNames = fieldNames; } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileAggregateFunction.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileAggregateFunction.java index 0ae26a8d..d20ff149 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileAggregateFunction.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileAggregateFunction.java @@ -13,14 +13,14 @@ package com.cloudera.cyber.profiler; import com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc; -import lombok.AllArgsConstructor; -import org.apache.flink.api.common.functions.AggregateFunction; - import java.text.DecimalFormat; import java.util.Map; +import lombok.AllArgsConstructor; +import org.apache.flink.api.common.functions.AggregateFunction; @AllArgsConstructor -public abstract class ProfileAggregateFunction implements AggregateFunction { +public abstract class ProfileAggregateFunction + implements AggregateFunction { public static final String PROFILE_GROUP_NAME_EXTENSION = "profile"; public static final String PROFILE_TOPIC_NAME = "profile"; public static final String PROFILE_SOURCE = "profile"; @@ -30,9 +30,9 @@ public abstract class ProfileAggregateFunction implements AggregateFunction measurementFormats; public ProfileAggregateFunction(ProfileGroupConfig profileGroupConfig, String profileGroupName) { - this.profileGroupConfig = profileGroupConfig; - this.profileGroupName = profileGroupName; - this.measurementFormats = getMeasurementFormats(); + this.profileGroupConfig = profileGroupConfig; + this.profileGroupName = profileGroupName; + this.measurementFormats = getMeasurementFormats(); } @Override @@ -46,15 +46,16 @@ public ProfileGroupAcc add(ProfileMessage message, ProfileGroupAcc profileAccumu @Override public ProfileMessage getResult(ProfileGroupAcc profileAccumulator) { - Map extensions = profileAccumulator.getProfileExtensions(profileGroupConfig, measurementFormats); + Map extensions = + profileAccumulator.getProfileExtensions(profileGroupConfig, measurementFormats); extensions.put(PROFILE_GROUP_NAME_EXTENSION, profileGroupName); return new ProfileMessage(profileAccumulator.getEndTimestamp(), extensions); } @Override public ProfileGroupAcc merge(ProfileGroupAcc acc1, ProfileGroupAcc acc2) { - acc1.merge(acc2); - return acc1; + acc1.merge(acc2); + return acc1; } protected abstract Map getMeasurementFormats(); diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileJob.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileJob.java index 39ae388b..c0e6c5bd 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileJob.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileJob.java @@ -35,17 +35,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import freemarker.template.TemplateException; -import java.util.function.BiFunction; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; -import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; -import org.apache.flink.streaming.api.windowing.time.Time; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; @@ -62,9 +51,19 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; +import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; +import org.apache.flink.streaming.api.windowing.time.Time; @Slf4j public abstract class ProfileJob { @@ -75,24 +74,33 @@ public abstract class ProfileJob { protected static final String PARAM_LATENESS_TOLERANCE_MILLIS = "profile.lateness"; protected static final String PARAMS_PHOENIX_DB_INIT = "phoenix.db.init"; protected static final String PARAMS_PHOENIX_DB_QUERY_PARAM = "phoenix.db.query.param."; - protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_DATA_TABLE_NAME = "phoenix.db.query.param.measurement_data_table_name"; - protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_METADATA_TABLE_NAME = "phoenix.db.query.param.measurement_metadata_table_name"; - protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_NAME = "phoenix.db.query.param.measurement_sequence_name"; - protected static final String PARAMS_PHOENIX_DB_QUERY_PROFILE_METADATA_TABLE_NAME = "phoenix.db.query.param.profile_metadata_table_name"; - protected static final String PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_NAME = "phoenix.db.query.param.profile_sequence_name"; - protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_START_WITH = "phoenix.db.query.param.measurement_sequence_start_with"; - protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_CACHE = "phoenix.db.query.param.measurement_sequence_cache"; - protected static final String PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_START_WITH = "phoenix.db.query.param.profile_sequence_start_with"; - protected static final String PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_CACHE = "phoenix.db.query.param.profile_sequence_cache"; + protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_DATA_TABLE_NAME = + "phoenix.db.query.param.measurement_data_table_name"; + protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_METADATA_TABLE_NAME = + "phoenix.db.query.param.measurement_metadata_table_name"; + protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_NAME = + "phoenix.db.query.param.measurement_sequence_name"; + protected static final String PARAMS_PHOENIX_DB_QUERY_PROFILE_METADATA_TABLE_NAME = + "phoenix.db.query.param.profile_metadata_table_name"; + protected static final String PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_NAME = + "phoenix.db.query.param.profile_sequence_name"; + protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_START_WITH = + "phoenix.db.query.param.measurement_sequence_start_with"; + protected static final String PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_CACHE = + "phoenix.db.query.param.measurement_sequence_cache"; + protected static final String PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_START_WITH = + "phoenix.db.query.param.profile_sequence_start_with"; + protected static final String PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_CACHE = + "phoenix.db.query.param.profile_sequence_cache"; protected static final String PARAMS_PHOENIX_DB_QUERY_KEY_COUNT = "phoenix.db.query.param.field_key_count"; protected static final ObjectMapper jsonObjectMapper = - new ObjectMapper() - .activateDefaultTyping(BasicPolymorphicTypeValidator.builder(). - allowIfSubType(Map.class). - allowIfSubType(List.class). - allowIfSubType(java.util.concurrent.TimeUnit.class). - build()) - .enable(SerializationFeature.INDENT_OUTPUT); + new ObjectMapper() + .activateDefaultTyping(BasicPolymorphicTypeValidator.builder() + .allowIfSubType(Map.class) + .allowIfSubType(List.class) + .allowIfSubType(java.util.concurrent.TimeUnit.class) + .build()) + .enable(SerializationFeature.INDENT_OUTPUT); private static final Function FIRST_VALUE_MAPPER = rs -> { try { @@ -105,17 +113,17 @@ public abstract class ProfileJob { private static final Function PROFILE_META_MAPPER = rs -> { try { ProfileDtoBuilder profileDtoBuilder = ProfileDto.builder().id(rs.getInt("ID")) - .profileGroupName(rs.getString("PROFILE_GROUP_NAME")) - .keyFieldNames(rs.getString("KEY_FIELD_NAMES")); + .profileGroupName(rs.getString("PROFILE_GROUP_NAME")) + .keyFieldNames(rs.getString("KEY_FIELD_NAMES")); long periodDuration = rs.getLong("PERIOD_DURATION"); if (!rs.wasNull()) { profileDtoBuilder.periodDuration(periodDuration) - .periodDurationUnit(rs.getString("PERIOD_DURATION_UNIT")); + .periodDurationUnit(rs.getString("PERIOD_DURATION_UNIT")); } long statsSlide = rs.getLong("STATS_SLIDE"); if (!rs.wasNull()) { profileDtoBuilder.statsSlide(statsSlide) - .statsSlideUnit(rs.getString("STATS_SLIDE_UNIT")); + .statsSlideUnit(rs.getString("STATS_SLIDE_UNIT")); } return profileDtoBuilder.build(); } catch (SQLException e) { @@ -127,16 +135,16 @@ public abstract class ProfileJob { private static final Function MEASUREMENT_META_MAPPER = rs -> { try { MeasurementDtoBuilder format = MeasurementDto.builder() - .id(rs.getInt("ID")) - .resultExtensionName(rs.getString("RESULT_EXTENSION_NAME")) - .calculateStats(rs.getBoolean("CALCULATE_STATS")) - .aggregationMethod(rs.getString("AGGREGATION_METHOD")) - .fieldName(rs.getString("FIELD_NAME")) - .format(rs.getString("FORMAT")); + .id(rs.getInt("ID")) + .resultExtensionName(rs.getString("RESULT_EXTENSION_NAME")) + .calculateStats(rs.getBoolean("CALCULATE_STATS")) + .aggregationMethod(rs.getString("AGGREGATION_METHOD")) + .fieldName(rs.getString("FIELD_NAME")) + .format(rs.getString("FORMAT")); long firstSeenExpirationDuration = rs.getLong("FIRST_SEEN_EXPIRATION_DURATION"); if (!rs.wasNull()) { format.firstSeenExpirationDuration(firstSeenExpirationDuration) - .firstSeenExpirationDurationUnit(rs.getString("FIRST_SEEN_EXPIRATION_DURATION_UNIT")); + .firstSeenExpirationDurationUnit(rs.getString("FIRST_SEEN_EXPIRATION_DURATION_UNIT")); } format.profileId(rs.getInt("PROFILE_ID")); return format.build(); @@ -146,52 +154,66 @@ public abstract class ProfileJob { } }; protected final FreemarkerImmediateGenerator freemarkerGenerator = new FreemarkerImmediateGenerator(); - private static final String SELECT_PROFILE_BY_NAME = "SELECT * FROM ${profile_metadata_table_name} WHERE PROFILE_GROUP_NAME = ?"; - private static final String SELECT_MEASUREMENT_BY_NAME = "SELECT * FROM ${measurement_metadata_table_name} WHERE RESULT_EXTENSION_NAME = ? AND PROFILE_ID = ?"; - private static final String UPSERT_PROFILE_META = "UPSERT INTO ${profile_metadata_table_name} (ID,PROFILE_GROUP_NAME,KEY_FIELD_NAMES,PERIOD_DURATION,PERIOD_DURATION_UNIT,STATS_SLIDE,STATS_SLIDE_UNIT) VALUES (?,?,?,?,?,?,?)"; - private static final String UPSERT_MEASUREMENT_META = "UPSERT INTO ${measurement_metadata_table_name} (ID,FIELD_NAME,RESULT_EXTENSION_NAME,AGGREGATION_METHOD,CALCULATE_STATS,FORMAT,FIRST_SEEN_EXPIRATION_DURATION, FIRST_SEEN_EXPIRATION_DURATION_UNIT, PROFILE_ID) VALUES (?,?,?,?,?,?,?,?,?)"; + private static final String SELECT_PROFILE_BY_NAME = + "SELECT * FROM ${profile_metadata_table_name} WHERE PROFILE_GROUP_NAME = ?"; + private static final String SELECT_MEASUREMENT_BY_NAME = + "SELECT * FROM ${measurement_metadata_table_name} WHERE RESULT_EXTENSION_NAME = ? AND PROFILE_ID = ?"; + private static final String UPSERT_PROFILE_META = + "UPSERT INTO ${profile_metadata_table_name} " + + "(ID,PROFILE_GROUP_NAME,KEY_FIELD_NAMES,PERIOD_DURATION,PERIOD_DURATION_UNIT,STATS_SLIDE,STATS_SLIDE_UNIT) " + + "VALUES (?,?,?,?,?,?,?)"; + private static final String UPSERT_MEASUREMENT_META = + "UPSERT INTO ${measurement_metadata_table_name} " + + "(ID,FIELD_NAME,RESULT_EXTENSION_NAME,AGGREGATION_METHOD,CALCULATE_STATS,FORMAT,FIRST_SEEN_EXPIRATION_DURATION, " + + "FIRST_SEEN_EXPIRATION_DURATION_UNIT, PROFILE_ID) VALUES (?,?,?,?,?,?,?,?,?)"; private static final String SELECT_PROFILE_ID = "SELECT NEXT VALUE FOR ${profile_sequence_name}"; private static final String SELECT_MEASUREMENT_ID = "SELECT NEXT VALUE FOR ${measurement_sequence_name}"; - private static final String SELECT_METADATA_FROM_MEASUREMENT_DATA_TABLE = "SELECT * FROM ${measurement_data_table_name} LIMIT 1"; - - private static final Function> PROFILE_PS_CONSUMER_FUNCTION = profileGroupName -> ps -> { - try { - ps.setString(1, profileGroupName); - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - }; - - private static final BiFunction> MEASUREMENT_PS_CONSUMER_FUNCTION = (resultExtensionName, profileId) -> ps -> { - try { - ps.setString(1, resultExtensionName); - ps.setInt(2, profileId); - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - }; + private static final String SELECT_METADATA_FROM_MEASUREMENT_DATA_TABLE = + "SELECT * FROM ${measurement_data_table_name} LIMIT 1"; + + private static final Function> PROFILE_PS_CONSUMER_FUNCTION = + profileGroupName -> ps -> { + try { + ps.setString(1, profileGroupName); + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + }; + + private static final BiFunction> MEASUREMENT_PS_CONSUMER_FUNCTION = + (resultExtensionName, profileId) -> ps -> { + try { + ps.setString(1, resultExtensionName); + ps.setInt(2, profileId); + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + }; protected List parseConfigFile(String configJson) throws JsonProcessingException { List profileGroupConfigs = jsonObjectMapper.readValue( - configJson, - new TypeReference>() { - }); + configJson, + new TypeReference>() { + }); ProfileGroupConfig.verify(profileGroupConfigs); return profileGroupConfigs; } - protected StreamExecutionEnvironment createPipeline(final ParameterTool params) throws IOException, SQLException, TemplateException { + protected StreamExecutionEnvironment createPipeline(final ParameterTool params) + throws IOException, SQLException, TemplateException { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FlinkUtils.setupEnv(env, params); // set the JdbcCatalog as the current catalog of the session // read the profiler config - List profileGroups = parseConfigFile(new String(Files.readAllBytes(Paths.get(params.getRequired(PARAM_PROFILE_CONFIG))))); + List profileGroups = + parseConfigFile(new String(Files.readAllBytes(Paths.get(params.getRequired(PARAM_PROFILE_CONFIG))))); List profileDtos = phoenixDataBaseInit(params, profileGroups); long allowedLatenessMillis = params.getLong(PARAM_LATENESS_TOLERANCE_MILLIS, Time.minutes(5).toMilliseconds()); - final DataStream messages = ScoredMessageWatermarkedStream.of(createSource(env, params), allowedLatenessMillis); + final DataStream messages = + ScoredMessageWatermarkedStream.of(createSource(env, params), allowedLatenessMillis); DataStream profiledStreams = null; for (ProfileGroupConfig profileGroupConfig : profileGroups) { @@ -199,13 +221,15 @@ protected StreamExecutionEnvironment createPipeline(final ParameterTool params) } Preconditions.checkNotNull(profiledStreams, "At least one profile must be specified"); - DataStream scoredMessages = score(profiledStreams.map(new ProfileMessageToMessageMap()), env, params); + DataStream scoredMessages = + score(profiledStreams.map(new ProfileMessageToMessageMap()), env, params); writeProfileMeasurementsResults(params, profileDtos, profiledStreams); writeResults(params, scoredMessages); return env; } - private List phoenixDataBaseInit(ParameterTool params, List profileGroups) throws SQLException, IOException, TemplateException { + private List phoenixDataBaseInit(ParameterTool params, List profileGroups) + throws SQLException, IOException, TemplateException { if (params.getBoolean(PARAMS_PHOENIX_DB_INIT)) { Properties properties = Utils.readProperties(params.getProperties(), PARAMS_PHOENIX_DB_QUERY_PARAM); client = new PhoenixThinClient(params); @@ -215,7 +239,8 @@ private List phoenixDataBaseInit(ParameterTool params, List persistProfileMeasurementConfigMeta(List profileGroups, PhoenixThinClient client, Properties properties) throws SQLException, IOException, TemplateException { + protected List persistProfileMeasurementConfigMeta(List profileGroups, + PhoenixThinClient client, Properties properties) + throws SQLException, IOException, TemplateException { ImmutableMap propertiesMap = Maps.fromProperties(properties); List resultProfiles = new ArrayList<>(); for (ProfileDto profile : profileGroups.stream().map(ProfileDto::of).collect(Collectors.toList())) { @@ -248,54 +278,78 @@ protected List persistProfileMeasurementConfigMeta(List propertiesMap, Integer profileId, MeasurementDto measurement) throws SQLException, IOException, TemplateException { - MeasurementDto measurementEntity = client.selectResult(freemarkerGenerator.replaceByTemplate(SELECT_MEASUREMENT_BY_NAME, propertiesMap), MEASUREMENT_META_MAPPER, - MEASUREMENT_PS_CONSUMER_FUNCTION.apply(measurement.getResultExtensionName(), profileId)); + private MeasurementDto processMeasurement(PhoenixThinClient client, ImmutableMap propertiesMap, + Integer profileId, MeasurementDto measurement) + throws SQLException, IOException, TemplateException { + MeasurementDto measurementEntity = + client.selectResult(freemarkerGenerator.replaceByTemplate(SELECT_MEASUREMENT_BY_NAME, propertiesMap), + MEASUREMENT_META_MAPPER, + MEASUREMENT_PS_CONSUMER_FUNCTION.apply(measurement.getResultExtensionName(), profileId)); if (measurementEntity == null) { return createMeasurement(client, propertiesMap, profileId, measurement); } else if (measurementEntity.getId() == null) { - throw new RuntimeException(String.format("Error when trying to map measurement from phoenix database with name %s and profile id %d", measurement.getResultExtensionName(), measurement.getProfileId())); + throw new RuntimeException(String.format( + "Error when trying to map measurement from phoenix database with name %s and profile id %d", + measurement.getResultExtensionName(), measurement.getProfileId())); } else { return updateMeasurement(client, propertiesMap, profileId, measurementEntity, measurement); } } - private MeasurementDto createMeasurement(PhoenixThinClient client, Map params, Integer profileId, MeasurementDto measurement) throws SQLException, IOException, TemplateException { - Integer measurementId = client.selectResult(freemarkerGenerator.replaceByTemplate(SELECT_MEASUREMENT_ID, params), FIRST_VALUE_MAPPER); - client.insertIntoTable(freemarkerGenerator.replaceByTemplate(UPSERT_MEASUREMENT_META, params), getUpdateMeasurementMetaConsumer(measurement, measurementId, profileId)); + private MeasurementDto createMeasurement(PhoenixThinClient client, Map params, Integer profileId, + MeasurementDto measurement) + throws SQLException, IOException, TemplateException { + Integer measurementId = + client.selectResult(freemarkerGenerator.replaceByTemplate(SELECT_MEASUREMENT_ID, params), + FIRST_VALUE_MAPPER); + client.insertIntoTable(freemarkerGenerator.replaceByTemplate(UPSERT_MEASUREMENT_META, params), + getUpdateMeasurementMetaConsumer(measurement, measurementId, profileId)); return measurement.toBuilder().id(measurementId).profileId(profileId).build(); } - private MeasurementDto updateMeasurement(PhoenixThinClient client, Map params, Integer profileId, MeasurementDto measurementDtoDb, MeasurementDto measurement) throws SQLException, IOException, TemplateException { + private MeasurementDto updateMeasurement(PhoenixThinClient client, Map params, Integer profileId, + MeasurementDto measurementDtoDb, MeasurementDto measurement) + throws SQLException, IOException, TemplateException { if (!ProfileUtils.compareMeasurementDto(measurementDtoDb, measurement)) { log.error("Measurement from configuration {} measurement from database {}", measurement, measurementDtoDb); - throw new IllegalStateException("Key fields cannot be changed for the measurement='" + measurement + "' measurementDtoDb='" + measurementDtoDb + "'"); + throw new IllegalStateException( + "Key fields cannot be changed for the measurement='" + measurement + "' measurementDtoDb='" + + measurementDtoDb + "'"); } Integer measurementId = measurementDtoDb.getId(); - client.insertIntoTable(freemarkerGenerator.replaceByTemplate(UPSERT_MEASUREMENT_META, params), getUpdateMeasurementMetaConsumer(measurement, measurementId, profileId)); + client.insertIntoTable(freemarkerGenerator.replaceByTemplate(UPSERT_MEASUREMENT_META, params), + getUpdateMeasurementMetaConsumer(measurement, measurementId, profileId)); return measurement.toBuilder().id(measurementId).profileId(profileId).build(); } - private ProfileDto processProfile(PhoenixThinClient client, ImmutableMap propertiesMap, ProfileDto profile) throws SQLException, IOException, TemplateException { + private ProfileDto processProfile(PhoenixThinClient client, ImmutableMap propertiesMap, + ProfileDto profile) throws SQLException, IOException, TemplateException { String profileGroupName = profile.getProfileGroupName(); - ProfileDto profileDtoDb = client.selectResult(freemarkerGenerator.replaceByTemplate(SELECT_PROFILE_BY_NAME, propertiesMap), PROFILE_META_MAPPER, PROFILE_PS_CONSUMER_FUNCTION.apply(profileGroupName)); + ProfileDto profileDtoDb = + client.selectResult(freemarkerGenerator.replaceByTemplate(SELECT_PROFILE_BY_NAME, propertiesMap), + PROFILE_META_MAPPER, PROFILE_PS_CONSUMER_FUNCTION.apply(profileGroupName)); if (profileDtoDb == null) { return createProfile(client, propertiesMap, profile); } else if (profileDtoDb.getId() == null) { - throw new SQLException(String.format("Error when trying to map profile from phoenix database with name %s", profileGroupName)); + throw new SQLException(String.format("Error when trying to map profile from phoenix database with name %s", + profileGroupName)); - }else { + } else { return updateProfile(client, propertiesMap, profile, profileDtoDb); } } - private ProfileDto updateProfile(PhoenixThinClient client, ImmutableMap propertiesMap, ProfileDto profile, ProfileDto profileDtoDb) throws SQLException, IOException, TemplateException { + private ProfileDto updateProfile(PhoenixThinClient client, ImmutableMap propertiesMap, + ProfileDto profile, ProfileDto profileDtoDb) + throws SQLException, IOException, TemplateException { if (!ProfileUtils.compareProfileDto(profileDtoDb, profile)) { log.error("Profile from configuration {} profile from database {}", profile, profileDtoDb); - throw new IllegalStateException("Key fields cannot be changed for the profile='" + profile + "' profileDtoDb='" + profileDtoDb + "'"); + throw new IllegalStateException( + "Key fields cannot be changed for the profile='" + profile + "' profileDtoDb='" + profileDtoDb + "'"); } Integer profileId = profileDtoDb.getId(); - client.insertIntoTable(freemarkerGenerator.replaceByTemplate(UPSERT_PROFILE_META, propertiesMap), getUpdateProfileConsumer(profile, profileId)); + client.insertIntoTable(freemarkerGenerator.replaceByTemplate(UPSERT_PROFILE_META, propertiesMap), + getUpdateProfileConsumer(profile, profileId)); ArrayList measurementDtos = new ArrayList<>(); for (MeasurementDto measurementDto : profile.getMeasurementDtos()) { measurementDtos.add(processMeasurement(client, propertiesMap, profileId, measurementDto)); @@ -303,9 +357,12 @@ private ProfileDto updateProfile(PhoenixThinClient client, ImmutableMap propertiesMap, ProfileDto profile) throws SQLException, IOException, TemplateException { - Integer profileId = client.selectResult(freemarkerGenerator.replaceByTemplate(SELECT_PROFILE_ID, propertiesMap), FIRST_VALUE_MAPPER); - client.insertIntoTable(freemarkerGenerator.replaceByTemplate(UPSERT_PROFILE_META, propertiesMap), getUpdateProfileConsumer(profile, profileId)); + private ProfileDto createProfile(PhoenixThinClient client, ImmutableMap propertiesMap, + ProfileDto profile) throws SQLException, IOException, TemplateException { + Integer profileId = client.selectResult(freemarkerGenerator.replaceByTemplate(SELECT_PROFILE_ID, propertiesMap), + FIRST_VALUE_MAPPER); + client.insertIntoTable(freemarkerGenerator.replaceByTemplate(UPSERT_PROFILE_META, propertiesMap), + getUpdateProfileConsumer(profile, profileId)); ArrayList measurementDtos = new ArrayList<>(); for (MeasurementDto measurementDto : profile.getMeasurementDtos()) { measurementDtos.add(createMeasurement(client, propertiesMap, profileId, measurementDto)); @@ -313,13 +370,15 @@ private ProfileDto createProfile(PhoenixThinClient client, ImmutableMap getUpdateMeasurementMetaConsumer(MeasurementDto measurement, Integer measurementId, Integer profileId) { + private Consumer getUpdateMeasurementMetaConsumer(MeasurementDto measurement, + Integer measurementId, Integer profileId) { return ps -> { try { ps.setInt(1, measurementId); ps.setString(2, measurement.getFieldName()); ps.setString(3, measurement.getResultExtensionName()); - ps.setString(4, Optional.ofNullable(measurement.getAggregationMethod()).map(Objects::toString).orElse(null)); + ps.setString(4, + Optional.ofNullable(measurement.getAggregationMethod()).map(Objects::toString).orElse(null)); if (measurement.getCalculateStats() == null) { ps.setNull(5, Types.BOOLEAN); } else { @@ -370,34 +429,47 @@ private ThrowingConsumer getUpdateProfileConsumer(ProfileDto * @param params Configuration parameters * @param messages Data stream of messages to profile. */ - protected DataStream profile(final ParameterTool params, DataStream messages, ProfileGroupConfig profileGroupConfig, long allowedLatenessMillis, DataStream profileMessageStreams) { - - Time profilePeriodDuration = Time.of(profileGroupConfig.getPeriodDuration(), TimeUnit.valueOf(profileGroupConfig.getPeriodDurationUnit())); - DataStream profileMessages = messages.filter(new ProfileMessageFilter(profileGroupConfig)). - map(new ScoredMessageToProfileMessageMap(profileGroupConfig)). - keyBy(new MessageKeySelector(profileGroupConfig.getKeyFieldNames())).window(TumblingEventTimeWindows.of(profilePeriodDuration)). - aggregate(new FieldValueProfileAggregateFunction(profileGroupConfig)); + protected DataStream profile(final ParameterTool params, DataStream messages, + ProfileGroupConfig profileGroupConfig, long allowedLatenessMillis, + DataStream profileMessageStreams) { + + Time profilePeriodDuration = Time.of(profileGroupConfig.getPeriodDuration(), + TimeUnit.valueOf(profileGroupConfig.getPeriodDurationUnit())); + DataStream profileMessages = messages.filter(new ProfileMessageFilter(profileGroupConfig)) + .map(new ScoredMessageToProfileMessageMap( + profileGroupConfig)) + .keyBy(new MessageKeySelector( + profileGroupConfig.getKeyFieldNames())).window( + TumblingEventTimeWindows.of(profilePeriodDuration)) + .aggregate(new FieldValueProfileAggregateFunction( + profileGroupConfig)); if (profileGroupConfig.hasFirstSeen()) { profileMessages = updateFirstSeen(params, profileMessages, profileGroupConfig); } if (profileGroupConfig.hasStats()) { - MessageKeySelector profileKeySelector = new MessageKeySelector(Collections.singletonList(ProfileAggregateFunction.PROFILE_GROUP_NAME_EXTENSION)); - Time statsSlide = Time.of(profileGroupConfig.getStatsSlide(), TimeUnit.valueOf(profileGroupConfig.getStatsSlideUnit())); - - DataStream statsStream = ProfileMessage.watermarkedStreamOf(profileMessages, allowedLatenessMillis). - keyBy(profileKeySelector). - window(SlidingEventTimeWindows.of(profilePeriodDuration, statsSlide)). - aggregate(new StatsProfileAggregateFunction(profileGroupConfig)); + MessageKeySelector profileKeySelector = new MessageKeySelector( + Collections.singletonList(ProfileAggregateFunction.PROFILE_GROUP_NAME_EXTENSION)); + Time statsSlide = + Time.of(profileGroupConfig.getStatsSlide(), TimeUnit.valueOf(profileGroupConfig.getStatsSlideUnit())); + + DataStream statsStream = + ProfileMessage.watermarkedStreamOf(profileMessages, allowedLatenessMillis) + .keyBy(profileKeySelector) + .window(SlidingEventTimeWindows.of(profilePeriodDuration, statsSlide)) + .aggregate(new StatsProfileAggregateFunction(profileGroupConfig)); StatsProfileKeySelector statsKeySelector = new StatsProfileKeySelector(); - profileMessages = profileMessages.join(statsStream).where(profileKeySelector).equalTo(statsKeySelector).window(TumblingEventTimeWindows.of(profilePeriodDuration)).apply(new ProfileStatsJoin()); + profileMessages = profileMessages.join(statsStream).where(profileKeySelector).equalTo(statsKeySelector) + .window(TumblingEventTimeWindows.of(profilePeriodDuration)) + .apply(new ProfileStatsJoin()); profileMessageStreams = unionProfileMessages(profileMessageStreams, statsStream); } return unionProfileMessages(profileMessageStreams, profileMessages); } - private DataStream unionProfileMessages(DataStream profileMessageUnion, DataStream newStream) { + private DataStream unionProfileMessages(DataStream profileMessageUnion, + DataStream newStream) { if (profileMessageUnion == null) { return newStream; } else { @@ -405,19 +477,26 @@ private DataStream unionProfileMessages(DataStream profileDtos, DataStream results) throws IOException, TemplateException; + protected abstract void writeProfileMeasurementsResults(ParameterTool params, List profileDtos, + DataStream results) + throws IOException, TemplateException; protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params); protected abstract void writeResults(ParameterTool params, DataStream results); - protected abstract DataStream updateFirstSeen(ParameterTool params, DataStream results, ProfileGroupConfig profileGroupConfig); + protected abstract DataStream updateFirstSeen(ParameterTool params, + DataStream results, + ProfileGroupConfig profileGroupConfig); - protected abstract DataStream createRulesSource(StreamExecutionEnvironment env, ParameterTool params); + protected abstract DataStream createRulesSource(StreamExecutionEnvironment env, + ParameterTool params); - protected abstract void writeScoredRuleCommandResult(ParameterTool params, DataStream results); + protected abstract void writeScoredRuleCommandResult(ParameterTool params, + DataStream results); - private DataStream score(DataStream in, StreamExecutionEnvironment env, ParameterTool params) { + private DataStream score(DataStream in, StreamExecutionEnvironment env, + ParameterTool params) { DataStream rulesSource = createRulesSource(env, params); SingleOutputStreamOperator results = ScoringJob.enrich(in, rulesSource, params); writeScoredRuleCommandResult(params, results.getSideOutput(ScoringJob.COMMAND_RESULT_OUTPUT_TAG)); diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileJobKafka.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileJobKafka.java index 1f290062..192118d8 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileJobKafka.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileJobKafka.java @@ -12,9 +12,23 @@ package com.cloudera.cyber.profiler; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_METRON; +import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; +import static com.cloudera.cyber.profiler.FirstSeenHbaseLookup.FIRST_SEEN_ENRICHMENT_TYPE; +import static com.cloudera.cyber.profiler.phoenix.PhoenixThinClient.PHOENIX_THIN_PROPERTY_AUTHENTICATION; +import static com.cloudera.cyber.profiler.phoenix.PhoenixThinClient.PHOENIX_THIN_PROPERTY_AVATICA_PASSWORD; +import static com.cloudera.cyber.profiler.phoenix.PhoenixThinClient.PHOENIX_THIN_PROPERTY_AVATICA_USER; +import static com.cloudera.cyber.profiler.phoenix.PhoenixThinClient.PHOENIX_THIN_PROPERTY_KEYTAB; +import static com.cloudera.cyber.profiler.phoenix.PhoenixThinClient.PHOENIX_THIN_PROPERTY_PRINCIPAL; +import static com.cloudera.cyber.profiler.phoenix.PhoenixThinClient.PHOENIX_THIN_PROPERTY_URL; + import com.cloudera.cyber.ValidateUtils; import com.cloudera.cyber.commands.EnrichmentCommand; -import com.cloudera.cyber.enrichment.hbase.config.*; +import com.cloudera.cyber.enrichment.hbase.config.EnrichmentConfig; +import com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfig; +import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; +import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat; +import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig; import com.cloudera.cyber.enrichment.hbase.writer.HbaseEnrichmentCommandSink; import com.cloudera.cyber.flink.ConfigConstants; import com.cloudera.cyber.flink.FlinkUtils; @@ -33,6 +47,14 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import freemarker.template.TemplateException; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; @@ -46,17 +68,6 @@ import org.apache.flink.util.OutputTag; import org.apache.flink.util.Preconditions; -import java.io.IOException; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.*; -import java.util.function.Consumer; - -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_METRON; -import static com.cloudera.cyber.flink.ConfigConstants.PARAMS_TOPIC_INPUT; -import static com.cloudera.cyber.profiler.FirstSeenHbaseLookup.FIRST_SEEN_ENRICHMENT_TYPE; -import static com.cloudera.cyber.profiler.phoenix.PhoenixThinClient.*; - public class ProfileJobKafka extends ProfileJob { private static final String PROFILE_GROUP_ID = "profile"; @@ -68,12 +79,17 @@ public class ProfileJobKafka extends ProfileJob { private static final String PARAMS_PHOENIX_DB_INTERVAL_MILLIS = "phoenix.db.interval_millis"; private static final String PARAMS_PHOENIX_DB_MAX_RETRY_TIMES = "phoenix.db.max_retries_times"; - private static final String UPSERT_SQL = "<#assign key_count = field_key_count?number >UPSERT INTO ${measurement_data_table_name} (MEASUREMENT_ID, <#list 1..key_count as i> KEY_${i}, MEASUREMENT_NAME, MEASUREMENT_TYPE, MEASUREMENT_TIME, MEASUREMENT_VALUE) VALUES(?, <#list 1..key_count as i> ?, ?, ?, ?, ?) ON DUPLICATE KEY IGNORE"; + private static final String UPSERT_SQL = + "<#assign key_count = field_key_count?number >UPSERT INTO ${measurement_data_table_name} " + + "(MEASUREMENT_ID, <#list 1..key_count as i> KEY_${i}, MEASUREMENT_NAME, " + + "MEASUREMENT_TYPE, MEASUREMENT_TIME, MEASUREMENT_VALUE)" + + " VALUES(?, <#list 1..key_count as i> ?, ?, ?, ?, ?) ON DUPLICATE KEY IGNORE"; private static final String PARAMS_GROUP_ID = "kafka.group.id"; private static final String EMPTY_ERROR_MESSAGE_TEMPLATE = "'%s' can not be empty."; - private static final String INCORRECT_NUMERIC_MESSAGE_TEMPLATE = "Property '%s' has incorrect value '%s'. It should be numeric"; + private static final String INCORRECT_NUMERIC_MESSAGE_TEMPLATE = + "Property '%s' has incorrect value '%s'. It should be numeric"; private KafkaSink sink; @@ -82,50 +98,60 @@ public static void main(String[] args) throws Exception { ParameterTool params = Utils.getParamToolsFromProperties(args); validatePhoenixParam(params); FlinkUtils.executeEnv(new ProfileJobKafka() - .createPipeline(params), "Flink Profiling", params); + .createPipeline(params), "Flink Profiling", params); } @Override protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params) { return env.fromSource( - new FlinkUtils<>(ScoredMessage.class).createKafkaGenericSource(params.getRequired(PARAMS_TOPIC_INPUT), params, params.get(PARAMS_GROUP_ID, PROFILE_GROUP_ID)), - WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); + new FlinkUtils<>(ScoredMessage.class).createKafkaGenericSource(params.getRequired(PARAMS_TOPIC_INPUT), + params, params.get(PARAMS_GROUP_ID, PROFILE_GROUP_ID)), + WatermarkStrategy.noWatermarks(), "Kafka Source").uid("kafka-source"); } @Override protected void writeResults(ParameterTool params, DataStream results) { if (sink == null) { sink = new FlinkUtils<>(ScoredMessage.class).createKafkaSink( - params.getRequired(ConfigConstants.PARAMS_TOPIC_OUTPUT), PROFILE_GROUP_ID, - params); + params.getRequired(ConfigConstants.PARAMS_TOPIC_OUTPUT), PROFILE_GROUP_ID, + params); } results.sinkTo(sink).name("Kafka Results").uid("kafka.results."); } - protected void writeProfileMeasurementsResults(ParameterTool params, List profileDtos, DataStream results) throws IOException, TemplateException { + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") + protected void writeProfileMeasurementsResults(ParameterTool params, List profileDtos, + DataStream results) + throws IOException, TemplateException { if (params.getBoolean(PARAMS_PHOENIX_DB_INIT)) { - DataStream measurementDtoDataStream = results.flatMap(new ProfileMessageToMeasurementDataDtoMapping(new ArrayList<>(profileDtos))); + DataStream measurementDtoDataStream = + results.flatMap(new ProfileMessageToMeasurementDataDtoMapping(new ArrayList<>(profileDtos))); Properties properties = Utils.readProperties(params.getProperties(), PARAMS_PHOENIX_DB_QUERY_PARAM); - JdbcStatementBuilder objectJdbcStatementBuilder = new PhoenixJdbcStatementBuilder(params.getInt(PARAMS_PHOENIX_DB_QUERY_KEY_COUNT, 0)); + JdbcStatementBuilder objectJdbcStatementBuilder = + new PhoenixJdbcStatementBuilder(params.getInt(PARAMS_PHOENIX_DB_QUERY_KEY_COUNT, 0)); JdbcExecutionOptions.Builder jdbcExecutionOptionsBuilder = JdbcExecutionOptions.builder(); - validateNumericParamAndApply(PARAMS_PHOENIX_DB_BATCH_SIZE, params.get(PARAMS_PHOENIX_DB_BATCH_SIZE), jdbcExecutionOptionsBuilder::withBatchSize); - validateNumericParamAndApply(PARAMS_PHOENIX_DB_INTERVAL_MILLIS, params.get(PARAMS_PHOENIX_DB_INTERVAL_MILLIS), jdbcExecutionOptionsBuilder::withBatchIntervalMs); - validateNumericParamAndApply(PARAMS_PHOENIX_DB_MAX_RETRY_TIMES, params.get(PARAMS_PHOENIX_DB_MAX_RETRY_TIMES), jdbcExecutionOptionsBuilder::withMaxRetries); + validateNumericParamAndApply(PARAMS_PHOENIX_DB_BATCH_SIZE, params.get(PARAMS_PHOENIX_DB_BATCH_SIZE), + jdbcExecutionOptionsBuilder::withBatchSize); + validateNumericParamAndApply(PARAMS_PHOENIX_DB_INTERVAL_MILLIS, + params.get(PARAMS_PHOENIX_DB_INTERVAL_MILLIS), jdbcExecutionOptionsBuilder::withBatchIntervalMs); + validateNumericParamAndApply(PARAMS_PHOENIX_DB_MAX_RETRY_TIMES, + params.get(PARAMS_PHOENIX_DB_MAX_RETRY_TIMES), jdbcExecutionOptionsBuilder::withMaxRetries); JdbcExecutionOptions executionOptions = jdbcExecutionOptionsBuilder.build(); JdbcConnectionOptions connectionOptions = new JdbcConnectionOptions.JdbcConnectionOptionsBuilder() - .withDriverName(PhoenixThinClient.getDRIVER()) - .withUrl(client.getDbUrl()) - .build(); + .withDriverName(PhoenixThinClient.getDRIVER()) + .withUrl(client.getDbUrl()) + .build(); SinkFunction jdbcSink = JdbcSink.sink( - freemarkerGenerator.replaceByTemplate(UPSERT_SQL, Maps.fromProperties(properties)), - objectJdbcStatementBuilder, - executionOptions, - connectionOptions); + freemarkerGenerator.replaceByTemplate(UPSERT_SQL, Maps.fromProperties(properties)), + objectJdbcStatementBuilder, + executionOptions, + connectionOptions); measurementDtoDataStream.addSink(jdbcSink).name("JDBC Sink").uid("jdbc.profile.group"); } } + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") @Override protected DataStream updateFirstSeen(ParameterTool params, DataStream results, ProfileGroupConfig profileGroupConfig) { @@ -136,21 +162,28 @@ protected DataStream updateFirstSeen(ParameterTool params, DataS if (HBASE_METRON.equals(storageFormat)) { columnFamilyName = params.getRequired(PARAMS_FIRST_SEEN_HBASE_COLUMN_FAMILY); } - EnrichmentStorageConfig enrichmentStorageConfig = new EnrichmentStorageConfig(storageFormat, tableName, columnFamilyName); + EnrichmentStorageConfig enrichmentStorageConfig = + new EnrichmentStorageConfig(storageFormat, tableName, columnFamilyName); // look up the previous first seen timestamp and update the profile message SingleOutputStreamOperator updatedProfileMessages = results - .process(new FirstSeenHbaseLookup(enrichmentStorageConfig, profileGroupConfig)); - final OutputTag firstSeenEnrichmentUpdatesTag = new OutputTag(FIRST_SEEN_ENRICHMENT_UPDATE) { - }; - DataStream firstSeenEnrichmentUpdates = updatedProfileMessages.getSideOutput(firstSeenEnrichmentUpdatesTag); + .process(new FirstSeenHbaseLookup(enrichmentStorageConfig, profileGroupConfig)); + final OutputTag firstSeenEnrichmentUpdatesTag = + new OutputTag(FIRST_SEEN_ENRICHMENT_UPDATE) { + }; + DataStream firstSeenEnrichmentUpdates = + updatedProfileMessages.getSideOutput(firstSeenEnrichmentUpdatesTag); EnrichmentsConfig enrichmentsConfig = new EnrichmentsConfig(); - enrichmentsConfig.getStorageConfigs().put(EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME, enrichmentStorageConfig); - enrichmentsConfig.getEnrichmentConfigs().put(FIRST_SEEN_ENRICHMENT_TYPE, new EnrichmentConfig(EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME, new EnrichmentFieldsConfig(null, null, null, null))); + enrichmentsConfig.getStorageConfigs() + .put(EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME, enrichmentStorageConfig); + enrichmentsConfig.getEnrichmentConfigs().put(FIRST_SEEN_ENRICHMENT_TYPE, + new EnrichmentConfig(EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME, + new EnrichmentFieldsConfig(null, null, null, null))); enrichmentsConfig.validate(); - HBaseSinkFunction hbaseSink = new HbaseEnrichmentCommandSink(tableName, enrichmentsConfig, params); + HBaseSinkFunction hbaseSink = + new HbaseEnrichmentCommandSink(tableName, enrichmentsConfig, params); firstSeenEnrichmentUpdates.addSink(hbaseSink); return updatedProfileMessages; @@ -159,15 +192,19 @@ protected DataStream updateFirstSeen(ParameterTool params, DataS @Override protected DataStream createRulesSource(StreamExecutionEnvironment env, ParameterTool params) { String topic = params.getRequired("query.input.topic"); - KafkaSource source = new FlinkUtils<>(ScoringRuleCommand.class).createKafkaGenericSource(topic, params, params.get(PARAMS_GROUP_ID, PROFILE_GROUP_ID)); + KafkaSource source = + new FlinkUtils<>(ScoringRuleCommand.class).createKafkaGenericSource(topic, params, + params.get(PARAMS_GROUP_ID, PROFILE_GROUP_ID)); return env.fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Score Rule Source") - .uid("kafka.input.rule.command"); + .uid("kafka.input.rule.command"); } @Override protected void writeScoredRuleCommandResult(ParameterTool params, DataStream results) { String topic = params.getRequired("query.output.topic"); - KafkaSink scoredSink = new FlinkUtils<>(ScoringRuleCommandResult.class).createKafkaSink(topic, params.get(PARAMS_GROUP_ID, PROFILE_GROUP_ID), params); + KafkaSink scoredSink = + new FlinkUtils<>(ScoringRuleCommandResult.class).createKafkaSink(topic, + params.get(PARAMS_GROUP_ID, PROFILE_GROUP_ID), params); results.sinkTo(scoredSink).name("Kafka Score Rule Command Results").uid("kafka.output.rule.command.results"); } @@ -184,7 +221,9 @@ public void accept(PreparedStatement ps, MeasurementDataDto measurementDataDto) int index = 1; ps.setInt(index++, measurementDataDto.getMeasurementId()); for (int i = 0; i < keyCount; i++) { - ps.setString(index++, Iterables.get(Optional.ofNullable(measurementDataDto.getKeys()).orElseGet(ArrayList::new), i, null)); + ps.setString(index++, + Iterables.get(Optional.ofNullable(measurementDataDto.getKeys()).orElseGet(ArrayList::new), i, + null)); } ps.setString(index++, measurementDataDto.getMeasurementName()); ps.setString(index++, measurementDataDto.getMeasurementType()); @@ -194,34 +233,62 @@ public void accept(PreparedStatement ps, MeasurementDataDto measurementDataDto) } private void validateNumericParamAndApply(String paramName, String paramValue, Consumer consumer) { - Preconditions.checkArgument(StringUtils.isNumeric(paramValue), INCORRECT_NUMERIC_MESSAGE_TEMPLATE, paramName, paramValue); + Preconditions.checkArgument(StringUtils.isNumeric(paramValue), INCORRECT_NUMERIC_MESSAGE_TEMPLATE, paramName, + paramValue); consumer.accept(Integer.parseInt(paramValue)); } private static void validatePhoenixParam(ParameterTool params) { String phoenixFlag = params.get(PARAMS_PHOENIX_DB_INIT); if (Boolean.parseBoolean(phoenixFlag)) { - Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_URL)), EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_URL); - if (!PhoenixAuthenticationType.SPNEGO.name().equalsIgnoreCase(params.get(PHOENIX_THIN_PROPERTY_AUTHENTICATION, ""))) { - Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_AVATICA_USER)), EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_AVATICA_USER); - Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_AVATICA_PASSWORD)), EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_AVATICA_PASSWORD); + Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_URL)), + EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_URL); + if (!PhoenixAuthenticationType.SPNEGO.name().equalsIgnoreCase( + params.get(PHOENIX_THIN_PROPERTY_AUTHENTICATION, ""))) { + Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_AVATICA_USER)), + EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_AVATICA_USER); + Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_AVATICA_PASSWORD)), + EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_AVATICA_PASSWORD); } else { - Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_PRINCIPAL)), EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_PRINCIPAL); - Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_KEYTAB)), EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_KEYTAB); + Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_PRINCIPAL)), + EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_PRINCIPAL); + Preconditions.checkArgument(StringUtils.isNotEmpty(params.get(PHOENIX_THIN_PROPERTY_KEYTAB)), + EMPTY_ERROR_MESSAGE_TEMPLATE, PHOENIX_THIN_PROPERTY_KEYTAB); } - ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_DATA_TABLE_NAME), PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_DATA_TABLE_NAME); - ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_METADATA_TABLE_NAME), PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_METADATA_TABLE_NAME); - ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_NAME), PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_NAME); - ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_METADATA_TABLE_NAME), PARAMS_PHOENIX_DB_QUERY_PROFILE_METADATA_TABLE_NAME); - ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_NAME), PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_NAME); - Preconditions.checkArgument(StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_START_WITH)), INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_START_WITH, params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_START_WITH)); - Preconditions.checkArgument(StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_CACHE)), INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_CACHE, params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_CACHE)); - Preconditions.checkArgument(StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_START_WITH)), INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_START_WITH, params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_START_WITH)); - Preconditions.checkArgument(StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_CACHE)), INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_CACHE, params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_CACHE)); - Preconditions.checkArgument(StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_KEY_COUNT)), INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_KEY_COUNT, params.get(PARAMS_PHOENIX_DB_QUERY_KEY_COUNT)); + ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_DATA_TABLE_NAME), + PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_DATA_TABLE_NAME); + ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_METADATA_TABLE_NAME), + PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_METADATA_TABLE_NAME); + ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_NAME), + PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_NAME); + ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_METADATA_TABLE_NAME), + PARAMS_PHOENIX_DB_QUERY_PROFILE_METADATA_TABLE_NAME); + ValidateUtils.validatePhoenixName(params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_NAME), + PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_NAME); + Preconditions.checkArgument( + StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_START_WITH)), + INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_START_WITH, + params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_START_WITH)); + Preconditions.checkArgument( + StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_CACHE)), + INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_CACHE, + params.get(PARAMS_PHOENIX_DB_QUERY_MEASUREMENT_SEQUENCE_CACHE)); + Preconditions.checkArgument( + StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_START_WITH)), + INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_START_WITH, + params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_START_WITH)); + Preconditions.checkArgument( + StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_CACHE)), + INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_CACHE, + params.get(PARAMS_PHOENIX_DB_QUERY_PROFILE_SEQUENCE_CACHE)); + Preconditions.checkArgument(StringUtils.isNumeric(params.get(PARAMS_PHOENIX_DB_QUERY_KEY_COUNT)), + INCORRECT_NUMERIC_MESSAGE_TEMPLATE, PARAMS_PHOENIX_DB_QUERY_KEY_COUNT, + params.get(PARAMS_PHOENIX_DB_QUERY_KEY_COUNT)); } else { - Preconditions.checkArgument(StringUtils.equals(phoenixFlag, "false"), "Invalid properties '%s' value %s (expected 'true' or 'false').", PARAMS_PHOENIX_DB_INIT, phoenixFlag); + Preconditions.checkArgument(StringUtils.equals(phoenixFlag, "false"), + "Invalid properties '%s' value %s (expected 'true' or 'false').", PARAMS_PHOENIX_DB_INIT, + phoenixFlag); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessage.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessage.java index b1bb3ab4..74ac8a52 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessage.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessage.java @@ -12,24 +12,24 @@ package com.cloudera.cyber.profiler; +import java.time.Duration; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Data; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.streaming.api.datastream.DataStream; -import java.time.Duration; -import java.util.Map; - @Data @AllArgsConstructor public class ProfileMessage { private final long ts; private final Map extensions; - public static DataStream watermarkedStreamOf(DataStream inputStream, long maximumLatenessMillis) { + public static DataStream watermarkedStreamOf(DataStream inputStream, + long maximumLatenessMillis) { WatermarkStrategy watermarkStrategy = WatermarkStrategy - .forBoundedOutOfOrderness(Duration.ofMillis(maximumLatenessMillis)) - .withTimestampAssigner((profileMessage, timestamp) -> profileMessage.getTs()); + .forBoundedOutOfOrderness(Duration.ofMillis(maximumLatenessMillis)) + .withTimestampAssigner((profileMessage, timestamp) -> profileMessage.getTs()); return inputStream.assignTimestampsAndWatermarks(watermarkStrategy); } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageFilter.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageFilter.java index eb49c90d..9e7f7213 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageFilter.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageFilter.java @@ -14,21 +14,22 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.scoring.ScoredMessage; -import org.apache.flink.api.common.functions.FilterFunction; - import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.flink.api.common.functions.FilterFunction; /** * Filters messages that can contribute to an aggregation of a profile. * + *

    * The message source must match one of the messages in the source list or if there are no sources specified * don't filter based on source. * + *

    * If the profile contains only measurements that need field values, filter the message if it doesn't contain one of these fields. - * Otherwise, if one of the profile measurement is cyber score details or does not need a field, return the message. The cyber score details - * field is always guaranteed to be specified. + * Otherwise, if one of the profile measurement is cyber score details or does not need a field, return the message. + * The cyber score details field is always guaranteed to be specified. */ public class ProfileMessageFilter implements FilterFunction { @@ -42,10 +43,14 @@ public ProfileMessageFilter(ProfileGroupConfig profileGroupConfig) { } else { this.sources = null; } - this.keyFieldNames = profileGroupConfig.getKeyFieldNames().stream().filter(fieldName -> !ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD.equals(fieldName)).collect(Collectors.toList()); - List optionalFields = profileGroupConfig.getMeasurementFieldNames().stream(). - filter(fieldName -> !ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD.equals(fieldName)). - collect(Collectors.toList()); + this.keyFieldNames = profileGroupConfig.getKeyFieldNames().stream().filter( + fieldName -> !ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD.equals(fieldName)) + .collect(Collectors.toList()); + List optionalFields = profileGroupConfig.getMeasurementFieldNames().stream() + .filter( + fieldName -> !ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD.equals( + fieldName)) + .collect(Collectors.toList()); if (optionalFields.size() == profileGroupConfig.getMeasurements().size()) { this.measurementFieldNames = optionalFields; } else { @@ -57,9 +62,9 @@ public ProfileMessageFilter(ProfileGroupConfig profileGroupConfig) { public boolean filter(ScoredMessage scoredMessage) { Message message = scoredMessage.getMessage(); Map extensions = scoredMessage.getMessage().getExtensions(); - return ((sources == null || sources.contains(message.getSource())) && - extensions != null && - extensions.keySet().containsAll(keyFieldNames) && - (measurementFieldNames == null || measurementFieldNames.stream().anyMatch(extensions::containsKey))); + return ((sources == null || sources.contains(message.getSource())) + && extensions != null + && extensions.keySet().containsAll(keyFieldNames) + && (measurementFieldNames == null || measurementFieldNames.stream().anyMatch(extensions::containsKey))); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageToMeasurementDataDtoMapping.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageToMeasurementDataDtoMapping.java index 4a94d6af..baea068c 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageToMeasurementDataDtoMapping.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageToMeasurementDataDtoMapping.java @@ -12,16 +12,14 @@ package com.cloudera.cyber.profiler; +import static com.cloudera.cyber.profiler.ProfileAggregateFunction.PROFILE_GROUP_NAME_EXTENSION; +import static com.cloudera.cyber.profiler.StatsProfileAggregateFunction.STATS_PROFILE_GROUP_SUFFIX; +import static com.cloudera.cyber.profiler.accumulator.StatsProfileGroupAcc.STATS_EXTENSION_SUFFIXES; + import com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc; import com.cloudera.cyber.profiler.dto.MeasurementDataDto; import com.cloudera.cyber.profiler.dto.MeasurementDto; import com.cloudera.cyber.profiler.dto.ProfileDto; -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.functions.RichFlatMapFunction; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.Collector; - import java.sql.Timestamp; import java.time.Instant; import java.util.ArrayList; @@ -30,10 +28,10 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; - -import static com.cloudera.cyber.profiler.ProfileAggregateFunction.PROFILE_GROUP_NAME_EXTENSION; -import static com.cloudera.cyber.profiler.StatsProfileAggregateFunction.STATS_PROFILE_GROUP_SUFFIX; -import static com.cloudera.cyber.profiler.accumulator.StatsProfileGroupAcc.STATS_EXTENSION_SUFFIXES; +import lombok.RequiredArgsConstructor; +import org.apache.flink.api.common.functions.RichFlatMapFunction; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.Collector; @RequiredArgsConstructor public class ProfileMessageToMeasurementDataDtoMapping extends RichFlatMapFunction { @@ -44,14 +42,16 @@ public class ProfileMessageToMeasurementDataDtoMapping extends RichFlatMapFuncti @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - this.profileNameToKeyFieldNames = this.profileDtos.stream().collect(Collectors.toMap(ProfileDto::getProfileGroupName - , profileDto -> profileDto.getKeyFieldNames().split(","), - (first, second) -> first, - HashMap::new)); - this.profileNameToProfileDto = this.profileDtos.stream().collect(Collectors.toMap(ProfileDto::getProfileGroupName - , Function.identity(), - (first, second) -> first, - HashMap::new)); + this.profileNameToKeyFieldNames = + this.profileDtos.stream().collect(Collectors.toMap(ProfileDto::getProfileGroupName, + profileDto -> profileDto.getKeyFieldNames().split(","), + (first, second) -> first, + HashMap::new)); + this.profileNameToProfileDto = + this.profileDtos.stream().collect(Collectors.toMap(ProfileDto::getProfileGroupName, + Function.identity(), + (first, second) -> first, + HashMap::new)); } @Override @@ -68,14 +68,20 @@ private void flatMapStatsProfile(ProfileMessage profileMessage, Collector { if (STATS_EXTENSION_SUFFIXES.stream().anyMatch(key::endsWith)) { String[] parts = key.split("\\."); - MeasurementDataDto measurementDto = MeasurementDataDto.builder() - .measurementId(getMeasurementIdByName(parts[0]).orElse(-1)) - .measurementName(parts[0]) - .measurementType(parts[1]) - .measurementTime(Timestamp.from(Instant.ofEpochMilli( - Long.parseLong(profileMessage.getExtensions().get(ProfileGroupAcc.START_PERIOD_EXTENSION))))) - .measurementValue(Double.parseDouble(Optional.ofNullable(value).orElse("0"))) - .build(); + MeasurementDataDto measurementDto = + MeasurementDataDto.builder() + .measurementId( + getMeasurementIdByName(parts[0]).orElse(-1)) + .measurementName(parts[0]) + .measurementType(parts[1]) + .measurementTime( + Timestamp.from(Instant.ofEpochMilli( + Long.parseLong( + profileMessage.getExtensions() + .get(ProfileGroupAcc.START_PERIOD_EXTENSION))))) + .measurementValue(Double.parseDouble( + Optional.ofNullable(value).orElse("0"))) + .build(); collector.collect(measurementDto); } }); @@ -83,28 +89,38 @@ private void flatMapStatsProfile(ProfileMessage profileMessage, Collector getMeasurementIdByName(String measurementName) { return profileDtos.stream().flatMap(profile -> profile.getMeasurementDtos().stream()) - .filter(measurementDto -> measurementDto.getResultExtensionName().equals(measurementName)) - .findFirst().map(MeasurementDto::getId); + .filter(measurementDto -> measurementDto.getResultExtensionName().equals(measurementName)) + .findFirst().map(MeasurementDto::getId); } private void flatMapProfile(ProfileMessage profileMessage, Collector collector) { String profileGroupName = profileMessage.getExtensions().get(PROFILE_GROUP_NAME_EXTENSION); ProfileDto profileDto = profileNameToProfileDto.get(profileGroupName); if (profileDto != null) { - final ArrayList keyFieldNamesValues = Arrays.stream(profileNameToKeyFieldNames.get(profileGroupName)) - .map(keyFieldName -> profileMessage.getExtensions().get(keyFieldName)) - .collect(Collectors.toCollection(ArrayList::new)); + final ArrayList keyFieldNamesValues = + Arrays.stream(profileNameToKeyFieldNames.get(profileGroupName)) + .map(keyFieldName -> profileMessage.getExtensions().get(keyFieldName)) + .collect(Collectors.toCollection(ArrayList::new)); for (MeasurementDto measurementDto : profileDto.getMeasurementDtos()) { String measurementName = measurementDto.getResultExtensionName(); - MeasurementDataDto measurementDataDto = MeasurementDataDto.builder() - .measurementId(measurementDto.getId()) - .keys(keyFieldNamesValues) - .measurementName(measurementName) - .profileId(profileDto.getId()) - .measurementTime(Timestamp.from(Instant.ofEpochMilli( - Long.parseLong(profileMessage.getExtensions().get(ProfileGroupAcc.START_PERIOD_EXTENSION))))) - .measurementValue(Double.parseDouble(Optional.ofNullable(profileMessage.getExtensions()).map(map -> map.get(measurementName)).orElse("0"))) - .build(); + MeasurementDataDto measurementDataDto = + MeasurementDataDto.builder() + .measurementId(measurementDto.getId()) + .keys(keyFieldNamesValues) + .measurementName(measurementName) + .profileId(profileDto.getId()) + .measurementTime( + Timestamp.from(Instant.ofEpochMilli( + Long.parseLong( + profileMessage.getExtensions() + .get(ProfileGroupAcc.START_PERIOD_EXTENSION))))) + .measurementValue(Double.parseDouble( + Optional.ofNullable( + profileMessage.getExtensions()) + .map(map -> map.get( + measurementName)) + .orElse("0"))) + .build(); collector.collect(measurementDataDto); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageToMessageMap.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageToMessageMap.java index 7116f0bc..a4f2e8b5 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageToMessageMap.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileMessageToMessageMap.java @@ -12,29 +12,29 @@ package com.cloudera.cyber.profiler; +import static com.cloudera.cyber.profiler.ProfileAggregateFunction.PROFILE_SOURCE; +import static com.cloudera.cyber.profiler.ProfileAggregateFunction.PROFILE_TOPIC_NAME; + import com.cloudera.cyber.Message; import com.cloudera.cyber.SignedSourceKey; import org.apache.flink.api.common.functions.MapFunction; -import static com.cloudera.cyber.profiler.ProfileAggregateFunction.PROFILE_SOURCE; -import static com.cloudera.cyber.profiler.ProfileAggregateFunction.PROFILE_TOPIC_NAME; - public class ProfileMessageToMessageMap implements MapFunction { @Override public Message map(ProfileMessage profileMessage) throws Exception { - SignedSourceKey sourceKey = SignedSourceKey.builder() - .topic(PROFILE_TOPIC_NAME) - .partition(0) - .offset(0) - .signature(new byte[1]) - .build(); + SignedSourceKey sourceKey = SignedSourceKey.builder() + .topic(PROFILE_TOPIC_NAME) + .partition(0) + .offset(0) + .signature(new byte[1]) + .build(); return Message.builder() - .extensions(profileMessage.getExtensions()) - .ts(profileMessage.getTs()) - .originalSource(sourceKey) - .source(PROFILE_SOURCE) - .build(); + .extensions(profileMessage.getExtensions()) + .ts(profileMessage.getTs()) + .originalSource(sourceKey) + .source(PROFILE_SOURCE) + .build(); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileStatsJoin.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileStatsJoin.java index 37e1dbea..204c9b92 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileStatsJoin.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileStatsJoin.java @@ -13,11 +13,10 @@ package com.cloudera.cyber.profiler; import com.cloudera.cyber.profiler.accumulator.StatsProfileGroupAcc; -import org.apache.flink.api.common.functions.JoinFunction; - import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; +import org.apache.flink.api.common.functions.JoinFunction; public class ProfileStatsJoin implements JoinFunction { @@ -31,8 +30,9 @@ public static boolean isStatsExtension(String extensionName) { @Override public ProfileMessage join(ProfileMessage profileMessage, ProfileMessage statsMessage) { - Map statsExtensions = statsMessage.getExtensions().entrySet().stream().filter(e -> isStatsExtension(e.getKey())). - collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map statsExtensions = + statsMessage.getExtensions().entrySet().stream().filter(e -> isStatsExtension(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Map extensionsWithStats = new HashMap<>(profileMessage.getExtensions()); extensionsWithStats.putAll(statsExtensions); return new ProfileMessage(profileMessage.getTs(), extensionsWithStats); diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileUtils.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileUtils.java index 26ad31b3..ea4c5625 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileUtils.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ProfileUtils.java @@ -17,10 +17,12 @@ public static boolean compareProfileDto(ProfileDto profileDto1, ProfileDto profi return true; } - return Objects.equals(profileDto1.getProfileGroupName(), profileDto2.getProfileGroupName()) && Objects.equals(profileDto1.getKeyFieldNames(), profileDto2.getKeyFieldNames()) - && Utils.isTimeEqual(profileDto1.getPeriodDuration(), profileDto1.getPeriodDurationUnit(), profileDto2.getPeriodDuration(), profileDto2.getPeriodDurationUnit()) - && Utils.isTimeEqual(profileDto1.getStatsSlide(), profileDto1.getStatsSlideUnit(), - profileDto2.getStatsSlide(), profileDto2.getStatsSlideUnit()); + return Objects.equals(profileDto1.getProfileGroupName(), profileDto2.getProfileGroupName()) + && Objects.equals(profileDto1.getKeyFieldNames(), profileDto2.getKeyFieldNames()) + && Utils.isTimeEqual(profileDto1.getPeriodDuration(), profileDto1.getPeriodDurationUnit(), + profileDto2.getPeriodDuration(), profileDto2.getPeriodDurationUnit()) + && Utils.isTimeEqual(profileDto1.getStatsSlide(), profileDto1.getStatsSlideUnit(), + profileDto2.getStatsSlide(), profileDto2.getStatsSlideUnit()); } @@ -31,10 +33,14 @@ public static boolean compareMeasurementDto(MeasurementDto measurementDto1, Meas if (measurementDto1 == measurementDto2) { return true; } - return Objects.equals(measurementDto1.getFieldName(), measurementDto2.getFieldName()) && Objects.equals(measurementDto1.getResultExtensionName(), measurementDto2.getResultExtensionName()) && Objects.equals( - measurementDto1.getAggregationMethod(), measurementDto2.getAggregationMethod()) && Objects.equals(measurementDto1.getFormat(), - measurementDto2.getFormat()) && Utils.isTimeEqual(measurementDto1.getFirstSeenExpirationDuration(), measurementDto1.getFirstSeenExpirationDurationUnit(), measurementDto2.getFirstSeenExpirationDuration(), - measurementDto2.getFirstSeenExpirationDurationUnit()); + return Objects.equals(measurementDto1.getFieldName(), measurementDto2.getFieldName()) + && Objects.equals(measurementDto1.getResultExtensionName(), measurementDto2.getResultExtensionName()) + && Objects.equals( + measurementDto1.getAggregationMethod(), measurementDto2.getAggregationMethod()) + && Objects.equals(measurementDto1.getFormat(), + measurementDto2.getFormat()) && Utils.isTimeEqual(measurementDto1.getFirstSeenExpirationDuration(), + measurementDto1.getFirstSeenExpirationDurationUnit(), measurementDto2.getFirstSeenExpirationDuration(), + measurementDto2.getFirstSeenExpirationDurationUnit()); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ScoredMessageToProfileMessageMap.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ScoredMessageToProfileMessageMap.java index 7a9e4d49..386661d2 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ScoredMessageToProfileMessageMap.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/ScoredMessageToProfileMessageMap.java @@ -13,11 +13,10 @@ package com.cloudera.cyber.profiler; import com.cloudera.cyber.scoring.ScoredMessage; -import org.apache.flink.api.common.functions.MapFunction; - import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.flink.api.common.functions.MapFunction; public class ScoredMessageToProfileMessageMap implements MapFunction { @@ -34,17 +33,19 @@ public ScoredMessageToProfileMessageMap(ProfileGroupConfig profileGroupConfig) { public ProfileMessage map(ScoredMessage scoredMessage) { Map reducedExtensions = new HashMap<>(); Map allExtensions = scoredMessage.getMessage().getExtensions(); - allFieldNames.forEach(fieldName -> {{ - if (CYBER_SCORE_FIELD.equals(fieldName)) { - reducedExtensions.put(CYBER_SCORE_FIELD, scoredMessage.getCyberScore().toString()); - } else { - String extensionValue = allExtensions.get(fieldName); - if (extensionValue != null) { - reducedExtensions.put(fieldName, extensionValue); + allFieldNames.forEach(fieldName -> { + { + if (CYBER_SCORE_FIELD.equals(fieldName)) { + reducedExtensions.put(CYBER_SCORE_FIELD, scoredMessage.getCyberScore().toString()); + } else { + String extensionValue = allExtensions.get(fieldName); + if (extensionValue != null) { + reducedExtensions.put(fieldName, extensionValue); + } } } - }}); + }); - return new ProfileMessage(scoredMessage.getTs(), reducedExtensions); + return new ProfileMessage(scoredMessage.getTs(), reducedExtensions); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/StatsProfileAggregateFunction.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/StatsProfileAggregateFunction.java index cb9384c5..19e08f45 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/StatsProfileAggregateFunction.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/StatsProfileAggregateFunction.java @@ -14,7 +14,6 @@ import com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc; import com.cloudera.cyber.profiler.accumulator.StatsProfileGroupAcc; - import java.text.DecimalFormat; import java.util.Map; import java.util.stream.Collectors; @@ -34,8 +33,9 @@ public ProfileGroupAcc createAccumulator() { } protected Map getMeasurementFormats() { - return profileGroupConfig.getMeasurements().stream().filter(ProfileMeasurementConfig::hasStats). - collect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName, v -> DEFAULT_FORMAT)); + return profileGroupConfig.getMeasurements().stream().filter(ProfileMeasurementConfig::hasStats) + .collect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName, + v -> DEFAULT_FORMAT)); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/StatsProfileKeySelector.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/StatsProfileKeySelector.java index f245d63f..f1748675 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/StatsProfileKeySelector.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/StatsProfileKeySelector.java @@ -12,7 +12,6 @@ package com.cloudera.cyber.profiler; -import com.cloudera.cyber.Message; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.functions.KeySelector; @@ -20,7 +19,9 @@ public class StatsProfileKeySelector implements KeySelector>> accFactory = - new HashMap>>() {{ - put(SUM, DoubleCounter::new); - put(COUNT, DoubleCounter::new); - put(MIN, DoubleMinimum::new); - put(MAX, DoubleMaximum::new); - put(COUNT_DISTINCT, CountDistinctAcc::new); - }}; + new HashMap>>() { + { + put(SUM, DoubleCounter::new); + put(COUNT, DoubleCounter::new); + put(MIN, DoubleMinimum::new); + put(MAX, DoubleMaximum::new); + put(COUNT_DISTINCT, CountDistinctAcc::new); + } + }; @FunctionalInterface interface ProfileUpdateFunction { - void update(ProfileMeasurementConfig config, ProfileMessage message, Accumulator acc); + void update(ProfileMeasurementConfig config, ProfileMessage message, + Accumulator acc); } private static final Map accUpdate = - new HashMap() {{ - put(SUM, FieldValueProfileGroupAcc::updateDoubleAccumulator); - put(COUNT, FieldValueProfileGroupAcc::updateCounterAccumulator); - put(MIN, FieldValueProfileGroupAcc::updateDoubleAccumulator); - put(MAX, FieldValueProfileGroupAcc::updateDoubleAccumulator); - put(COUNT_DISTINCT, FieldValueProfileGroupAcc::updateStringAccumulator); - }}; + new HashMap() { + { + put(SUM, FieldValueProfileGroupAcc::updateDoubleAccumulator); + put(COUNT, FieldValueProfileGroupAcc::updateCounterAccumulator); + put(MIN, FieldValueProfileGroupAcc::updateDoubleAccumulator); + put(MAX, FieldValueProfileGroupAcc::updateDoubleAccumulator); + put(COUNT_DISTINCT, FieldValueProfileGroupAcc::updateStringAccumulator); + } + }; @FunctionalInterface interface ProfileExtensionFunction { - void getExtensions(ProfileMeasurementConfig config , Accumulator acc, Map extensions, DecimalFormat format); + void getExtensions(ProfileMeasurementConfig config, Accumulator acc, + Map extensions, DecimalFormat format); } private static Stream getAccumulatedMeasurements(ProfileGroupConfig profileGroupConfig) { @@ -65,30 +82,34 @@ private static Stream getAccumulatedMeasurements(Profi } private static final Map extensionUpdate = - new HashMap() {{ - put(SUM, FieldValueProfileGroupAcc::getDoubleAccumulatorExtensions); - put(COUNT, FieldValueProfileGroupAcc::getDoubleAccumulatorExtensions); - put(MIN, FieldValueProfileGroupAcc::getDoubleAccumulatorExtensions); - put(MAX, FieldValueProfileGroupAcc::getDoubleAccumulatorExtensions); - put(COUNT_DISTINCT, FieldValueProfileGroupAcc::getCountDistinctAccumulatorExtensions); - }}; + new HashMap() { + { + put(SUM, FieldValueProfileGroupAcc::getDoubleAccumulatorExtensions); + put(COUNT, FieldValueProfileGroupAcc::getDoubleAccumulatorExtensions); + put(MIN, FieldValueProfileGroupAcc::getDoubleAccumulatorExtensions); + put(MAX, FieldValueProfileGroupAcc::getDoubleAccumulatorExtensions); + put(COUNT_DISTINCT, FieldValueProfileGroupAcc::getCountDistinctAccumulatorExtensions); + } + }; List keyFieldValues = new ArrayList<>(); public FieldValueProfileGroupAcc(ProfileGroupConfig profileGroupConfig) { - super(getAccumulatedMeasurements(profileGroupConfig). - map(config -> accFactory.get(config.getAggregationMethod()).get()). - collect(Collectors.toList())); + super(getAccumulatedMeasurements(profileGroupConfig) + .map(config -> accFactory.get(config.getAggregationMethod()).get()) + .collect(Collectors.toList())); } @Override protected void updateAccumulators(ProfileMessage message, ProfileGroupConfig profileGroupConfig) { Map extensions = message.getExtensions(); if (keyFieldValues.isEmpty()) { - keyFieldValues = profileGroupConfig.getKeyFieldNames().stream().map(extensions::get).collect(Collectors.toList()); + keyFieldValues = + profileGroupConfig.getKeyFieldNames().stream().map(extensions::get).collect(Collectors.toList()); } - Iterator measurementConfigIter = getAccumulatedMeasurements(profileGroupConfig).iterator(); + Iterator measurementConfigIter = + getAccumulatedMeasurements(profileGroupConfig).iterator(); Iterator> accumulatorIter = accumulators.iterator(); while (measurementConfigIter.hasNext() && accumulatorIter.hasNext()) { ProfileMeasurementConfig measurementConfig = measurementConfigIter.next(); @@ -99,20 +120,22 @@ protected void updateAccumulators(ProfileMessage message, ProfileGroupConfig pro } @Override - protected void addExtensions(ProfileGroupConfig profileGroupConfig, Map extensions, Map measurementFormats) { + protected void addExtensions(ProfileGroupConfig profileGroupConfig, Map extensions, + Map measurementFormats) { Iterator keyFieldNameIter = profileGroupConfig.getKeyFieldNames().iterator(); Iterator keyFieldValueIter = keyFieldValues.iterator(); - while(keyFieldNameIter.hasNext() && keyFieldValueIter.hasNext()) { + while (keyFieldNameIter.hasNext() && keyFieldValueIter.hasNext()) { extensions.put(keyFieldNameIter.next(), keyFieldValueIter.next()); } Iterator> myAccIter = accumulators.iterator(); Iterator measurementIter = profileGroupConfig.getMeasurements().iterator(); - while(myAccIter.hasNext() && measurementIter.hasNext()) { + while (myAccIter.hasNext() && measurementIter.hasNext()) { ProfileMeasurementConfig measurementConfig = measurementIter.next(); if (!measurementConfig.getAggregationMethod().equals(FIRST_SEEN)) { Accumulator acc = myAccIter.next(); DecimalFormat measurementFormat = measurementFormats.get(measurementConfig.getResultExtensionName()); - extensionUpdate.get(measurementConfig.getAggregationMethod()).getExtensions(measurementConfig, acc, extensions, measurementFormat); + extensionUpdate.get(measurementConfig.getAggregationMethod()) + .getExtensions(measurementConfig, acc, extensions, measurementFormat); } } } @@ -121,35 +144,43 @@ protected void addExtensions(ProfileGroupConfig profileGroupConfig, Map acc, Map extensions, DecimalFormat format) { - extensions.put(config.getResultExtensionName(), format.format(((Accumulator)acc).getLocalValue())); + private static void getDoubleAccumulatorExtensions(ProfileMeasurementConfig config, + Accumulator acc, + Map extensions, DecimalFormat format) { + extensions.put(config.getResultExtensionName(), format.format(((Accumulator) acc).getLocalValue())); } - private static void getCountDistinctAccumulatorExtensions(ProfileMeasurementConfig config, Accumulator acc, Map extensions, DecimalFormat format) { - extensions.put(config.getResultExtensionName(), format.format(((CountDistinctAcc)acc).getLocalValue().getUnion().getResult().getEstimate())); + private static void getCountDistinctAccumulatorExtensions(ProfileMeasurementConfig config, + Accumulator acc, + Map extensions, DecimalFormat format) { + extensions.put(config.getResultExtensionName(), + format.format(((CountDistinctAcc) acc).getLocalValue().getUnion().getResult().getEstimate())); } - private static void updateDoubleAccumulator(ProfileMeasurementConfig config, ProfileMessage message, Accumulator acc) { + private static void updateDoubleAccumulator(ProfileMeasurementConfig config, ProfileMessage message, + Accumulator acc) { Double fieldValueDouble = getFieldValueAsDouble(message, config.getFieldName()); if (fieldValueDouble != null) { ((Accumulator) acc).add(fieldValueDouble); } } - private static void updateCounterAccumulator(ProfileMeasurementConfig config, ProfileMessage message, Accumulator acc) { - ((Accumulator)acc).add(1D); + private static void updateCounterAccumulator(ProfileMeasurementConfig config, ProfileMessage message, + Accumulator acc) { + ((Accumulator) acc).add(1D); } - private static void updateStringAccumulator(ProfileMeasurementConfig config, ProfileMessage message, Accumulator acc) { + private static void updateStringAccumulator(ProfileMeasurementConfig config, ProfileMessage message, + Accumulator acc) { String stringFieldValue = message.getExtensions().get(config.getFieldName()); if (stringFieldValue != null) { - ((Accumulator)acc).add(stringFieldValue); + ((Accumulator) acc).add(stringFieldValue); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/ProfileGroupAcc.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/ProfileGroupAcc.java index 1aef97b0..82957120 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/ProfileGroupAcc.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/ProfileGroupAcc.java @@ -14,16 +14,15 @@ import com.cloudera.cyber.profiler.ProfileGroupConfig; import com.cloudera.cyber.profiler.ProfileMessage; -import org.apache.flink.api.common.accumulators.Accumulator; -import org.apache.flink.api.common.accumulators.LongMaximum; -import org.apache.flink.api.common.accumulators.LongMinimum; - import java.io.Serializable; import java.text.DecimalFormat; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import org.apache.flink.api.common.accumulators.Accumulator; +import org.apache.flink.api.common.accumulators.LongMaximum; +import org.apache.flink.api.common.accumulators.LongMinimum; public abstract class ProfileGroupAcc { public static final String START_PERIOD_EXTENSION = "start_period"; @@ -39,7 +38,8 @@ public ProfileGroupAcc(List> accumulators protected abstract void updateAccumulators(ProfileMessage message, ProfileGroupConfig profileGroupConfig); - protected abstract void addExtensions(ProfileGroupConfig profileGroupConfig, Map extensions, Map measurementFormats); + protected abstract void addExtensions(ProfileGroupConfig profileGroupConfig, Map extensions, + Map measurementFormats); protected static Double getFieldValueAsDouble(ProfileMessage message, String fieldName) { String extensionValue = message.getExtensions().get(fieldName); @@ -59,7 +59,8 @@ public void addMessage(ProfileMessage message, ProfileGroupConfig profileGroupCo updateAccumulators(message, profileGroupConfig); } - public Map getProfileExtensions(ProfileGroupConfig profileGroupConfig, Map measurementFormats) { + public Map getProfileExtensions(ProfileGroupConfig profileGroupConfig, + Map measurementFormats) { Map extensions = new HashMap<>(); extensions.put(START_PERIOD_EXTENSION, Long.toString(startPeriodTimestamp.getLocalValuePrimitive())); extensions.put(END_PERIOD_EXTENSION, Long.toString(endPeriodTimestamp.getLocalValuePrimitive())); diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/SerializableUnion.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/SerializableUnion.java index 31a5766d..efdff8c3 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/SerializableUnion.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/SerializableUnion.java @@ -12,18 +12,17 @@ package com.cloudera.cyber.profiler.accumulator; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.theta.SetOperation; -import org.apache.datasketches.theta.Sketch; -import org.apache.datasketches.theta.Sketches; -import org.apache.datasketches.theta.Union; +import static org.apache.datasketches.theta.SetOperation.builder; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; - -import static org.apache.datasketches.theta.SetOperation.builder; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.theta.SetOperation; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.Union; public class SerializableUnion implements Serializable { @@ -38,7 +37,7 @@ Union getUnion() { } private void writeObject(ObjectOutputStream oos) - throws IOException { + throws IOException { oos.defaultWriteObject(); byte[] unionBytes = union.getResult().toByteArray(); oos.writeInt(unionBytes.length); @@ -46,7 +45,7 @@ private void writeObject(ObjectOutputStream oos) } private void readObject(ObjectInputStream ois) - throws ClassNotFoundException, IOException { + throws ClassNotFoundException, IOException { ois.defaultReadObject(); int unionByteLength = ois.readInt(); byte[] unionBytes = new byte[unionByteLength]; diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/StatsAcc.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/StatsAcc.java index 7081aef1..82b2e989 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/StatsAcc.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/StatsAcc.java @@ -12,14 +12,13 @@ package com.cloudera.cyber.profiler.accumulator; +import java.util.ArrayList; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.flink.api.common.accumulators.Accumulator; -import java.util.ArrayList; - public class StatsAcc implements Accumulator> { - ArrayList mergedStatistics; + ArrayList mergedStatistics; public StatsAcc() { resetLocal(); @@ -48,7 +47,7 @@ public void resetLocal() { @Override public void merge(Accumulator> other) { - mergedStatistics.addAll(((StatsAcc)other).mergedStatistics); + mergedStatistics.addAll(((StatsAcc) other).mergedStatistics); } @Override diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/StatsProfileGroupAcc.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/StatsProfileGroupAcc.java index f9bd5df2..7075494f 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/StatsProfileGroupAcc.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/accumulator/StatsProfileGroupAcc.java @@ -16,38 +16,40 @@ import com.cloudera.cyber.profiler.ProfileMeasurementConfig; import com.cloudera.cyber.profiler.ProfileMessage; import com.google.common.collect.Lists; -import org.apache.commons.math3.stat.descriptive.AggregateSummaryStatistics; -import org.apache.commons.math3.stat.descriptive.StatisticalSummaryValues; -import org.apache.flink.api.common.accumulators.Accumulator; - import java.io.Serializable; import java.text.DecimalFormat; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.math3.stat.descriptive.AggregateSummaryStatistics; +import org.apache.commons.math3.stat.descriptive.StatisticalSummaryValues; +import org.apache.flink.api.common.accumulators.Accumulator; public class StatsProfileGroupAcc extends ProfileGroupAcc { public static final String MIN_RESULT_SUFFIX = ".min"; public static final String MAX_RESULT_SUFFIX = ".max"; public static final String MEAN_RESULT_SUFFIX = ".mean"; public static final String STDDEV_RESULT_SUFFIX = ".stddev"; - public static final List STATS_EXTENSION_SUFFIXES = Lists.newArrayList(MIN_RESULT_SUFFIX, MAX_RESULT_SUFFIX, MEAN_RESULT_SUFFIX, STDDEV_RESULT_SUFFIX); + public static final List STATS_EXTENSION_SUFFIXES = + Lists.newArrayList(MIN_RESULT_SUFFIX, MAX_RESULT_SUFFIX, MEAN_RESULT_SUFFIX, STDDEV_RESULT_SUFFIX); private static final DecimalFormat DEFAULT_FORMAT = new DecimalFormat("0.00"); public StatsProfileGroupAcc(ProfileGroupConfig profileGroupConfig) { - super(profileGroupConfig.getMeasurements().stream().filter(ProfileMeasurementConfig::hasStats). - map(config -> new StatsAcc()).collect(Collectors.toList())); + super(profileGroupConfig.getMeasurements().stream().filter(ProfileMeasurementConfig::hasStats) + .map(config -> new StatsAcc()).collect(Collectors.toList())); } @Override protected void updateAccumulators(ProfileMessage message, ProfileGroupConfig profileGroupConfig) { - Iterator measurementIter = profileGroupConfig.getMeasurements().stream(). - filter(ProfileMeasurementConfig::hasStats).iterator(); - Iterator> accumulatorIter= accumulators.iterator(); - while(measurementIter.hasNext() && accumulatorIter.hasNext()) { + Iterator measurementIter = profileGroupConfig.getMeasurements().stream() + .filter( + ProfileMeasurementConfig::hasStats) + .iterator(); + Iterator> accumulatorIter = accumulators.iterator(); + while (measurementIter.hasNext() && accumulatorIter.hasNext()) { ProfileMeasurementConfig measurementConfig = measurementIter.next(); - StatsAcc accumulator = (StatsAcc)accumulatorIter.next(); + StatsAcc accumulator = (StatsAcc) accumulatorIter.next(); Double fieldValue = getFieldValueAsDouble(message, measurementConfig.getResultExtensionName()); if (fieldValue != null) { accumulator.getLocalValue().get(0).addValue(fieldValue); @@ -56,20 +58,24 @@ protected void updateAccumulators(ProfileMessage message, ProfileGroupConfig pro } @Override - protected void addExtensions(ProfileGroupConfig profileGroupConfig, Map extensions, Map ignored) { - Iterator measurementIter = profileGroupConfig.getMeasurements().stream(). - filter(ProfileMeasurementConfig::hasStats).iterator(); - Iterator> accumulatorIter= accumulators.iterator(); - while(measurementIter.hasNext() && accumulatorIter.hasNext()) { + protected void addExtensions(ProfileGroupConfig profileGroupConfig, Map extensions, + Map ignored) { + Iterator measurementIter = profileGroupConfig.getMeasurements().stream() + .filter( + ProfileMeasurementConfig::hasStats) + .iterator(); + Iterator> accumulatorIter = accumulators.iterator(); + while (measurementIter.hasNext() && accumulatorIter.hasNext()) { ProfileMeasurementConfig measurementConfig = measurementIter.next(); - StatsAcc accumulator = (StatsAcc)accumulatorIter.next(); + StatsAcc accumulator = (StatsAcc) accumulatorIter.next(); StatisticalSummaryValues stats = AggregateSummaryStatistics.aggregate(accumulator.getLocalValue()); String resultExtensionName = measurementConfig.getResultExtensionName(); extensions.put(resultExtensionName.concat(MIN_RESULT_SUFFIX), DEFAULT_FORMAT.format(stats.getMin())); extensions.put(resultExtensionName.concat(MAX_RESULT_SUFFIX), DEFAULT_FORMAT.format(stats.getMax())); extensions.put(resultExtensionName.concat(MEAN_RESULT_SUFFIX), DEFAULT_FORMAT.format(stats.getMean())); - extensions.put(resultExtensionName.concat(STDDEV_RESULT_SUFFIX), DEFAULT_FORMAT.format(stats.getStandardDeviation())); + extensions.put(resultExtensionName.concat(STDDEV_RESULT_SUFFIX), + DEFAULT_FORMAT.format(stats.getStandardDeviation())); } } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/MeasurementDataDto.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/MeasurementDataDto.java index 7988a372..05bea9aa 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/MeasurementDataDto.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/MeasurementDataDto.java @@ -12,17 +12,15 @@ package com.cloudera.cyber.profiler.dto; +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.ArrayList; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.io.Serializable; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.List; - @Data @NoArgsConstructor(force = true, access = AccessLevel.PUBLIC) @AllArgsConstructor diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/MeasurementDto.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/MeasurementDto.java index aae0d9d2..d2a109b7 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/MeasurementDto.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/MeasurementDto.java @@ -37,13 +37,13 @@ public class MeasurementDto implements Serializable { public static MeasurementDto of(ProfileMeasurementConfig config) { return MeasurementDto.builder() - .fieldName(config.getFieldName()) - .resultExtensionName(config.getResultExtensionName()) - .aggregationMethod(config.getAggregationMethod().toString()) - .calculateStats(config.getCalculateStats()) - .format(config.getFormat()) - .firstSeenExpirationDuration(config.getFirstSeenExpirationDuration()) - .firstSeenExpirationDurationUnit(config.getFirstSeenExpirationDurationUnit()) - .build(); + .fieldName(config.getFieldName()) + .resultExtensionName(config.getResultExtensionName()) + .aggregationMethod(config.getAggregationMethod().toString()) + .calculateStats(config.getCalculateStats()) + .format(config.getFormat()) + .firstSeenExpirationDuration(config.getFirstSeenExpirationDuration()) + .firstSeenExpirationDurationUnit(config.getFirstSeenExpirationDurationUnit()) + .build(); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/ProfileDto.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/ProfileDto.java index 07fd9cc2..c86df8d1 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/ProfileDto.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/dto/ProfileDto.java @@ -41,13 +41,15 @@ public class ProfileDto implements Serializable { public static ProfileDto of(ProfileGroupConfig profileGroup) { ArrayList keyFieldNames = new ArrayList<>(profileGroup.getKeyFieldNames()); return ProfileDto.builder() - .profileGroupName(profileGroup.getProfileGroupName()) - .keyFieldNames(keyFieldNames.stream().sorted().collect(Collectors.joining(","))) - .measurementDtos(new ArrayList<>(profileGroup.getMeasurements().stream().map(MeasurementDto::of).collect(Collectors.toList()))) - .periodDuration(profileGroup.getPeriodDuration()) - .periodDurationUnit(profileGroup.getPeriodDurationUnit()) - .statsSlide(profileGroup.getStatsSlide()) - .statsSlideUnit(profileGroup.getStatsSlideUnit()) - .build(); + .profileGroupName(profileGroup.getProfileGroupName()) + .keyFieldNames(keyFieldNames.stream().sorted().collect(Collectors.joining(","))) + .measurementDtos(new ArrayList<>( + profileGroup.getMeasurements().stream().map(MeasurementDto::of) + .collect(Collectors.toList()))) + .periodDuration(profileGroup.getPeriodDuration()) + .periodDurationUnit(profileGroup.getPeriodDurationUnit()) + .statsSlide(profileGroup.getStatsSlide()) + .statsSlideUnit(profileGroup.getStatsSlideUnit()) + .build(); } } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixAuthenticationType.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixAuthenticationType.java index fe1aed96..2c0f32f3 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixAuthenticationType.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixAuthenticationType.java @@ -13,5 +13,5 @@ package com.cloudera.cyber.profiler.phoenix; public enum PhoenixAuthenticationType { - NONE,BASIC,DIGEST,SPNEGO + NONE, BASIC, DIGEST, SPNEGO } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixThickClient.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixThickClient.java index 06bd1bc6..8cb691d7 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixThickClient.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixThickClient.java @@ -13,9 +13,6 @@ package com.cloudera.cyber.profiler.phoenix; import com.cloudera.cyber.flink.Utils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -25,9 +22,9 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; @Slf4j public class PhoenixThickClient { @@ -79,7 +76,8 @@ public void insertIntoTable(String sql, Consumer consumer) th } - public List selectListResultWithParams(String sql, Function mapper, Consumer consumer) throws SQLException { + public List selectListResultWithParams(String sql, Function mapper, + Consumer consumer) throws SQLException { List results = new ArrayList<>(); try (Connection conn = DriverManager.getConnection(dbUrl, userName, password)) { try (PreparedStatement ps = conn.prepareStatement(sql)) { @@ -98,11 +96,12 @@ public List selectListResultWithParams(String sql, Function } } - public T selectResultWithParams(String sql, Function mapper, Consumer consumer) throws SQLException { + public T selectResultWithParams(String sql, Function mapper, Consumer consumer) + throws SQLException { try (Connection conn = DriverManager.getConnection(dbUrl, userName, password)) { try (PreparedStatement ps = conn.prepareStatement(sql)) { consumer.accept(ps); - try(ResultSet resultSet = ps.executeQuery(sql)) { + try (ResultSet resultSet = ps.executeQuery(sql)) { if (resultSet.next()) { return mapper.apply(resultSet); } diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixThinClient.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixThinClient.java index 9aaef769..defc64f4 100644 --- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixThinClient.java +++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/phoenix/PhoenixThinClient.java @@ -13,9 +13,6 @@ package com.cloudera.cyber.profiler.phoenix; import com.cloudera.cyber.flink.Utils; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.api.java.utils.ParameterTool; - import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -27,6 +24,8 @@ import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.java.utils.ParameterTool; @Slf4j public class PhoenixThinClient { @@ -49,16 +48,22 @@ public class PhoenixThinClient { public PhoenixThinClient(ParameterTool params) { this.userName = params.get(PHOENIX_THIN_PROPERTY_AVATICA_USER); this.password = params.get(PHOENIX_THIN_PROPERTY_AVATICA_PASSWORD); - dbUrl = "jdbc:phoenix:thin:url=" + - params.get(PHOENIX_THIN_PROPERTY_URL) + ";" + - Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_SERIALIZATION)).map(str -> String.format("serialization=%s;", str)).orElse("serialization=PROTOBUF;") + - Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_AUTHENTICATION)).map(str -> String.format("authentication=%s;", str)).orElse("authentication=BASIC;") + - Optional.ofNullable(userName).map(str -> String.format("avatica_user=%s;", str)).orElse("") + - Optional.ofNullable(password).map(str -> String.format("avatica_password=%s;", str)).orElse("") + - Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_PRINCIPAL)).map(str -> String.format("principal=%s;", str)).orElse("") + - Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_KEYTAB)).map(str -> String.format("keytab=%s;", str)).orElse("") + - Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_TRUSTSTORE)).map(str -> String.format("truststore=%s;", str)).orElse("") + - Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_TRUSTSTORE_PASSWORD)).map(str -> String.format("truststore_password=%s;", str)).orElse(""); + dbUrl = "jdbc:phoenix:thin:url=" + + params.get(PHOENIX_THIN_PROPERTY_URL) + ";" + + Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_SERIALIZATION)) + .map(str -> String.format("serialization=%s;", str)).orElse("serialization=PROTOBUF;") + + Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_AUTHENTICATION)) + .map(str -> String.format("authentication=%s;", str)).orElse("authentication=BASIC;") + + Optional.ofNullable(userName).map(str -> String.format("avatica_user=%s;", str)).orElse("") + + Optional.ofNullable(password).map(str -> String.format("avatica_password=%s;", str)).orElse("") + + Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_PRINCIPAL)) + .map(str -> String.format("principal=%s;", str)).orElse("") + + Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_KEYTAB)) + .map(str -> String.format("keytab=%s;", str)).orElse("") + + Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_TRUSTSTORE)) + .map(str -> String.format("truststore=%s;", str)).orElse("") + + Optional.ofNullable(params.get(PHOENIX_THIN_PROPERTY_TRUSTSTORE_PASSWORD)) + .map(str -> String.format("truststore_password=%s;", str)).orElse(""); } private Object connectionResultMetaData(Function function) { @@ -118,7 +123,8 @@ public List selectListResult(String sql, Function mapper) t return selectListResult(sql, mapper, null); } - public List selectListResult(String sql, Function mapper, Consumer consumer) throws SQLException { + public List selectListResult(String sql, Function mapper, Consumer consumer) + throws SQLException { List results = new ArrayList<>(); Optional> optionalConsumer = Optional.ofNullable(consumer); try (Connection conn = DriverManager.getConnection(dbUrl)) { @@ -141,7 +147,8 @@ public T selectResult(String sql, Function mapper) throws SQLE return selectResult(sql, mapper, null); } - public T selectResult(String sql, Function mapper, Consumer consumer) throws SQLException { + public T selectResult(String sql, Function mapper, Consumer consumer) + throws SQLException { Optional> optionalConsumer = Optional.ofNullable(consumer); try (Connection conn = DriverManager.getConnection(dbUrl)) { try (PreparedStatement ps = conn.prepareStatement(sql)) { diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/FirstSeenHbaseLookupTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/FirstSeenHbaseLookupTest.java index 35835803..788a4157 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/FirstSeenHbaseLookupTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/FirstSeenHbaseLookupTest.java @@ -24,6 +24,12 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; import org.apache.flink.metrics.SimpleCounter; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; @@ -31,8 +37,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.*; - public class FirstSeenHbaseLookupTest extends FirstSeenHbaseLookup { private static final long CURRENT_TIMESTAMP = MessageUtils.getCurrentTimestamp(); @@ -135,11 +139,11 @@ private void verifyFirstSeen(String key1, String key2, long startPeriodTimestamp } private static ProfileGroupConfig createProfileGroupConfig() { - ArrayList measurements = Lists.newArrayList(ProfileMeasurementConfig.builder().aggregationMethod(ProfileAggregationMethod.FIRST_SEEN).resultExtensionName(FIRST_SEEN_RESULT_NAME). - firstSeenExpirationDuration(5L).firstSeenExpirationDurationUnit("SECONDS").build()); - return ProfileGroupConfig.builder().profileGroupName(PROFILE_GROUP_NAME). - keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)).periodDuration(1L). - periodDurationUnit("SECONDS").sources(Lists.newArrayList("ANY")).measurements(measurements).build(); + ArrayList measurements = Lists.newArrayList(ProfileMeasurementConfig.builder().aggregationMethod(ProfileAggregationMethod.FIRST_SEEN).resultExtensionName(FIRST_SEEN_RESULT_NAME) + .firstSeenExpirationDuration(5L).firstSeenExpirationDurationUnit("SECONDS").build()); + return ProfileGroupConfig.builder().profileGroupName(PROFILE_GROUP_NAME) + .keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)).periodDuration(1L) + .periodDurationUnit("SECONDS").sources(Lists.newArrayList("ANY")).measurements(measurements).build(); } protected void connectHbase() { diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/FirstSeenHbaseTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/FirstSeenHbaseTest.java index 831a1a14..1b091686 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/FirstSeenHbaseTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/FirstSeenHbaseTest.java @@ -12,6 +12,10 @@ package com.cloudera.cyber.profiler; +import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_SIMPLE; +import static com.cloudera.cyber.profiler.FirstSeenHbaseLookup.FIRST_SEEN_ENRICHMENT_TYPE; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.createMeasurement; + import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageConfig; import com.cloudera.cyber.hbase.LookupKey; @@ -19,15 +23,10 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.junit.Assert; -import org.junit.Test; - import java.util.ArrayList; import java.util.Map; - -import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_SIMPLE; -import static com.cloudera.cyber.profiler.FirstSeenHbaseLookup.FIRST_SEEN_ENRICHMENT_TYPE; -import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.createMeasurement; +import org.junit.Assert; +import org.junit.Test; public class FirstSeenHbaseTest { @@ -70,9 +69,9 @@ public void testFirstSeenHbase() { private static ProfileGroupConfig createProfileGroupConfig() { ArrayList measurements = Lists.newArrayList(createMeasurement(ProfileAggregationMethod.FIRST_SEEN, FIRST_SEEN_RESULT_NAME, null)); - return ProfileGroupConfig.builder(). - profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)). - periodDuration(5L).periodDurationUnit("MINUTES"). - sources(Lists.newArrayList("ANY")).measurements(measurements).build(); + return ProfileGroupConfig.builder() + .profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)) + .periodDuration(5L).periodDurationUnit("MINUTES") + .sources(Lists.newArrayList("ANY")).measurements(measurements).build(); } } diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileGroupTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileGroupTest.java index d139c067..33ff2a1b 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileGroupTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileGroupTest.java @@ -15,7 +15,6 @@ import com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; - import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -30,10 +29,10 @@ public class ProfileGroupTest { protected ProfileGroupConfig getProfileGroupConfig(List measurements) { return ProfileGroupConfig.builder().profileGroupName(ProfileGroupTest.PROFILE_GROUP_NAME).keyFieldNames(Lists.newArrayList(KEY_FIELD_NAME)) - .sources(Lists.newArrayList("ANY")). - periodDuration(1L).periodDurationUnit(TimeUnit.MINUTES.name()). - measurements(Lists.newArrayList(measurements)). - build(); + .sources(Lists.newArrayList("ANY")) + .periodDuration(1L).periodDurationUnit(TimeUnit.MINUTES.name()) + .measurements(Lists.newArrayList(measurements)) + .build(); } protected void addMessage(ProfileGroupAcc profileGroupAccumulator, long timestamp, long aggregationFieldValue, ProfileGroupConfig profileGroupConfig) { diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileJobTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileJobTest.java index 9f5460d2..d35290df 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileJobTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileJobTest.java @@ -12,13 +12,44 @@ package com.cloudera.cyber.profiler; +import static com.cloudera.cyber.flink.FlinkUtils.PARAMS_PARALLELISM; +import static com.cloudera.cyber.profiler.ProfileAggregateFunction.PROFILE_GROUP_NAME_EXTENSION; +import static com.cloudera.cyber.profiler.StatsProfileAggregateFunction.STATS_PROFILE_GROUP_SUFFIX; +import static com.cloudera.cyber.profiler.accumulator.StatsProfileGroupAcc.END_PERIOD_EXTENSION; +import static com.cloudera.cyber.profiler.accumulator.StatsProfileGroupAcc.START_PERIOD_EXTENSION; +import static com.cloudera.cyber.profiler.accumulator.StatsProfileGroupAcc.STATS_EXTENSION_SUFFIXES; +import static com.cloudera.cyber.rules.DynamicRuleCommandType.UPSERT; +import static com.cloudera.cyber.rules.RuleType.JS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.withPrecision; +import static org.hamcrest.Matchers.equalTo; + import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.TestUtils; import com.cloudera.cyber.profiler.dto.ProfileDto; import com.cloudera.cyber.rules.DynamicRuleCommandResult; -import com.cloudera.cyber.scoring.*; +import com.cloudera.cyber.scoring.ScoredMessage; +import com.cloudera.cyber.scoring.Scores; +import com.cloudera.cyber.scoring.ScoringProcessFunction; +import com.cloudera.cyber.scoring.ScoringRule; +import com.cloudera.cyber.scoring.ScoringRuleCommand; +import com.cloudera.cyber.scoring.ScoringRuleCommandResult; +import com.cloudera.cyber.scoring.ScoringSummarizationMode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeoutException; +import java.util.stream.IntStream; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.datastream.DataStream; @@ -29,23 +60,6 @@ import org.hamcrest.MatcherAssert; import org.junit.Test; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.TimeoutException; -import java.util.stream.IntStream; - -import static com.cloudera.cyber.flink.FlinkUtils.PARAMS_PARALLELISM; -import static com.cloudera.cyber.profiler.ProfileAggregateFunction.PROFILE_GROUP_NAME_EXTENSION; -import static com.cloudera.cyber.profiler.StatsProfileAggregateFunction.STATS_PROFILE_GROUP_SUFFIX; -import static com.cloudera.cyber.profiler.accumulator.StatsProfileGroupAcc.*; -import static com.cloudera.cyber.rules.DynamicRuleCommandType.UPSERT; -import static com.cloudera.cyber.rules.RuleType.JS; -import static org.assertj.core.api.Assertions.*; -import static org.hamcrest.Matchers.equalTo; - public class ProfileJobTest extends ProfileJob { private static final String KEY_FIELD_NAME = "key_field"; @@ -119,11 +133,11 @@ public void testPipeline() throws Exception { // send the messages to the bytes profile JobTester.stopTest(); - ImmutableMap> possibleKeyValues = ImmutableMap.>builder(). - put(TEST_PROFILE_GROUP, Lists.newArrayList(KEY_1, KEY_2)). - put(SUM_SCORES_PROFILE_GROUP, Lists.newArrayList(IP_SRC_1, IP_SRC_2, IP_SRC_3)). - put(BYTE_COUNT_PROFILE_GROUP, Lists.newArrayList(REGION_1, REGION_2)). - build(); + ImmutableMap> possibleKeyValues = ImmutableMap.>builder() + .put(TEST_PROFILE_GROUP, Lists.newArrayList(KEY_1, KEY_2)) + .put(SUM_SCORES_PROFILE_GROUP, Lists.newArrayList(IP_SRC_1, IP_SRC_2, IP_SRC_3)) + .put(BYTE_COUNT_PROFILE_GROUP, Lists.newArrayList(REGION_1, REGION_2)) + .build(); List messages = new ArrayList<>(); @@ -214,7 +228,7 @@ private void sendSumMessage(long timestamp, String keyFieldValue, String secondK IP_SRC_3, 31.0); List scores = Collections.singletonList(Scores.builder().ruleId(RULE_UUID).reason("my reason").score(ipToScore.get(secondKeyField)).build()); - ScoredMessage message = ScoringProcessFunction.scoreMessage(TestUtils.createMessage(timestamp, "netflow", extensions), scores, ScoringSummarizationMode.DEFAULT()); + ScoredMessage message = ScoringProcessFunction.scoreMessage(TestUtils.createMessage(timestamp, "netflow", extensions), scores, ScoringSummarizationMode.defaultValue()); source.sendRecord(message, timestamp); } diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileMessageFilterTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileMessageFilterTest.java index 3f918f82..7067f3e7 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileMessageFilterTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileMessageFilterTest.java @@ -12,6 +12,9 @@ package com.cloudera.cyber.profiler; +import static com.cloudera.cyber.profiler.ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.createMeasurement; + import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.TestUtils; import com.cloudera.cyber.scoring.ScoredMessage; @@ -19,15 +22,15 @@ import com.cloudera.cyber.scoring.ScoringSummarizationMode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.junit.Assert; -import org.junit.Test; - -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static com.cloudera.cyber.profiler.ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD; -import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.createMeasurement; +import org.junit.Assert; +import org.junit.Test; public class ProfileMessageFilterTest { @@ -53,10 +56,10 @@ private void testMessageFilter(String source, boolean includeNullFieldMeasuremen ProfileGroupConfig profileGroupConfig = createProfileGroupConfig(Lists.newArrayList(source),includeNullFieldMeasurements, includeScoreMeasurement); ProfileMessageFilter filter = new ProfileMessageFilter(profileGroupConfig); - Map keyFieldValues = profileGroupConfig.getKeyFieldNames().stream(). - collect(Collectors.toMap(keyFieldName -> keyFieldName, value ->"value")); - String requiredMeasurementFieldName = profileGroupConfig.getMeasurementFieldNames().stream().filter(name -> !name.equals(CYBER_SCORE_FIELD)). - findFirst().orElseThrow(() -> new RuntimeException("test should contain at least one required field")); + Map keyFieldValues = profileGroupConfig.getKeyFieldNames().stream() + .collect(Collectors.toMap(keyFieldName -> keyFieldName, value ->"value")); + String requiredMeasurementFieldName = profileGroupConfig.getMeasurementFieldNames().stream().filter(name -> !name.equals(CYBER_SCORE_FIELD)) + .findFirst().orElseThrow(() -> new RuntimeException("test should contain at least one required field")); Map allFields = new HashMap<>(keyFieldValues); allFields.put(requiredMeasurementFieldName, "value"); @@ -78,7 +81,7 @@ private void testMessageFilter(String source, boolean includeNullFieldMeasuremen } private void verifyFilter(ProfileMessageFilter filter, String source, Map fieldValues, boolean matches) { - ScoredMessage scoredMessage = ScoringProcessFunction.scoreMessage(TestUtils.createMessage(MessageUtils.getCurrentTimestamp(), source, fieldValues), Collections.emptyList(), ScoringSummarizationMode.DEFAULT()); + ScoredMessage scoredMessage = ScoringProcessFunction.scoreMessage(TestUtils.createMessage(MessageUtils.getCurrentTimestamp(), source, fieldValues), Collections.emptyList(), ScoringSummarizationMode.defaultValue()); Assert.assertEquals(matches, filter.filter(scoredMessage)); } @@ -100,10 +103,10 @@ private static ProfileGroupConfig createProfileGroupConfig(ArrayList sou measurements.add(createMeasurement(ProfileAggregationMethod.SUM, "total_score", CYBER_SCORE_FIELD, null)); } - return ProfileGroupConfig.builder(). - profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)). - periodDuration(5L).periodDurationUnit("MINUTES"). - sources(sources).measurements(measurements).build(); + return ProfileGroupConfig.builder() + .profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)) + .periodDuration(5L).periodDurationUnit("MINUTES") + .sources(sources).measurements(measurements).build(); } } diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileStatsJoinTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileStatsJoinTest.java index 2e175c19..74e06b74 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileStatsJoinTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ProfileStatsJoinTest.java @@ -14,16 +14,12 @@ import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc; -import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; -import org.junit.Assert; -import org.junit.Test; - import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.stream.Collectors; +import org.junit.Assert; +import org.junit.Test; public class ProfileStatsJoinTest extends ProfileGroupTest { @@ -42,9 +38,9 @@ public void testProfileStatsJoin() { } private ProfileMessage getExpectedJoinedMessage( ProfileMessage message1, ProfileMessage message2) { - Map mergedExtensions = message2.getExtensions().entrySet().stream(). - filter(e -> ProfileStatsJoin.isStatsExtension(e.getKey())). - collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map mergedExtensions = message2.getExtensions().entrySet().stream() + .filter(e -> ProfileStatsJoin.isStatsExtension(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); message1.getExtensions().forEach(mergedExtensions::putIfAbsent); return new ProfileMessage(message1.getTs(), mergedExtensions); } @@ -69,9 +65,9 @@ private ProfileMessage createStatsMessage(ProfileGroupConfig profileGroupConfig, } private ProfileGroupConfig getProfileGroupConfig() { - List measurements = Collections.singletonList(ProfileMeasurementConfig.builder().resultExtensionName(RESULT_EXTENSION_NAME). - aggregationMethod(ProfileAggregationMethod.SUM).fieldName(SUM_FIELD_NAME).format("0.000000").calculateStats(true). - build()); + List measurements = Collections.singletonList(ProfileMeasurementConfig.builder().resultExtensionName(RESULT_EXTENSION_NAME) + .aggregationMethod(ProfileAggregationMethod.SUM).fieldName(SUM_FIELD_NAME).format("0.000000").calculateStats(true) + .build()); return getProfileGroupConfig(measurements); } diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ScoredMessageToProfileMessageMapTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ScoredMessageToProfileMessageMapTest.java index 68a91df2..c7ba9ef0 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ScoredMessageToProfileMessageMapTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/ScoredMessageToProfileMessageMapTest.java @@ -12,6 +12,8 @@ package com.cloudera.cyber.profiler; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.createMeasurement; + import com.cloudera.cyber.Message; import com.cloudera.cyber.TestUtils; import com.cloudera.cyber.scoring.ScoredMessage; @@ -19,14 +21,15 @@ import com.cloudera.cyber.scoring.ScoringProcessFunction; import com.cloudera.cyber.scoring.ScoringSummarizationMode; import com.google.common.collect.Lists; -import org.junit.Assert; -import org.junit.Test; - -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.createMeasurement; +import org.junit.Assert; +import org.junit.Test; public class ScoredMessageToProfileMessageMapTest { private static final String TEST_PROFILE_GROUP = "test_profile"; @@ -36,9 +39,9 @@ public class ScoredMessageToProfileMessageMapTest { public void testScoredToProfileMessageConversion() { double expectedScore = 55.0; ProfileGroupConfig profileGroupConfig = createAllAggregationMethodProfileGroupConfig(); - Map inputExtensions = Stream.concat(profileGroupConfig.getKeyFieldNames().stream(), profileGroupConfig.getMeasurementFieldNames().stream()). - filter(fieldName -> !fieldName.equals(ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD)). - collect(Collectors.toMap(k -> k, v -> v.concat("_value"))); + Map inputExtensions = Stream.concat(profileGroupConfig.getKeyFieldNames().stream(), profileGroupConfig.getMeasurementFieldNames().stream()) + .filter(fieldName -> !fieldName.equals(ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD)) + .collect(Collectors.toMap(k -> k, v -> v.concat("_value"))); String notRequiredFieldName = "not required"; inputExtensions.put(notRequiredFieldName, "should be removed"); @@ -46,7 +49,7 @@ public void testScoredToProfileMessageConversion() { Message inputMessage = TestUtils.createMessage(inputExtensions); List scoreDetails = Collections.singletonList(Scores.builder().reason("because").score(expectedScore).build()); - ScoredMessage inputScoredMessage = ScoringProcessFunction.scoreMessage(inputMessage, scoreDetails, ScoringSummarizationMode.DEFAULT()); + ScoredMessage inputScoredMessage = ScoringProcessFunction.scoreMessage(inputMessage, scoreDetails, ScoringSummarizationMode.defaultValue()); ScoredMessageToProfileMessageMap map = new ScoredMessageToProfileMessageMap(profileGroupConfig); ProfileMessage outputProfileMessage = map.map(inputScoredMessage); @@ -71,10 +74,10 @@ private static ProfileGroupConfig createAllAggregationMethodProfileGroupConfig() measurements.add(createMeasurement(ProfileAggregationMethod.MAX, "max_score", ScoredMessageToProfileMessageMap.CYBER_SCORE_FIELD, null)); - return ProfileGroupConfig.builder(). - profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1)). - periodDuration(5L).periodDurationUnit("MINUTES"). - sources(Lists.newArrayList("ANY")).measurements(measurements).build(); + return ProfileGroupConfig.builder() + .profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1)) + .periodDuration(5L).periodDurationUnit("MINUTES") + .sources(Lists.newArrayList("ANY")).measurements(measurements).build(); } } diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/FieldValueProfileGroupAccTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/FieldValueProfileGroupAccTest.java index dce212b3..84b0cbad 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/FieldValueProfileGroupAccTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/FieldValueProfileGroupAccTest.java @@ -12,24 +12,23 @@ package com.cloudera.cyber.profiler.accumulator; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc.END_PERIOD_EXTENSION; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc.START_PERIOD_EXTENSION; + import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.profiler.ProfileAggregationMethod; import com.cloudera.cyber.profiler.ProfileGroupConfig; import com.cloudera.cyber.profiler.ProfileMeasurementConfig; import com.cloudera.cyber.profiler.ProfileMessage; import com.google.common.collect.Lists; -import org.junit.Assert; -import org.junit.Test; - import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; - -import static com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc.END_PERIOD_EXTENSION; -import static com.cloudera.cyber.profiler.accumulator.ProfileGroupAcc.START_PERIOD_EXTENSION; +import org.junit.Assert; +import org.junit.Test; public class FieldValueProfileGroupAccTest extends ProfileGroupConfigTestUtils { public static final String SUM_FIELD = "sum_field"; @@ -63,10 +62,10 @@ public void testMerge() { createMeasurement(ProfileAggregationMethod.MAX, MAX_RESULT, MAX_FIELD), createMeasurement(ProfileAggregationMethod.MIN, MIN_RESULT, MIN_FIELD), createMeasurement(ProfileAggregationMethod.FIRST_SEEN, FIRST_SEEN_RESULT, null)); - ProfileGroupConfig profileGroupConfig = ProfileGroupConfig.builder(). - profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)). - periodDuration(5L).periodDurationUnit("MINUTES"). - sources(Lists.newArrayList("ANY")).measurements(measurements).build(); + ProfileGroupConfig profileGroupConfig = ProfileGroupConfig.builder() + .profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)) + .periodDuration(5L).periodDurationUnit("MINUTES") + .sources(Lists.newArrayList("ANY")).measurements(measurements).build(); // create the first accumulator FieldValueProfileGroupAcc acc1 = new FieldValueProfileGroupAcc(profileGroupConfig); @@ -155,10 +154,10 @@ public static ProfileGroupConfig createProfileGroupConfig(String format) { createMeasurement(ProfileAggregationMethod.MAX, MAX_RESULT, MAX_FIELD, format), createMeasurement(ProfileAggregationMethod.MIN, MIN_RESULT, MIN_FIELD, format), createMeasurement(ProfileAggregationMethod.FIRST_SEEN, FIRST_SEEN_RESULT, null, format)); - return ProfileGroupConfig.builder(). - profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)). - periodDuration(5L).periodDurationUnit("MINUTES"). - sources(Lists.newArrayList("ANY")).measurements(measurements).build(); + return ProfileGroupConfig.builder() + .profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)) + .periodDuration(5L).periodDurationUnit("MINUTES") + .sources(Lists.newArrayList("ANY")).measurements(measurements).build(); } @@ -189,8 +188,8 @@ public static ProfileMessage createMessage(long timestamp, double sum_field, Str } public static Map getFormats(ProfileGroupConfig profileGroupConfig){ - return profileGroupConfig.getMeasurements().stream(). - collect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName, + return profileGroupConfig.getMeasurements().stream() + .collect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName, ProfileMeasurementConfig::getDecimalFormat)); } diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/ProfileGroupConfigTestUtils.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/ProfileGroupConfigTestUtils.java index ec56e6a0..52132368 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/ProfileGroupConfigTestUtils.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/ProfileGroupConfigTestUtils.java @@ -28,14 +28,14 @@ public static ProfileMeasurementConfig createMeasurement(ProfileAggregationMetho } public static ProfileMeasurementConfig createMeasurement(ProfileAggregationMethod aggMethod, String resultName, String fieldName, String format) { - return ProfileMeasurementConfig.builder().aggregationMethod(aggMethod). - resultExtensionName(resultName).fieldName(fieldName).format(format).build(); + return ProfileMeasurementConfig.builder().aggregationMethod(aggMethod) + .resultExtensionName(resultName).fieldName(fieldName).format(format).build(); } public static ProfileMeasurementConfig createMeasurement(ProfileAggregationMethod aggMethod, String resultName, String fieldName, String format, boolean calculateStats) { - return ProfileMeasurementConfig.builder().aggregationMethod(aggMethod). - resultExtensionName(resultName).fieldName(fieldName).format(format). - calculateStats(calculateStats).build(); + return ProfileMeasurementConfig.builder().aggregationMethod(aggMethod) + .resultExtensionName(resultName).fieldName(fieldName).format(format) + .calculateStats(calculateStats).build(); } } diff --git a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/StatsProfileGroupAccTest.java b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/StatsProfileGroupAccTest.java index 368556ea..3fc068b9 100644 --- a/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/StatsProfileGroupAccTest.java +++ b/flink-cyber/flink-profiler-java/src/test/java/com/cloudera/cyber/profiler/accumulator/StatsProfileGroupAccTest.java @@ -12,6 +12,11 @@ package com.cloudera.cyber.profiler.accumulator; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.KEY_1; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.KEY_2; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.TEST_PROFILE_GROUP; +import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.createMeasurement; + import com.cloudera.cyber.MessageUtils; import com.cloudera.cyber.profiler.ProfileAggregationMethod; import com.cloudera.cyber.profiler.ProfileGroupConfig; @@ -19,16 +24,13 @@ import com.cloudera.cyber.profiler.ProfileMessage; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.junit.Assert; -import org.junit.Test; - import java.text.DecimalFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; - -import static com.cloudera.cyber.profiler.accumulator.ProfileGroupConfigTestUtils.*; +import org.junit.Assert; +import org.junit.Test; public class StatsProfileGroupAccTest { @@ -38,8 +40,8 @@ public class StatsProfileGroupAccTest { private static final DecimalFormat DEFAULT_FORMAT = new DecimalFormat("0.00"); private Map getMeasurementFormats(ProfileGroupConfig profileGroupConfig) { - return profileGroupConfig.getMeasurements().stream().filter(ProfileMeasurementConfig::hasStats). - collect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName, v -> DEFAULT_FORMAT)); + return profileGroupConfig.getMeasurements().stream().filter(ProfileMeasurementConfig::hasStats) + .collect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName, v -> DEFAULT_FORMAT)); } @Test @@ -79,10 +81,10 @@ public static ProfileGroupConfig createProfileGroup() { createMeasurement(ProfileAggregationMethod.COUNT, NO_STATS_RESULT_NAME, null, "0.00", false), createMeasurement(ProfileAggregationMethod.SUM, STATS_RESULT_NAME, STATS_FIELD_NAME, "0.00", true)); - return ProfileGroupConfig.builder(). - profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)). - periodDuration(5L).periodDurationUnit("MINUTES"). - sources(Lists.newArrayList("ANY")).measurements(measurements).build(); + return ProfileGroupConfig.builder() + .profileGroupName(TEST_PROFILE_GROUP).keyFieldNames(Lists.newArrayList(KEY_1, KEY_2)) + .periodDuration(5L).periodDurationUnit("MINUTES") + .sources(Lists.newArrayList("ANY")).measurements(measurements).build(); } diff --git a/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileAggregationMethod.java b/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileAggregationMethod.java index 565cd3b6..ace3c2d5 100644 --- a/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileAggregationMethod.java +++ b/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileAggregationMethod.java @@ -14,7 +14,6 @@ import com.google.common.collect.ImmutableMap; - import java.text.DecimalFormat; import java.util.Map; @@ -26,21 +25,23 @@ public enum ProfileAggregationMethod { COUNT_DISTINCT, FIRST_SEEN; - public static final Map defaultFormat = ImmutableMap.builder(). - put(COUNT, new DecimalFormat("#")). - put(SUM, new DecimalFormat("#.##")). - put(MIN, new DecimalFormat("#.##")). - put(MAX, new DecimalFormat("#.##")). - put(COUNT_DISTINCT, new DecimalFormat("#")). - put(FIRST_SEEN, new DecimalFormat("#")). - build(); + public static final Map defaultFormat = + ImmutableMap.builder() + .put(COUNT, new DecimalFormat("#")) + .put(SUM, new DecimalFormat("#.##")) + .put(MIN, new DecimalFormat("#.##")) + .put(MAX, new DecimalFormat("#.##")) + .put(COUNT_DISTINCT, new DecimalFormat("#")) + .put(FIRST_SEEN, new DecimalFormat("#")) + .build(); - public static final Map usesFieldValue = ImmutableMap.builder(). - put(COUNT, false). - put(SUM, true). - put(MIN, true). - put(MAX, true). - put(COUNT_DISTINCT, true). - put(FIRST_SEEN, false). - build(); + public static final Map usesFieldValue = + ImmutableMap.builder() + .put(COUNT, false) + .put(SUM, true) + .put(MIN, true) + .put(MAX, true) + .put(COUNT_DISTINCT, true) + .put(FIRST_SEEN, false) + .build(); } diff --git a/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileConfig.java b/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileConfig.java index 5b995e98..c02e9f03 100644 --- a/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileConfig.java +++ b/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileConfig.java @@ -13,11 +13,10 @@ package com.cloudera.cyber.profiler; -import lombok.Builder; -import lombok.Data; - import java.util.Map; import java.util.concurrent.TimeUnit; +import lombok.Builder; +import lombok.Data; @Data @Builder diff --git a/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileGroupConfig.java b/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileGroupConfig.java index 292b8f65..1b3586aa 100644 --- a/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileGroupConfig.java +++ b/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileGroupConfig.java @@ -12,14 +12,21 @@ package com.cloudera.cyber.profiler; -import lombok.*; -import org.apache.flink.util.Preconditions; - import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.flink.util.Preconditions; @Data @NoArgsConstructor(force = true, access = AccessLevel.PUBLIC) @@ -30,16 +37,23 @@ public class ProfileGroupConfig implements Serializable { static final String NULL_PROFILE_GROUP_NAME_ERROR = "profileGroupName is null"; static final String EMPTY_PROFILE_GROUP_NAME_ERROR = "profileGroupName cannot be empty"; - public static final String EMPTY_SOURCES_ERROR = "Profile group '%s': %s sources list - specify a list of source names or ANY matching any source"; - public static final String EMPTY_KEY_FIELDS_ERROR = "Profile group '%s': %s key field list - specify the name of one or more key fields for the profile"; + public static final String EMPTY_SOURCES_ERROR = + "Profile group '%s': %s sources list - specify a list of source names or ANY matching any source"; + public static final String EMPTY_KEY_FIELDS_ERROR = + "Profile group '%s': %s key field list - specify the name of one or more key fields for the profile"; public static final String PROFILE_TIME_ERROR = "Profile group '%s': %s is %s"; - public static final String NULL_EMPTY_MEASUREMENTS_ERROR = "Profile group '%s': %s measurements list - specify at least on measurement for the profile group"; - public static final String MISSING_STATS_SLIDE = "Profile group '%s' has calculateStats enabled but does not specify statsSlide"; - public static final String UNNECESSARY_STATS_SLIDE_ERROR = "Profile group '%s' does not have calculateStats enabled but specifies statsSlide"; - public static final String MISSING_STATS_SLIDE_UNIT = "Profile group '%s' has calculateStates enabled but does not specify statsSlideUnit"; + public static final String NULL_EMPTY_MEASUREMENTS_ERROR = + "Profile group '%s': %s measurements list - specify at least on measurement for the profile group"; + public static final String MISSING_STATS_SLIDE = + "Profile group '%s' has calculateStats enabled but does not specify statsSlide"; + public static final String UNNECESSARY_STATS_SLIDE_ERROR = + "Profile group '%s' does not have calculateStats enabled but specifies statsSlide"; + public static final String MISSING_STATS_SLIDE_UNIT = + "Profile group '%s' has calculateStates enabled but does not specify statsSlideUnit"; public static final String ILLEGAL_TIME_UNIT_ERROR = "Profile group '%s' %s has an undefined time unit."; public static final String DUPLICATE_PROFILE_GROUP_NAMES_ERROR = "Duplicate profile group names '%s'."; - public static final String DUPLICATE_RESULT_EXTENSIONS_NAMES = "Profile group '%s' has duplicate result extension names '%s'."; + public static final String DUPLICATE_RESULT_EXTENSIONS_NAMES = + "Profile group '%s' has duplicate result extension names '%s'."; private String profileGroupName; private ArrayList sources; @@ -56,15 +70,17 @@ public boolean hasStats() { } public boolean needsSourceFilter() { - return !sources.contains(ANY_SOURCE); + return !sources.contains(ANY_SOURCE); } public List getMeasurementFieldNames() { - return measurements.stream().map(ProfileMeasurementConfig::getFieldName).filter(Objects::nonNull).collect(Collectors.toList()); + return measurements.stream().map(ProfileMeasurementConfig::getFieldName).filter(Objects::nonNull) + .collect(Collectors.toList()); } public boolean hasFirstSeen() { - return measurements.stream().anyMatch(m -> m.getAggregationMethod().equals(ProfileAggregationMethod.FIRST_SEEN)); + return measurements.stream() + .anyMatch(m -> m.getAggregationMethod().equals(ProfileAggregationMethod.FIRST_SEEN)); } public void verify() { @@ -77,45 +93,54 @@ public void verify() { if (hasStats()) { verifyTime("statsSlide", statsSlide, statsSlideUnit); } else { - Preconditions.checkState(statsSlide == null && statsSlideUnit == null, String.format(UNNECESSARY_STATS_SLIDE_ERROR, getProfileGroupName())); + Preconditions.checkState(statsSlide == null && statsSlideUnit == null, + String.format(UNNECESSARY_STATS_SLIDE_ERROR, getProfileGroupName())); } int offset = 1; Set uniqueMeasurementResults = new HashSet<>(); Set duplicateMeasurementResults = new HashSet<>(); - for(ProfileMeasurementConfig measurement : measurements) { + for (ProfileMeasurementConfig measurement : measurements) { measurement.verify(this, offset++); String resultExtensionName = measurement.getResultExtensionName(); if (!uniqueMeasurementResults.add(resultExtensionName)) { duplicateMeasurementResults.add(resultExtensionName); } } - Preconditions.checkState(duplicateMeasurementResults.isEmpty(), String.format(DUPLICATE_RESULT_EXTENSIONS_NAMES, getProfileGroupName(), String.join(", ", duplicateMeasurementResults))); + Preconditions.checkState(duplicateMeasurementResults.isEmpty(), + String.format(DUPLICATE_RESULT_EXTENSIONS_NAMES, getProfileGroupName(), + String.join(", ", duplicateMeasurementResults))); } public static void verify(List profileGroupConfigs) { profileGroupConfigs.forEach(ProfileGroupConfig::verify); Set uniqueProfileGroupNames = new HashSet<>(); - Set duplicateProfileGroupNames = profileGroupConfigs.stream().map(ProfileGroupConfig::getProfileGroupName) - .filter(n -> !uniqueProfileGroupNames.add(n)) - .collect(Collectors.toSet()); - Preconditions.checkState(duplicateProfileGroupNames.isEmpty(), String.format(DUPLICATE_PROFILE_GROUP_NAMES_ERROR, String.join(", ", duplicateProfileGroupNames))); + Set duplicateProfileGroupNames = + profileGroupConfigs.stream().map(ProfileGroupConfig::getProfileGroupName) + .filter(n -> !uniqueProfileGroupNames.add(n)) + .collect(Collectors.toSet()); + Preconditions.checkState(duplicateProfileGroupNames.isEmpty(), + String.format(DUPLICATE_PROFILE_GROUP_NAMES_ERROR, String.join(", ", duplicateProfileGroupNames))); } private void verifyList(List listToVerify, String messageFormat) { Preconditions.checkNotNull(listToVerify, String.format(messageFormat, getProfileGroupName(), "null")); - Preconditions.checkArgument(!listToVerify.isEmpty(), String.format(messageFormat, getProfileGroupName(), "empty")); + Preconditions.checkArgument(!listToVerify.isEmpty(), + String.format(messageFormat, getProfileGroupName(), "empty")); } public void verifyTime(String baseTimeFieldName, Long profileTimeDuration, String profileTimeUnit) { final String timeUnitFieldName = baseTimeFieldName.concat("Unit"); // check missing duration and unit - Preconditions.checkNotNull(profileTimeDuration, String.format(PROFILE_TIME_ERROR, getProfileGroupName(), baseTimeFieldName, "null")); - Preconditions.checkNotNull(profileTimeUnit, String.format(PROFILE_TIME_ERROR, getProfileGroupName(), timeUnitFieldName, "null")); + Preconditions.checkNotNull(profileTimeDuration, + String.format(PROFILE_TIME_ERROR, getProfileGroupName(), baseTimeFieldName, "null")); + Preconditions.checkNotNull(profileTimeUnit, + String.format(PROFILE_TIME_ERROR, getProfileGroupName(), timeUnitFieldName, "null")); // time unit must match one of the define java time units Preconditions.checkState(Stream.of(TimeUnit.values()).anyMatch(unit -> unit.name().equals(profileTimeUnit)), - String.format(PROFILE_TIME_ERROR, getProfileGroupName(), timeUnitFieldName, "not a legal time unit")); - Preconditions.checkState(profileTimeDuration > 0, String.format(PROFILE_TIME_ERROR, getProfileGroupName(), baseTimeFieldName, "0 or negative")); + String.format(PROFILE_TIME_ERROR, getProfileGroupName(), timeUnitFieldName, "not a legal time unit")); + Preconditions.checkState(profileTimeDuration > 0, + String.format(PROFILE_TIME_ERROR, getProfileGroupName(), baseTimeFieldName, "0 or negative")); } } diff --git a/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileMeasurementConfig.java b/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileMeasurementConfig.java index 24d51730..136dc9ae 100644 --- a/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileMeasurementConfig.java +++ b/flink-cyber/flink-profiler/src/main/java/com/cloudera/cyber/profiler/ProfileMeasurementConfig.java @@ -12,11 +12,15 @@ package com.cloudera.cyber.profiler; -import lombok.*; -import org.apache.flink.util.Preconditions; - import java.io.Serializable; import java.text.DecimalFormat; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.flink.util.Preconditions; @Data @Builder @@ -26,7 +30,8 @@ public class ProfileMeasurementConfig implements Serializable { public static final String NULL_FIELD_VALUE_ERROR = "Profile group %s: measurement %d has a null %s"; public static final String EMPTY_FIELD_VALUE_ERROR = "Profile group %s: measurement %d has an empty %s"; - public static final String FIRST_SEEN_ON_NUMERIC = "Profile group %s: measurement offset %d has firstSeenExpiration but the aggregationMethod is not FIRST_SEEN."; + public static final String FIRST_SEEN_ON_NUMERIC = + "Profile group %s: measurement offset %d has firstSeenExpiration but the aggregationMethod is not FIRST_SEEN."; private Integer id; private String fieldName; private String resultExtensionName; @@ -50,23 +55,28 @@ public DecimalFormat getDecimalFormat() { public void verify(ProfileGroupConfig profileGroupConfig, int offset) { String profileGroupName = profileGroupConfig.getProfileGroupName(); - Preconditions.checkNotNull(aggregationMethod, String.format(NULL_FIELD_VALUE_ERROR, profileGroupName, offset, "aggregationMethod")); - if (!aggregationMethod.equals(ProfileAggregationMethod.COUNT) && !aggregationMethod.equals(ProfileAggregationMethod.FIRST_SEEN)) { + Preconditions.checkNotNull(aggregationMethod, + String.format(NULL_FIELD_VALUE_ERROR, profileGroupName, offset, "aggregationMethod")); + if (!aggregationMethod.equals(ProfileAggregationMethod.COUNT) + && !aggregationMethod.equals(ProfileAggregationMethod.FIRST_SEEN)) { checkString(profileGroupName, offset, "fieldName", fieldName); } checkString(profileGroupName, offset, "resultExtensionName", resultExtensionName); if (ProfileAggregationMethod.FIRST_SEEN.equals(aggregationMethod)) { - if (firstSeenExpirationDuration != null || firstSeenExpirationDurationUnit != null ) { - profileGroupConfig.verifyTime("firstSeenExpirationDuration", firstSeenExpirationDuration, firstSeenExpirationDurationUnit); + if (firstSeenExpirationDuration != null || firstSeenExpirationDurationUnit != null) { + profileGroupConfig.verifyTime("firstSeenExpirationDuration", firstSeenExpirationDuration, + firstSeenExpirationDurationUnit); } - } else { - Preconditions.checkState(firstSeenExpirationDuration == null && firstSeenExpirationDurationUnit == null, String.format(FIRST_SEEN_ON_NUMERIC, profileGroupName, offset)); + } else { + Preconditions.checkState(firstSeenExpirationDuration == null && firstSeenExpirationDurationUnit == null, + String.format(FIRST_SEEN_ON_NUMERIC, profileGroupName, offset)); } } private void checkString(String profileGroupName, int offset, String name, String value) { Preconditions.checkNotNull(value, String.format(NULL_FIELD_VALUE_ERROR, profileGroupName, offset, name)); - Preconditions.checkState(!value.isEmpty(), String.format(EMPTY_FIELD_VALUE_ERROR, profileGroupName, offset, name)); + Preconditions.checkState(!value.isEmpty(), + String.format(EMPTY_FIELD_VALUE_ERROR, profileGroupName, offset, name)); } } diff --git a/flink-cyber/flink-profiler/src/test/java/com/cloudera/cyber/profiler/ProfileGroupConfigTest.java b/flink-cyber/flink-profiler/src/test/java/com/cloudera/cyber/profiler/ProfileGroupConfigTest.java index 6b700016..e91bd9b1 100644 --- a/flink-cyber/flink-profiler/src/test/java/com/cloudera/cyber/profiler/ProfileGroupConfigTest.java +++ b/flink-cyber/flink-profiler/src/test/java/com/cloudera/cyber/profiler/ProfileGroupConfigTest.java @@ -12,28 +12,31 @@ package com.cloudera.cyber.profiler; -import org.apache.commons.compress.utils.Lists; -import org.junit.Assert; -import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.apache.commons.compress.utils.Lists; +import org.junit.Assert; +import org.junit.Test; public class ProfileGroupConfigTest { private static final ArrayList TEST_SOURCES = new ArrayList<>(Collections.singletonList("test_source")); private static final ArrayList ANY_SOURCES = new ArrayList<>(Collections.singletonList("ANY")); private static final ArrayList TEST_KEY_FIELDS = new ArrayList<>(Collections.singletonList("key_field")); - private static final ProfileMeasurementConfig GOOD_MEASUREMENT = ProfileMeasurementConfig.builder(). - aggregationMethod(ProfileAggregationMethod.MAX).fieldName("field").resultExtensionName("field_count").build(); - private static final ProfileMeasurementConfig GOOD_STATS_MEASUREMENT = ProfileMeasurementConfig.builder(). - aggregationMethod(ProfileAggregationMethod.MAX).fieldName("field").resultExtensionName("field_count"). - calculateStats(true).build(); - private static final ProfileMeasurementConfig GOOD_FIRST_SEEN_MEASUREMENT = ProfileMeasurementConfig.builder(). - aggregationMethod(ProfileAggregationMethod.FIRST_SEEN).fieldName("field").resultExtensionName("first_field").build(); + private static final ProfileMeasurementConfig GOOD_MEASUREMENT = ProfileMeasurementConfig.builder() + .aggregationMethod(ProfileAggregationMethod.MAX).fieldName("field").resultExtensionName("field_count").build(); + private static final ProfileMeasurementConfig GOOD_STATS_MEASUREMENT = ProfileMeasurementConfig.builder() + .aggregationMethod(ProfileAggregationMethod.MAX).fieldName("field").resultExtensionName("field_count") + .calculateStats(true).build(); + private static final ProfileMeasurementConfig GOOD_FIRST_SEEN_MEASUREMENT = ProfileMeasurementConfig.builder() + .aggregationMethod(ProfileAggregationMethod.FIRST_SEEN).fieldName("field").resultExtensionName("first_field").build(); private static final ArrayList MEASUREMENTS = new ArrayList<>(Collections.singletonList(GOOD_MEASUREMENT)); private static final ArrayList MEASUREMENTS_WITH_STATS = new ArrayList<>(Collections.singletonList(GOOD_STATS_MEASUREMENT)); @@ -42,31 +45,31 @@ public class ProfileGroupConfigTest { @Test public void testGoodProfile() { - ProfileGroupConfig goodNoStatsProfile = ProfileGroupConfig.builder(). - profileGroupName("good_name").sources(TEST_SOURCES). - keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS). - periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileGroupConfig goodNoStatsProfile = ProfileGroupConfig.builder() + .profileGroupName("good_name").sources(TEST_SOURCES) + .keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS) + .periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()) + .build(); goodNoStatsProfile.verify(); Assert.assertFalse(goodNoStatsProfile.hasStats()); Assert.assertFalse(goodNoStatsProfile.hasFirstSeen()); Assert.assertEquals(Collections.singletonList("field"), goodNoStatsProfile.getMeasurementFieldNames()); - ProfileGroupConfig goodStatsProfile = ProfileGroupConfig.builder(). - profileGroupName("good_name").sources(TEST_SOURCES). - keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS_WITH_STATS). - periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()). - statsSlide(10L).statsSlideUnit(TimeUnit.HOURS.name()). - build(); + ProfileGroupConfig goodStatsProfile = ProfileGroupConfig.builder() + .profileGroupName("good_name").sources(TEST_SOURCES) + .keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS_WITH_STATS) + .periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()) + .statsSlide(10L).statsSlideUnit(TimeUnit.HOURS.name()) + .build(); goodStatsProfile.verify(); Assert.assertTrue(goodStatsProfile.hasStats()); Assert.assertFalse(goodStatsProfile.hasFirstSeen()); - ProfileGroupConfig goodFirstSeen = ProfileGroupConfig.builder(). - profileGroupName("good_name").sources(TEST_SOURCES). - keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS_WITH_FIRST). - periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileGroupConfig goodFirstSeen = ProfileGroupConfig.builder() + .profileGroupName("good_name").sources(TEST_SOURCES) + .keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS_WITH_FIRST) + .periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()) + .build(); goodFirstSeen.verify(); Assert.assertFalse(goodFirstSeen.hasStats()); Assert.assertTrue(goodFirstSeen.hasFirstSeen()); @@ -77,23 +80,23 @@ public void testDuplicateProfileGroupNames() { String dup1Name = "dup_1"; String dup2Name = "dup_2"; - ProfileGroupConfig duplicateProfile1 = ProfileGroupConfig.builder(). - profileGroupName(dup1Name).sources(TEST_SOURCES). - keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS). - periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileGroupConfig duplicateProfile1 = ProfileGroupConfig.builder() + .profileGroupName(dup1Name).sources(TEST_SOURCES) + .keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS) + .periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()) + .build(); - ProfileGroupConfig duplicateProfile2 = ProfileGroupConfig.builder(). - profileGroupName(dup2Name).sources(TEST_SOURCES). - keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS). - periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileGroupConfig duplicateProfile2 = ProfileGroupConfig.builder() + .profileGroupName(dup2Name).sources(TEST_SOURCES) + .keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS) + .periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()) + .build(); - ProfileGroupConfig uniqueProfile = ProfileGroupConfig.builder(). - profileGroupName("unique").sources(TEST_SOURCES). - keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS). - periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileGroupConfig uniqueProfile = ProfileGroupConfig.builder() + .profileGroupName("unique").sources(TEST_SOURCES) + .keyFieldNames(TEST_KEY_FIELDS).measurements(MEASUREMENTS) + .periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()) + .build(); // test with one duplicate assertThatThrownBy(() -> ProfileGroupConfig.verify(Arrays.asList(duplicateProfile1, uniqueProfile, duplicateProfile1))) @@ -125,11 +128,11 @@ public void testDuplicateMeasurementResultNames() { private void testDuplicateMeasurementResultNames(ArrayList dupMeasurements, List expectedDuplicates) { String profileGroupName = "dup_measurements"; - ProfileGroupConfig duplicateMeasurementResults = ProfileGroupConfig.builder(). - profileGroupName(profileGroupName).sources(TEST_SOURCES). - keyFieldNames(TEST_KEY_FIELDS).measurements(dupMeasurements). - periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileGroupConfig duplicateMeasurementResults = ProfileGroupConfig.builder() + .profileGroupName(profileGroupName).sources(TEST_SOURCES) + .keyFieldNames(TEST_KEY_FIELDS).measurements(dupMeasurements) + .periodDuration(20L).periodDurationUnit(TimeUnit.HOURS.name()) + .build(); assertThatThrownBy(duplicateMeasurementResults::verify) .isInstanceOf(IllegalStateException.class) @@ -138,19 +141,19 @@ private void testDuplicateMeasurementResultNames(ArrayList measurements = new ArrayList<>(Collections.singleton(measurementConfig)); - ProfileGroupConfig config = ProfileGroupConfig.builder().profileGroupName(TEST_PROFILE_GROUP). - measurements(measurements).build(); + ProfileGroupConfig config = ProfileGroupConfig.builder().profileGroupName(TEST_PROFILE_GROUP) + .measurements(measurements).build(); measurementConfig.verify(config, TEST_OFFSET); } @@ -108,63 +107,63 @@ public void testIllegalResultExtension() { @Test public void testNullAggregationMethod() { - ProfileMeasurementConfig nullAggregation = ProfileMeasurementConfig.builder().fieldName("valid_field"). - resultExtensionName("valid_extension_name").build(); + ProfileMeasurementConfig nullAggregation = ProfileMeasurementConfig.builder().fieldName("valid_field") + .resultExtensionName("valid_extension_name").build(); testNullField(nullAggregation, String.format(ProfileMeasurementConfig.NULL_FIELD_VALUE_ERROR, TEST_PROFILE_GROUP, TEST_OFFSET, "aggregationMethod")); } @Test public void testFirstSeenDurationOnNumeric() { - ProfileMeasurementConfig numericWithFirstSeenDuration = ProfileMeasurementConfig.builder(). - fieldName("valid_field").resultExtensionName("valid_result_ext"). - aggregationMethod(ProfileAggregationMethod.COUNT). - firstSeenExpirationDuration(10L). - build(); + ProfileMeasurementConfig numericWithFirstSeenDuration = ProfileMeasurementConfig.builder() + .fieldName("valid_field").resultExtensionName("valid_result_ext") + .aggregationMethod(ProfileAggregationMethod.COUNT) + .firstSeenExpirationDuration(10L) + .build(); testIllegalField(numericWithFirstSeenDuration, String.format(ProfileMeasurementConfig.FIRST_SEEN_ON_NUMERIC, TEST_PROFILE_GROUP, TEST_OFFSET)); - ProfileMeasurementConfig numericWithFirstSeenDurationUnit = ProfileMeasurementConfig.builder(). - fieldName("valid_field").resultExtensionName("valid_result_ext"). - aggregationMethod(ProfileAggregationMethod.COUNT). - firstSeenExpirationDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileMeasurementConfig numericWithFirstSeenDurationUnit = ProfileMeasurementConfig.builder() + .fieldName("valid_field").resultExtensionName("valid_result_ext") + .aggregationMethod(ProfileAggregationMethod.COUNT) + .firstSeenExpirationDurationUnit(TimeUnit.HOURS.name()) + .build(); testIllegalField(numericWithFirstSeenDurationUnit, String.format(ProfileMeasurementConfig.FIRST_SEEN_ON_NUMERIC, TEST_PROFILE_GROUP, TEST_OFFSET)); - ProfileMeasurementConfig numericWithFirstSeenDurationAndUnit = ProfileMeasurementConfig.builder(). - fieldName("valid_field").resultExtensionName("valid_result_ext"). - aggregationMethod(ProfileAggregationMethod.COUNT). - firstSeenExpirationDuration(10L).firstSeenExpirationDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileMeasurementConfig numericWithFirstSeenDurationAndUnit = ProfileMeasurementConfig.builder() + .fieldName("valid_field").resultExtensionName("valid_result_ext") + .aggregationMethod(ProfileAggregationMethod.COUNT) + .firstSeenExpirationDuration(10L).firstSeenExpirationDurationUnit(TimeUnit.HOURS.name()) + .build(); testIllegalField(numericWithFirstSeenDurationAndUnit, String.format(ProfileMeasurementConfig.FIRST_SEEN_ON_NUMERIC, TEST_PROFILE_GROUP, TEST_OFFSET)); } @Test public void testIllegalFirstSeenDurationAndUnit() { - ProfileMeasurementConfig missingUnit = ProfileMeasurementConfig.builder(). - fieldName("valid_field").resultExtensionName("valid_result_ext"). - aggregationMethod(ProfileAggregationMethod.FIRST_SEEN). - firstSeenExpirationDuration(10L). - build(); + ProfileMeasurementConfig missingUnit = ProfileMeasurementConfig.builder() + .fieldName("valid_field").resultExtensionName("valid_result_ext") + .aggregationMethod(ProfileAggregationMethod.FIRST_SEEN) + .firstSeenExpirationDuration(10L) + .build(); testNullField(missingUnit, String.format(ProfileGroupConfig.PROFILE_TIME_ERROR, TEST_PROFILE_GROUP, "firstSeenExpirationDurationUnit", "null")); - ProfileMeasurementConfig missingDuration = ProfileMeasurementConfig.builder(). - fieldName("valid_field").resultExtensionName("valid_result_ext"). - aggregationMethod(ProfileAggregationMethod.FIRST_SEEN). - firstSeenExpirationDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileMeasurementConfig missingDuration = ProfileMeasurementConfig.builder() + .fieldName("valid_field").resultExtensionName("valid_result_ext") + .aggregationMethod(ProfileAggregationMethod.FIRST_SEEN) + .firstSeenExpirationDurationUnit(TimeUnit.HOURS.name()) + .build(); testNullField(missingDuration, String.format(ProfileGroupConfig.PROFILE_TIME_ERROR, TEST_PROFILE_GROUP, "firstSeenExpirationDuration", "null")); - ProfileMeasurementConfig illegalUnits = ProfileMeasurementConfig.builder(). - fieldName("valid_field").resultExtensionName("valid_result_ext"). - aggregationMethod(ProfileAggregationMethod.FIRST_SEEN). - firstSeenExpirationDuration(10L).firstSeenExpirationDurationUnit("illegal unit"). - build(); + ProfileMeasurementConfig illegalUnits = ProfileMeasurementConfig.builder() + .fieldName("valid_field").resultExtensionName("valid_result_ext") + .aggregationMethod(ProfileAggregationMethod.FIRST_SEEN) + .firstSeenExpirationDuration(10L).firstSeenExpirationDurationUnit("illegal unit") + .build(); testIllegalField(illegalUnits, String.format(ProfileGroupConfig.PROFILE_TIME_ERROR, TEST_PROFILE_GROUP, "firstSeenExpirationDurationUnit", "not a legal time unit")); - ProfileMeasurementConfig negativeDuration = ProfileMeasurementConfig.builder(). - fieldName("valid_field").resultExtensionName("valid_result_ext"). - aggregationMethod(ProfileAggregationMethod.FIRST_SEEN). - firstSeenExpirationDuration(-8L).firstSeenExpirationDurationUnit(TimeUnit.HOURS.name()). - build(); + ProfileMeasurementConfig negativeDuration = ProfileMeasurementConfig.builder() + .fieldName("valid_field").resultExtensionName("valid_result_ext") + .aggregationMethod(ProfileAggregationMethod.FIRST_SEEN) + .firstSeenExpirationDuration(-8L).firstSeenExpirationDurationUnit(TimeUnit.HOURS.name()) + .build(); testIllegalField(negativeDuration, String.format(ProfileGroupConfig.PROFILE_TIME_ERROR, TEST_PROFILE_GROUP, "firstSeenExpirationDuration", "0 or negative")); } diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/BaseDynamicRule.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/BaseDynamicRule.java index 3e98bec3..1ee08d90 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/BaseDynamicRule.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/BaseDynamicRule.java @@ -13,15 +13,14 @@ package com.cloudera.cyber.rules; import com.cloudera.cyber.Message; +import java.time.Instant; +import java.util.Map; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; import org.apache.avro.specific.SpecificRecordBase; -import java.time.Instant; -import java.util.Map; - @Getter @Setter @@ -59,7 +58,7 @@ public Map apply(Message message) { return getType().engine(ruleScript).feed(message); } - public static abstract class BaseDynamicRuleBuilder> { + public abstract static class BaseDynamicRuleBuilder> { private String name; private int order; private Instant tsStart; @@ -68,7 +67,8 @@ public static abstract class BaseDynamicRuleBuilder b) { + private static void $fillValuesFromInstanceIntoBuilder(BaseDynamicRule instance, + BaseDynamicRuleBuilder b) { b.name(instance.name); b.order(instance.order); b.tsStart(instance.tsStart); @@ -118,7 +118,9 @@ public B version(int version) { public abstract C build(); public String toString() { - return "BaseDynamicRule.BaseDynamicRuleBuilder(super=" + super.toString() + ", name=" + this.name + ", order=" + this.order + ", tsStart=" + this.tsStart + ", tsEnd=" + this.tsEnd + ", type=" + this.type + ", ruleScript=" + this.ruleScript + ", version=" + this.version + ")"; + return "BaseDynamicRule.BaseDynamicRuleBuilder(super=" + super.toString() + ", name=" + this.name + + ", order=" + this.order + ", tsStart=" + this.tsStart + ", tsEnd=" + this.tsEnd + ", type=" + + this.type + ", ruleScript=" + this.ruleScript + ", version=" + this.version + ")"; } protected B $fillValuesFrom(C instance) { diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRule.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRule.java index c15bba11..71c564ce 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRule.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRule.java @@ -13,12 +13,11 @@ package com.cloudera.cyber.rules; import com.cloudera.cyber.Message; - import java.util.Map; -import java.util.UUID; public interface DynamicRule { boolean isEnabled(); + boolean isValid(); int getOrder(); @@ -28,7 +27,9 @@ public interface DynamicRule { int getVersion(); T withId(String uuid); + T withEnabled(boolean enabled); + T withVersion(int version); Map apply(Message message); diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleCommand.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleCommand.java index 72f2db4c..cc4a38bc 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleCommand.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleCommand.java @@ -12,25 +12,31 @@ package com.cloudera.cyber.rules; +import static com.cloudera.cyber.AvroTypes.utf8toStringMap; + import com.cloudera.cyber.Timestamped; import com.cloudera.cyber.flink.HasHeaders; -import lombok.*; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.ToString; import org.apache.avro.AvroRuntimeException; import org.apache.avro.specific.SpecificRecordBase; -import java.util.Map; -import java.util.UUID; - -import static com.cloudera.cyber.AvroTypes.utf8toStringMap; - @Data @NoArgsConstructor @AllArgsConstructor @ToString -public abstract class DynamicRuleCommand extends SpecificRecordBase implements Timestamped, HasHeaders { - @NonNull protected String id; - @NonNull protected DynamicRuleCommandType type; - @NonNull protected long ts; +public abstract class DynamicRuleCommand extends SpecificRecordBase + implements Timestamped, HasHeaders { + @NonNull + protected String id; + @NonNull + protected DynamicRuleCommandType type; + @NonNull + protected long ts; protected String ruleId = null; protected T rule = null; protected Map headers; @@ -47,30 +53,53 @@ protected DynamicRuleCommand(DynamicRuleCommandBuilder b) { @Override public Object get(int field$) { switch (field$) { - case 0: return id; - case 1: return type; - case 2: return ts; - case 3: return ruleId; - case 4: return rule; - case 5: return headers; - default: throw new AvroRuntimeException("Bad index"); + case 0: + return id; + case 1: + return type; + case 2: + return ts; + case 3: + return ruleId; + case 4: + return rule; + case 5: + return headers; + default: + throw new AvroRuntimeException("Bad index"); } } @Override public void put(int field$, Object value$) { switch (field$) { - case 0: id = value$.toString(); break; - case 1: type = DynamicRuleCommandType.valueOf(value$.toString()); break; - case 2: ts = (long) value$; break; - case 3: ruleId = (value$ != null ) ? (value$.toString()) : null; break; - case 4: rule = (T)value$; break; - case 5: headers = utf8toStringMap(value$); break; - default: throw new AvroRuntimeException("Bad index"); + case 0: + id = value$.toString(); + break; + case 1: + type = DynamicRuleCommandType.valueOf(value$.toString()); + break; + case 2: + ts = (long) value$; + break; + case 3: + ruleId = (value$ != null) ? (value$.toString()) : null; + break; + case 4: + rule = (T) value$; + break; + case 5: + headers = utf8toStringMap(value$); + break; + default: + throw new AvroRuntimeException("Bad index"); } } - public static abstract class DynamicRuleCommandBuilder, B extends DynamicRuleCommandBuilder> { + public abstract static class DynamicRuleCommandBuilder< + T extends DynamicRule, + C extends DynamicRuleCommand, + B extends DynamicRuleCommandBuilder> { private @NonNull String id; private @NonNull DynamicRuleCommandType type; private @NonNull long ts; @@ -113,7 +142,9 @@ public B headers(Map headers) { public abstract C build(); public String toString() { - return "DynamicRuleCommand.DynamicRuleCommandBuilder(super=" + super.toString() + ", id=" + this.id + ", type=" + this.type + ", ts=" + this.ts + ", ruleId=" + this.ruleId + ", rule=" + this.rule + ", headers=" + this.headers + ")"; + return "DynamicRuleCommand.DynamicRuleCommandBuilder(super=" + super.toString() + ", id=" + this.id + + ", type=" + this.type + ", ts=" + this.ts + ", ruleId=" + this.ruleId + ", rule=" + this.rule + + ", headers=" + this.headers + ")"; } } } diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleCommandResult.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleCommandResult.java index b3915b9d..2e0f0786 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleCommandResult.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleCommandResult.java @@ -105,7 +105,8 @@ public DynamicRuleCommandResultBuilder subtaskNumber(int numberOfPara public abstract RESULT build(); public String toString() { - return "DynamicRuleCommandResult.DynamicRuleCommandResultBuilder(super=" + super.toString() + ", cmdId=" + this.cmdId + ", success=" + this.success + ", rule=" + this.rule + ")"; + return "DynamicRuleCommandResult.DynamicRuleCommandResultBuilder(super=" + super.toString() + ", cmdId=" + + this.cmdId + ", success=" + this.success + ", rule=" + this.rule + ")"; } } } diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleProcessFunction.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleProcessFunction.java index 198dec32..8d7eae0d 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleProcessFunction.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/DynamicRuleProcessFunction.java @@ -13,6 +13,15 @@ package com.cloudera.cyber.rules; import com.cloudera.cyber.Message; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -22,15 +31,13 @@ import org.apache.flink.util.Collector; import org.apache.flink.util.OutputTag; -import java.util.*; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @RequiredArgsConstructor @Slf4j -public abstract class DynamicRuleProcessFunction, C extends DynamicRuleCommand, CR extends DynamicRuleCommandResult, T> - extends BroadcastProcessFunction { +public abstract class DynamicRuleProcessFunction< + R extends DynamicRule, + C extends DynamicRuleCommand, + CR extends DynamicRuleCommandResult, T> + extends BroadcastProcessFunction { @NonNull protected OutputTag outputSink; @NonNull @@ -38,21 +45,27 @@ public abstract class DynamicRuleProcessFunction, C ext protected transient Supplier> resultBuilderSupplier; @Override - public void processElement(Message message, ReadOnlyContext readOnlyContext, Collector collector) throws Exception { - log.debug("{}, time: {}, processing: {}, watermark: {}: {}", Thread.currentThread().getId(), readOnlyContext.timestamp(), readOnlyContext.currentProcessingTime(), readOnlyContext.currentWatermark(), message); + public void processElement(Message message, ReadOnlyContext readOnlyContext, Collector collector) + throws Exception { + log.debug("{}, time: {}, processing: {}, watermark: {}: {}", Thread.currentThread().getId(), + readOnlyContext.timestamp(), readOnlyContext.currentProcessingTime(), readOnlyContext.currentWatermark(), + message); List rules = readOnlyContext.getBroadcastState(rulesStateDescriptor).get(RulesForm.ACTIVE); processMessages(message, collector, rules); } - protected void setResultBuilderSupplier(Supplier> resultBuilderSupplier) { + protected void setResultBuilderSupplier( + Supplier> resultBuilderSupplier) { this.resultBuilderSupplier = resultBuilderSupplier; } protected abstract void processMessages(Message message, Collector collector, List rules); @Override - public void processBroadcastElement(C dynamicRuleCommand, Context context, Collector collector) throws Exception { - log.info("Rule Command {}, time: {}, processing: %{}, watermark: {}: {}", Thread.currentThread().getId(), context.timestamp(), context.currentProcessingTime(), context.currentWatermark(), dynamicRuleCommand); + public void processBroadcastElement(C dynamicRuleCommand, Context context, Collector collector) + throws Exception { + log.info("Rule Command {}, time: {}, processing: %{}, watermark: {}: {}", Thread.currentThread().getId(), + context.timestamp(), context.currentProcessingTime(), context.currentWatermark(), dynamicRuleCommand); BroadcastState> state = context.getBroadcastState(rulesStateDescriptor); List scoringRules = processRulesCommands(state.get(RulesForm.ALL), dynamicRuleCommand, context); @@ -60,9 +73,9 @@ public void processBroadcastElement(C dynamicRuleCommand, Context context, Colle state.put(RulesForm.ALL, scoringRules); List activeRules = scoringRules.stream() - .filter(DynamicRule::isEnabled) - .sorted(Comparator.comparingInt(DynamicRule::getOrder)) - .collect(Collectors.toList()); + .filter(DynamicRule::isEnabled) + .sorted(Comparator.comparingInt(DynamicRule::getOrder)) + .collect(Collectors.toList()); // update the ordered version with just the active rules for processing state.put(RulesForm.ACTIVE, activeRules); @@ -76,23 +89,24 @@ private List processRulesCommands(List stateRules, C ruleCommand, Context switch (ruleCommand.getType()) { case DELETE: final List newRules = rules.stream() - .filter(r -> { - if (r.getId().equals(ruleCommand.getRuleId())) { - log.info("Successfully removed rule with id '{}'.", ruleCommand.getRuleId()); - outputRule(context, r, ruleCommand.getId()); - return false; - } else { - return true; - } - }) - .collect(Collectors.toList()); + .filter(r -> { + if (r.getId().equals(ruleCommand.getRuleId())) { + log.info("Successfully removed rule with id '{}'.", ruleCommand.getRuleId()); + outputRule(context, r, ruleCommand.getId()); + return false; + } else { + return true; + } + }) + .collect(Collectors.toList()); if (newRules.size() == rules.size()) { log.info("Unable to find rule with id '{}' to remove.", ruleCommand.getRuleId()); outputRule(context, null, ruleCommand.getId()); } return newRules; case UPSERT: - String ruleId = (ruleCommand.getRule().getId() == null) ? UUID.randomUUID().toString() : ruleCommand.getRule().getId(); + String ruleId = (ruleCommand.getRule().getId() == null) ? UUID.randomUUID().toString() : + ruleCommand.getRule().getId(); Optional matchingRule = rules.stream().filter(r -> r.getId().equals(ruleId)).findFirst(); int newRuleVersion = 0; if (matchingRule.isPresent()) { @@ -106,10 +120,11 @@ private List processRulesCommands(List stateRules, C ruleCommand, Context Boolean success = newRule.isValid(); outputRule(context, newRule, ruleCommand.getId(), success); - if(success) { - return Stream.concat(rules.stream().filter(r -> !r.getId().equals(ruleCommand.getRuleId())), - Stream.of(newRule)) - .collect(Collectors.toList());} else { + if (success) { + return Stream.concat(rules.stream().filter(r -> !r.getId().equals(ruleCommand.getRuleId())), + Stream.of(newRule)) + .collect(Collectors.toList()); + } else { return rules; } @@ -118,7 +133,8 @@ private List processRulesCommands(List stateRules, C ruleCommand, Context case DISABLE: return setEnable(context, ruleCommand.getId(), rules, ruleCommand.getRuleId(), false); case GET: - R getRuleResult = rules.stream().filter(r -> r.getId().equals(ruleCommand.getRuleId())).findFirst().orElse(null); + R getRuleResult = + rules.stream().filter(r -> r.getId().equals(ruleCommand.getRuleId())).findFirst().orElse(null); log.info("Get rule with id '{}' ", ruleCommand.getRuleId()); outputRule(context, getRuleResult, ruleCommand.getId()); return rules; @@ -135,16 +151,16 @@ private List processRulesCommands(List stateRules, C ruleCommand, Context private void outputRule(Context context, R r, String cmdId) { - outputRule(context, r, cmdId, true); + outputRule(context, r, cmdId, true); } private void outputRule(Context context, R r, String cmdId, Boolean success) { context.output(this.outputSink, resultBuilderSupplier.get() - .cmdId(cmdId) - .rule(r) - .success(success) - .subtaskNumber(getRuntimeContext().getIndexOfThisSubtask()) - .build()); + .cmdId(cmdId) + .rule(r) + .success(success) + .subtaskNumber(getRuntimeContext().getIndexOfThisSubtask()) + .build()); } private List setEnable(Context context, String id, List allScoringRules, String ruleId, boolean state) { diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/RuleType.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/RuleType.java index 2ee11852..12403204 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/RuleType.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/RuleType.java @@ -12,13 +12,17 @@ package com.cloudera.cyber.rules; -import com.cloudera.cyber.rules.engines.*; +import com.cloudera.cyber.rules.engines.JavaScriptGraaljsEngineBuilder; +import com.cloudera.cyber.rules.engines.JavaScriptNashornEngineBuilder; +import com.cloudera.cyber.rules.engines.PythonEngineBuilder; +import com.cloudera.cyber.rules.engines.RuleEngine; +import com.cloudera.cyber.rules.engines.RuleEngineBuilder; +import com.cloudera.cyber.rules.engines.StellarEngineBuilder; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; -import lombok.Getter; - import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import lombok.Getter; public enum RuleType { JS_GRAAL(new JavaScriptGraaljsEngineBuilder()), @@ -28,10 +32,10 @@ public enum RuleType { JS_NASHORN(new JavaScriptNashornEngineBuilder()), //will use the first valid engine JS(Stream.of(JS_NASHORN, JS_GRAAL) - .map(RuleType::getEngineBuilder) - .filter(RuleEngineBuilder::isValid) - .findFirst() - .orElseThrow(() -> new RuntimeException("No valid JS engine was found!"))), + .map(RuleType::getEngineBuilder) + .filter(RuleEngineBuilder::isValid) + .findFirst() + .orElseThrow(() -> new RuntimeException("No valid JS engine was found!"))), PYTHON(new PythonEngineBuilder()), STELLAR(new StellarEngineBuilder()); @@ -42,16 +46,13 @@ public enum RuleType { private RuleType(RuleEngineBuilder engineBuilder) { this.engineBuilder = engineBuilder; engineCache = Caffeine.newBuilder() - .expireAfterWrite(60, TimeUnit.MINUTES) - .maximumSize(100) - .build(); + .expireAfterWrite(60, TimeUnit.MINUTES) + .maximumSize(100) + .build(); } /** - * Create a cached rule engine based on the script - * - * @param ruleScript - * @return + * Create a cached rule engine based on the script. */ public RuleEngine engine(String ruleScript) { return engineCache.get(ruleScript, s -> engineBuilder.script(s).build()); diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptEngine.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptEngine.java index 2cd7b681..0b24fa40 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptEngine.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptEngine.java @@ -2,11 +2,6 @@ import com.cloudera.cyber.Message; import com.cloudera.cyber.libs.CyberFunctionDefinition; -import lombok.Data; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -import javax.script.ScriptException; import java.lang.reflect.Parameter; import java.util.HashMap; import java.util.Map; @@ -14,13 +9,18 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Collectors; +import javax.script.ScriptException; +import lombok.Data; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; @Data @Slf4j public abstract class JavaScriptEngine implements RuleEngine { protected static final String SCORE_FUNCTION = "score"; - protected static final String setupScript = CyberFunctionDefinition.findAll().map(JavaScriptEngine::generateJavascript).collect(Collectors.joining(";")); + protected static final String setupScript = + CyberFunctionDefinition.findAll().map(JavaScriptEngine::generateJavascript).collect(Collectors.joining(";")); @NonNull protected final String script; @@ -28,13 +28,14 @@ public abstract class JavaScriptEngine implements RuleEngine { public JavaScriptEngine(String script) { this.script = script; - this.functionName = SCORE_FUNCTION + "_" + - Math.abs(this.script.hashCode()) + "_" - + ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE); + this.functionName = SCORE_FUNCTION + "_" + + Math.abs(this.script.hashCode()) + "_" + + ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE); } private static String generateJavascript(CyberFunctionDefinition functionDefinition) { - String paramNames = functionDefinition.getParameters().stream().map(Parameter::getName).collect(Collectors.joining(",")); + String paramNames = + functionDefinition.getParameters().stream().map(Parameter::getName).collect(Collectors.joining(",")); String functionName = functionDefinition.getFunctionName(); return functionName + "=function(" + paramNames + "){ return _" + functionName + ".eval(" + paramNames + ");}"; } @@ -45,7 +46,8 @@ protected static void initBindings(BiConsumer bindingConsumer) { Class implementationClass = func.getImplementationClass(); try { log.info("Registering {} as {}", implementationClass, udfFunctionName); - bindingConsumer.accept("_".concat(udfFunctionName), implementationClass.getDeclaredConstructor().newInstance()); + bindingConsumer.accept("_".concat(udfFunctionName), + implementationClass.getDeclaredConstructor().newInstance()); log.info("Successfully registered {} as {}", implementationClass, udfFunctionName); } catch (Exception e) { log.debug(String.format("Could not register %s as %s", implementationClass, udfFunctionName), e); diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptGraaljsEngine.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptGraaljsEngine.java index 4fddef84..7e1f8944 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptGraaljsEngine.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptGraaljsEngine.java @@ -1,5 +1,8 @@ package com.cloudera.cyber.rules.engines; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import javax.script.ScriptException; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -10,10 +13,6 @@ import org.graalvm.polyglot.HostAccess; import org.graalvm.polyglot.Value; -import javax.script.ScriptException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; - @Slf4j public class JavaScriptGraaljsEngine extends JavaScriptEngine { @@ -39,10 +38,10 @@ public JavaScriptGraaljsEngine(String script) { public void open() { if (!isOpen.get()) { Context context = Context.newBuilder() - .engine(ENGINE) - .allowHostAccess(HostAccess.ALL) - .allowHostClassLookup(className -> true) - .build(); + .engine(ENGINE) + .allowHostAccess(HostAccess.ALL) + .allowHostClassLookup(className -> true) + .build(); Value bindings = context.getBindings(LANGUAGE_ID); initBindings(bindings::putMember); @@ -61,7 +60,7 @@ public void open() { isValid = false; } - if (isValid){ + if (isValid) { context.initialize(LANGUAGE_ID); } diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptNashornEngine.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptNashornEngine.java index f5dcfe4d..dfca8866 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptNashornEngine.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/JavaScriptNashornEngine.java @@ -12,6 +12,12 @@ package com.cloudera.cyber.rules.engines; +import javax.script.Bindings; +import javax.script.Invocable; +import javax.script.ScriptContext; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -20,8 +26,6 @@ import org.apache.commons.lang3.concurrent.LazyInitializer; import org.apache.flink.annotation.VisibleForTesting; -import javax.script.*; - @Slf4j @Deprecated @@ -53,7 +57,8 @@ public JavaScriptNashornEngine(String script) { private static ValidatedScriptEngine create(String functionName, String script) { ScriptEngine engine = mgr.getEngineByName(ENGINE_NAME); if (engine == null) { - log.error("Wasn't able to create Nashorn JavaScript engine. It's likely related to the Java version being higher than 14."); + log.error( + "Wasn't able to create Nashorn JavaScript engine. It's likely related to the Java version being higher than 14."); new ValidatedScriptEngine(false, null); } boolean isValid = true; @@ -102,7 +107,8 @@ public synchronized void eval(String script) throws ScriptException { } @Override - public synchronized Object invokeFunction(String function, Object... args) throws ScriptException, NoSuchMethodException { + public synchronized Object invokeFunction(String function, Object... args) + throws ScriptException, NoSuchMethodException { try { return ((Invocable) getScriptEngine()).invokeFunction(function, args); } catch (ConcurrentException e) { diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/RuleEngine.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/RuleEngine.java index c12acb7c..1ce2cc0b 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/RuleEngine.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/RuleEngine.java @@ -13,15 +13,18 @@ package com.cloudera.cyber.rules.engines; import com.cloudera.cyber.Message; - -import javax.script.ScriptException; import java.util.Map; +import javax.script.ScriptException; public interface RuleEngine { String getScript(); + void open(); + void close(); + boolean validate(); + Map feed(Message message); void eval(String script) throws ScriptException; diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/StellarEngine.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/StellarEngine.java index 8053537c..93b74c1b 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/StellarEngine.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/StellarEngine.java @@ -13,13 +13,11 @@ package com.cloudera.cyber.rules.engines; import com.cloudera.cyber.Message; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; -import javax.script.ScriptException; -import java.util.Map; - @Getter @AllArgsConstructor @EqualsAndHashCode diff --git a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/UnimplementedEngine.java b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/UnimplementedEngine.java index 024d94cc..adb3cd7b 100644 --- a/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/UnimplementedEngine.java +++ b/flink-cyber/flink-rules/src/main/java/com/cloudera/cyber/rules/engines/UnimplementedEngine.java @@ -13,8 +13,6 @@ package com.cloudera.cyber.rules.engines; import com.cloudera.cyber.Message; - -import javax.script.ScriptException; import java.util.Map; public class UnimplementedEngine implements RuleEngine { diff --git a/flink-cyber/flink-rules/src/test/java/com/cloudera/cyber/rules/engines/TestJavascriptEngine.java b/flink-cyber/flink-rules/src/test/java/com/cloudera/cyber/rules/engines/TestJavascriptEngine.java index 1c94c4fb..3e95f5db 100644 --- a/flink-cyber/flink-rules/src/test/java/com/cloudera/cyber/rules/engines/TestJavascriptEngine.java +++ b/flink-cyber/flink-rules/src/test/java/com/cloudera/cyber/rules/engines/TestJavascriptEngine.java @@ -12,31 +12,31 @@ package com.cloudera.cyber.rules.engines; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; + import com.cloudera.cyber.Message; import com.cloudera.cyber.TestUtils; import com.cloudera.cyber.rules.RuleType; import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import javax.script.ScriptException; import java.time.Instant; import java.util.Map; import java.util.stream.Stream; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasKey; +import javax.script.ScriptException; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; public class TestJavascriptEngine { - private static final String scoreScript = "return { local: ip_local(message.local), remote: ip_local(message.remote) }"; + private static final String scoreScript = + "return { local: ip_local(message.local), remote: ip_local(message.remote) }"; public static Stream jsBuilders() { return Stream.of(RuleType.JS, RuleType.JS_GRAAL, RuleType.JS_NASHORN) - .map(RuleType::getEngineBuilder) - .filter(RuleEngineBuilder::isValid) - .map(Arguments::of); + .map(RuleType::getEngineBuilder) + .filter(RuleEngineBuilder::isValid) + .map(Arguments::of); } @ParameterizedTest @@ -46,12 +46,12 @@ public void testJavascriptExecution(RuleEngineBuilder results = engine.feed( - createMessage(Message.builder() - .ts(Instant.now().toEpochMilli()) - .extensions(ImmutableMap.of( - "a", "1.0", - "b", "2.0" - ))) + createMessage(Message.builder() + .ts(Instant.now().toEpochMilli()) + .extensions(ImmutableMap.of( + "a", "1.0", + "b", "2.0" + ))) ); assertThat("Results are produced", results, hasKey("score")); @@ -70,16 +70,17 @@ public void testJavascriptEval(RuleEngineBuilder jsB @ParameterizedTest @MethodSource("jsBuilders") public void testExtractHostname(RuleEngineBuilder jsBuilder) { - String extractHostnameScript = "return { tld: extract_hostname(message.domain, Java.type(\"com.cloudera.cyber.libs.hostnames.ExtractHostname.HostnameFeature\").TLD) }"; + String extractHostnameScript = + "return { tld: extract_hostname(message.domain, Java.type(\"com.cloudera.cyber.libs.hostnames.ExtractHostname.HostnameFeature\").TLD) }"; RuleEngine engine = jsBuilder.script(extractHostnameScript).build(); engine.open(); Map results = engine.feed( - createMessage(Message.builder() - .ts(Instant.now().toEpochMilli()) - .extensions(ImmutableMap.of( - "domain", "google.com" - ))) + createMessage(Message.builder() + .ts(Instant.now().toEpochMilli()) + .extensions(ImmutableMap.of( + "domain", "google.com" + ))) ); assertThat("top level domain correct", results.get("tld"), equalTo("com")); @@ -92,12 +93,12 @@ public void testJavascriptWithUDF(RuleEngineBuilder engine.open(); Map results = engine.feed( - createMessage(Message.builder() - .ts(Instant.now().toEpochMilli()) - .extensions(ImmutableMap.of( - "local", "192.168.0.1", - "remote", "8.8.8.8" - ))) + createMessage(Message.builder() + .ts(Instant.now().toEpochMilli()) + .extensions(ImmutableMap.of( + "local", "192.168.0.1", + "remote", "8.8.8.8" + ))) ); assertThat("Results are produced", results, hasKey("local")); @@ -112,12 +113,12 @@ public void testJavascriptWithMultiArgUDF(RuleEngineBuilder results = engine.feed( - createMessage(Message.builder() - .ts(Instant.now().toEpochMilli()) - .extensions(ImmutableMap.of( - "local", "192.168.0.1", - "remote", "8.8.8.8" - ))) + createMessage(Message.builder() + .ts(Instant.now().toEpochMilli()) + .extensions(ImmutableMap.of( + "local", "192.168.0.1", + "remote", "8.8.8.8" + ))) ); assertThat("Results are produced", results, hasKey("local")); @@ -132,11 +133,11 @@ public void testJavascriptWithUDFcache(RuleEngineBuilder results = engine.feed( - createMessage(Message.builder() - .ts(Instant.now().toEpochMilli()) - .extensions(ImmutableMap.of( - "local", "192.168.0.1" - ))) + createMessage(Message.builder() + .ts(Instant.now().toEpochMilli()) + .extensions(ImmutableMap.of( + "local", "192.168.0.1" + ))) ); assertThat(results.get("local"), equalTo(true)); } @@ -145,18 +146,20 @@ public void testJavascriptWithUDFcache(RuleEngineBuilder jsBuilder) { - RuleEngine invalidEngine = jsBuilder.script("111return { local: in_subnet(message.local, '192.168.0.1/24') }").build(); + RuleEngine invalidEngine = + jsBuilder.script("111return { local: in_subnet(message.local, '192.168.0.1/24') }").build(); assertThat(invalidEngine.validate(), equalTo(false)); - RuleEngine validEngine = jsBuilder.script("return { local: in_subnet(message.local, '192.168.0.1/24') }").build(); + RuleEngine validEngine = + jsBuilder.script("return { local: in_subnet(message.local, '192.168.0.1/24') }").build(); assertThat(validEngine.validate(), equalTo(true)); } private Message createMessage(Message.MessageBuilder builder) { return builder - .message("") - .source("test") - .originalSource(TestUtils.source()) - .build(); + .message("") + .source("test") + .originalSource(TestUtils.source()) + .build(); } } diff --git a/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/Session.java b/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/Session.java index d0a41da9..4fb05750 100644 --- a/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/Session.java +++ b/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/Session.java @@ -15,29 +15,30 @@ import com.cloudera.cyber.GroupedMessage; import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.operators.MessageConcatenate; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows; import org.apache.flink.streaming.api.windowing.time.Time; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - public class Session { - public static SingleOutputStreamOperator sessionize(DataStream source, final List sessionKey, final Long sessionTimeout) { + public static SingleOutputStreamOperator sessionize(DataStream source, + final List sessionKey, + final Long sessionTimeout) { return source - .keyBy(new KeySelector>() { - @Override - public Map getKey(Message message) throws Exception { - return sessionKey.stream().collect(Collectors.toMap( - f -> f.toString(), - f -> message.getExtensions().get(f).toString())); - } - }) - .window(EventTimeSessionWindows.withGap(Time.milliseconds(sessionTimeout))) - .aggregate(new MessageConcatenate()); + .keyBy(new KeySelector>() { + @Override + public Map getKey(Message message) throws Exception { + return sessionKey.stream().collect(Collectors.toMap( + f -> f, + f -> message.getExtensions().get(f))); + } + }) + .window(EventTimeSessionWindows.withGap(Time.milliseconds(sessionTimeout))) + .aggregate(new MessageConcatenate()); } } diff --git a/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/SessionJob.java b/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/SessionJob.java index 447522f5..f2adbf56 100644 --- a/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/SessionJob.java +++ b/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/SessionJob.java @@ -12,19 +12,18 @@ package com.cloudera.cyber.sessions; +import static com.cloudera.cyber.sessions.Session.sessionize; + import com.cloudera.cyber.GroupedMessage; import com.cloudera.cyber.Message; +import java.util.Arrays; +import java.util.List; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import java.util.Arrays; -import java.util.List; - -import static com.cloudera.cyber.sessions.Session.sessionize; - public abstract class SessionJob { private static final long DEFAULT_SESSION_TIMEOUT = 1000; protected static final String PARAM_SESSION_KEY = "session.key"; @@ -46,6 +45,8 @@ public StreamExecutionEnvironment createPipeline(ParameterTool params) throws Ex } protected abstract void writeResults(ParameterTool params, SingleOutputStreamOperator results); - protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, List sessionKey, Long sessionTimeout); + + protected abstract DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, + List sessionKey, Long sessionTimeout); } \ No newline at end of file diff --git a/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/SessionJobKafka.java b/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/SessionJobKafka.java index 58503acf..ec83060d 100644 --- a/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/SessionJobKafka.java +++ b/flink-cyber/flink-sessions/src/main/java/com/cloudera/cyber/sessions/SessionJobKafka.java @@ -12,10 +12,14 @@ package com.cloudera.cyber.sessions; +import static com.cloudera.cyber.flink.FlinkUtils.createKafkaSource; + import com.cloudera.cyber.GroupedMessage; import com.cloudera.cyber.Message; import com.cloudera.cyber.flink.FlinkUtils; import com.cloudera.cyber.flink.Utils; +import java.util.Arrays; +import java.util.List; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.connector.kafka.sink.KafkaSink; @@ -24,11 +28,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Preconditions; -import java.util.Arrays; -import java.util.List; - -import static com.cloudera.cyber.flink.FlinkUtils.createKafkaSource; - public class SessionJobKafka extends SessionJob { public static void main(String[] args) throws Exception { @@ -39,37 +38,38 @@ public static void main(String[] args) throws Exception { } /** - * Returns a consumer group id for the sessioniser ensuring that each topic is only processed once with the same keys + * Returns a consumer group id for the sessioniser ensuring that each topic is only processed once with the same keys. * - * @param inputTopic topic to read from - * @param sessionKey the keys being used to sessionise + * @param inputTopic topic to read from + * @param sessionKey the keys being used to sessionise * @param sessionTimeout duration of time window for session * @return Generated group id for Kakfa */ private String createGroupId(String inputTopic, List sessionKey, Long sessionTimeout) { List parts = Arrays.asList("sessionizer", - inputTopic, - String.valueOf(sessionKey.hashCode()), - String.valueOf(sessionTimeout)); + inputTopic, + String.valueOf(sessionKey.hashCode()), + String.valueOf(sessionTimeout)); return String.join(".", parts); } @Override protected void writeResults(ParameterTool params, SingleOutputStreamOperator results) { KafkaSink sink = new FlinkUtils<>(GroupedMessage.class).createKafkaSink( - params.getRequired("topic.enrichment"), "sessionizer", - params); + params.getRequired("topic.enrichment"), "sessionizer", + params); results.sinkTo(sink).name("Kafka Results").uid("kafka.results"); } @Override - protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, List sessionKey, Long sessionTimeout) { + protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params, + List sessionKey, Long sessionTimeout) { String inputTopic = params.getRequired("topic.input"); String groupId = createGroupId(inputTopic, sessionKey, sessionTimeout); return - env.fromSource(createKafkaSource(inputTopic, - params, - groupId), WatermarkStrategy.noWatermarks(), "Kafka Source") - .uid("kafka.input"); + env.fromSource(createKafkaSource(inputTopic, + params, + groupId), WatermarkStrategy.noWatermarks(), "Kafka Source") + .uid("kafka.input"); } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java index e7444b1b..30a6ad2c 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java @@ -18,14 +18,12 @@ package org.apache.metron.stellar.common; +import static org.apache.commons.lang3.StringUtils.isEmpty; + import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.base.Joiner; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.TokenStream; - import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; @@ -33,7 +31,14 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; - +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.TokenStream; +import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator; +import org.apache.metron.stellar.common.evaluators.ComparisonExpressionWithOperatorEvaluator; +import org.apache.metron.stellar.common.evaluators.NumberLiteralEvaluator; +import org.apache.metron.stellar.common.generated.StellarLexer; +import org.apache.metron.stellar.common.generated.StellarParser; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.Context.ActivityType; import org.apache.metron.stellar.dsl.DefaultVariableResolver; @@ -42,13 +47,6 @@ import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator; -import org.apache.metron.stellar.common.evaluators.ComparisonExpressionWithOperatorEvaluator; -import org.apache.metron.stellar.common.evaluators.NumberLiteralEvaluator; -import org.apache.metron.stellar.common.generated.StellarLexer; -import org.apache.metron.stellar.common.generated.StellarParser; - -import static org.apache.commons.lang3.StringUtils.isEmpty; /** * The base implementation of a Stellar processor. This is used to evaluate Stellar expressions. @@ -58,208 +56,211 @@ * @see StellarPredicateProcessor */ public class BaseStellarProcessor { - public static final int DEFAULT_CACHE_SIZE = 500; - public static final int DEFAULT_EXPIRY_TIME = 10; - public static final TimeUnit DEFAULT_EXPIRY_TIME_UNITS = TimeUnit.MINUTES; - - /** - * The default expression cache. This is used when the expression cache is not otherwise specified. - */ - private static Cache defaultExpressionCache; - static { - defaultExpressionCache = createCache(DEFAULT_CACHE_SIZE, DEFAULT_EXPIRY_TIME, DEFAULT_EXPIRY_TIME_UNITS); - } - /** - * The class containing the type that the Stellar expression being processed will evaluate to. - */ - private Class clazz; + public static final int DEFAULT_CACHE_SIZE = 500; + public static final int DEFAULT_EXPIRY_TIME = 10; + public static final TimeUnit DEFAULT_EXPIRY_TIME_UNITS = TimeUnit.MINUTES; - /** - * @param clazz The class containing the type that the Stellar expression being processed will evaluate to. - */ - Cache expressionCache; + /** + * The default expression cache. This is used when the expression cache is not otherwise specified. + */ + private static final Cache defaultExpressionCache; - /** - * Create a default stellar processor. This processor uses the static expression cache. - */ - BaseStellarProcessor(final Class clazz) { - this(clazz, defaultExpressionCache); - } - - BaseStellarProcessor(final Class clazz, int cacheSize, int expiryTime, TimeUnit expiryUnit) { - this(clazz, createCache(cacheSize, expiryTime, expiryUnit)); - } + static { + defaultExpressionCache = createCache(DEFAULT_CACHE_SIZE, DEFAULT_EXPIRY_TIME, DEFAULT_EXPIRY_TIME_UNITS); + } - BaseStellarProcessor(final Class clazz, Cache expressionCache) { - this.clazz = clazz; - this.expressionCache = expressionCache; - } + /** + * The class containing the type that the Stellar expression being processed will evaluate to. + */ + private final Class clazz; - static Cache createCache( int cacheSize - , int expiryTime - , TimeUnit expiryUnit - ) { - CacheLoader loader = key -> compile(key); - return Caffeine.newBuilder() - .maximumSize(cacheSize) - .expireAfterAccess(expiryTime, expiryUnit) - .build(loader); - } + Cache expressionCache; - /** - * Parses the given rule and returns a set of variables that are used in the given Stellar expression, {@code rule}. - * - * @param rule The Stellar expression to find out what variables are used. - * @return A set of variables used in the given Stellar expression. - */ - public Set variablesUsed(final String rule) { - if (rule == null || isEmpty(rule.trim())) { - return null; + /** + * Create a default stellar processor. This processor uses the static expression cache. + */ + BaseStellarProcessor(final Class clazz) { + this(clazz, defaultExpressionCache); } - StellarCompiler.Expression expression = null; - try { - expression = expressionCache.get(rule, r -> compile(r)); - } catch (Throwable e) { - throw new ParseException("Unable to parse: " + rule + " due to: " + e.getMessage(), e); - } - return expression.variablesUsed; - } - /** - * Parses and evaluates the given Stellar expression, {@code rule}. - * @param rule The Stellar expression to parse and evaluate. - * @param variableResolver The {@link VariableResolver} to determine values of variables used in the Stellar expression, {@code rule}. - * @param functionResolver The {@link FunctionResolver} to determine values of functions used in the Stellar expression, {@code rule}. - * @param context The context used during validation. - * @return The value of the evaluated Stellar expression, {@code rule}. - */ - public T parse(final String rule, final VariableResolver variableResolver, final FunctionResolver functionResolver, final Context context) { - StellarCompiler.Expression expression = null; - if (rule == null || isEmpty(rule.trim())) { - return null; - } - if(context.getActivityType() == null) { - context.setActivityType(ActivityType.PARSE_ACTIVITY); + BaseStellarProcessor(final Class clazz, int cacheSize, int expiryTime, TimeUnit expiryUnit) { + this(clazz, createCache(cacheSize, expiryTime, expiryUnit)); } - try { - expression = expressionCache.get(rule, r -> compile(r)); - } catch (Throwable e) { - throw createException(rule, variableResolver, e); - } - try { - return clazz.cast(expression - .apply(new StellarCompiler.ExpressionState(context, functionResolver, variableResolver))); - } - catch(Throwable e) { - throw createException(rule, variableResolver, e); + + BaseStellarProcessor(final Class clazz, Cache expressionCache) { + this.clazz = clazz; + this.expressionCache = expressionCache; } - finally { - // always reset the activity type - context.setActivityType(null); + + static Cache createCache(int cacheSize, + int expiryTime, + TimeUnit expiryUnit + ) { + CacheLoader loader = key -> compile(key); + return Caffeine.newBuilder() + .maximumSize(cacheSize) + .expireAfterAccess(expiryTime, expiryUnit) + .build(loader); } - } - private ParseException createException(String rule, VariableResolver resolver, Throwable t) { - String message = "Unable to parse: " + rule + " due to: " + t.getMessage(); - Set variablesUsed = variablesUsed(rule); - if(variablesUsed == null || variablesUsed.isEmpty()) { - return new ParseException(message, t); + /** + * Parses the given rule and returns a set of variables that are used in the given Stellar expression, {@code rule}. + * + * @param rule The Stellar expression to find out what variables are used. + * @return A set of variables used in the given Stellar expression. + */ + public Set variablesUsed(final String rule) { + if (rule == null || isEmpty(rule.trim())) { + return null; + } + StellarCompiler.Expression expression = null; + try { + expression = expressionCache.get(rule, r -> compile(r)); + } catch (Throwable e) { + throw new ParseException("Unable to parse: " + rule + " due to: " + e.getMessage(), e); + } + return expression.variablesUsed; } - List> messagesUsed = new ArrayList<>(variablesUsed.size()); - for(String v : variablesUsed) { - Optional resolved = Optional.ofNullable(resolver.resolve(v)); - messagesUsed.add(new AbstractMap.SimpleEntry<>(v, resolved.orElse("missing"))); + + /** + * Parses and evaluates the given Stellar expression, {@code rule}. + * + * @param rule The Stellar expression to parse and evaluate. + * @param variableResolver The {@link VariableResolver} to determine values of variables used in the Stellar expression, {@code rule}. + * @param functionResolver The {@link FunctionResolver} to determine values of functions used in the Stellar expression, {@code rule}. + * @param context The context used during validation. + * @return The value of the evaluated Stellar expression, {@code rule}. + */ + public T parse(final String rule, final VariableResolver variableResolver, final FunctionResolver functionResolver, + final Context context) { + StellarCompiler.Expression expression = null; + if (rule == null || isEmpty(rule.trim())) { + return null; + } + if (context.getActivityType() == null) { + context.setActivityType(ActivityType.PARSE_ACTIVITY); + } + try { + expression = expressionCache.get(rule, r -> compile(r)); + } catch (Throwable e) { + throw createException(rule, variableResolver, e); + } + try { + return clazz.cast(expression + .apply(new StellarCompiler.ExpressionState(context, functionResolver, variableResolver))); + } catch (Throwable e) { + throw createException(rule, variableResolver, e); + } finally { + // always reset the activity type + context.setActivityType(null); + } } - return new ParseException(message + " with relevant variables " + Joiner.on(",").join(messagesUsed), t); - } - /** - * Parses and evaluates the given Stellar expression, {@code rule}. - * @param rule The Stellar expression to parse and evaluate. - * @return The Expression, which can be reevaluated without reparsing in different Contexts and Resolvers. - */ - public static StellarCompiler.Expression compile(final String rule) { - if (rule == null || isEmpty(rule.trim())) { - return null; + private ParseException createException(String rule, VariableResolver resolver, Throwable t) { + String message = "Unable to parse: " + rule + " due to: " + t.getMessage(); + Set variablesUsed = variablesUsed(rule); + if (variablesUsed == null || variablesUsed.isEmpty()) { + return new ParseException(message, t); + } + List> messagesUsed = new ArrayList<>(variablesUsed.size()); + for (String v : variablesUsed) { + Optional resolved = Optional.ofNullable(resolver.resolve(v)); + messagesUsed.add(new AbstractMap.SimpleEntry<>(v, resolved.orElse("missing"))); + } + return new ParseException(message + " with relevant variables " + Joiner.on(",").join(messagesUsed), t); } - ANTLRInputStream input = new ANTLRInputStream(rule); - StellarLexer lexer = new StellarLexer(input); - lexer.removeErrorListeners(); - lexer.addErrorListener(new ErrorListener()); - TokenStream tokens = new CommonTokenStream(lexer); - StellarParser parser = new StellarParser(tokens); + /** + * Parses and evaluates the given Stellar expression, {@code rule}. + * + * @param rule The Stellar expression to parse and evaluate. + * @return The Expression, which can be reevaluated without reparsing in different Contexts and Resolvers. + */ + public static StellarCompiler.Expression compile(final String rule) { + if (rule == null || isEmpty(rule.trim())) { + return null; + } - StellarCompiler treeBuilder = new StellarCompiler( - ArithmeticEvaluator.INSTANCE, - NumberLiteralEvaluator.INSTANCE, - ComparisonExpressionWithOperatorEvaluator.INSTANCE - ); - parser.addParseListener(treeBuilder); - parser.removeErrorListeners(); - parser.addErrorListener(new ErrorListener()); - parser.transformation(); - return treeBuilder.getExpression(); - } + ANTLRInputStream input = new ANTLRInputStream(rule); + StellarLexer lexer = new StellarLexer(input); + lexer.removeErrorListeners(); + lexer.addErrorListener(new ErrorListener()); + TokenStream tokens = new CommonTokenStream(lexer); + StellarParser parser = new StellarParser(tokens); - /** - * This method determines if a given rule is valid or not. If the given rule is valid then true - * will be returned otherwise a {@link ParseException} is thrown. If it is desired to return a boolean - * whether the rule is valid or not, use the {@link #validate(String, boolean, Context) validate} method. It is important - * to note that all variables will resolve to 'null.' - * - * @param rule The rule to validate. - * @return If the given rule is valid then true otherwise an exception is thrown. - * @throws ParseException If the rule is invalid an exception of this type is thrown. - */ - public boolean validate(final String rule) throws ParseException { - return validate(rule, true, Context.EMPTY_CONTEXT()); - } + StellarCompiler treeBuilder = new StellarCompiler( + ArithmeticEvaluator.INSTANCE, + NumberLiteralEvaluator.INSTANCE, + ComparisonExpressionWithOperatorEvaluator.INSTANCE + ); + parser.addParseListener(treeBuilder); + parser.removeErrorListeners(); + parser.addErrorListener(new ErrorListener()); + parser.transformation(); + return treeBuilder.getExpression(); + } - /** - * Validates a given Stellar expression based on given context. - * @param rule The Stellar expression to validate. - * @param context The context used to validate the Stellar expression. - * @return If valid Stellar expression true, otherwise an exception will be thrown. - * @throws ParseException The exception containing the information as to why the expression is not valid. - */ - public boolean validate(final String rule, final Context context) throws ParseException { - return validate(rule, true, context); - } + /** + * This method determines if a given rule is valid or not. If the given rule is valid then true + * will be returned otherwise a {@link ParseException} is thrown. If it is desired to return a boolean + * whether the rule is valid or not, use the {@link #validate(String, boolean, Context) validate} method. It is important + * to note that all variables will resolve to 'null.' + * + * @param rule The rule to validate. + * @return If the given rule is valid then true otherwise an exception is thrown. + * @throws ParseException If the rule is invalid an exception of this type is thrown. + */ + public boolean validate(final String rule) throws ParseException { + return validate(rule, true, Context.EMPTY_CONTEXT()); + } - /** - * Here it is not desirable to add our custom listener. It is not the intent to evaluate the rule. - * The rule is only meant to be validated. Validate in this instance means check whether or not the - * rule is syntactically valid and whether the functions used exist. For example, it will not check - * for variables that are not defined. Currently all variables resolve to 'null.' This is mainly to - * make sure things function as expected when values are null. - * - * @param rule The Stellar transformation to validate. - * @param throwException If true an invalid Stellar transformation will throw a {@link ParseException} otherwise a boolean will be returned. - * @param context The Stellar context to be used when validating the Stellar transformation. - * @return If {@code throwException} is true and {@code rule} is invalid a {@link ParseException} is thrown. If - * {@code throwException} is false and {@code rule} is invalid then false is returned. Otherwise true if {@code rule} is valid, - * false if {@code rule} is invalid. - * @throws ParseException Thrown if {@code rule} is invalid and {@code throwException} is true. - */ - public boolean validate(final String rule, final boolean throwException, final Context context) throws ParseException { - if (rule == null || isEmpty(rule.trim())) { - return true; + /** + * Validates a given Stellar expression based on given context. + * + * @param rule The Stellar expression to validate. + * @param context The context used to validate the Stellar expression. + * @return If valid Stellar expression true, otherwise an exception will be thrown. + * @throws ParseException The exception containing the information as to why the expression is not valid. + */ + public boolean validate(final String rule, final Context context) throws ParseException { + return validate(rule, true, context); } - // set the context to validation - // it will be reset in parse() - context.setActivityType(ActivityType.VALIDATION_ACTIVITY); - try { - parse(rule, DefaultVariableResolver.NULL_RESOLVER(), StellarFunctions.FUNCTION_RESOLVER(), context); - } catch (Throwable t) { - if (throwException) { - throw new ParseException("Unable to parse " + rule + ": " + t.getMessage(), t); - } else { - return false; - } + /** + * Here it is not desirable to add our custom listener. It is not the intent to evaluate the rule. + * The rule is only meant to be validated. Validate in this instance means check whether or not the + * rule is syntactically valid and whether the functions used exist. For example, it will not check + * for variables that are not defined. Currently all variables resolve to 'null.' This is mainly to + * make sure things function as expected when values are null. + * + * @param rule The Stellar transformation to validate. + * @param throwException If true an invalid Stellar transformation will throw a {@link ParseException} + * otherwise a boolean will be returned. + * @param context The Stellar context to be used when validating the Stellar transformation. + * @return If {@code throwException} is true and {@code rule} is invalid a {@link ParseException} is thrown. If + * {@code throwException} is false and {@code rule} is invalid then false is returned. Otherwise true if {@code rule} is valid, + * false if {@code rule} is invalid. + * @throws ParseException Thrown if {@code rule} is invalid and {@code throwException} is true. + */ + public boolean validate(final String rule, final boolean throwException, final Context context) + throws ParseException { + if (rule == null || isEmpty(rule.trim())) { + return true; + } + + // set the context to validation + // it will be reset in parse() + context.setActivityType(ActivityType.VALIDATION_ACTIVITY); + try { + parse(rule, DefaultVariableResolver.NULL_RESOLVER(), StellarFunctions.FUNCTION_RESOLVER(), context); + } catch (Throwable t) { + if (throwException) { + throw new ParseException("Unable to parse " + rule + ": " + t.getMessage(), t); + } else { + return false; + } + } + return true; } - return true; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/BooleanOp.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/BooleanOp.java index 925131e8..8e11c685 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/BooleanOp.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/BooleanOp.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,5 +21,5 @@ package org.apache.metron.stellar.common; public interface BooleanOp { - boolean op(boolean left, boolean right); + boolean op(boolean left, boolean right); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/CachingStellarProcessor.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/CachingStellarProcessor.java index fa43c137..513e2caf 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/CachingStellarProcessor.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/CachingStellarProcessor.java @@ -7,18 +7,27 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -28,211 +37,207 @@ import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; - /** * The Caching Stellar Processor is a stellar processor that optionally fronts stellar with an expression-by-expression * LFU cache. */ public class CachingStellarProcessor extends StellarProcessor { - private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static ThreadLocal> > variableCache = ThreadLocal.withInitial(() -> new HashMap<>()); - - /** - * A property that defines the maximum cache size. - */ - public static String MAX_CACHE_SIZE_PARAM = "stellar.cache.maxSize"; - - /** - * A property that defines the max time in minutes that elements are retained in the cache. - */ - public static String MAX_TIME_RETAIN_PARAM = "stellar.cache.maxTimeRetain"; - - /** - * A property that defines if cache usage stats should be recorded. - */ - public static String RECORD_STATS = "stellar.cache.record.stats"; - - /** - * The cache key is based on the expression and input values. - */ - public static class Key { + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final ThreadLocal>> variableCache = ThreadLocal.withInitial(() -> new HashMap<>()); /** - * The expression to execute. + * A property that defines the maximum cache size. */ - private String expression; + public static String MAX_CACHE_SIZE_PARAM = "stellar.cache.maxSize"; /** - * The variables that serve as input to the expression. + * A property that defines the max time in minutes that elements are retained in the cache. */ - private Map input; - - public Key(String expression, Map input) { - this.expression = expression; - this.input = input; - } - - public String getExpression() { - return expression; - } + public static String MAX_TIME_RETAIN_PARAM = "stellar.cache.maxTimeRetain"; - public Map getInput() { - return input; - } + /** + * A property that defines if cache usage stats should be recorded. + */ + public static String RECORD_STATS = "stellar.cache.record.stats"; - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - Key key = (Key) o; - return new EqualsBuilder() - .append(expression, key.expression) - .append(input, key.input) - .isEquals(); + /** + * The cache key is based on the expression and input values. + */ + public static class Key { + + /** + * The expression to execute. + */ + private final String expression; + + /** + * The variables that serve as input to the expression. + */ + private final Map input; + + public Key(String expression, Map input) { + this.expression = expression; + this.input = input; + } + + public String getExpression() { + return expression; + } + + public Map getInput() { + return input; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + Key key = (Key) o; + return new EqualsBuilder() + .append(expression, key.expression) + .append(input, key.input) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(expression) + .append(input) + .toHashCode(); + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .append("expression", expression) + .append("input", input) + .toString(); + } } + /** + * Parses and evaluates the given Stellar expression, {@code expression}. Results will be taken + * from a cache if possible. + * + * @param expression The Stellar expression to parse and evaluate. + * @param variableResolver The {@link VariableResolver} to determine values of variables used in + * the Stellar expression, {@code expression}. + * @param functionResolver The {@link FunctionResolver} to determine values of functions used in + * the Stellar expression, {@code expression}. + * @param context The context used during validation. + * @return The value of the evaluated Stellar expression, {@code expression}. + */ @Override - public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(expression) - .append(input) - .toHashCode(); + @SuppressWarnings("unchecked") + public Object parse( + String expression, + VariableResolver variableResolver, + FunctionResolver functionResolver, + Context context) { + + Optional cacheOpt = context.getCapability(Context.Capabilities.CACHE, false); + if (cacheOpt.isPresent()) { + + // use the cache + Cache cache = (Cache) cacheOpt.get(); + Key k = toKey(expression, variableResolver); + return cache.get(k, x -> parseUncached(x.expression, variableResolver, functionResolver, context)); + + } else { + + LOG.debug("No cache present."); + return parseUncached(expression, variableResolver, functionResolver, context); + } } - @Override - public String toString() { - return new ToStringBuilder(this) - .append("expression", expression) - .append("input", input) - .toString(); - } - } - - /** - * Parses and evaluates the given Stellar expression, {@code expression}. Results will be taken - * from a cache if possible. - * - * @param expression The Stellar expression to parse and evaluate. - * @param variableResolver The {@link VariableResolver} to determine values of variables used in - * the Stellar expression, {@code expression}. - * @param functionResolver The {@link FunctionResolver} to determine values of functions used in - * the Stellar expression, {@code expression}. - * @param context The context used during validation. - * @return The value of the evaluated Stellar expression, {@code expression}. - */ - @Override - @SuppressWarnings("unchecked") - public Object parse( - String expression, - VariableResolver variableResolver, - FunctionResolver functionResolver, - Context context) { - - Optional cacheOpt = context.getCapability(Context.Capabilities.CACHE, false); - if(cacheOpt.isPresent()) { - - // use the cache - Cache cache = (Cache) cacheOpt.get(); - Key k = toKey(expression, variableResolver); - return cache.get(k, x -> parseUncached(x.expression, variableResolver, functionResolver, context)); - - } else { - - LOG.debug("No cache present."); - return parseUncached(expression, variableResolver, functionResolver, context); - } - } - - protected Object parseUncached(String expression, VariableResolver variableResolver, FunctionResolver functionResolver, Context context) { - LOG.debug("Executing Stellar; expression={}", expression); - return super.parse(expression, variableResolver, functionResolver, context); - } - - /** - * Create a cache key using the expression and all variables used by that expression. - * - * @param expression The Stellar expression. - * @param resolver The variable resolver. - * @return A key with which to do a cache lookup. - */ - protected Key toKey(String expression, VariableResolver resolver) { - - // fetch only the variables used in the expression - Set variablesUsed = variableCache.get().computeIfAbsent(expression, this::variablesUsed); - - // resolve each of the variables used by the expression - Map input = new HashMap<>(); - if (variablesUsed != null) { - for (String v : variablesUsed) { - input.computeIfAbsent(v, resolver::resolve); - } + protected Object parseUncached(String expression, VariableResolver variableResolver, + FunctionResolver functionResolver, Context context) { + LOG.debug("Executing Stellar; expression={}", expression); + return super.parse(expression, variableResolver, functionResolver, context); } - Key cacheKey = new Key(expression, input); - LOG.debug("Created cache key; {}", cacheKey); - return cacheKey; - } - - /** - * Create a cache given a config. Note that if the cache size is {@literal <}= 0, then no cache will be returned. - * @param config configuration for the cache - * @return A cache. - */ - public static Cache createCache(Map config) { - - // the cache configuration is required - if(config == null) { - LOG.debug("Cannot create cache; missing cache configuration"); - return null; - } - - // max cache size is required - Long maxSize = getParam(config, MAX_CACHE_SIZE_PARAM, null, Long.class); - if(maxSize == null || maxSize <= 0) { - LOG.error("Cannot create cache; missing or invalid configuration; {} = {}", MAX_CACHE_SIZE_PARAM, maxSize); - return null; - } - - // max time retain is required - Integer maxTimeRetain = getParam(config, MAX_TIME_RETAIN_PARAM, null, Integer.class); - if(maxTimeRetain == null || maxTimeRetain <= 0) { - LOG.error("Cannot create cache; missing or invalid configuration; {} = {}", MAX_TIME_RETAIN_PARAM, maxTimeRetain); - return null; + /** + * Create a cache key using the expression and all variables used by that expression. + * + * @param expression The Stellar expression. + * @param resolver The variable resolver. + * @return A key with which to do a cache lookup. + */ + protected Key toKey(String expression, VariableResolver resolver) { + + // fetch only the variables used in the expression + Set variablesUsed = variableCache.get().computeIfAbsent(expression, this::variablesUsed); + + // resolve each of the variables used by the expression + Map input = new HashMap<>(); + if (variablesUsed != null) { + for (String v : variablesUsed) { + input.computeIfAbsent(v, resolver::resolve); + } + } + + Key cacheKey = new Key(expression, input); + LOG.debug("Created cache key; {}", cacheKey); + return cacheKey; } - Caffeine cache = Caffeine - .newBuilder() - .maximumSize(maxSize) - .expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES); - - // record stats is optional - Boolean recordStats = getParam(config, RECORD_STATS, false, Boolean.class); - if(recordStats) { - cache.recordStats(); + /** + * Create a cache given a config. Note that if the cache size is {@literal <}= 0, then no cache will be returned. + * + * @param config configuration for the cache + * @return A cache. + */ + public static Cache createCache(Map config) { + + // the cache configuration is required + if (config == null) { + LOG.debug("Cannot create cache; missing cache configuration"); + return null; + } + + // max cache size is required + Long maxSize = getParam(config, MAX_CACHE_SIZE_PARAM, null, Long.class); + if (maxSize == null || maxSize <= 0) { + LOG.error("Cannot create cache; missing or invalid configuration; {} = {}", MAX_CACHE_SIZE_PARAM, maxSize); + return null; + } + + // max time retain is required + Integer maxTimeRetain = getParam(config, MAX_TIME_RETAIN_PARAM, null, Integer.class); + if (maxTimeRetain == null || maxTimeRetain <= 0) { + LOG.error("Cannot create cache; missing or invalid configuration; {} = {}", MAX_TIME_RETAIN_PARAM, + maxTimeRetain); + return null; + } + + Caffeine cache = Caffeine + .newBuilder() + .maximumSize(maxSize) + .expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES); + + // record stats is optional + Boolean recordStats = getParam(config, RECORD_STATS, false, Boolean.class); + if (recordStats) { + cache.recordStats(); + } + + return cache.build(); } - return cache.build(); - } - - private static T getParam(Map config, String key, T defaultVal, Class clazz) { - Object o = config.get(key); - if(o == null) { - return defaultVal; + private static T getParam(Map config, String key, T defaultVal, Class clazz) { + Object o = config.get(key); + if (o == null) { + return defaultVal; + } + T ret = ConversionUtils.convert(o, clazz); + return ret == null ? defaultVal : ret; } - T ret = ConversionUtils.convert(o, clazz); - return ret == null?defaultVal:ret; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/Constants.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/Constants.java index 0eb5e80d..dee3081c 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/Constants.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/Constants.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common; import java.util.HashMap; @@ -22,106 +25,85 @@ public class Constants { - public static final String ZOOKEEPER_ROOT = "/metron"; - public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology"; - public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; - public static final String SENSOR_TYPE = Fields.SENSOR_TYPE.getName(); - public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field"; - public static final String THREAT_SCORE_FIELD_PROPERTY = "threat.triage.score.field"; - public static final String ENRICHMENT_TOPIC = "enrichments"; - public static final String INDEXING_TOPIC = "indexing"; - public static final String ERROR_STREAM = "error"; - public static final String ERROR_TYPE = "error"; - public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment"; - public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel"; - public static final String STELLAR_CONTEXT_CONF = "stellarContext"; - public static final String GUID = Fields.GUID.getName(); - - public interface Field { - String getName(); - } - public enum Fields implements Field { - SRC_ADDR("ip_src_addr") - ,SRC_PORT("ip_src_port") - ,DST_ADDR("ip_dst_addr") - ,DST_PORT("ip_dst_port") - ,PROTOCOL("protocol") - ,TIMESTAMP("timestamp") - ,ORIGINAL("original_string") - ,GUID("guid") - ,SENSOR_TYPE("source.type") - ,INCLUDES_REVERSE_TRAFFIC("includes_reverse_traffic") - ; - private static Map nameToField; - - static { - nameToField = new HashMap<>(); - for (Fields f : Fields.values()) { - nameToField.put(f.getName(), f); - } + public static final String ZOOKEEPER_ROOT = "/metron"; + public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology"; + public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; + public static final String SENSOR_TYPE = Fields.SENSOR_TYPE.getName(); + public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field"; + public static final String THREAT_SCORE_FIELD_PROPERTY = "threat.triage.score.field"; + public static final String ENRICHMENT_TOPIC = "enrichments"; + public static final String INDEXING_TOPIC = "indexing"; + public static final String ERROR_STREAM = "error"; + public static final String ERROR_TYPE = "error"; + public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment"; + public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel"; + public static final String STELLAR_CONTEXT_CONF = "stellarContext"; + public static final String GUID = Fields.GUID.getName(); + + public interface Field { + String getName(); } - private String name; + public enum Fields implements Field { + SRC_ADDR("ip_src_addr"), SRC_PORT("ip_src_port"), DST_ADDR("ip_dst_addr"), DST_PORT("ip_dst_port"), + PROTOCOL("protocol"), TIMESTAMP("timestamp"), ORIGINAL("original_string"), GUID("guid"), + SENSOR_TYPE("source.type"), INCLUDES_REVERSE_TRAFFIC("includes_reverse_traffic"); + private static final Map nameToField; - Fields(String name) { - this.name = name; - } + static { + nameToField = new HashMap<>(); + for (Fields f : Fields.values()) { + nameToField.put(f.getName(), f); + } + } - public String getName() { - return name; - } + private final String name; - public static Fields fromString(String fieldName) { - return nameToField.get(fieldName); - } - } - - public enum ErrorFields { - MESSAGE("message") - ,FAILED_SENSOR_TYPE("failed_sensor_type") - ,ERROR_TYPE("error_type") - ,EXCEPTION("exception") - ,STACK("stack") - ,TIMESTAMP("timestamp") - ,HOSTNAME("hostname") - ,RAW_MESSAGE("raw_message") - ,RAW_MESSAGE_BYTES("raw_message_bytes") - ,ERROR_FIELDS("error_fields") - ,ERROR_HASH("error_hash") - ,METADATA("metadata") - ; - - private String name; - - ErrorFields(String name) { - this.name = name; - } + Fields(String name) { + this.name = name; + } - public String getName() { - return name; + public String getName() { + return name; + } + + public static Fields fromString(String fieldName) { + return nameToField.get(fieldName); + } } - } - public enum ErrorType { + public enum ErrorFields { + MESSAGE("message"), FAILED_SENSOR_TYPE("failed_sensor_type"), ERROR_TYPE("error_type"), EXCEPTION("exception"), + STACK("stack"), TIMESTAMP("timestamp"), HOSTNAME("hostname"), RAW_MESSAGE("raw_message"), + RAW_MESSAGE_BYTES("raw_message_bytes"), ERROR_FIELDS("error_fields"), ERROR_HASH("error_hash"), + METADATA("metadata"); - PARSER_ERROR("parser_error") - ,PARSER_INVALID("parser_invalid") - ,ENRICHMENT_ERROR("enrichments_error") - ,THREAT_INTEL_ERROR("threatintel_error") - ,INDEXING_ERROR("indexing_error") - ,DEFAULT_ERROR("error") - ; + private final String name; - private String type; + ErrorFields(String name) { + this.name = name; + } - ErrorType(String type) { - this.type = type; + public String getName() { + return name; + } } - public String getType() { - return type; + public enum ErrorType { + + PARSER_ERROR("parser_error"), PARSER_INVALID("parser_invalid"), ENRICHMENT_ERROR("enrichments_error"), + THREAT_INTEL_ERROR("threatintel_error"), INDEXING_ERROR("indexing_error"), DEFAULT_ERROR("error"); + + private final String type; + + ErrorType(String type) { + this.type = type; + } + + public String getType() { + return type; + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/DefaultStellarStatefulExecutor.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/DefaultStellarStatefulExecutor.java index 86222c6d..dfee65f6 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/DefaultStellarStatefulExecutor.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/DefaultStellarStatefulExecutor.java @@ -21,142 +21,144 @@ package org.apache.metron.stellar.common; import com.google.common.collect.ImmutableMap; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.lang.ClassUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.stellar.dsl.VariableResolver; -import org.apache.metron.stellar.common.utils.ConversionUtils; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; /** * The default implementation of a StellarStatefulExecutor. */ public class DefaultStellarStatefulExecutor implements StellarStatefulExecutor, Serializable { - /** - * The current state of the Stellar execution environment. - */ - private Map state; - - /** - * Provides additional context for initializing certain Stellar functions. For - * example, references to Zookeeper or HBase. - */ - private Context context; - - /** - * Responsible for function resolution. - */ - private FunctionResolver functionResolver; - - public DefaultStellarStatefulExecutor() { - this(StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); - } - - public DefaultStellarStatefulExecutor(FunctionResolver functionResolver, Context context) { - clearState(); - this.context = context; - this.functionResolver = functionResolver; - } - - /** - * @param initialState Initial state loaded into the execution environment. - */ - public DefaultStellarStatefulExecutor(Map initialState) { - this(); - this.state = new HashMap<>(initialState); - } - - /** - * The current state of the Stellar execution environment. - */ - @Override - public Map getState() { - return ImmutableMap.copyOf(state); - } - - /** - * Execute an expression and assign the result to a variable. The variable is maintained - * in the context of this executor and is available to all subsequent expressions. - * - * @param variable The name of the variable to assign to. - * @param expression The expression to execute. - * @param transientState Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - */ - @Override - public void assign(String variable, String expression, Map transientState) { - Object result = execute(expression, transientState); - if(result == null || variable == null) { - return; + /** + * The current state of the Stellar execution environment. + */ + private Map state; + + /** + * Provides additional context for initializing certain Stellar functions. For + * example, references to Zookeeper or HBase. + */ + private Context context; + + /** + * Responsible for function resolution. + */ + private FunctionResolver functionResolver; + + public DefaultStellarStatefulExecutor() { + this(StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); + } + + public DefaultStellarStatefulExecutor(FunctionResolver functionResolver, Context context) { + clearState(); + this.context = context; + this.functionResolver = functionResolver; } - state.put(variable, result); - } - @Override - public void assign(String variable, Object value) { - if(value == null || variable == null) { - return; + /** + * Initial state constructor. + * + * @param initialState Initial state loaded into the execution environment. + */ + public DefaultStellarStatefulExecutor(Map initialState) { + this(); + this.state = new HashMap<>(initialState); } - state.put(variable, value); - } - - /** - * Execute a Stellar expression and return the result. The internal state of the executor - * is not modified. - * - * @param expression The expression to execute. - * @param state Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - * @param clazz The expected type of the expression's result. - * @param The expected type of the expression's result. - */ - @Override - public T execute(String expression, Map state, Class clazz) { - Object resultObject = execute(expression, state); - - // perform type conversion, if necessary - T result = ConversionUtils.convert(resultObject, clazz); - if (result == null) { - throw new IllegalArgumentException(String.format("Unexpected type: expected=%s, actual=%s, expression=%s", - clazz.getSimpleName(), ClassUtils.getShortClassName(resultObject,"null"), expression)); + + /** + * The current state of the Stellar execution environment. + */ + @Override + public Map getState() { + return ImmutableMap.copyOf(state); + } + + /** + * Execute an expression and assign the result to a variable. The variable is maintained + * in the context of this executor and is available to all subsequent expressions. + * + * @param variable The name of the variable to assign to. + * @param expression The expression to execute. + * @param transientState Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + */ + @Override + public void assign(String variable, String expression, Map transientState) { + Object result = execute(expression, transientState); + if (result == null || variable == null) { + return; + } + state.put(variable, result); + } + + @Override + public void assign(String variable, Object value) { + if (value == null || variable == null) { + return; + } + state.put(variable, value); + } + + /** + * Execute a Stellar expression and return the result. The internal state of the executor + * is not modified. + * + * @param expression The expression to execute. + * @param state Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + * @param clazz The expected type of the expression's result. + * @param The expected type of the expression's result. + */ + @Override + public T execute(String expression, Map state, Class clazz) { + Object resultObject = execute(expression, state); + + // perform type conversion, if necessary + T result = ConversionUtils.convert(resultObject, clazz); + if (result == null) { + throw new IllegalArgumentException(String.format("Unexpected type: expected=%s, actual=%s, expression=%s", + clazz.getSimpleName(), ClassUtils.getShortClassName(resultObject, "null"), expression)); + } + + return result; + } + + /** + * Execute a Stellar expression. + * + * @param expression The expression to execute. + * @param transientState Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + */ + private Object execute(String expression, Map transientState) { + VariableResolver variableResolver = new MapVariableResolver(state, transientState); + StellarProcessor processor = new StellarProcessor(); + return processor.parse(expression, variableResolver, functionResolver, context); + } + + @Override + public void clearState() { + this.state = new HashMap<>(); + } + + @Override + public void setContext(Context context) { + this.context = context; + } + + public void setFunctionResolver(FunctionResolver functionResolver) { + this.functionResolver = functionResolver; } - return result; - } - - @Override - public void clearState() { - this.state = new HashMap<>(); - } - - @Override - public void setContext(Context context) { - this.context = context; - } - - public void setFunctionResolver(FunctionResolver functionResolver) { - this.functionResolver = functionResolver; - } - - /** - * Execute a Stellar expression. - * - * @param expression The expression to execute. - * @param transientState Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - */ - private Object execute(String expression, Map transientState) { - VariableResolver variableResolver = new MapVariableResolver(state, transientState); - StellarProcessor processor = new StellarProcessor(); - return processor.parse(expression, variableResolver, functionResolver, context); - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/FrameContext.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/FrameContext.java index 9e068375..bf8e76c9 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/FrameContext.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/FrameContext.java @@ -7,39 +7,43 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common; public enum FrameContext { - BOOLEAN_AND, - BOOLEAN_OR; + BOOLEAN_AND, + BOOLEAN_OR; - public static class Context { - private FrameContext variety; - public Context(FrameContext variety) { - this.variety = variety; - } + public static class Context { + private final FrameContext variety; - public FrameContext getVariety() { - return variety; - } + public Context(FrameContext variety) { + this.variety = variety; + } - @Override - public String toString() { - return "Context{" + - "variety=" + variety + - '}'; + public FrameContext getVariety() { + return variety; + } + + @Override + public String toString() { + return "Context{" + + "variety=" + variety + + '}'; + } } - } - public Context create() { - return new Context(this); - } + public Context create() { + return new Context(this); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/LambdaExpression.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/LambdaExpression.java index 4f6570ed..cc0056e6 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/LambdaExpression.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/LambdaExpression.java @@ -18,52 +18,53 @@ package org.apache.metron.stellar.common; -import org.apache.metron.stellar.dsl.DefaultVariableResolver; -import org.apache.metron.stellar.dsl.Token; -import org.apache.metron.stellar.dsl.VariableResolver; - import java.util.ArrayDeque; import java.util.Deque; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.HashMap; +import org.apache.metron.stellar.dsl.DefaultVariableResolver; +import org.apache.metron.stellar.dsl.Token; +import org.apache.metron.stellar.dsl.VariableResolver; public class LambdaExpression extends StellarCompiler.Expression { - StellarCompiler.ExpressionState state; - List variables; - public LambdaExpression(List variables, Deque> tokenDeque, StellarCompiler.ExpressionState state) { - super(tokenDeque); - this.state = state; - this.variables = variables; - } + StellarCompiler.ExpressionState state; + List variables; - @Override - public Deque> getTokenDeque() { - Deque> ret = new ArrayDeque<>(super.getTokenDeque().size()); - for(Token token : super.getTokenDeque()) { - ret.add(token); + public LambdaExpression(List variables, Deque> tokenDeque, StellarCompiler.ExpressionState state) { + super(tokenDeque); + this.state = state; + this.variables = variables; } - return ret; - } - public Object apply(List variableArgs) { - Map lambdaVariables = new HashMap<>(); - int i = 0; - for(;i < Math.min(variables.size(),variableArgs.size()) ;++i) { - lambdaVariables.put(variables.get(i), variableArgs.get(i)); - } - for(;i < variables.size();++i) { - lambdaVariables.put(variables.get(i), null); + @Override + public Deque> getTokenDeque() { + Deque> ret = new ArrayDeque<>(super.getTokenDeque().size()); + for (Token token : super.getTokenDeque()) { + ret.add(token); + } + return ret; } - VariableResolver variableResolver = new DefaultVariableResolver(variable -> lambdaVariables.getOrDefault(variable - , state.variableResolver.resolve(variable) - ), variable -> true); - StellarCompiler.ExpressionState localState = new StellarCompiler.ExpressionState( - state.context - , state.functionResolver - , variableResolver); - return apply(localState); - } + public Object apply(List variableArgs) { + Map lambdaVariables = new HashMap<>(); + int i = 0; + for (; i < Math.min(variables.size(), variableArgs.size()); ++i) { + lambdaVariables.put(variables.get(i), variableArgs.get(i)); + } + for (; i < variables.size(); ++i) { + lambdaVariables.put(variables.get(i), null); + } + + VariableResolver variableResolver = + new DefaultVariableResolver(variable -> lambdaVariables.getOrDefault(variable, + state.variableResolver.resolve(variable) + ), variable -> true); + StellarCompiler.ExpressionState localState = new StellarCompiler.ExpressionState( + state.context, + state.functionResolver, + variableResolver); + return apply(localState); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java index 587b5585..b8ee5967 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java @@ -15,123 +15,124 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common; import java.util.Map; -public class StellarAssignment implements Map.Entry{ - private String variable; - private String statement; - - public StellarAssignment(String variable, String statement) { - this.variable = variable; - this.statement = statement; - } - - public String getVariable() { - return variable; - } +public class StellarAssignment implements Map.Entry { + private final String variable; + private final String statement; - public String getStatement() { - return statement; - } - - public static boolean isAssignment(String statement) { - return statement != null && - statement.contains(":=") && // has the assignment operator - !statement.trim().startsWith("%"); // not a magic like %define x := 2 - } - - public static StellarAssignment from(String statement) { - if(statement == null || statement.length() == 0) { - return new StellarAssignment(null, null); + public StellarAssignment(String variable, String statement) { + this.variable = variable; + this.statement = statement; } - char prev = statement.charAt(0); - char curr; - String variable = "" + prev; - String s = null; - boolean isAssignment = false; - for(int i = 1;i < statement.length();++i,prev=curr) { - curr = statement.charAt(i); - if(prev == ':' && curr == '=') { - isAssignment = true; - variable = variable.substring(0, variable.length() - 1); - s = ""; - continue; - } - if(!isAssignment) { - variable += curr; - } - else { - s += curr; - } + + public String getVariable() { + return variable; } - if(!isAssignment) { - s = variable; - variable = null; + public String getStatement() { + return statement; } - if(s != null) { - s = s.trim(); + public static boolean isAssignment(String statement) { + return statement != null + && statement.contains(":=") + // has the assignment operator + && !statement.trim().startsWith("%"); // not a magic like %define x := 2 } - if(variable != null) { - variable = variable.trim(); + + public static StellarAssignment from(String statement) { + if (statement == null || statement.length() == 0) { + return new StellarAssignment(null, null); + } + char prev = statement.charAt(0); + char curr; + String variable = "" + prev; + String s = null; + boolean isAssignment = false; + for (int i = 1; i < statement.length(); ++i, prev = curr) { + curr = statement.charAt(i); + if (prev == ':' && curr == '=') { + isAssignment = true; + variable = variable.substring(0, variable.length() - 1); + s = ""; + continue; + } + if (!isAssignment) { + variable += curr; + } else { + s += curr; + } + } + + if (!isAssignment) { + s = variable; + variable = null; + } + + if (s != null) { + s = s.trim(); + } + if (variable != null) { + variable = variable.trim(); + } + return new StellarAssignment(variable, s); } - return new StellarAssignment(variable, s); - } - /** - * Returns the key corresponding to this entry. - * - * @return the key corresponding to this entry - * @throws IllegalStateException implementations may, but are not - * required to, throw this exception if the entry has been - * removed from the backing map. - */ - @Override - public String getKey() { - return variable; - } + /** + * Returns the key corresponding to this entry. + * + * @return the key corresponding to this entry + * @throws IllegalStateException implementations may, but are not + * required to, throw this exception if the entry has been + * removed from the backing map. + */ + @Override + public String getKey() { + return variable; + } - /** - * Returns the value corresponding to this entry. If the mapping - * has been removed from the backing map (by the iterator's - * remove operation), the results of this call are undefined. - * - * @return the value corresponding to this entry - * @throws IllegalStateException implementations may, but are not - * required to, throw this exception if the entry has been - * removed from the backing map. - */ - @Override - public Object getValue() { - return statement; - } + /** + * Returns the value corresponding to this entry. If the mapping + * has been removed from the backing map (by the iterator's + * remove operation), the results of this call are undefined. + * + * @return the value corresponding to this entry + * @throws IllegalStateException implementations may, but are not + * required to, throw this exception if the entry has been + * removed from the backing map. + */ + @Override + public Object getValue() { + return statement; + } - /** - * Replaces the value corresponding to this entry with the specified - * value (optional operation). (Writes through to the map.) The - * behavior of this call is undefined if the mapping has already been - * removed from the map (by the iterator's remove operation). - * - * @param value new value to be stored in this entry - * @return old value corresponding to the entry - * @throws UnsupportedOperationException if the put operation - * is not supported by the backing map - * @throws ClassCastException if the class of the specified value - * prevents it from being stored in the backing map - * @throws NullPointerException if the backing map does not permit - * null values, and the specified value is null - * @throws IllegalArgumentException if some property of this value - * prevents it from being stored in the backing map - * @throws IllegalStateException implementations may, but are not - * required to, throw this exception if the entry has been - * removed from the backing map. - */ - @Override - public String setValue(Object value) { - throw new UnsupportedOperationException("Assignments are immutable."); - } + /** + * Replaces the value corresponding to this entry with the specified + * value (optional operation). (Writes through to the map.) The + * behavior of this call is undefined if the mapping has already been + * removed from the map (by the iterator's remove operation). + * + * @param value new value to be stored in this entry + * @return old value corresponding to the entry + * @throws UnsupportedOperationException if the put operation + * is not supported by the backing map + * @throws ClassCastException if the class of the specified value + * prevents it from being stored in the backing map + * @throws NullPointerException if the backing map does not permit + * null values, and the specified value is null + * @throws IllegalArgumentException if some property of this value + * prevents it from being stored in the backing map + * @throws IllegalStateException implementations may, but are not + * required to, throw this exception if the entry has been + * removed from the backing map. + */ + @Override + public String setValue(Object value) { + throw new UnsupportedOperationException("Assignments are immutable."); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java index 1a507c8d..de948b94 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java @@ -15,9 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common; +import static java.lang.String.format; + import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; import java.io.Serializable; import java.util.ArrayDeque; import java.util.ArrayList; @@ -31,8 +35,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; - -import com.google.common.collect.Iterables; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator; @@ -42,6 +44,7 @@ import org.apache.metron.stellar.common.generated.StellarParser; import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.Context.ActivityType; import org.apache.metron.stellar.dsl.FunctionMarker; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.StellarFunction; @@ -49,838 +52,852 @@ import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import static java.lang.String.format; -import org.apache.metron.stellar.dsl.Context.ActivityType; - public class StellarCompiler extends StellarBaseListener { - private static Token EXPRESSION_REFERENCE = new Token<>(null, Object.class); - private static Token LAMBDA_VARIABLES = new Token<>(null, Object.class); - - private Expression expression; - private final ArithmeticEvaluator arithmeticEvaluator; - private final NumberLiteralEvaluator numberLiteralEvaluator; - private final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator; - - public interface ShortCircuitOp {} - - public static class ShortCircuitFrame {} - public static class BooleanArg implements ShortCircuitOp {} - public static class IfExpr implements ShortCircuitOp {} - public static class ThenExpr implements ShortCircuitOp {} - public static class ElseExpr implements ShortCircuitOp {} - public static class EndConditional implements ShortCircuitOp {} - public static class MatchClauseCheckExpr implements ShortCircuitOp {} - public static class MatchClauseEnd implements ShortCircuitOp {} - public static class MatchClausesEnd implements ShortCircuitOp {} - - public static class ExpressionState { - Context context; - FunctionResolver functionResolver; - VariableResolver variableResolver; - public ExpressionState(Context context - , FunctionResolver functionResolver - , VariableResolver variableResolver - ) { - this.context = context; - this.variableResolver = variableResolver; - this.functionResolver = functionResolver; - } - } - - public static class Expression implements Serializable { - final Deque> tokenDeque; - final Deque multiArgumentState; - final Set variablesUsed; - public Expression(Deque> tokenDeque) { - this.tokenDeque = tokenDeque; - this.variablesUsed = new HashSet<>(); - this.multiArgumentState = new ArrayDeque<>(); - } - - public void clear() { - tokenDeque.clear(); - variablesUsed.clear(); - multiArgumentState.clear(); - } - - public Deque> getTokenDeque() { - return tokenDeque; + private static final Token EXPRESSION_REFERENCE = new Token<>(null, Object.class); + private static final Token LAMBDA_VARIABLES = new Token<>(null, Object.class); + + private final Expression expression; + private final ArithmeticEvaluator arithmeticEvaluator; + private final NumberLiteralEvaluator numberLiteralEvaluator; + private final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator; + + public interface ShortCircuitOp { } - /** - * When treating empty or missing values as false, we need to ensure we ONLY do so in a conditional context. - * @param tokenValueType - * @return - */ - private boolean isConditionalContext(Class tokenValueType) { - return tokenValueType != null && ( - tokenValueType == BooleanArg.class - || tokenValueType == IfExpr.class - || tokenValueType == MatchClauseCheckExpr.class - ); + public static class ShortCircuitFrame { } - /** - * Determine if a token and value is an empty list in the appropriate conditional context - * @param token - * @param value - * @return - */ - private boolean isEmptyList(Token token, Object value) { - if(value != null && isConditionalContext(token.getUnderlyingType())) { - if (value instanceof Iterable) { - return Iterables.isEmpty((Iterable) value); - } else if (value instanceof Map) { - return ((Map) value).isEmpty(); - } - else { - return false; + public static class BooleanArg implements ShortCircuitOp { + } + + public static class IfExpr implements ShortCircuitOp { + } + + public static class ThenExpr implements ShortCircuitOp { + } + + public static class ElseExpr implements ShortCircuitOp { + } + + public static class EndConditional implements ShortCircuitOp { + } + + public static class MatchClauseCheckExpr implements ShortCircuitOp { + } + + public static class MatchClauseEnd implements ShortCircuitOp { + } + + public static class MatchClausesEnd implements ShortCircuitOp { + } + + public static class ExpressionState { + Context context; + FunctionResolver functionResolver; + VariableResolver variableResolver; + + public ExpressionState(Context context, + FunctionResolver functionResolver, + VariableResolver variableResolver + ) { + this.context = context; + this.variableResolver = variableResolver; + this.functionResolver = functionResolver; } - }else { - return false; - } } - /** - * Determine if a token is missing in a conditional context. - * @param token - * @return - */ - private boolean isBoolean(Token token, Object value) { - if(token == null || token.getValue() == null) { - return false; - } - return value == null && isConditionalContext(token.getValue().getClass()); - } - - public Object apply(ExpressionState state) { - Deque> instanceDeque = new ArrayDeque<>(); - { - int skipElseCount = 0; - boolean skipMatchClauses = false; - Token token = null; - for (Iterator> it = getTokenDeque().descendingIterator(); it.hasNext(); ) { - token = it.next(); - //if we've skipped an else previously, then we need to skip the deferred tokens associated with the else. - if (skipElseCount > 0 && token.getUnderlyingType() == ElseExpr.class) { - while (it.hasNext()) { - token = it.next(); - if (token.getUnderlyingType() == EndConditional.class) { - break; - } + public static class Expression implements Serializable { + final Deque> tokenDeque; + final Deque multiArgumentState; + final Set variablesUsed; + + public Expression(Deque> tokenDeque) { + this.tokenDeque = tokenDeque; + this.variablesUsed = new HashSet<>(); + this.multiArgumentState = new ArrayDeque<>(); + } + + public void clear() { + tokenDeque.clear(); + variablesUsed.clear(); + multiArgumentState.clear(); + } + + public Deque> getTokenDeque() { + return tokenDeque; + } + + /** + * When treating empty or missing values as false, we need to ensure we ONLY do so in a conditional context. + */ + private boolean isConditionalContext(Class tokenValueType) { + return tokenValueType != null + && ( + tokenValueType == BooleanArg.class + || tokenValueType == IfExpr.class + || tokenValueType == MatchClauseCheckExpr.class + ); + } + + /** + * Determine if a token and value is an empty list in the appropriate conditional context. + */ + private boolean isEmptyList(Token token, Object value) { + if (value != null && isConditionalContext(token.getUnderlyingType())) { + if (value instanceof Iterable) { + return Iterables.isEmpty((Iterable) value); + } else if (value instanceof Map) { + return ((Map) value).isEmpty(); + } else { + return false; + } + } else { + return false; } - // We've completed a single else. - skipElseCount--; - } - if (skipMatchClauses && (token.getUnderlyingType() == MatchClauseEnd.class - || token.getUnderlyingType() == MatchClauseCheckExpr.class)) { - while (it.hasNext()) { - token = it.next(); - if (token.getUnderlyingType() == MatchClausesEnd.class) { - break; - } + } + + /** + * Determine if a token is missing in a conditional context. + */ + private boolean isBoolean(Token token, Object value) { + if (token == null || token.getValue() == null) { + return false; } - skipMatchClauses = false; - } - /* - curr is the current value on the stack. This is the non-deferred actual evaluation for this expression - and with the current context. - */ - Token curr = instanceDeque.peek(); - boolean isFalsey = curr != null && - (isBoolean(token, curr.getValue()) || isEmptyList(token, curr.getValue())); - if(isFalsey){ - //If we're in a situation where the token is a boolean token and the current value is one of the implicitly falsey scenarios - //* null or missing variable - //* empty list - // then we want to treat it as explicitly false by replacing the current token. - curr = new Token<>(false, Boolean.class, curr.getMultiArgContext()); - instanceDeque.removeFirst(); - instanceDeque.addFirst(curr); - } - if (curr != null && curr.getValue() != null && curr.getValue() instanceof Boolean - && ShortCircuitOp.class.isAssignableFrom(token.getUnderlyingType())) { - //if we have a boolean as the current value and the next non-contextual token is a short circuit op - //then we need to short circuit possibly - if (token.getUnderlyingType() == BooleanArg.class) { - if (token.getMultiArgContext() != null - && token.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_OR - && (Boolean) (curr.getValue())) { - //short circuit the or - FrameContext.Context context = curr.getMultiArgContext(); - shortCircuit(it, context); - } else if (token.getMultiArgContext() != null - && token.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_AND - && !(Boolean) (curr.getValue())) { - //short circuit the and - FrameContext.Context context = curr.getMultiArgContext(); - shortCircuit(it, context); - } - } else if (token.getUnderlyingType() == IfExpr.class) { - //short circuit the if/then/else - instanceDeque.pop(); - if((Boolean)curr.getValue()) { - //choose then. Need to make sure we're keeping track of nesting. - skipElseCount++; - } else { - //choose else - // Need to count in case we see another if-else, to avoid breaking on wrong else. - int innerIfCount = 0; - while (it.hasNext()) { - Token t = it.next(); - if (t.getUnderlyingType() == IfExpr.class) { - innerIfCount++; - } else if (t.getUnderlyingType() == ElseExpr.class) { - if (innerIfCount == 0) { - break; - } else { - innerIfCount--; + return value == null && isConditionalContext(token.getValue().getClass()); + } + + public Object apply(ExpressionState state) { + Deque> instanceDeque = new ArrayDeque<>(); + { + int skipElseCount = 0; + boolean skipMatchClauses = false; + Token token = null; + for (Iterator> it = getTokenDeque().descendingIterator(); it.hasNext(); ) { + token = it.next(); + //if we've skipped an else previously, then we need to skip the deferred tokens associated with the else. + if (skipElseCount > 0 && token.getUnderlyingType() == ElseExpr.class) { + while (it.hasNext()) { + token = it.next(); + if (token.getUnderlyingType() == EndConditional.class) { + break; + } + } + // We've completed a single else. + skipElseCount--; } - } - } - } - } else if (token.getUnderlyingType() == MatchClauseCheckExpr.class) { - instanceDeque.pop(); - if ((Boolean) curr.getValue()) { - //skip everything else after lambda - skipMatchClauses = true; - } else { - while (it.hasNext()) { - Token t = it.next(); - if (t.getUnderlyingType() == MatchClauseEnd.class) { - break; - } + if (skipMatchClauses && (token.getUnderlyingType() == MatchClauseEnd.class + || token.getUnderlyingType() == MatchClauseCheckExpr.class)) { + while (it.hasNext()) { + token = it.next(); + if (token.getUnderlyingType() == MatchClausesEnd.class) { + break; + } + } + skipMatchClauses = false; + } + /* + curr is the current value on the stack. This is the non-deferred actual evaluation for this expression + and with the current context. + */ + Token curr = instanceDeque.peek(); + boolean isFalsey = curr != null + && (isBoolean(token, curr.getValue()) || isEmptyList(token, curr.getValue())); + if (isFalsey) { + //If we're in a situation where the token is a boolean token + // and the current value is one of the implicitly falsey scenarios + //* null or missing variable + //* empty list + // then we want to treat it as explicitly false by replacing the current token. + curr = new Token<>(false, Boolean.class, curr.getMultiArgContext()); + instanceDeque.removeFirst(); + instanceDeque.addFirst(curr); + } + if (curr != null && curr.getValue() != null && curr.getValue() instanceof Boolean + && ShortCircuitOp.class.isAssignableFrom(token.getUnderlyingType())) { + //if we have a boolean as the current value and the next non-contextual token is a short circuit op + //then we need to short circuit possibly + if (token.getUnderlyingType() == BooleanArg.class) { + if (token.getMultiArgContext() != null + && token.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_OR + && (Boolean) (curr.getValue())) { + //short circuit the or + FrameContext.Context context = curr.getMultiArgContext(); + shortCircuit(it, context); + } else if (token.getMultiArgContext() != null + && token.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_AND + && !(Boolean) (curr.getValue())) { + //short circuit the and + FrameContext.Context context = curr.getMultiArgContext(); + shortCircuit(it, context); + } + } else if (token.getUnderlyingType() == IfExpr.class) { + //short circuit the if/then/else + instanceDeque.pop(); + if ((Boolean) curr.getValue()) { + //choose then. Need to make sure we're keeping track of nesting. + skipElseCount++; + } else { + //choose else + // Need to count in case we see another if-else, to avoid breaking on wrong else. + int innerIfCount = 0; + while (it.hasNext()) { + Token t = it.next(); + if (t.getUnderlyingType() == IfExpr.class) { + innerIfCount++; + } else if (t.getUnderlyingType() == ElseExpr.class) { + if (innerIfCount == 0) { + break; + } else { + innerIfCount--; + } + } + } + } + } else if (token.getUnderlyingType() == MatchClauseCheckExpr.class) { + instanceDeque.pop(); + if ((Boolean) curr.getValue()) { + //skip everything else after lambda + skipMatchClauses = true; + } else { + while (it.hasNext()) { + Token t = it.next(); + if (t.getUnderlyingType() == MatchClauseEnd.class) { + break; + } + } + } + } + } + if (token.getUnderlyingType() == DeferredFunction.class) { + DeferredFunction func = (DeferredFunction) token.getValue(); + func.apply(instanceDeque, state); + } else if (token.getUnderlyingType() != ShortCircuitFrame.class + && !ShortCircuitOp.class.isAssignableFrom(token.getUnderlyingType()) + ) { + instanceDeque.push(token); + } + } - } } - } - if (token.getUnderlyingType() == DeferredFunction.class) { - DeferredFunction func = (DeferredFunction) token.getValue(); - func.apply(instanceDeque, state); - } - else if(token.getUnderlyingType() != ShortCircuitFrame.class - && !ShortCircuitOp.class.isAssignableFrom(token.getUnderlyingType()) - ) { - instanceDeque.push(token); - } + if (instanceDeque.isEmpty()) { + throw new ParseException("Invalid predicate: Empty stack."); + } + Token token = instanceDeque.pop(); + if (instanceDeque.isEmpty()) { + return token.getValue(); + } + if (instanceDeque.isEmpty()) { + throw new ParseException("Invalid parse, stack not empty: " + Joiner.on(',').join(instanceDeque)); + } else { + throw new ParseException("Invalid parse, found " + token); + } } - } - - if (instanceDeque.isEmpty()) { - throw new ParseException("Invalid predicate: Empty stack."); - } - Token token = instanceDeque.pop(); - if (instanceDeque.isEmpty()) { - return token.getValue(); - } - if (instanceDeque.isEmpty()) { - throw new ParseException("Invalid parse, stack not empty: " + Joiner.on(',').join(instanceDeque)); - } else { - throw new ParseException("Invalid parse, found " + token); - } - } - - public void shortCircuit(Iterator> it, FrameContext.Context context) { - while (it.hasNext()) { - Token token = it.next(); - if (token.getUnderlyingType() == ShortCircuitFrame.class && token.getMultiArgContext() == context) { - break; + public void shortCircuit(Iterator> it, FrameContext.Context context) { + while (it.hasNext()) { + Token token = it.next(); + if (token.getUnderlyingType() == ShortCircuitFrame.class && token.getMultiArgContext() == context) { + break; + } + } } - } } - } - interface DeferredFunction { - void apply( Deque> tokenDeque - , ExpressionState state - ); - } + interface DeferredFunction { + void apply(Deque> tokenDeque, + ExpressionState state + ); + } - public StellarCompiler( + public StellarCompiler( final ArithmeticEvaluator arithmeticEvaluator, final NumberLiteralEvaluator numberLiteralEvaluator, final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator - ){ - this(new Expression(new ArrayDeque<>()), arithmeticEvaluator, numberLiteralEvaluator, comparisonExpressionWithOperatorEvaluator); - } + ) { + this(new Expression(new ArrayDeque<>()), arithmeticEvaluator, numberLiteralEvaluator, + comparisonExpressionWithOperatorEvaluator); + } - public StellarCompiler( + public StellarCompiler( final Expression expression, final ArithmeticEvaluator arithmeticEvaluator, final NumberLiteralEvaluator numberLiteralEvaluator, final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator - ){ - this.expression = expression; - this.arithmeticEvaluator = arithmeticEvaluator; - this.numberLiteralEvaluator = numberLiteralEvaluator; - this.comparisonExpressionWithOperatorEvaluator = comparisonExpressionWithOperatorEvaluator; - } - - @Override - public void enterTransformation(StellarParser.TransformationContext ctx) { - expression.clear(); - } - - private boolean handleIn(final Token left, final Token right) { - Object key = right.getValue(); - - - if (left.getValue() != null) { - if (left.getValue() instanceof String && key instanceof String) { - return ((String) left.getValue()).contains(key.toString()); - } - else if (left.getValue() instanceof Collection) { - return ((Collection) left.getValue()).contains(key); - } - else if (left.getValue() instanceof Map) { - return ((Map) left.getValue()).containsKey(key); - } - else { - if (key == null) { - return key == left.getValue(); + ) { + this.expression = expression; + this.arithmeticEvaluator = arithmeticEvaluator; + this.numberLiteralEvaluator = numberLiteralEvaluator; + this.comparisonExpressionWithOperatorEvaluator = comparisonExpressionWithOperatorEvaluator; + } + + @Override + public void enterTransformation(StellarParser.TransformationContext ctx) { + expression.clear(); + } + + private boolean handleIn(final Token left, final Token right) { + Object key = right.getValue(); + + + if (left.getValue() != null) { + if (left.getValue() instanceof String && key instanceof String) { + return ((String) left.getValue()).contains(key.toString()); + } else if (left.getValue() instanceof Collection) { + return ((Collection) left.getValue()).contains(key); + } else if (left.getValue() instanceof Map) { + return ((Map) left.getValue()).containsKey(key); + } else { + if (key == null) { + return key == left.getValue(); + } else { + return key.equals(left.getValue()); + } + } + } else { + return false; + } + } + + @Override + public void exitNullConst(StellarParser.NullConstContext ctx) { + expression.tokenDeque.push(new Token<>(null, Object.class, getArgContext())); + } + + @Override + public void exitNaNArith(StellarParser.NaNArithContext ctx) { + expression.tokenDeque.push(new Token<>(Double.NaN, Double.class, getArgContext())); + } + + @Override + public void exitArithExpr_plus(StellarParser.ArithExpr_plusContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Pair, Token> p = getArithExpressionPair(tokenDeque); + tokenDeque.push( + arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.addition(context), p)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitArithExpr_minus(StellarParser.ArithExpr_minusContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Pair, Token> p = getArithExpressionPair(tokenDeque); + tokenDeque.push( + arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.subtraction(context), + p)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitArithExpr_div(StellarParser.ArithExpr_divContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Pair, Token> p = getArithExpressionPair(tokenDeque); + tokenDeque.push( + arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.division(context), p)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitArithExpr_mul(StellarParser.ArithExpr_mulContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Pair, Token> p = getArithExpressionPair(tokenDeque); + tokenDeque.push( + arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.multiplication(context), + p)); + }, DeferredFunction.class, context)); + } + + @SuppressWarnings("unchecked") + private Pair, Token> getArithExpressionPair(Deque> tokenDeque) { + Token right = (Token) popDeque(tokenDeque); + Token left = (Token) popDeque(tokenDeque); + return Pair.of(left, right); + } + + @Override + public void exitIf_expr(StellarParser.If_exprContext ctx) { + expression.tokenDeque.push(new Token<>(new IfExpr(), IfExpr.class, getArgContext())); + } + + @Override + public void enterThen_expr(StellarParser.Then_exprContext ctx) { + expression.tokenDeque.push(new Token<>(new ThenExpr(), ThenExpr.class, getArgContext())); + } + + @Override + public void enterElse_expr(StellarParser.Else_exprContext ctx) { + expression.tokenDeque.push(new Token<>(new ElseExpr(), ElseExpr.class, getArgContext())); + } + + @Override + public void exitElse_expr(StellarParser.Else_exprContext ctx) { + expression.tokenDeque.push(new Token<>(new EndConditional(), EndConditional.class, getArgContext())); + } + + @Override + public void exitInExpressionStatement(StellarParser.InExpressionStatementContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Token left = popDeque(tokenDeque); + Token right = popDeque(tokenDeque); + tokenDeque.push(new Token<>(handleIn(left, right), Boolean.class, context)); + }, DeferredFunction.class, context)); + } + + + @Override + public void exitNInExpressionStatement(StellarParser.NInExpressionStatementContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Token left = popDeque(tokenDeque); + Token right = popDeque(tokenDeque); + tokenDeque.push(new Token<>(!handleIn(left, right), Boolean.class, context)); + }, DeferredFunction.class, context)); + } + + @Override + @SuppressWarnings("unchecked") + public void exitNotFunc(StellarParser.NotFuncContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Token arg = (Token) popDeque(tokenDeque); + Boolean v = Optional.ofNullable(ConversionUtils.convert(arg.getValue(), Boolean.class)).orElse(false); + tokenDeque.push(new Token<>(!v, Boolean.class, context)); + }, DeferredFunction.class, context)); + } + + + @Override + public void exitVariable(StellarParser.VariableContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + String varName = ctx.getText(); + if (state.context.getActivityType().equals(ActivityType.PARSE_ACTIVITY) + && !state.variableResolver.exists(varName)) { + // when parsing, missing variables are an error! + throw new ParseException(String.format("variable: %s is not defined", varName)); + } + Object resolved = state.variableResolver.resolve(varName); + tokenDeque.push(new Token<>(resolved, Object.class, context)); + }, DeferredFunction.class, context)); + expression.variablesUsed.add(ctx.getText()); + } + + @Override + public void exitStringLiteral(StellarParser.StringLiteralContext ctx) { + String rawToken = ctx.getText(); + String literal = StringEscapeUtils.UNESCAPE_JSON.translate(rawToken); + expression.tokenDeque.push( + new Token<>(literal.substring(1, literal.length() - 1), String.class, getArgContext())); + } + + @Override + public void exitIntLiteral(StellarParser.IntLiteralContext ctx) { + expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); + } + + @Override + public void exitDoubleLiteral(StellarParser.DoubleLiteralContext ctx) { + expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); + } + + @Override + public void exitFloatLiteral(StellarParser.FloatLiteralContext ctx) { + expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); + } + + @Override + public void exitLongLiteral(StellarParser.LongLiteralContext ctx) { + expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); + } + + @Override + public void enterB_expr(StellarParser.B_exprContext ctx) { + //Enter is not guaranteed to be called by Antlr for logical labels, so we need to + //emulate it like this. See https://github.com/antlr/antlr4/issues/802 + if (ctx.getParent() instanceof StellarParser.LogicalExpressionOrContext) { + expression.multiArgumentState.push(FrameContext.BOOLEAN_OR.create()); + } else if (ctx.getParent() instanceof StellarParser.LogicalExpressionAndContext) { + expression.multiArgumentState.push(FrameContext.BOOLEAN_AND.create()); } - else { - return key.equals(left.getValue()); + } + + @Override + public void exitB_expr(StellarParser.B_exprContext ctx) { + if (ctx.getParent() instanceof StellarParser.LogicalExpressionOrContext + || ctx.getParent() instanceof StellarParser.LogicalExpressionAndContext + ) { + //we want to know when the argument to the boolean expression is complete + expression.tokenDeque.push(new Token<>(new BooleanArg(), BooleanArg.class, getArgContext())); } - } - } else { - return false; - } - } - - @Override - public void exitNullConst(StellarParser.NullConstContext ctx) { - expression.tokenDeque.push(new Token<>(null, Object.class, getArgContext())); - } - - @Override - public void exitNaNArith(StellarParser.NaNArithContext ctx) { - expression.tokenDeque.push(new Token<>(Double.NaN, Double.class, getArgContext())); - } - - @Override - public void exitArithExpr_plus(StellarParser.ArithExpr_plusContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { - Pair, Token> p = getArithExpressionPair(tokenDeque); - tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.addition(context), p)); - }, DeferredFunction.class, context)); - } - - @Override - public void exitArithExpr_minus(StellarParser.ArithExpr_minusContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - Pair, Token> p = getArithExpressionPair(tokenDeque); - tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.subtraction(context), p)); - }, DeferredFunction.class, context)); - } - - @Override - public void exitArithExpr_div(StellarParser.ArithExpr_divContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - Pair, Token> p = getArithExpressionPair(tokenDeque); - tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.division(context), p)); - }, DeferredFunction.class, context)); - } - - @Override - public void exitArithExpr_mul(StellarParser.ArithExpr_mulContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - Pair, Token> p = getArithExpressionPair(tokenDeque); - tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.multiplication(context), p)); - }, DeferredFunction.class, context)); - } - - @SuppressWarnings("unchecked") - private Pair, Token> getArithExpressionPair(Deque> tokenDeque) { - Token right = (Token) popDeque(tokenDeque); - Token left = (Token) popDeque(tokenDeque); - return Pair.of(left, right); - } - - @Override - public void exitIf_expr(StellarParser.If_exprContext ctx) { - expression.tokenDeque.push(new Token<>(new IfExpr(), IfExpr.class, getArgContext())); - } - - @Override - public void enterThen_expr(StellarParser.Then_exprContext ctx) { - expression.tokenDeque.push(new Token<>(new ThenExpr(), ThenExpr.class, getArgContext())); - } - - @Override - public void enterElse_expr(StellarParser.Else_exprContext ctx) { - expression.tokenDeque.push(new Token<>(new ElseExpr(), ElseExpr.class, getArgContext())); - } - - @Override - public void exitElse_expr(StellarParser.Else_exprContext ctx) { - expression.tokenDeque.push(new Token<>(new EndConditional(), EndConditional.class, getArgContext())); - } - - @Override - public void exitInExpressionStatement(StellarParser.InExpressionStatementContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - Token left = popDeque(tokenDeque); - Token right = popDeque(tokenDeque); - tokenDeque.push(new Token<>(handleIn(left, right), Boolean.class, context)); - }, DeferredFunction.class, context)); - } - - - @Override - public void exitNInExpressionStatement(StellarParser.NInExpressionStatementContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - Token left = popDeque(tokenDeque); - Token right = popDeque(tokenDeque); - tokenDeque.push(new Token<>(!handleIn(left, right), Boolean.class, context)); - }, DeferredFunction.class, context)); - } - - @Override - @SuppressWarnings("unchecked") - public void exitNotFunc(StellarParser.NotFuncContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { - Token arg = (Token) popDeque(tokenDeque); - Boolean v = Optional.ofNullable(ConversionUtils.convert(arg.getValue(), Boolean.class)).orElse(false); - tokenDeque.push(new Token<>(!v, Boolean.class, context)); - }, DeferredFunction.class, context)); - } - - - @Override - public void exitVariable(StellarParser.VariableContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - String varName = ctx.getText(); - if(state.context.getActivityType().equals(ActivityType.PARSE_ACTIVITY) && !state.variableResolver.exists(varName)) { - // when parsing, missing variables are an error! - throw new ParseException(String.format("variable: %s is not defined",varName)); - } - Object resolved = state.variableResolver.resolve(varName); - tokenDeque.push(new Token<>(resolved, Object.class, context)); - }, DeferredFunction.class, context)); - expression.variablesUsed.add(ctx.getText()); - } - - @Override - public void exitStringLiteral(StellarParser.StringLiteralContext ctx) { - String rawToken = ctx.getText(); - String literal = StringEscapeUtils.UNESCAPE_JSON.translate(rawToken); - expression.tokenDeque.push(new Token<>(literal.substring(1, literal.length()-1), String.class, getArgContext())); - } - - @Override - public void exitIntLiteral(StellarParser.IntLiteralContext ctx) { - expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); - } - - @Override - public void exitDoubleLiteral(StellarParser.DoubleLiteralContext ctx) { - expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); - } - - @Override - public void exitFloatLiteral(StellarParser.FloatLiteralContext ctx) { - expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); - } - - @Override - public void exitLongLiteral(StellarParser.LongLiteralContext ctx) { - expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); - } - - @Override - public void enterB_expr(StellarParser.B_exprContext ctx) { - //Enter is not guaranteed to be called by Antlr for logical labels, so we need to - //emulate it like this. See https://github.com/antlr/antlr4/issues/802 - if(ctx.getParent() instanceof StellarParser.LogicalExpressionOrContext) { - expression.multiArgumentState.push(FrameContext.BOOLEAN_OR.create()); - } - else if(ctx.getParent() instanceof StellarParser.LogicalExpressionAndContext) { - expression.multiArgumentState.push(FrameContext.BOOLEAN_AND.create()); - } - } - - @Override - public void exitB_expr(StellarParser.B_exprContext ctx) { - if(ctx.getParent() instanceof StellarParser.LogicalExpressionOrContext - || ctx.getParent() instanceof StellarParser.LogicalExpressionAndContext - ) - { - //we want to know when the argument to the boolean expression is complete - expression.tokenDeque.push(new Token<>(new BooleanArg(), BooleanArg.class, getArgContext())); - } - } - - @Override - public void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) { - final FrameContext.Context context = getArgContext(); - popArgContext(); - final FrameContext.Context parentContext = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - Token left = popDeque(tokenDeque); - Token right = popDeque(tokenDeque); - tokenDeque.push(new Token<>(booleanOp(left, right, (l, r) -> l && r, "&&"), Boolean.class, parentContext)); - }, DeferredFunction.class, context)); - expression.tokenDeque.push(new Token<>(new ShortCircuitFrame(), ShortCircuitFrame.class, context)); - } - - @Override - public void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) { - final FrameContext.Context context = getArgContext(); - popArgContext(); - final FrameContext.Context parentContext = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - Token left = popDeque(tokenDeque); - Token right = popDeque(tokenDeque); - - tokenDeque.push(new Token<>(booleanOp(left, right, (l, r) -> l || r, "||"), Boolean.class, parentContext)); - }, DeferredFunction.class, context)); - expression.tokenDeque.push(new Token<>(new ShortCircuitFrame(), ShortCircuitFrame.class, context)); - } - - @Override - public void exitLogicalConst(StellarParser.LogicalConstContext ctx) { - Boolean b; - switch (ctx.getText().toUpperCase()) { - case "TRUE": - b = true; - break; - case "FALSE": - b = false; - break; - default: - throw new ParseException("Unable to process " + ctx.getText() + " as a boolean constant"); - } - expression.tokenDeque.push(new Token<>(b, Boolean.class, getArgContext())); - } - - private boolean booleanOp(final Token left, final Token right, final BooleanOp op, final String opName) { - Boolean l = Optional.ofNullable(ConversionUtils.convert(left.getValue(), Boolean.class)).orElse(false); - Boolean r = Optional.ofNullable(ConversionUtils.convert(right.getValue(), Boolean.class)).orElse(false); - return op.op(l, r); - } - - - @Override - public void enterSingle_lambda_variable(StellarParser.Single_lambda_variableContext ctx) { - enterLambdaVariables(); - } - - @Override - public void exitSingle_lambda_variable(StellarParser.Single_lambda_variableContext ctx) { - exitLambdaVariables(); - } - - @Override - public void enterLambda_variables(StellarParser.Lambda_variablesContext ctx) { - enterLambdaVariables(); - } - - @Override - public void exitLambda_variables(StellarParser.Lambda_variablesContext ctx) { - exitLambdaVariables(); - } - - @Override - public void exitLambda_variable(StellarParser.Lambda_variableContext ctx) { - expression.tokenDeque.push(new Token<>(ctx.getText(), String.class, getArgContext())); - } - - private void enterLambdaVariables() { - expression.tokenDeque.push(LAMBDA_VARIABLES); - } - - @SuppressWarnings("ReferenceEquality") - private void exitLambdaVariables() { - Token t = expression.tokenDeque.pop(); - LinkedList variables = new LinkedList<>(); - for (; !expression.tokenDeque.isEmpty() && t != LAMBDA_VARIABLES; t = expression.tokenDeque.pop()) { - variables.addFirst(t.getValue().toString()); - } - expression.tokenDeque.push(new Token<>(variables, List.class, getArgContext())); - } - - private void enterLambda() { - expression.tokenDeque.push(EXPRESSION_REFERENCE); - } - - @SuppressWarnings({"unchecked","ReferenceEquality"}) - private void exitLambda(boolean hasArgs) { - final FrameContext.Context context = getArgContext(); - Token t = expression.tokenDeque.pop(); - final Deque> instanceDeque = new ArrayDeque<>(); - for(; !expression.tokenDeque.isEmpty() && t != EXPRESSION_REFERENCE; t = expression.tokenDeque.pop()) { - instanceDeque.addLast(t); - } - final List variables = hasArgs ? (List) instanceDeque.removeLast().getValue() :new ArrayList<>(); - expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { - LambdaExpression expr = new LambdaExpression(variables, instanceDeque, state); - tokenDeque.push(new Token<>(expr, Object.class, context)); - }, DeferredFunction.class, context)); - } - - @Override - public void enterLambda_with_args(StellarParser.Lambda_with_argsContext ctx) { - enterLambda(); - } - - @Override - public void exitLambda_with_args(StellarParser.Lambda_with_argsContext ctx) { - exitLambda(true); - } - - @Override - public void enterLambda_without_args(StellarParser.Lambda_without_argsContext ctx) { - enterLambda(); - } - - @Override - public void exitLambda_without_args(StellarParser.Lambda_without_argsContext ctx) { - exitLambda(false); - } - - @Override - public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - // resolve and initialize the function - String functionName = ctx.getChild(0).getText(); - StellarFunction function = resolveFunction(state.functionResolver, functionName); - initializeFunction(state.context, function, functionName); - - // fetch the args, execute, and push result onto the stack - List args = getFunctionArguments(popDeque(tokenDeque)); - Object result = function.apply(args, state.context); - tokenDeque.push(new Token<>(result, Object.class, context)); - }, DeferredFunction.class, context)); - } - - /** - * Get function arguments. - * @param token The token containing the function arguments. - * @return - */ - @SuppressWarnings("unchecked") - private List getFunctionArguments(final Token token) { - if (token.getUnderlyingType().equals(List.class)) { - return (List) token.getValue(); - - } else { - throw new ParseException("Unable to process in clause because " + token.getValue() + " is not a set"); - } - } - - /** - * Resolves a function by name. - * @param funcName - * @return - */ - private StellarFunction resolveFunction(FunctionResolver functionResolver, String funcName) { - try { - return functionResolver.apply(funcName); - - } catch (Exception e) { - String valid = Joiner.on(',').join(functionResolver.getFunctions()); - String error = format("Unable to resolve function named '%s'. Valid functions are %s", funcName, valid); - throw new ParseException(error, e); - } - } - - /** - * Initialize a Stellar function. - * @param function The function to initialize. - * @param functionName The name of the functions. - */ - private void initializeFunction(Context context, StellarFunction function, String functionName) { - try { - if (!function.isInitialized()) { - function.initialize(context); - } - } catch (Throwable t) { - String error = format("Unable to initialize function '%s'", functionName); - throw new ParseException(error, t); - } - } - - @Override - public void exitExistsFunc(StellarParser.ExistsFuncContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - String variable = ctx.getChild(2).getText(); - boolean exists = state.variableResolver.resolve(variable) != null; - tokenDeque.push(new Token<>(exists, Boolean.class, context)); - }, DeferredFunction.class, context)); - String variable = ctx.getChild(2).getText(); - expression.variablesUsed.add(variable); - } - - @Override - public void enterFunc_args(StellarParser.Func_argsContext ctx) { - expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); - } - - @Override - public void exitFunc_args(StellarParser.Func_argsContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { - LinkedList args = new LinkedList<>(); - while (true) { - Token token = popDeque(tokenDeque); - if (token.getUnderlyingType().equals(FunctionMarker.class)) { - break; - } else { - args.addFirst(token.getValue()); + } + + @Override + public void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) { + final FrameContext.Context context = getArgContext(); + popArgContext(); + final FrameContext.Context parentContext = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Token left = popDeque(tokenDeque); + Token right = popDeque(tokenDeque); + tokenDeque.push(new Token<>(booleanOp(left, right, (l, r) -> l && r, "&&"), Boolean.class, parentContext)); + }, DeferredFunction.class, context)); + expression.tokenDeque.push(new Token<>(new ShortCircuitFrame(), ShortCircuitFrame.class, context)); + } + + @Override + public void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) { + final FrameContext.Context context = getArgContext(); + popArgContext(); + final FrameContext.Context parentContext = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Token left = popDeque(tokenDeque); + Token right = popDeque(tokenDeque); + + tokenDeque.push(new Token<>(booleanOp(left, right, (l, r) -> l || r, "||"), Boolean.class, parentContext)); + }, DeferredFunction.class, context)); + expression.tokenDeque.push(new Token<>(new ShortCircuitFrame(), ShortCircuitFrame.class, context)); + } + + @Override + public void exitLogicalConst(StellarParser.LogicalConstContext ctx) { + Boolean b; + switch (ctx.getText().toUpperCase()) { + case "TRUE": + b = true; + break; + case "FALSE": + b = false; + break; + default: + throw new ParseException("Unable to process " + ctx.getText() + " as a boolean constant"); } - } - tokenDeque.push(new Token<>(args, List.class, context)); - }, DeferredFunction.class, context)); - } - - @Override - public void enterMap_entity(StellarParser.Map_entityContext ctx) { - expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); - } - - @Override - public void exitMap_entity(StellarParser.Map_entityContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - HashMap args = new HashMap<>(); - Object value = null; - for (int i = 0; true; i++) { - Token token = popDeque(tokenDeque); - if (token.getUnderlyingType().equals(FunctionMarker.class)) { - break; - } else { - if (i % 2 == 0) { - value = token.getValue(); - } else { - args.put(token.getValue(), value); - } + expression.tokenDeque.push(new Token<>(b, Boolean.class, getArgContext())); + } + + private boolean booleanOp(final Token left, final Token right, final BooleanOp op, final String opName) { + Boolean l = Optional.ofNullable(ConversionUtils.convert(left.getValue(), Boolean.class)).orElse(false); + Boolean r = Optional.ofNullable(ConversionUtils.convert(right.getValue(), Boolean.class)).orElse(false); + return op.op(l, r); + } + + + @Override + public void enterSingle_lambda_variable(StellarParser.Single_lambda_variableContext ctx) { + enterLambdaVariables(); + } + + @Override + public void exitSingle_lambda_variable(StellarParser.Single_lambda_variableContext ctx) { + exitLambdaVariables(); + } + + @Override + public void enterLambda_variables(StellarParser.Lambda_variablesContext ctx) { + enterLambdaVariables(); + } + + @Override + public void exitLambda_variables(StellarParser.Lambda_variablesContext ctx) { + exitLambdaVariables(); + } + + @Override + public void exitLambda_variable(StellarParser.Lambda_variableContext ctx) { + expression.tokenDeque.push(new Token<>(ctx.getText(), String.class, getArgContext())); + } + + private void enterLambdaVariables() { + expression.tokenDeque.push(LAMBDA_VARIABLES); + } + + @SuppressWarnings("ReferenceEquality") + private void exitLambdaVariables() { + Token t = expression.tokenDeque.pop(); + LinkedList variables = new LinkedList<>(); + for (; !expression.tokenDeque.isEmpty() && t != LAMBDA_VARIABLES; t = expression.tokenDeque.pop()) { + variables.addFirst(t.getValue().toString()); } - } - tokenDeque.push(new Token<>(args, Map.class, context)); - }, DeferredFunction.class, context)); - } - - @Override - public void exitList_entity(StellarParser.List_entityContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - LinkedList args = new LinkedList<>(); - while (true) { - Token token = popDeque(tokenDeque); - if (token.getUnderlyingType().equals(FunctionMarker.class)) { - break; + expression.tokenDeque.push(new Token<>(variables, List.class, getArgContext())); + } + + private void enterLambda() { + expression.tokenDeque.push(EXPRESSION_REFERENCE); + } + + @SuppressWarnings({"unchecked", "ReferenceEquality"}) + private void exitLambda(boolean hasArgs) { + final FrameContext.Context context = getArgContext(); + Token t = expression.tokenDeque.pop(); + final Deque> instanceDeque = new ArrayDeque<>(); + for (; !expression.tokenDeque.isEmpty() && t != EXPRESSION_REFERENCE; t = expression.tokenDeque.pop()) { + instanceDeque.addLast(t); + } + final List variables = + hasArgs ? (List) instanceDeque.removeLast().getValue() : new ArrayList<>(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + LambdaExpression expr = new LambdaExpression(variables, instanceDeque, state); + tokenDeque.push(new Token<>(expr, Object.class, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void enterLambda_with_args(StellarParser.Lambda_with_argsContext ctx) { + enterLambda(); + } + + @Override + public void exitLambda_with_args(StellarParser.Lambda_with_argsContext ctx) { + exitLambda(true); + } + + @Override + public void enterLambda_without_args(StellarParser.Lambda_without_argsContext ctx) { + enterLambda(); + } + + @Override + public void exitLambda_without_args(StellarParser.Lambda_without_argsContext ctx) { + exitLambda(false); + } + + @Override + public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + // resolve and initialize the function + String functionName = ctx.getChild(0).getText(); + StellarFunction function = resolveFunction(state.functionResolver, functionName); + initializeFunction(state.context, function, functionName); + + // fetch the args, execute, and push result onto the stack + List args = getFunctionArguments(popDeque(tokenDeque)); + Object result = function.apply(args, state.context); + tokenDeque.push(new Token<>(result, Object.class, context)); + }, DeferredFunction.class, context)); + } + + /** + * Get function arguments. + * + * @param token The token containing the function arguments. + */ + @SuppressWarnings("unchecked") + private List getFunctionArguments(final Token token) { + if (token.getUnderlyingType().equals(List.class)) { + return (List) token.getValue(); + } else { - args.addFirst(token.getValue()); + throw new ParseException("Unable to process in clause because " + token.getValue() + " is not a set"); } - } - tokenDeque.push(new Token<>(args, List.class, context)); - }, DeferredFunction.class, context)); - } - - @Override - public void exitDefault(StellarParser.DefaultContext ctx) { - expression.tokenDeque.push(new Token<>(true, Boolean.class, getArgContext())); - } - - @Override - public void exitMatchClauseCheckExpr(StellarParser.MatchClauseCheckExprContext ctx) { - final FrameContext.Context context = getArgContext(); - // if we are validating, and we have a single variable then we will get - // a null and we need to protect against that - if(ctx.getStart() == ctx.getStop()) { - expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { - if (tokenDeque.size() == 1 && (tokenDeque.peek().getValue() == null - || tokenDeque.peek().getUnderlyingType() == Boolean.class)) { - tokenDeque.pop(); - tokenDeque.add(new Token<>(false, Boolean.class, getArgContext())); - } - }, DeferredFunction.class, context)); - } - expression.tokenDeque.push(new Token<>(new MatchClauseCheckExpr(), MatchClauseCheckExpr.class, getArgContext())); - } - - @Override - public void exitMatchClauseAction(StellarParser.MatchClauseActionContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - Token token = popDeque(tokenDeque); - Object value = token.getValue(); - if (value != null && LambdaExpression.class.isAssignableFrom(value.getClass())) { - LambdaExpression expr = (LambdaExpression) value; - // at this time we don't support lambdas with arguments - // there is no context for it as such here - // it is possible that we add match variables, and use those - // as the context, but that could also be done with plain variables - // so it remains to be determined - Object result = expr.apply(new ArrayList<>()); - tokenDeque.push(new Token<>(result, Object.class, context)); - } else { - tokenDeque.push(new Token<>(value, Object.class, context)); - } - - }, DeferredFunction.class, context)); - } - - @Override - public void exitMatch_clause(StellarParser.Match_clauseContext ctx) { - expression.tokenDeque.push(new Token<>(new MatchClauseEnd(), MatchClauseEnd.class, getArgContext())); - } - - @Override - public void exitMatchClauses(StellarParser.MatchClausesContext ctx) { - expression.tokenDeque.push(new Token<>(new MatchClausesEnd(),MatchClausesEnd.class, getArgContext())); - } - - @Override - public void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) { - final FrameContext.Context context = getArgContext(); - expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { - StellarParser.Comp_operatorContext op = ctx.comp_operator(); - Token right = popDeque(tokenDeque); - Token left = popDeque(tokenDeque); - - tokenDeque.push(comparisonExpressionWithOperatorEvaluator.evaluate(left, right, (StellarParser.ComparisonOpContext) op, context)); - }, DeferredFunction.class, context)); - } - - @Override - public void enterList_entity(StellarParser.List_entityContext ctx) { - expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); - } - - private void popArgContext() { - if(!expression.multiArgumentState.isEmpty()) { - expression.multiArgumentState.pop(); - } - } - - private FrameContext.Context getArgContext() { - return expression.multiArgumentState.isEmpty() ? null : expression.multiArgumentState.peek(); - } - - private Token popDeque(Deque> tokenDeque) { - if (tokenDeque.isEmpty()) { - throw new ParseException("Unable to pop an empty stack"); - } - return tokenDeque.pop(); - } - - public Expression getExpression() { - return expression; - } + } + + /** + * Resolves a function by name. + */ + private StellarFunction resolveFunction(FunctionResolver functionResolver, String funcName) { + try { + return functionResolver.apply(funcName); + + } catch (Exception e) { + String valid = Joiner.on(',').join(functionResolver.getFunctions()); + String error = format("Unable to resolve function named '%s'. Valid functions are %s", funcName, valid); + throw new ParseException(error, e); + } + } + + /** + * Initialize a Stellar function. + * + * @param function The function to initialize. + * @param functionName The name of the functions. + */ + private void initializeFunction(Context context, StellarFunction function, String functionName) { + try { + if (!function.isInitialized()) { + function.initialize(context); + } + } catch (Throwable t) { + String error = format("Unable to initialize function '%s'", functionName); + throw new ParseException(error, t); + } + } + + @Override + public void exitExistsFunc(StellarParser.ExistsFuncContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + String variable = ctx.getChild(2).getText(); + boolean exists = state.variableResolver.resolve(variable) != null; + tokenDeque.push(new Token<>(exists, Boolean.class, context)); + }, DeferredFunction.class, context)); + String variable = ctx.getChild(2).getText(); + expression.variablesUsed.add(variable); + } + + @Override + public void enterFunc_args(StellarParser.Func_argsContext ctx) { + expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); + } + + @Override + public void exitFunc_args(StellarParser.Func_argsContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + LinkedList args = new LinkedList<>(); + while (true) { + Token token = popDeque(tokenDeque); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + args.addFirst(token.getValue()); + } + } + tokenDeque.push(new Token<>(args, List.class, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void enterMap_entity(StellarParser.Map_entityContext ctx) { + expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); + } + + @Override + public void exitMap_entity(StellarParser.Map_entityContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + HashMap args = new HashMap<>(); + Object value = null; + for (int i = 0; true; i++) { + Token token = popDeque(tokenDeque); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + if (i % 2 == 0) { + value = token.getValue(); + } else { + args.put(token.getValue(), value); + } + } + } + tokenDeque.push(new Token<>(args, Map.class, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitList_entity(StellarParser.List_entityContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + LinkedList args = new LinkedList<>(); + while (true) { + Token token = popDeque(tokenDeque); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + args.addFirst(token.getValue()); + } + } + tokenDeque.push(new Token<>(args, List.class, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitDefault(StellarParser.DefaultContext ctx) { + expression.tokenDeque.push(new Token<>(true, Boolean.class, getArgContext())); + } + + @Override + public void exitMatchClauseCheckExpr(StellarParser.MatchClauseCheckExprContext ctx) { + final FrameContext.Context context = getArgContext(); + // if we are validating, and we have a single variable then we will get + // a null and we need to protect against that + if (ctx.getStart() == ctx.getStop()) { + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + if (tokenDeque.size() == 1 && (tokenDeque.peek().getValue() == null + || tokenDeque.peek().getUnderlyingType() == Boolean.class)) { + tokenDeque.pop(); + tokenDeque.add(new Token<>(false, Boolean.class, getArgContext())); + } + }, DeferredFunction.class, context)); + } + expression.tokenDeque.push( + new Token<>(new MatchClauseCheckExpr(), MatchClauseCheckExpr.class, getArgContext())); + } + + @Override + public void exitMatchClauseAction(StellarParser.MatchClauseActionContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Token token = popDeque(tokenDeque); + Object value = token.getValue(); + if (value != null && LambdaExpression.class.isAssignableFrom(value.getClass())) { + LambdaExpression expr = (LambdaExpression) value; + // at this time we don't support lambdas with arguments + // there is no context for it as such here + // it is possible that we add match variables, and use those + // as the context, but that could also be done with plain variables + // so it remains to be determined + Object result = expr.apply(new ArrayList<>()); + tokenDeque.push(new Token<>(result, Object.class, context)); + } else { + tokenDeque.push(new Token<>(value, Object.class, context)); + } + + }, DeferredFunction.class, context)); + } + + @Override + public void exitMatch_clause(StellarParser.Match_clauseContext ctx) { + expression.tokenDeque.push(new Token<>(new MatchClauseEnd(), MatchClauseEnd.class, getArgContext())); + } + + @Override + public void exitMatchClauses(StellarParser.MatchClausesContext ctx) { + expression.tokenDeque.push(new Token<>(new MatchClausesEnd(), MatchClausesEnd.class, getArgContext())); + } + + @Override + public void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + StellarParser.Comp_operatorContext op = ctx.comp_operator(); + Token right = popDeque(tokenDeque); + Token left = popDeque(tokenDeque); + + tokenDeque.push(comparisonExpressionWithOperatorEvaluator.evaluate(left, right, + (StellarParser.ComparisonOpContext) op, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void enterList_entity(StellarParser.List_entityContext ctx) { + expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); + } + + private void popArgContext() { + if (!expression.multiArgumentState.isEmpty()) { + expression.multiArgumentState.pop(); + } + } + + private FrameContext.Context getArgContext() { + return expression.multiArgumentState.isEmpty() ? null : expression.multiArgumentState.peek(); + } + + private Token popDeque(Deque> tokenDeque) { + if (tokenDeque.isEmpty()) { + throw new ParseException("Unable to pop an empty stack"); + } + return tokenDeque.pop(); + } + + public Expression getExpression() { + return expression; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarPredicateProcessor.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarPredicateProcessor.java index a8586b93..63578263 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarPredicateProcessor.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarPredicateProcessor.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,13 +21,12 @@ package org.apache.metron.stellar.common; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.stellar.dsl.VariableResolver; +import static org.apache.commons.lang3.StringUtils.isEmpty; import java.util.concurrent.TimeUnit; - -import static org.apache.commons.lang3.StringUtils.isEmpty; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; /** * The Stellar Predicate Processor is intended to allow for specific predicate transformations using the Stellar @@ -36,37 +37,38 @@ public class StellarPredicateProcessor extends BaseStellarProcessor { - /** - * Create a default stellar processor. This processor uses the static expression cache. - */ - public StellarPredicateProcessor() { - super(Boolean.class); - } - - public StellarPredicateProcessor(int cacheSize, int expiryTime, TimeUnit expiryUnit) { - super(Boolean.class, cacheSize, expiryTime, expiryUnit); - } - @Override - public Boolean parse( String rule - , VariableResolver variableResolver - , FunctionResolver functionResolver - , Context context - ) - { - if(rule == null || isEmpty(rule.trim())) { - return true; + /** + * Create a default stellar processor. This processor uses the static expression cache. + */ + public StellarPredicateProcessor() { + super(Boolean.class); } - try { - return super.parse(rule, variableResolver, functionResolver, context); - } catch (ClassCastException e) { - // predicate must return boolean - throw new IllegalArgumentException(String.format("The rule '%s' does not return a boolean value.", rule), e); + + public StellarPredicateProcessor(int cacheSize, int expiryTime, TimeUnit expiryUnit) { + super(Boolean.class, cacheSize, expiryTime, expiryUnit); } - catch(Exception e) { - if(e.getCause() != null && e.getCause() instanceof ClassCastException) { - throw new IllegalArgumentException(String.format("The rule '%s' does not return a boolean value.", rule), e.getCause()); - } - throw e; + + @Override + public Boolean parse(String rule, + VariableResolver variableResolver, + FunctionResolver functionResolver, + Context context + ) { + if (rule == null || isEmpty(rule.trim())) { + return true; + } + try { + return super.parse(rule, variableResolver, functionResolver, context); + } catch (ClassCastException e) { + // predicate must return boolean + throw new IllegalArgumentException(String.format("The rule '%s' does not return a boolean value.", rule), + e); + } catch (Exception e) { + if (e.getCause() != null && e.getCause() instanceof ClassCastException) { + throw new IllegalArgumentException( + String.format("The rule '%s' does not return a boolean value.", rule), e.getCause()); + } + throw e; + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarProcessor.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarProcessor.java index 946a2609..123b5e2b 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarProcessor.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarProcessor.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,21 +31,18 @@ */ public class StellarProcessor extends BaseStellarProcessor { - /** - * Create a default stellar processor. This processor uses the static expression cache. - */ - public StellarProcessor() { - super(Object.class); - } + /** + * Create a default stellar processor. This processor uses the static expression cache. + */ + public StellarProcessor() { + super(Object.class); + } - /** - * Create a stellar processor with a new expression cache. NOTE: This object should be reused to prevent - * performance regressions. - * @param cacheSize - * @param expiryTime - * @param expiryUnit - */ - public StellarProcessor(int cacheSize, int expiryTime, TimeUnit expiryUnit) { - super(Object.class, cacheSize, expiryTime, expiryUnit); - } + /** + * Create a stellar processor with a new expression cache. NOTE: This object should be reused to prevent + * performance regressions. + */ + public StellarProcessor(int cacheSize, int expiryTime, TimeUnit expiryUnit) { + super(Object.class, cacheSize, expiryTime, expiryUnit); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarStatefulExecutor.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarStatefulExecutor.java index 9595d6e5..9a5e0d26 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarStatefulExecutor.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/StellarStatefulExecutor.java @@ -20,62 +20,62 @@ package org.apache.metron.stellar.common; -import org.apache.metron.stellar.dsl.Context; - import java.util.Map; +import org.apache.metron.stellar.dsl.Context; /** * Executes Stellar expressions and maintains state across multiple invocations. */ public interface StellarStatefulExecutor { - /** - * Assign a variable a specific value. - * @param variable The variable name. - * @param value The value to assign to the variable. - */ - void assign(String variable, Object value); + /** + * Assign a variable a specific value. + * + * @param variable The variable name. + * @param value The value to assign to the variable. + */ + void assign(String variable, Object value); - /** - * Execute an expression and assign the result to a variable. The variable is maintained - * in the context of this executor and is available to all subsequent expressions. - * - * @param variable The name of the variable to assign to. - * @param expression The expression to execute. - * @param state Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - */ - void assign(String variable, String expression, Map state); + /** + * Execute an expression and assign the result to a variable. The variable is maintained + * in the context of this executor and is available to all subsequent expressions. + * + * @param variable The name of the variable to assign to. + * @param expression The expression to execute. + * @param state Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + */ + void assign(String variable, String expression, Map state); - /** - * Execute a Stellar expression and return the result. The internal state of the executor - * is not modified. - * - * @param expression The expression to execute. - * @param state Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - * @param clazz The expected type of the expression's result. - * @param The expected type of the expression's result. - */ - T execute(String expression, Map state, Class clazz); + /** + * Execute a Stellar expression and return the result. The internal state of the executor + * is not modified. + * + * @param expression The expression to execute. + * @param state Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + * @param clazz The expected type of the expression's result. + * @param The expected type of the expression's result. + */ + T execute(String expression, Map state, Class clazz); - /** - * The current state of the Stellar execution environment. - */ - Map getState(); + /** + * The current state of the Stellar execution environment. + */ + Map getState(); - /** - * Removes all state from the execution environment. - */ - void clearState(); + /** + * Removes all state from the execution environment. + */ + void clearState(); - /** - * Sets the Context for the Stellar execution environment. This provides global data used - * to initialize Stellar functions. - * - * @param context The Stellar context. - */ - void setContext(Context context); + /** + * Sets the Context for the Stellar execution environment. This provides global data used + * to initialize Stellar functions. + * + * @param context The Stellar context. + */ + void setContext(Context context); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/benchmark/Microbenchmark.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/benchmark/Microbenchmark.java index 3a1b4b9d..c8b9fad5 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/benchmark/Microbenchmark.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/benchmark/Microbenchmark.java @@ -15,53 +15,56 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.benchmark; +import java.util.function.Consumer; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.stellar.common.StellarProcessor; - -import java.util.function.Consumer; public class Microbenchmark { - public static class StellarStatement { - String expression; - VariableResolver variableResolver; - FunctionResolver functionResolver; - Context context; - } + public static class StellarStatement { + String expression; + VariableResolver variableResolver; + FunctionResolver functionResolver; + Context context; + } - public static DescriptiveStatistics run(StellarStatement statement, int warmupRounds, int benchmarkRounds ) - { - run(warmupRounds, statement, ts -> {}); - final DescriptiveStatistics stats = new DescriptiveStatistics(); - run(benchmarkRounds, statement, ts -> { stats.addValue(ts);}); - return stats; - } + public static DescriptiveStatistics run(StellarStatement statement, int warmupRounds, int benchmarkRounds) { + run(warmupRounds, statement, ts -> { + }); + final DescriptiveStatistics stats = new DescriptiveStatistics(); + run(benchmarkRounds, statement, ts -> { + stats.addValue(ts); + }); + return stats; + } - private static void run(int numTimes, StellarStatement statement, Consumer func) { - StellarProcessor processor = new StellarProcessor(); - for(int i = 0;i < numTimes;++i) { - long start = System.nanoTime(); - processor.parse(statement.expression, statement.variableResolver, statement.functionResolver, statement.context); - func.accept((System.nanoTime() - start)/1000); + private static void run(int numTimes, StellarStatement statement, Consumer func) { + StellarProcessor processor = new StellarProcessor(); + for (int i = 0; i < numTimes; ++i) { + long start = System.nanoTime(); + processor.parse(statement.expression, statement.variableResolver, statement.functionResolver, + statement.context); + func.accept((System.nanoTime() - start) / 1000); + } } - } - public static String describe(DescriptiveStatistics stats, Double[] percentiles){ - StringBuilder sb = new StringBuilder(); - sb.append(String.format("round: mean of %dms [+-%d], measured %d rounds;\n", - (long)stats.getMean(), - (long)stats.getStandardDeviation(), stats.getN() )); - sb.append("\tMin - " + (long)stats.getMin() + "\n"); - for(double pctile : percentiles) { - sb.append("\t" + pctile + " - " + stats.getPercentile(pctile) + "\n"); + public static String describe(DescriptiveStatistics stats, Double[] percentiles) { + StringBuilder sb = new StringBuilder(); + sb.append(String.format("round: mean of %dms [+-%d], measured %d rounds;\n", + (long) stats.getMean(), + (long) stats.getStandardDeviation(), stats.getN())); + sb.append("\tMin - " + (long) stats.getMin() + "\n"); + for (double pctile : percentiles) { + sb.append("\t" + pctile + " - " + stats.getPercentile(pctile) + "\n"); + } + sb.append("\tMax - " + (long) stats.getMax()); + return sb.toString(); } - sb.append("\tMax - " + (long)stats.getMax()); - return sb.toString(); - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java index 50ee403d..ab8af25b 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java @@ -15,12 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.benchmark; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.io.Files; - import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; @@ -52,229 +52,233 @@ public class StellarMicrobenchmark { - public static int DEFAULT_WARMUP = 100; - public static int DEFAULT_NUM_TIMES = 1000; - public static Double[] DEFAULT_PERCENTILES = new Double[] { - 50d, 75d, 95d, 99d - }; + public static int DEFAULT_WARMUP = 100; + public static int DEFAULT_NUM_TIMES = 1000; + public static Double[] DEFAULT_PERCENTILES = new Double[] { + 50d, 75d, 95d, 99d + }; - enum BenchmarkOptions { - HELP("h", new OptionHandler() { + enum BenchmarkOptions { + HELP("h", new OptionHandler() { - @Nullable - @Override - public Option apply(@Nullable String s) { - return new Option(s, "help", false, "Generate Help screen"); - } - }), - WARMUP("w", new OptionHandler() { - @Nullable - @Override - public Option apply(@Nullable String s) { - Option o = new Option(s, "warmup", true, "Number of times for warmup per expression. Default: " + DEFAULT_WARMUP); - o.setArgName("NUM"); - o.setRequired(false); - return o; - } + @Nullable + @Override + public Option apply(@Nullable String s) { + return new Option(s, "help", false, "Generate Help screen"); + } + }), + WARMUP("w", new OptionHandler() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "warmup", true, + "Number of times for warmup per expression. Default: " + DEFAULT_WARMUP); + o.setArgName("NUM"); + o.setRequired(false); + return o; + } - @Override - public Optional getValue(BenchmarkOptions option, CommandLine cli) { - return Optional.ofNullable(option.get(cli).trim()); - } - }), - PERCENTILES("p", new OptionHandler() { - @Nullable - @Override - public Option apply(@Nullable String s) { - Option o = new Option(s, "percentiles", true - , "Percentiles to calculate per run. Default: " + Joiner.on(",").join(Arrays.asList(DEFAULT_PERCENTILES)) - ); - o.setArgName("NUM"); - o.setRequired(false); - return o; - } + @Override + public Optional getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + PERCENTILES("p", new OptionHandler() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "percentiles", true, + "Percentiles to calculate per run. Default: " + + Joiner.on(",").join(Arrays.asList(DEFAULT_PERCENTILES)) + ); + o.setArgName("NUM"); + o.setRequired(false); + return o; + } - @Override - public Optional getValue(BenchmarkOptions option, CommandLine cli) { - return Optional.ofNullable(option.get(cli).trim()); - } - }), - NUM_TIMES("n", new OptionHandler() { - @Nullable - @Override - public Option apply(@Nullable String s) { - Option o = new Option(s, "num_times", true, "Number of times to run per expression (after warmup). Default: " + DEFAULT_NUM_TIMES); - o.setArgName("NUM"); - o.setRequired(false); - return o; - } + @Override + public Optional getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + NUM_TIMES("n", new OptionHandler() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "num_times", true, + "Number of times to run per expression (after warmup). Default: " + DEFAULT_NUM_TIMES); + o.setArgName("NUM"); + o.setRequired(false); + return o; + } - @Override - public Optional getValue(BenchmarkOptions option, CommandLine cli) { - return Optional.ofNullable(option.get(cli).trim()); - } - }), - EXPRESSIONS("e", new OptionHandler() { - @Nullable - @Override - public Option apply(@Nullable String s) { - Option o = new Option(s, "expressions", true, "Stellar expressions"); - o.setArgName("FILE"); - o.setRequired(false); - return o; - } + @Override + public Optional getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + EXPRESSIONS("e", new OptionHandler() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "expressions", true, "Stellar expressions"); + o.setArgName("FILE"); + o.setRequired(false); + return o; + } - @Override - public Optional getValue(BenchmarkOptions option, CommandLine cli) { - return Optional.ofNullable(option.get(cli).trim()); - } - }), - VARIABLES("v", new OptionHandler() { - @Nullable - @Override - public Option apply(@Nullable String s) { - Option o = new Option(s, "variables", true, "File containing a JSON Map of variables to use"); - o.setArgName("FILE"); - o.setRequired(false); - return o; - } + @Override + public Optional getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + VARIABLES("v", new OptionHandler() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "variables", true, "File containing a JSON Map of variables to use"); + o.setArgName("FILE"); + o.setRequired(false); + return o; + } - @Override - public Optional getValue(BenchmarkOptions option, CommandLine cli) { - return Optional.ofNullable(option.get(cli).trim()); - } - }), - OUTPUT("o", new OptionHandler() { - @Nullable - @Override - public Option apply(@Nullable String s) { - Option o = new Option(s, "output", true, "File to write output."); - o.setArgName("FILE"); - o.setRequired(false); - return o; - } + @Override + public Optional getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + OUTPUT("o", new OptionHandler() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "output", true, "File to write output."); + o.setArgName("FILE"); + o.setRequired(false); + return o; + } - @Override - public Optional getValue(BenchmarkOptions option, CommandLine cli) { - return Optional.ofNullable(option.get(cli).trim()); - } - }) - ; - ; - Option option; - String shortCode; - OptionHandler handler; - BenchmarkOptions(String shortCode, OptionHandler optionHandler) { - this.shortCode = shortCode; - this.handler = optionHandler; - this.option = optionHandler.apply(shortCode); - } + @Override + public Optional getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }); + Option option; + String shortCode; + OptionHandler handler; - public boolean has(CommandLine cli) { - return cli.hasOption(shortCode); - } + BenchmarkOptions(String shortCode, OptionHandler optionHandler) { + this.shortCode = shortCode; + this.handler = optionHandler; + this.option = optionHandler.apply(shortCode); + } - public String get(CommandLine cli) { - return cli.getOptionValue(shortCode); - } + public boolean has(CommandLine cli) { + return cli.hasOption(shortCode); + } - public static CommandLine parse(CommandLineParser parser, String[] args) { - try { - CommandLine cli = parser.parse(getOptions(), args); - if(HELP.has(cli)) { - printHelp(); - System.exit(0); + public String get(CommandLine cli) { + return cli.getOptionValue(shortCode); } - return cli; - } catch (org.apache.commons.cli.ParseException e) { - System.err.println("Unable to parse args: " + Joiner.on(' ').join(args)); - e.printStackTrace(System.err); - printHelp(); - System.exit(-1); - return null; - } - } - public static EnumMap> createConfig(CommandLine cli) { - EnumMap > ret = new EnumMap<>(BenchmarkOptions.class); - for(BenchmarkOptions option : values()) { - ret.put(option, option.handler.getValue(option, cli)); - } - return ret; - } + public static CommandLine parse(CommandLineParser parser, String[] args) { + try { + CommandLine cli = parser.parse(getOptions(), args); + if (HELP.has(cli)) { + printHelp(); + System.exit(0); + } + return cli; + } catch (org.apache.commons.cli.ParseException e) { + System.err.println("Unable to parse args: " + Joiner.on(' ').join(args)); + e.printStackTrace(System.err); + printHelp(); + System.exit(-1); + return null; + } + } - public static void printHelp() { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp( "StellarBenchmark", getOptions()); - } + public static EnumMap> createConfig(CommandLine cli) { + EnumMap> ret = new EnumMap<>(BenchmarkOptions.class); + for (BenchmarkOptions option : values()) { + ret.put(option, option.handler.getValue(option, cli)); + } + return ret; + } - public static Options getOptions() { - Options ret = new Options(); - for(BenchmarkOptions o : BenchmarkOptions.values()) { - ret.addOption(o.option); - } - return ret; - } - } + public static void printHelp() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("StellarBenchmark", getOptions()); + } - public static void main(String... argv) throws IOException { - CommandLine cli = BenchmarkOptions.parse(new PosixParser(), argv); - if(!BenchmarkOptions.EXPRESSIONS.has(cli)) { - throw new IllegalStateException("You must at least specify an expressions file."); - } - File expressionsFile = new File(BenchmarkOptions.EXPRESSIONS.get(cli)); - Optional variablesFile = Optional.ofNullable(!BenchmarkOptions.VARIABLES.has(cli) - ?null - :new File(BenchmarkOptions.VARIABLES.get(cli)) - ); - Optional output = Optional.ofNullable(!BenchmarkOptions.OUTPUT.has(cli) - ?null - :new File(BenchmarkOptions.OUTPUT.get(cli)) - ); - List lines = Files.readLines(expressionsFile, Charset.defaultCharset()); - Map variables = new HashMap<>(); - if (variablesFile.isPresent()) { - variables = JSONUtils.INSTANCE.load(new FileInputStream(variablesFile.get()), JSONUtils.MAP_SUPPLIER); - } - int numTimes = DEFAULT_NUM_TIMES; - if (BenchmarkOptions.NUM_TIMES.has(cli)) { - numTimes = Integer.parseInt(BenchmarkOptions.NUM_TIMES.get(cli)); - } - int warmup = DEFAULT_WARMUP; - if (BenchmarkOptions.WARMUP.has(cli)) { - warmup = Integer.parseInt(BenchmarkOptions.WARMUP.get(cli)); - } - Double[] percentiles = DEFAULT_PERCENTILES; - if (BenchmarkOptions.PERCENTILES.has(cli)) { - List percentileList = new ArrayList<>(); - for (String token : Splitter.on(",").split(BenchmarkOptions.PERCENTILES.get(cli))) { - if (token.trim().isEmpty()) { - continue; + public static Options getOptions() { + Options ret = new Options(); + for (BenchmarkOptions o : BenchmarkOptions.values()) { + ret.addOption(o.option); + } + return ret; } - Double d = Double.parseDouble(token.trim()); - percentileList.add(d); - } - percentiles = (Double[])percentileList.toArray(); - } - PrintWriter out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8))); - if (output.isPresent()) { - out = new PrintWriter(output.get(), StandardCharsets.UTF_8.name()); } - for (String statement : lines) { - if (statement.trim().startsWith("#") || statement.trim().isEmpty()) { - continue; - } - Microbenchmark.StellarStatement s = new Microbenchmark.StellarStatement(); - s.context = Context.EMPTY_CONTEXT(); - s.expression = statement; - s.functionResolver = StellarFunctions.FUNCTION_RESOLVER(); - s.variableResolver = new MapVariableResolver(variables); - DescriptiveStatistics stats = Microbenchmark.run(s, warmup, numTimes); - out.println("Expression: " + statement); - out.println(Microbenchmark.describe(stats, percentiles)); - } - if (argv.length > 2) { - out.close(); + + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") + public static void main(String... argv) throws IOException { + CommandLine cli = BenchmarkOptions.parse(new PosixParser(), argv); + if (!BenchmarkOptions.EXPRESSIONS.has(cli)) { + throw new IllegalStateException("You must at least specify an expressions file."); + } + File expressionsFile = new File(BenchmarkOptions.EXPRESSIONS.get(cli)); + Optional variablesFile = Optional.ofNullable(!BenchmarkOptions.VARIABLES.has(cli) + ? null + : new File(BenchmarkOptions.VARIABLES.get(cli)) + ); + Optional output = Optional.ofNullable(!BenchmarkOptions.OUTPUT.has(cli) + ? null + : new File(BenchmarkOptions.OUTPUT.get(cli)) + ); + List lines = Files.readLines(expressionsFile, Charset.defaultCharset()); + Map variables = new HashMap<>(); + if (variablesFile.isPresent()) { + variables = JSONUtils.INSTANCE.load(new FileInputStream(variablesFile.get()), JSONUtils.MAP_SUPPLIER); + } + int numTimes = DEFAULT_NUM_TIMES; + if (BenchmarkOptions.NUM_TIMES.has(cli)) { + numTimes = Integer.parseInt(BenchmarkOptions.NUM_TIMES.get(cli)); + } + int warmup = DEFAULT_WARMUP; + if (BenchmarkOptions.WARMUP.has(cli)) { + warmup = Integer.parseInt(BenchmarkOptions.WARMUP.get(cli)); + } + Double[] percentiles = DEFAULT_PERCENTILES; + if (BenchmarkOptions.PERCENTILES.has(cli)) { + List percentileList = new ArrayList<>(); + for (String token : Splitter.on(",").split(BenchmarkOptions.PERCENTILES.get(cli))) { + if (token.trim().isEmpty()) { + continue; + } + Double d = Double.parseDouble(token.trim()); + percentileList.add(d); + } + percentiles = (Double[]) percentileList.toArray(); + } + PrintWriter out = + new PrintWriter(new BufferedWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8))); + if (output.isPresent()) { + out = new PrintWriter(output.get(), StandardCharsets.UTF_8.name()); + } + for (String statement : lines) { + if (statement.trim().startsWith("#") || statement.trim().isEmpty()) { + continue; + } + Microbenchmark.StellarStatement s = new Microbenchmark.StellarStatement(); + s.context = Context.EMPTY_CONTEXT(); + s.expression = statement; + s.functionResolver = StellarFunctions.FUNCTION_RESOLVER(); + s.variableResolver = new MapVariableResolver(variables); + DescriptiveStatistics stats = Microbenchmark.run(s, warmup, numTimes); + out.println("Expression: " + statement); + out.println(Microbenchmark.describe(stats, percentiles)); + } + if (argv.length > 2) { + out.close(); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java index 99859b29..8f266dfb 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,52 +21,50 @@ package org.apache.metron.stellar.common.configuration; import com.google.common.base.Function; +import java.io.IOException; import org.apache.metron.stellar.common.Constants; import org.apache.metron.stellar.common.utils.JSONUtils; -import java.io.IOException; -import java.util.Map; - public enum ConfigurationType implements Function { - GLOBAL("global",".", s -> { - try { - return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER); - } catch (IOException e) { - throw new RuntimeException("Unable to load " + s, e); - } - }); + GLOBAL("global", ".", s -> { + try { + return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER); + } catch (IOException e) { + throw new RuntimeException("Unable to load " + s, e); + } + }); - String name; - String directory; - String zookeeperRoot; - Function deserializer; + String name; + String directory; + String zookeeperRoot; + Function deserializer; - ConfigurationType(String name, String directory, Function deserializer) { - this.name = name; - this.directory = directory; - this.zookeeperRoot = Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name; - this.deserializer = deserializer; - } + ConfigurationType(String name, String directory, Function deserializer) { + this.name = name; + this.directory = directory; + this.zookeeperRoot = Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name; + this.deserializer = deserializer; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public String getDirectory() { - return directory; - } + public String getDirectory() { + return directory; + } - public Object deserialize(String s) { - return deserializer.apply(s); - } + public Object deserialize(String s) { + return deserializer.apply(s); + } - @Override - public Object apply(String s) { - return deserialize(s); - } + @Override + public Object apply(String s) { + return deserialize(s); + } - public String getZookeeperRoot() { - return zookeeperRoot; - } + public String getZookeeperRoot() { + return zookeeperRoot; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationsUtils.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationsUtils.java index 399e5d15..41abb59a 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationsUtils.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationsUtils.java @@ -6,17 +6,30 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.configuration; +import static org.apache.metron.stellar.common.configuration.ConfigurationType.GLOBAL; + +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -27,147 +40,139 @@ import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.zookeeper.KeeperException; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +public class ConfigurationsUtils { -import static org.apache.metron.stellar.common.configuration.ConfigurationType.GLOBAL; + public static CuratorFramework getClient(String zookeeperUrl) { + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + return CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy); + } -public class ConfigurationsUtils { + public static void writeGlobalConfigToZookeeper(Map globalConfig, String zookeeperUrl) + throws Exception { + try (CuratorFramework client = getClient(zookeeperUrl)) { + client.start(); + writeGlobalConfigToZookeeper(globalConfig, client); + } + } + + public static void writeGlobalConfigToZookeeper(Map globalConfig, CuratorFramework client) + throws Exception { + writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSON(globalConfig), client); + } - public static CuratorFramework getClient(String zookeeperUrl) { - RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); - return CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy); - } - - public static void writeGlobalConfigToZookeeper(Map globalConfig, String zookeeperUrl) throws Exception { - try(CuratorFramework client = getClient(zookeeperUrl)) { - client.start(); - writeGlobalConfigToZookeeper(globalConfig, client); - } - } - public static void writeGlobalConfigToZookeeper(Map globalConfig, CuratorFramework client) throws Exception { - writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSON(globalConfig), client); - } - - public static void writeGlobalConfigToZookeeper(byte[] globalConfig, String zookeeperUrl) throws Exception { - try(CuratorFramework client = getClient(zookeeperUrl)) { - client.start(); - writeGlobalConfigToZookeeper(globalConfig, client); - } - } - - public static void writeGlobalConfigToZookeeper(byte[] globalConfig, CuratorFramework client) throws Exception { - GLOBAL.deserialize(new String(globalConfig, StandardCharsets.UTF_8)); - writeToZookeeper(GLOBAL.getZookeeperRoot(), globalConfig, client); - } - - public static void writeConfigToZookeeper(String name, Map config, String zookeeperUrl) throws Exception { - writeConfigToZookeeper(name, JSONUtils.INSTANCE.toJSON(config), zookeeperUrl); - } - - public static void writeConfigToZookeeper(String name, byte[] config, String zookeeperUrl) throws Exception { - try(CuratorFramework client = getClient(zookeeperUrl)) { - client.start(); - writeToZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, config, client); - } - } - - public static void writeToZookeeper(String path, byte[] configData, CuratorFramework client) throws Exception { - try { - client.setData().forPath(path, configData); - } catch (KeeperException.NoNodeException e) { - client.create().creatingParentsIfNeeded().forPath(path, configData); - } - } - - public static byte[] readGlobalConfigBytesFromZookeeper(CuratorFramework client) throws Exception { - return readFromZookeeper(GLOBAL.getZookeeperRoot(), client); - } - - public static byte[] readConfigBytesFromZookeeper(String name, CuratorFramework client) throws Exception { - return readFromZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, client); - } - - public static byte[] readFromZookeeper(String path, CuratorFramework client) throws Exception { - if(client != null && client.getData() != null && path != null) { - return client.getData().forPath(path); - } - return new byte[]{}; - } - - public static void setupStellarStatically(CuratorFramework client) throws Exception { - byte[] ret = null; - try { - ret = readGlobalConfigBytesFromZookeeper(client); - } catch (KeeperException.NoNodeException nne) { - //can't find the node - } - if (ret == null || ret.length == 0) { - setupStellarStatically(client, Optional.empty()); - } else { - setupStellarStatically(client, Optional.of(new String(ret, StandardCharsets.UTF_8))); - } - } - - public static void setupStellarStatically(CuratorFramework client, Optional globalConfig) { - /* - In order to validate stellar functions, the function resolver must be initialized. Otherwise, - those utilities that require validation cannot validate the stellar expressions necessarily. - */ - Context.Builder builder = new Context.Builder().with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) - ; - if(globalConfig.isPresent()) { - builder = builder.with(Context.Capabilities.GLOBAL_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())) - .with(Context.Capabilities.STELLAR_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())); - } - else { - builder = builder.with(Context.Capabilities.STELLAR_CONFIG, () -> new HashMap<>()); - } - Context stellarContext = builder.build(); - StellarFunctions.FUNCTION_RESOLVER().initialize(stellarContext); - } - - public static byte[] readGlobalConfigFromFile(String rootPath) throws IOException { - byte[] globalConfig = new byte[0]; - File configPath = new File(rootPath, GLOBAL.getName() + ".json"); - if (configPath.exists()) { - globalConfig = Files.readAllBytes(configPath.toPath()); - } - return globalConfig; - } - - public interface ConfigurationVisitor { - void visit(ConfigurationType configurationType, String name, String data); - } - - public static void visitConfigs(CuratorFramework client, final ConfigurationVisitor callback) throws Exception { - visitConfigs(client, (type, name, data) -> { - setupStellarStatically(client, Optional.ofNullable(data)); - callback.visit(type, name, data); - }, GLOBAL); - } - - public static void visitConfigs(CuratorFramework client, ConfigurationVisitor callback, ConfigurationType configType) throws Exception { - - if (client.checkExists().forPath(configType.getZookeeperRoot()) != null) { - - if (configType.equals(GLOBAL)) { - byte[] globalConfigData = client.getData().forPath(configType.getZookeeperRoot()); - callback.visit(configType, "global", new String(globalConfigData, StandardCharsets.UTF_8)); - } - } - } - - public static void dumpConfigs(PrintStream out, CuratorFramework client) throws Exception { - ConfigurationsUtils.visitConfigs(client, (type, name, data) -> { - type.deserialize(data); - out.println(type + " Config: " + name + "\n" + data); - }); - } + public static void writeGlobalConfigToZookeeper(byte[] globalConfig, String zookeeperUrl) throws Exception { + try (CuratorFramework client = getClient(zookeeperUrl)) { + client.start(); + writeGlobalConfigToZookeeper(globalConfig, client); + } + } + + public static void writeGlobalConfigToZookeeper(byte[] globalConfig, CuratorFramework client) throws Exception { + GLOBAL.deserialize(new String(globalConfig, StandardCharsets.UTF_8)); + writeToZookeeper(GLOBAL.getZookeeperRoot(), globalConfig, client); + } + + public static void writeConfigToZookeeper(String name, Map config, String zookeeperUrl) + throws Exception { + writeConfigToZookeeper(name, JSONUtils.INSTANCE.toJSON(config), zookeeperUrl); + } + + public static void writeConfigToZookeeper(String name, byte[] config, String zookeeperUrl) throws Exception { + try (CuratorFramework client = getClient(zookeeperUrl)) { + client.start(); + writeToZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, config, client); + } + } + + public static void writeToZookeeper(String path, byte[] configData, CuratorFramework client) throws Exception { + try { + client.setData().forPath(path, configData); + } catch (KeeperException.NoNodeException e) { + client.create().creatingParentsIfNeeded().forPath(path, configData); + } + } + + public static byte[] readGlobalConfigBytesFromZookeeper(CuratorFramework client) throws Exception { + return readFromZookeeper(GLOBAL.getZookeeperRoot(), client); + } + + public static byte[] readConfigBytesFromZookeeper(String name, CuratorFramework client) throws Exception { + return readFromZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, client); + } + + public static byte[] readFromZookeeper(String path, CuratorFramework client) throws Exception { + if (client != null && client.getData() != null && path != null) { + return client.getData().forPath(path); + } + return new byte[] {}; + } + + public static void setupStellarStatically(CuratorFramework client) throws Exception { + byte[] ret = null; + try { + ret = readGlobalConfigBytesFromZookeeper(client); + } catch (KeeperException.NoNodeException nne) { + //can't find the node + } + if (ret == null || ret.length == 0) { + setupStellarStatically(client, Optional.empty()); + } else { + setupStellarStatically(client, Optional.of(new String(ret, StandardCharsets.UTF_8))); + } + } + + public static void setupStellarStatically(CuratorFramework client, Optional globalConfig) { + /* + In order to validate stellar functions, the function resolver must be initialized. Otherwise, + those utilities that require validation cannot validate the stellar expressions necessarily. + */ + Context.Builder builder = new Context.Builder().with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client); + if (globalConfig.isPresent()) { + builder = builder.with(Context.Capabilities.GLOBAL_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())) + .with(Context.Capabilities.STELLAR_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())); + } else { + builder = builder.with(Context.Capabilities.STELLAR_CONFIG, () -> new HashMap<>()); + } + Context stellarContext = builder.build(); + StellarFunctions.FUNCTION_RESOLVER().initialize(stellarContext); + } + + public static byte[] readGlobalConfigFromFile(String rootPath) throws IOException { + byte[] globalConfig = new byte[0]; + File configPath = new File(rootPath, GLOBAL.getName() + ".json"); + if (configPath.exists()) { + globalConfig = Files.readAllBytes(configPath.toPath()); + } + return globalConfig; + } + + public interface ConfigurationVisitor { + void visit(ConfigurationType configurationType, String name, String data); + } + + public static void visitConfigs(CuratorFramework client, final ConfigurationVisitor callback) throws Exception { + visitConfigs(client, (type, name, data) -> { + setupStellarStatically(client, Optional.ofNullable(data)); + callback.visit(type, name, data); + }, GLOBAL); + } + + public static void visitConfigs(CuratorFramework client, ConfigurationVisitor callback, + ConfigurationType configType) throws Exception { + + if (client.checkExists().forPath(configType.getZookeeperRoot()) != null) { + + if (configType.equals(GLOBAL)) { + byte[] globalConfigData = client.getData().forPath(configType.getZookeeperRoot()); + callback.visit(configType, "global", new String(globalConfigData, StandardCharsets.UTF_8)); + } + } + } + + public static void dumpConfigs(PrintStream out, CuratorFramework client) throws Exception { + ConfigurationsUtils.visitConfigs(client, (type, name, data) -> { + type.deserialize(data); + out.println(type + " Config: " + name + "\n" + data); + }); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/encoding/Encodings.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/encoding/Encodings.java index aceda1f7..d37d73e3 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/encoding/Encodings.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/encoding/Encodings.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,124 +37,126 @@ * Endodings utility enum for Stellar supported encodings. */ public enum Encodings { - BASE32((possible) -> new Base32().isInAlphabet(possible), - (possible) -> new String(new Base32().decode(possible), StandardCharsets.UTF_8), - (possible) -> new String( new Base32().encode(possible.getBytes(StandardCharsets.UTF_8)),StandardCharsets.UTF_8)), - BASE32HEX((possible) -> new Base32(true).isInAlphabet(possible), - (possible) -> new String(new Base32(true).decode(possible.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8), - (possible) -> new String(new Base32(true).encode(possible.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8)), - BASE64((possible) -> Base64.isBase64(possible), - (possible) -> new String(new Base64().decode(possible.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8), - (possible) -> new String(new Base64().encode(possible.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8)), - BINARY((possible) -> { - for (byte b : possible.getBytes(StandardCharsets.UTF_8)) { - if ((b != 48 && b != 49)) { - return false; + BASE32((possible) -> new Base32().isInAlphabet(possible), + (possible) -> new String(new Base32().decode(possible), StandardCharsets.UTF_8), + (possible) -> new String(new Base32().encode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)), + BASE32HEX((possible) -> new Base32(true).isInAlphabet(possible), + (possible) -> new String(new Base32(true).decode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8), + (possible) -> new String(new Base32(true).encode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)), + BASE64((possible) -> Base64.isBase64(possible), + (possible) -> new String(new Base64().decode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8), + (possible) -> new String(new Base64().encode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)), + BINARY((possible) -> { + for (byte b : possible.getBytes(StandardCharsets.UTF_8)) { + if ((b != 48 && b != 49)) { + return false; + } } - } - return true; - }, - (possible) -> { - String str = new String(BinaryCodec.fromAscii(possible.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8); - if (StringUtils.isEmpty(str.trim())) { - return possible; - } - return str; - }, - (possible) -> BinaryCodec.toAsciiString(possible.getBytes(StandardCharsets.UTF_8))), - HEX((possible) -> { - try { - Hex hex = new Hex(StandardCharsets.UTF_8); - hex.decode(possible.getBytes(StandardCharsets.UTF_8)); return true; - } catch (DecoderException e) { - return false; - } }, - (possible) -> { + (possible) -> { + String str = new String(BinaryCodec.fromAscii(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + if (StringUtils.isEmpty(str.trim())) { + return possible; + } + return str; + }, + (possible) -> BinaryCodec.toAsciiString(possible.getBytes(StandardCharsets.UTF_8))), + HEX((possible) -> { try { - Hex hex = new Hex(StandardCharsets.UTF_8); - return new String(hex.decode(possible.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8); + Hex hex = new Hex(StandardCharsets.UTF_8); + hex.decode(possible.getBytes(StandardCharsets.UTF_8)); + return true; } catch (DecoderException e) { - return possible; + return false; } - }, - (possible) -> new String(new Hex(StandardCharsets.UTF_8).encode(possible.getBytes(StandardCharsets.UTF_8)),StandardCharsets.UTF_8)); + }, + (possible) -> { + try { + Hex hex = new Hex(StandardCharsets.UTF_8); + return new String(hex.decode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + } catch (DecoderException e) { + return possible; + } + }, + (possible) -> new String(new Hex(StandardCharsets.UTF_8).encode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); - Predicate is; - Function decode; - Function encode; + Predicate is; + Function decode; + Function encode; - /** - * Create a specialed Endodings enum member. - * - * @param is function for detecting - * @param decode funtion for decoding - */ - Encodings(Predicate is, Function decode, Function encode) { - this.is = is; - this.decode = decode; - this.encode = encode; - } + /** + * Create a specialed Endodings enum member. + * + * @param is function for detecting + * @param decode funtion for decoding + */ + Encodings(Predicate is, Function decode, Function encode) { + this.is = is; + this.decode = decode; + this.encode = encode; + } - public static final List SUPPORTED_LIST = new ArrayList<>(Arrays.asList(BASE32.name(), - BASE32HEX.name(), BASE64.name(), BINARY.name() , HEX.name())); + public static final List SUPPORTED_LIST = new ArrayList<>(Arrays.asList(BASE32.name(), + BASE32HEX.name(), BASE64.name(), BINARY.name(), HEX.name())); - /** - * Determines if the passed String is encoded in this encoding. - * A given String may be compatible with the encoding, but not actually encoded. - * - * @param possible the String to test - * @return true or false - */ - public boolean is(String possible) { - return is.test(possible); - } + /** + * Determines if the passed String is encoded in this encoding. + * A given String may be compatible with the encoding, but not actually encoded. + * + * @param possible the String to test + * @return true or false + */ + public boolean is(String possible) { + return is.test(possible); + } - /** - * Attempts to decode a given String without verification. - * Any failure or compatibility issues will result in the original - * String being returned. - * - * @param encoded the String to decode - * @return The String decoded, or the original String - */ - public String decode(String encoded) { - return decode(encoded, false); - } + /** + * Attempts to decode a given String without verification. + * Any failure or compatibility issues will result in the original + * String being returned. + * + * @param encoded the String to decode + * @return The String decoded, or the original String + */ + public String decode(String encoded) { + return decode(encoded, false); + } - /** - * Attempts to decode a given String, optionally verifying first. - * Any failure or compatibility issues will result in the original - * String being returned. - * - * @param encoded the String to decode - * @param verify flag to perform verification - * @return The String decoded, or the original String - */ - public String decode(String encoded, boolean verify) { - if (verify) { - if (is.test(encoded)) { + /** + * Attempts to decode a given String, optionally verifying first. + * Any failure or compatibility issues will result in the original + * String being returned. + * + * @param encoded the String to decode + * @param verify flag to perform verification + * @return The String decoded, or the original String + */ + public String decode(String encoded, boolean verify) { + if (verify) { + if (is.test(encoded)) { + return decode.apply(encoded); + } else { + return encoded; + } + } return decode.apply(encoded); - } else { - return encoded; - } } - return decode.apply(encoded); - } - /** - * Encodes the given String - * @param toEncode - * @return an encoded String - */ - public String encode(String toEncode) { - return encode.apply(toEncode); - } + /** + * Encodes the given String. + * + * @return an encoded String. + */ + public String encode(String toEncode) { + return encode.apply(toEncode); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ArithmeticEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ArithmeticEvaluator.java index b89f40c9..a800dec8 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ArithmeticEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ArithmeticEvaluator.java @@ -18,86 +18,87 @@ package org.apache.metron.stellar.common.evaluators; +import java.util.function.BiFunction; import org.apache.commons.lang3.tuple.Pair; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.FrameContext; - -import java.util.function.BiFunction; +import org.apache.metron.stellar.dsl.Token; public enum ArithmeticEvaluator { - INSTANCE; + INSTANCE; - public Token evaluate(BiFunction> function, - Pair, Token> p) { - if (p == null || p.getKey() == null || p.getValue() == null) { - throw new IllegalArgumentException(); - } + public Token evaluate(BiFunction> function, + Pair, Token> p) { + if (p == null || p.getKey() == null || p.getValue() == null) { + throw new IllegalArgumentException(); + } - final Number l = p.getKey().getValue(); - final Number r = p.getValue().getValue(); + final Number l = p.getKey().getValue(); + final Number r = p.getValue().getValue(); - return function.apply(l == null ? 0 : l, r == null ? 0 : r); - } + return function.apply(l == null ? 0 : l, r == null ? 0 : r); + } - /** - * This is a helper class that defines how to handle arithmetic operations. The conversion between number - * types is taken for the Java spec: http://docs.oracle.com/javase/specs/jls/se8/html/jls-5.html#jls-5.6.2 - */ - public static class ArithmeticEvaluatorFunctions { - public static BiFunction> addition(final FrameContext.Context context) { - return (Number l, Number r) -> { - if (l instanceof Double || r instanceof Double) { - return new Token<>(l.doubleValue() + r.doubleValue(), Double.class, context); - } else if (l instanceof Float || r instanceof Float) { - return new Token<>(l.floatValue() + r.floatValue(), Float.class, context); - } else if (l instanceof Long || r instanceof Long) { - return new Token<>(l.longValue() + r.longValue(), Long.class, context); - } else { - return new Token<>(l.intValue() + r.intValue(), Integer.class, context); + /** + * This is a helper class that defines how to handle arithmetic operations. The conversion between number + * types is taken for the Java spec: http://docs.oracle.com/javase/specs/jls/se8/html/jls-5.html#jls-5.6.2 + */ + public static class ArithmeticEvaluatorFunctions { + public static BiFunction> addition(final FrameContext.Context context) { + return (Number l, Number r) -> { + if (l instanceof Double || r instanceof Double) { + return new Token<>(l.doubleValue() + r.doubleValue(), Double.class, context); + } else if (l instanceof Float || r instanceof Float) { + return new Token<>(l.floatValue() + r.floatValue(), Float.class, context); + } else if (l instanceof Long || r instanceof Long) { + return new Token<>(l.longValue() + r.longValue(), Long.class, context); + } else { + return new Token<>(l.intValue() + r.intValue(), Integer.class, context); + } + }; } - }; - } - public static BiFunction> multiplication(final FrameContext.Context context) { - return (Number l, Number r) -> { - if (l instanceof Double || r instanceof Double) { - return new Token<>(l.doubleValue() * r.doubleValue(), Double.class, context); - } else if (l instanceof Float || r instanceof Float) { - return new Token<>(l.floatValue() * r.floatValue(), Float.class, context); - } else if (l instanceof Long || r instanceof Long) { - return new Token<>(l.longValue() * r.longValue(), Long.class, context); - } else { - return new Token<>(l.intValue() * r.intValue(), Integer.class, context); + public static BiFunction> multiplication( + final FrameContext.Context context) { + return (Number l, Number r) -> { + if (l instanceof Double || r instanceof Double) { + return new Token<>(l.doubleValue() * r.doubleValue(), Double.class, context); + } else if (l instanceof Float || r instanceof Float) { + return new Token<>(l.floatValue() * r.floatValue(), Float.class, context); + } else if (l instanceof Long || r instanceof Long) { + return new Token<>(l.longValue() * r.longValue(), Long.class, context); + } else { + return new Token<>(l.intValue() * r.intValue(), Integer.class, context); + } + }; } - }; - } - public static BiFunction> subtraction(final FrameContext.Context context) { - return (Number l, Number r) -> { - if (l instanceof Double || r instanceof Double) { - return new Token<>(l.doubleValue() - r.doubleValue(), Double.class, context); - } else if (l instanceof Float || r instanceof Float) { - return new Token<>(l.floatValue() - r.floatValue(), Float.class, context); - } else if (l instanceof Long || r instanceof Long) { - return new Token<>(l.longValue() - r.longValue(), Long.class, context); - } else { - return new Token<>(l.intValue() - r.intValue(), Integer.class, context); + public static BiFunction> subtraction( + final FrameContext.Context context) { + return (Number l, Number r) -> { + if (l instanceof Double || r instanceof Double) { + return new Token<>(l.doubleValue() - r.doubleValue(), Double.class, context); + } else if (l instanceof Float || r instanceof Float) { + return new Token<>(l.floatValue() - r.floatValue(), Float.class, context); + } else if (l instanceof Long || r instanceof Long) { + return new Token<>(l.longValue() - r.longValue(), Long.class, context); + } else { + return new Token<>(l.intValue() - r.intValue(), Integer.class, context); + } + }; } - }; - } - public static BiFunction> division(FrameContext.Context context) { - return (Number l, Number r) -> { - if (l instanceof Double || r instanceof Double) { - return new Token<>(l.doubleValue() / r.doubleValue(), Double.class, context); - } else if (l instanceof Float || r instanceof Float) { - return new Token<>(l.floatValue() / r.floatValue(), Float.class, context); - } else if (l instanceof Long || r instanceof Long) { - return new Token<>(l.longValue() / r.longValue(), Long.class, context); - } else { - return new Token<>(l.intValue() / r.intValue(), Integer.class, context); + public static BiFunction> division(FrameContext.Context context) { + return (Number l, Number r) -> { + if (l instanceof Double || r instanceof Double) { + return new Token<>(l.doubleValue() / r.doubleValue(), Double.class, context); + } else if (l instanceof Float || r instanceof Float) { + return new Token<>(l.floatValue() / r.floatValue(), Float.class, context); + } else if (l instanceof Long || r instanceof Long) { + return new Token<>(l.longValue() / r.longValue(), Long.class, context); + } else { + return new Token<>(l.intValue() / r.intValue(), Integer.class, context); + } + }; } - }; } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonExpressionEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonExpressionEvaluator.java index f2449818..644bdfb5 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonExpressionEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonExpressionEvaluator.java @@ -18,8 +18,8 @@ package org.apache.metron.stellar.common.evaluators; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.generated.StellarParser; +import org.apache.metron.stellar.dsl.Token; /** * This is used to determine what is needed to evaluate a Stellar comparison expression. A Stellar comparison @@ -29,14 +29,15 @@ */ public interface ComparisonExpressionEvaluator { - /** - * This will compare the values of {@code left} and {@code right} using the {@code op} input to determine a value - * to return. - * @param left The token representing the left side of a comparison expression. - * @param right The token representing the right side of a comparison expression. - * @param op This is a representation of a comparison operator {@literal (eg. <, <=, >, >=, ==, !=) } - * @return True if the expression is evaluated to be true, otherwise false. An example of expressions that - * should be true are {@code 1 == 1}, {@code 1f > 0}, etc. - */ - boolean evaluate(Token left, Token right, StellarParser.ComparisonOpContext op); + /** + * This will compare the values of {@code left} and {@code right} using the {@code op} input to determine a value + * to return. + * + * @param left The token representing the left side of a comparison expression. + * @param right The token representing the right side of a comparison expression. + * @param op This is a representation of a comparison operator {@literal (eg. <, <=, >, >=, ==, !=) } + * @return True if the expression is evaluated to be true, otherwise false. An example of expressions that + * should be true are {@code 1 == 1}, {@code 1f > 0}, etc. + */ + boolean evaluate(Token left, Token right, StellarParser.ComparisonOpContext op); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonExpressionWithOperatorEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonExpressionWithOperatorEvaluator.java index 70bc4ca8..23b2f729 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonExpressionWithOperatorEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonExpressionWithOperatorEvaluator.java @@ -18,10 +18,10 @@ package org.apache.metron.stellar.common.evaluators; -import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.FrameContext; import org.apache.metron.stellar.common.generated.StellarParser; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Token; /** * This is the evaluator used when evaluating Stellar comparison operators. @@ -30,67 +30,76 @@ * @see ComparisonOperatorsEvaluator */ public enum ComparisonExpressionWithOperatorEvaluator { - /** - * The instance of {@link ComparisonExpressionWithOperatorEvaluator} used in - * order to evaluate Stellar comparison expressions. - */ - INSTANCE; - - /** - * The different strategies used to evaluate a Stellar comparison operator. They are broken into - * two categories: equality operator comparisons and comparison operator comparisons. - */ - enum Strategy { - /** - * The evaluator used to evaluate comparison operator expressions. - */ - COMPARISON_OPERATORS(new ComparisonOperatorsEvaluator()), /** - * The evaluator used to evaluate equality operator expressions. + * The instance of {@link ComparisonExpressionWithOperatorEvaluator} used in + * order to evaluate Stellar comparison expressions. */ - EQUALITY_OPERATORS(new EqualityOperatorsEvaluator()), - ; + INSTANCE; /** - * The evaluator to be used when evaluating Stellar expressions. + * The different strategies used to evaluate a Stellar comparison operator. They are broken into + * two categories: equality operator comparisons and comparison operator comparisons. */ - private ComparisonExpressionEvaluator evaluator; + enum Strategy { + /** + * The evaluator used to evaluate comparison operator expressions. + */ + COMPARISON_OPERATORS(new ComparisonOperatorsEvaluator()), + /** + * The evaluator used to evaluate equality operator expressions. + */ + EQUALITY_OPERATORS(new EqualityOperatorsEvaluator()), + ; + + /** + * The evaluator to be used when evaluating Stellar expressions. + */ + private final ComparisonExpressionEvaluator evaluator; + + Strategy(final ComparisonExpressionEvaluator evaluator) { + this.evaluator = evaluator; + } - Strategy(final ComparisonExpressionEvaluator evaluator) { - this.evaluator = evaluator; + /** + * evaluator getter. + * + * @return The evaluator needed to evaluate Stellar comparison expressions. + */ + public ComparisonExpressionEvaluator evaluator() { + return evaluator; + } } /** + * When evaluating comparison expressions with operators, they are broken into four cases: * - * @return The evaluator needed to evaluate Stellar comparison expressions. + *

    + * 1. Testing equality, see {@link EqualityOperatorsEvaluator} + * 2. Testing not equal, see {@link EqualityOperatorsEvaluator}. + * This will be the negation of {@link EqualityOperatorsEvaluator#evaluate(Token, Token, StellarParser.ComparisonOpContext)}. + * 3. Testing less than, less than or equal, greater than, and greater than or equal {@link ComparisonOperatorsEvaluator} + * 4. Otherwise thrown {@link ParseException}. + * + * @param left The value of the left side of the Stellar expression. + * @param right The value of the right side of the Stellar expression. + * @param op The operator in the Stellar expression. + * @return A token with type boolean. This is based on the comparison of the {@code right} and {@code left} values. */ - public ComparisonExpressionEvaluator evaluator() { - return evaluator; - } - } + public Token evaluate(final Token left, final Token right, + final StellarParser.ComparisonOpContext op, FrameContext.Context context) { + if (op.EQ() != null) { + return new Token<>(Strategy.EQUALITY_OPERATORS.evaluator().evaluate(left, right, op), Boolean.class, + context); + } else if (op.NEQ() != null) { + return new Token<>(!Strategy.EQUALITY_OPERATORS.evaluator().evaluate(left, right, op), Boolean.class, + context); + } else if (op.LT() != null || op.GT() != null || op.LTE() != null || op.GTE() != null) { + return new Token<>(Strategy.COMPARISON_OPERATORS.evaluator().evaluate(left, right, op), Boolean.class, + context); + } - /** - * When evaluating comparison expressions with operators, they are broken into four cases: - * - * 1. Testing equality, see {@link EqualityOperatorsEvaluator} - * 2. Testing not equal, see {@link EqualityOperatorsEvaluator}. This will be the negation of {@link EqualityOperatorsEvaluator#evaluate(Token, Token, StellarParser.ComparisonOpContext)}. - * 3. Testing less than, less than or equal, greater than, and greater than or equal {@link ComparisonOperatorsEvaluator} - * 4. Otherwise thrown {@link ParseException}. - * - * @param left The value of the left side of the Stellar expression. - * @param right The value of the right side of the Stellar expression. - * @param op The operator in the Stellar expression. - * @return A token with type boolean. This is based on the comparison of the {@code right} and {@code left} values. - */ - public Token evaluate(final Token left, final Token right, final StellarParser.ComparisonOpContext op, FrameContext.Context context) { - if (op.EQ() != null) { - return new Token<>(Strategy.EQUALITY_OPERATORS.evaluator().evaluate(left, right, op), Boolean.class, context); - } else if (op.NEQ() != null) { - return new Token<>(!Strategy.EQUALITY_OPERATORS.evaluator().evaluate(left, right, op), Boolean.class, context); - } else if (op.LT() != null || op.GT() != null || op.LTE() != null || op.GTE() != null) { - return new Token<>(Strategy.COMPARISON_OPERATORS.evaluator().evaluate(left, right, op), Boolean.class, context); + throw new ParseException( + "Unsupported operations. The following expression is invalid: " + left.getValue() + op.getText() + + right.getValue()); } - - throw new ParseException("Unsupported operations. The following expression is invalid: " + left.getValue() + op.getText() + right.getValue()); - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonOperatorsEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonOperatorsEvaluator.java index 1dc80218..10083f1e 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonOperatorsEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/ComparisonOperatorsEvaluator.java @@ -18,9 +18,9 @@ package org.apache.metron.stellar.common.evaluators; +import org.apache.metron.stellar.common.generated.StellarParser; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Token; -import org.apache.metron.stellar.common.generated.StellarParser; /** * {@link ComparisonOperatorsEvaluator} is used to evaluate comparison expressions using the following @@ -28,158 +28,162 @@ */ public class ComparisonOperatorsEvaluator implements ComparisonExpressionEvaluator { - /** - * 1. If either the left or right's value is null then return false. - * 2. If both sides of the expression are instances of {@link Number} then: - * 1. If either side is a {@link Double} then get {@link Number#doubleValue()} from both sides and compare using given operator. - * 2. Else if either side is a {@link Float} then get {@link Number#floatValue()} from both sides and compare using given operator. - * 3. Else if either side is a {@link Long} then get {@link Number#longValue()} from both sides and compare using given operator. - * 4. Otherwise get {@link Number#intValue()} from both sides and compare using given operator. - * 3. If both sides are of the same type and implement the {@link Comparable} interface then use {@code compareTo} method. - * 4. If none of the above are met then a {@link ParseException} is thrown. - * - * @param left The token representing the left side of a comparison expression. - * @param right The token representing the right side of a comparison expression. - * @param op This is a representation of a comparison operator {@literal (eg. <, <=, >, >=, ==, !=) } - * @return A boolean value based on the comparison of {@code left} and {@code right}. - */ - @Override - public boolean evaluate(final Token left, final Token right, final StellarParser.ComparisonOpContext op) { - if (left.getValue() == null || right.getValue() == null) { - return false; - } else if (left.getValue() instanceof Number && right.getValue() instanceof Number) { - return compareNumbers((Number) left.getValue(), (Number) right.getValue(), op); - } else if (left.getValue().getClass() == right.getValue().getClass() - && left.getValue() instanceof Comparable && right.getValue() instanceof Comparable) { - return compare((Comparable) left.getValue(), (Comparable) right.getValue(), op); + /** + * 1. If either the left or right's value is null then return false. + * 2. If both sides of the expression are instances of {@link Number} then: + * 1. If either side is a {@link Double} then get {@link Number#doubleValue()} from both sides and compare using given operator. + * 2. Else if either side is a {@link Float} then get {@link Number#floatValue()} from both sides and compare using given operator. + * 3. Else if either side is a {@link Long} then get {@link Number#longValue()} from both sides and compare using given operator. + * 4. Otherwise get {@link Number#intValue()} from both sides and compare using given operator. + * 3. If both sides are of the same type and implement the {@link Comparable} interface then use {@code compareTo} method. + * 4. If none of the above are met then a {@link ParseException} is thrown. + * + * @param left The token representing the left side of a comparison expression. + * @param right The token representing the right side of a comparison expression. + * @param op This is a representation of a comparison operator {@literal (eg. <, <=, >, >=, ==, !=) } + * @return A boolean value based on the comparison of {@code left} and {@code right}. + */ + @Override + public boolean evaluate(final Token left, final Token right, final StellarParser.ComparisonOpContext op) { + if (left.getValue() == null || right.getValue() == null) { + return false; + } else if (left.getValue() instanceof Number && right.getValue() instanceof Number) { + return compareNumbers((Number) left.getValue(), (Number) right.getValue(), op); + } else if (left.getValue().getClass() == right.getValue().getClass() + && left.getValue() instanceof Comparable && right.getValue() instanceof Comparable) { + return compare((Comparable) left.getValue(), (Comparable) right.getValue(), op); + } + + throw new ParseException( + "Unsupported operations. The following expression is invalid: " + left.getValue() + op + + right.getValue()); } - throw new ParseException("Unsupported operations. The following expression is invalid: " + left.getValue() + op + right.getValue()); - } + /** + * This method uses the inputs' ability to compare with one another's values by using the {@code compareTo} method. It will use this and + * the operator to evaluate the output. + * + * @param l The value of the left side of the expression. + * @param r The value of the right side of the expression. + * @param op The operator to use when comparing. + * @param The type of values being compared. + * @return A boolean value representing the comparison of the two values with the given operator. + * For example, {@code 1 <= 1} would be true. + */ + @SuppressWarnings("unchecked") + private boolean compare(final T l, final T r, final StellarParser.ComparisonOpContext op) { + int compareTo = l.compareTo(r); - /** - * This method uses the inputs' ability to compare with one another's values by using the {@code compareTo} method. It will use this and - * the operator to evaluate the output. - * - * @param l The value of the left side of the expression. - * @param r The value of the right side of the expression. - * @param op The operator to use when comparing. - * @param The type of values being compared. - * @return A boolean value representing the comparison of the two values with the given operator. For example, {@code 1 <= 1} would be true. - */ - @SuppressWarnings("unchecked") - private boolean compare(final T l, final T r, final StellarParser.ComparisonOpContext op) { - int compareTo = l.compareTo(r); + if (op.LT() != null) { + return compareTo < 0; + } else if (op.LTE() != null) { + return compareTo <= 0; + } else if (op.GT() != null) { + return compareTo > 0; + } else if (op.GTE() != null) { + return compareTo >= 0; + } - if (op.LT() != null) { - return compareTo < 0; - } else if (op.LTE() != null) { - return compareTo <= 0; - } else if (op.GT() != null) { - return compareTo > 0; - } else if (op.GTE() != null) { - return compareTo >= 0; + throw new ParseException("Unsupported operator: " + op); } - throw new ParseException("Unsupported operator: " + op); - } + /** + * This method uses the inputs' ability to compare with one another's values by using the {@code compareTo} method. It will use this and + * the operator to evaluate the output. + * + * @param l The left side of the expression. + * @param r The right side of the expression + * @param op The operator used in the expression. + * @return A boolean value representing the comparison of the two values with the given operator. + * For example, {@code 1 <= 1} would be true. + */ + private boolean compareNumbers(final Number l, final Number r, final StellarParser.ComparisonOpContext op) { + if (op.LT() != null) { + return lessThan(l, r); + } else if (op.LTE() != null) { + return lessThanEqual(l, r); + } else if (op.GT() != null) { + return greaterThan(l, r); + } else if (op.GTE() != null) { + return greaterThanEqual(l, r); + } - /** - * This method uses the inputs' ability to compare with one another's values by using the {@code compareTo} method. It will use this and - * the operator to evaluate the output. - * - * @param l The left side of the expression. - * @param r The right side of the expression - * @param op The operator used in the expression. - * @return A boolean value representing the comparison of the two values with the given operator. For example, {@code 1 <= 1} would be true. - */ - private boolean compareNumbers(final Number l, final Number r, final StellarParser.ComparisonOpContext op) { - if (op.LT() != null) { - return lessThan(l, r); - } else if (op.LTE() != null) { - return lessThanEqual(l, r); - } else if (op.GT() != null) { - return greaterThan(l, r); - } else if (op.GTE() != null) { - return greaterThanEqual(l, r); + throw new ParseException("Unsupported operator: " + op); } - throw new ParseException("Unsupported operator: " + op); - } - - /** - * If the left side of the expression is less than the right then true otherwise false. - * - * @param l The value of the left side of the expression. - * @param r The value of the right side of the expression. - * @return If the left side of the expression is less than the right then true otherwise false. - */ - private boolean lessThan(final Number l, final Number r) { - if (l instanceof Double || r instanceof Double) { - return l.doubleValue() < r.doubleValue(); - } else if (l instanceof Float || r instanceof Float) { - return l.floatValue() < r.floatValue(); - } else if (l instanceof Long || r instanceof Long) { - return l.longValue() < r.longValue(); - } else { - return l.intValue() < r.intValue(); + /** + * If the left side of the expression is less than the right then true otherwise false. + * + * @param l The value of the left side of the expression. + * @param r The value of the right side of the expression. + * @return If the left side of the expression is less than the right then true otherwise false. + */ + private boolean lessThan(final Number l, final Number r) { + if (l instanceof Double || r instanceof Double) { + return l.doubleValue() < r.doubleValue(); + } else if (l instanceof Float || r instanceof Float) { + return l.floatValue() < r.floatValue(); + } else if (l instanceof Long || r instanceof Long) { + return l.longValue() < r.longValue(); + } else { + return l.intValue() < r.intValue(); + } } - } - /** - * If the left side of the expression is less than or equal to the right then true otherwise false. - * - * @param l The value of the left side of the expression. - * @param r The value of the right side of the expression. - * @return If the left side of the expression is less than or equal to the right then true otherwise false. - */ - private boolean lessThanEqual(final Number l, final Number r) { - if (l instanceof Double || r instanceof Double) { - return l.doubleValue() <= r.doubleValue(); - } else if (l instanceof Float || r instanceof Float) { - return l.floatValue() <= r.floatValue(); - } else if (l instanceof Long || r instanceof Long) { - return l.longValue() <= r.longValue(); - } else { - return l.intValue() <= r.intValue(); + /** + * If the left side of the expression is less than or equal to the right then true otherwise false. + * + * @param l The value of the left side of the expression. + * @param r The value of the right side of the expression. + * @return If the left side of the expression is less than or equal to the right then true otherwise false. + */ + private boolean lessThanEqual(final Number l, final Number r) { + if (l instanceof Double || r instanceof Double) { + return l.doubleValue() <= r.doubleValue(); + } else if (l instanceof Float || r instanceof Float) { + return l.floatValue() <= r.floatValue(); + } else if (l instanceof Long || r instanceof Long) { + return l.longValue() <= r.longValue(); + } else { + return l.intValue() <= r.intValue(); + } } - } - /** - * If the left side of the expression is greater than the right then true otherwise false. - * - * @param l The value of the left side of the expression. - * @param r The value of the right side of the expression. - * @return If the left side of the expression is greater than the right then true otherwise false. - */ - private boolean greaterThan(final Number l, final Number r) { - if (l instanceof Double || r instanceof Double) { - return l.doubleValue() > r.doubleValue(); - } else if (l instanceof Float || r instanceof Float) { - return l.floatValue() > r.floatValue(); - } else if (l instanceof Long || r instanceof Long) { - return l.longValue() > r.longValue(); - } else { - return l.intValue() > r.intValue(); + /** + * If the left side of the expression is greater than the right then true otherwise false. + * + * @param l The value of the left side of the expression. + * @param r The value of the right side of the expression. + * @return If the left side of the expression is greater than the right then true otherwise false. + */ + private boolean greaterThan(final Number l, final Number r) { + if (l instanceof Double || r instanceof Double) { + return l.doubleValue() > r.doubleValue(); + } else if (l instanceof Float || r instanceof Float) { + return l.floatValue() > r.floatValue(); + } else if (l instanceof Long || r instanceof Long) { + return l.longValue() > r.longValue(); + } else { + return l.intValue() > r.intValue(); + } } - } - /** - * If the left side of the expression is greater than or equal to the right then true otherwise false. - * - * @param l The value of the left side of the expression. - * @param r The value of the right side of the expression. - * @return If the left side of the expression is greater than or equal to the right then true otherwise false. - */ - private boolean greaterThanEqual(final Number l, final Number r) { - if (l instanceof Double || r instanceof Double) { - return l.doubleValue() >= r.doubleValue(); - } else if (l instanceof Float || r instanceof Float) { - return l.floatValue() >= r.floatValue(); - } else if (l instanceof Long || r instanceof Long) { - return l.longValue() >= r.longValue(); - } else { - return l.intValue() >= r.intValue(); + /** + * If the left side of the expression is greater than or equal to the right then true otherwise false. + * + * @param l The value of the left side of the expression. + * @param r The value of the right side of the expression. + * @return If the left side of the expression is greater than or equal to the right then true otherwise false. + */ + private boolean greaterThanEqual(final Number l, final Number r) { + if (l instanceof Double || r instanceof Double) { + return l.doubleValue() >= r.doubleValue(); + } else if (l instanceof Float || r instanceof Float) { + return l.floatValue() >= r.floatValue(); + } else if (l instanceof Long || r instanceof Long) { + return l.longValue() >= r.longValue(); + } else { + return l.intValue() >= r.intValue(); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/DoubleLiteralEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/DoubleLiteralEvaluator.java index 30167aeb..41e123b4 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/DoubleLiteralEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/DoubleLiteralEvaluator.java @@ -18,17 +18,17 @@ package org.apache.metron.stellar.common.evaluators; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.FrameContext; import org.apache.metron.stellar.common.generated.StellarParser; +import org.apache.metron.stellar.dsl.Token; public class DoubleLiteralEvaluator implements NumberEvaluator { - @Override - public Token evaluate(StellarParser.DoubleLiteralContext context, FrameContext.Context contextVariety) { - if (context == null) { - throw new IllegalArgumentException("Cannot evaluate a context that is null."); - } + @Override + public Token evaluate(StellarParser.DoubleLiteralContext context, FrameContext.Context contextVariety) { + if (context == null) { + throw new IllegalArgumentException("Cannot evaluate a context that is null."); + } - return new Token<>(Double.parseDouble(context.getText()), Double.class, contextVariety); - } + return new Token<>(Double.parseDouble(context.getText()), Double.class, contextVariety); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/EqualityOperatorsEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/EqualityOperatorsEvaluator.java index c95eb342..bc9f272f 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/EqualityOperatorsEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/EqualityOperatorsEvaluator.java @@ -18,58 +18,60 @@ package org.apache.metron.stellar.common.evaluators; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.generated.StellarParser; +import org.apache.metron.stellar.dsl.Token; /** * {@link EqualityOperatorsEvaluator} is used to evaluate equality expressions using the following operator '=='. There are - * four major cases when evaluating a equality expression. See {@link EqualityOperatorsEvaluator#evaluate(Token, Token, StellarParser.ComparisonOpContext)} + * four major cases when evaluating a equality expression. + * See {@link EqualityOperatorsEvaluator#evaluate(Token, Token, StellarParser.ComparisonOpContext)} * for a description. */ public class EqualityOperatorsEvaluator implements ComparisonExpressionEvaluator { - /** - * 1. If either side of the expression is null then check equality using Java's '==' expression. - * 2. Else if both sides of the expression are of type {@link Number} then: - * 1. If either side of the expression is a {@link Double} then use {@link Number#doubleValue()} to test equality. - * 2. Else if either side of the expression is a {@link Float} then use {@link Number#floatValue()} to test equality. - * 3. Else if either side of the expression is a {@link Long} then use {@link Number#longValue()} to test equality. - * 4. Otherwise use {@link Number#intValue()} to test equality - * 3. Otherwise use {@code equals} method compare the left side with the right side. - * @param left The token representing the left side of a comparison expression. - * @param right The token representing the right side of a comparison expression. - * @param op This is a representation of a comparison operator {@literal (eg. <, <=, >, >=, ==, !=) } - * @return A boolean value based on the comparison of {@code left} and {@code right}. - */ - @Override - public boolean evaluate(final Token left, final Token right, final StellarParser.ComparisonOpContext op) { - if (left.getValue() == null || right.getValue() == null) { - return left.getValue() == right.getValue(); - } else if (left.getValue() instanceof Number && right.getValue() instanceof Number) { - return eq((Number) left.getValue(), (Number) right.getValue()); - } else { - return left.getValue().equals(right.getValue()); + /** + * 1. If either side of the expression is null then check equality using Java's '==' expression. + * 2. Else if both sides of the expression are of type {@link Number} then: + * 1. If either side of the expression is a {@link Double} then use {@link Number#doubleValue()} to test equality. + * 2. Else if either side of the expression is a {@link Float} then use {@link Number#floatValue()} to test equality. + * 3. Else if either side of the expression is a {@link Long} then use {@link Number#longValue()} to test equality. + * 4. Otherwise use {@link Number#intValue()} to test equality + * 3. Otherwise use {@code equals} method compare the left side with the right side. + * + * @param left The token representing the left side of a comparison expression. + * @param right The token representing the right side of a comparison expression. + * @param op This is a representation of a comparison operator {@literal (eg. <, <=, >, >=, ==, !=) } + * @return A boolean value based on the comparison of {@code left} and {@code right}. + */ + @Override + public boolean evaluate(final Token left, final Token right, final StellarParser.ComparisonOpContext op) { + if (left.getValue() == null || right.getValue() == null) { + return left.getValue() == right.getValue(); + } else if (left.getValue() instanceof Number && right.getValue() instanceof Number) { + return eq((Number) left.getValue(), (Number) right.getValue()); + } else { + return left.getValue().equals(right.getValue()); + } } - } - /** - * This method follows Java's number promotions when comparing numbers. - * - * @param l The left side of the equality expression. - * @param r The right side of the equality expression. - * @return All comparisons use the '==' operator from Java. If either input is a double then compare double values. - * If either side is a float compare float values. If either side is a long compare long values. Otherwise compare - * int values. - */ - private boolean eq(final Number l, final Number r) { - if (l instanceof Double || r instanceof Double) { - return l.doubleValue() == r.doubleValue(); - } else if (l instanceof Float || r instanceof Float) { - return l.floatValue() == r.floatValue(); - } else if (l instanceof Long || r instanceof Long) { - return l.longValue() == r.longValue(); - } else { - return l.intValue() == r.intValue(); + /** + * This method follows Java's number promotions when comparing numbers. + * + * @param l The left side of the equality expression. + * @param r The right side of the equality expression. + * @return All comparisons use the '==' operator from Java. If either input is a double then compare double values. + * If either side is a float compare float values. If either side is a long compare long values. Otherwise compare + * int values. + */ + private boolean eq(final Number l, final Number r) { + if (l instanceof Double || r instanceof Double) { + return l.doubleValue() == r.doubleValue(); + } else if (l instanceof Float || r instanceof Float) { + return l.floatValue() == r.floatValue(); + } else if (l instanceof Long || r instanceof Long) { + return l.longValue() == r.longValue(); + } else { + return l.intValue() == r.intValue(); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/FloatLiteralEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/FloatLiteralEvaluator.java index bfa0bc91..a5ee6b6b 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/FloatLiteralEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/FloatLiteralEvaluator.java @@ -18,17 +18,17 @@ package org.apache.metron.stellar.common.evaluators; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.FrameContext; import org.apache.metron.stellar.common.generated.StellarParser; +import org.apache.metron.stellar.dsl.Token; public class FloatLiteralEvaluator implements NumberEvaluator { - @Override - public Token evaluate(StellarParser.FloatLiteralContext context, FrameContext.Context contextVariety) { - if (context == null) { - throw new IllegalArgumentException("Cannot evaluate a context that is null."); - } + @Override + public Token evaluate(StellarParser.FloatLiteralContext context, FrameContext.Context contextVariety) { + if (context == null) { + throw new IllegalArgumentException("Cannot evaluate a context that is null."); + } - return new Token<>(Float.parseFloat(context.getText()), Float.class, contextVariety); - } + return new Token<>(Float.parseFloat(context.getText()), Float.class, contextVariety); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/IntLiteralEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/IntLiteralEvaluator.java index ad557009..681353d6 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/IntLiteralEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/IntLiteralEvaluator.java @@ -18,17 +18,17 @@ package org.apache.metron.stellar.common.evaluators; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.FrameContext; import org.apache.metron.stellar.common.generated.StellarParser; +import org.apache.metron.stellar.dsl.Token; public class IntLiteralEvaluator implements NumberEvaluator { - @Override - public Token evaluate(StellarParser.IntLiteralContext context, FrameContext.Context contextVariety) { - if (context == null) { - throw new IllegalArgumentException("Cannot evaluate a context that is null."); - } + @Override + public Token evaluate(StellarParser.IntLiteralContext context, FrameContext.Context contextVariety) { + if (context == null) { + throw new IllegalArgumentException("Cannot evaluate a context that is null."); + } - return new Token<>(Integer.parseInt(context.getText()), Integer.class, contextVariety); - } + return new Token<>(Integer.parseInt(context.getText()), Integer.class, contextVariety); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/LongLiteralEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/LongLiteralEvaluator.java index f5461adc..34886afe 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/LongLiteralEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/LongLiteralEvaluator.java @@ -18,25 +18,27 @@ package org.apache.metron.stellar.common.evaluators; -import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.FrameContext; import org.apache.metron.stellar.common.generated.StellarParser; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Token; public class LongLiteralEvaluator implements NumberEvaluator { - @Override - public Token evaluate(StellarParser.LongLiteralContext context, FrameContext.Context contextVariety) { - if (context == null) { - throw new IllegalArgumentException("Cannot evaluate a context that is null."); - } + @Override + public Token evaluate(StellarParser.LongLiteralContext context, FrameContext.Context contextVariety) { + if (context == null) { + throw new IllegalArgumentException("Cannot evaluate a context that is null."); + } - String value = context.getText(); - if (value.endsWith("l") || value.endsWith("L")) { - value = value.substring(0, value.length() - 1); // Drop the 'L' or 'l'. Long.parseLong does not accept a string with either of these. - return new Token<>(Long.parseLong(value), Long.class, contextVariety); - } else { - // Technically this should never happen, but just being safe. - throw new ParseException("Invalid format for long. Failed trying to parse a long with the following value: " + value); + String value = context.getText(); + if (value.endsWith("l") || value.endsWith("L")) { + value = value.substring(0, value.length() + - 1); // Drop the 'L' or 'l'. Long.parseLong does not accept a string with either of these. + return new Token<>(Long.parseLong(value), Long.class, contextVariety); + } else { + // Technically this should never happen, but just being safe. + throw new ParseException( + "Invalid format for long. Failed trying to parse a long with the following value: " + value); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/NumberEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/NumberEvaluator.java index 83aed622..f161e2a6 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/NumberEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/NumberEvaluator.java @@ -18,10 +18,10 @@ package org.apache.metron.stellar.common.evaluators; -import org.apache.metron.stellar.dsl.Token; import org.apache.metron.stellar.common.FrameContext; import org.apache.metron.stellar.common.generated.StellarParser; +import org.apache.metron.stellar.dsl.Token; public interface NumberEvaluator { - Token evaluate(T context, FrameContext.Context contextVariety); + Token evaluate(T context, FrameContext.Context contextVariety); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/NumberLiteralEvaluator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/NumberLiteralEvaluator.java index 2c5d5403..3b25d5d2 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/NumberLiteralEvaluator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/evaluators/NumberLiteralEvaluator.java @@ -18,54 +18,55 @@ package org.apache.metron.stellar.common.evaluators; +import java.util.HashMap; +import java.util.Map; import org.apache.metron.stellar.common.FrameContext; import org.apache.metron.stellar.common.generated.StellarParser; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Token; -import java.util.HashMap; -import java.util.Map; - public enum NumberLiteralEvaluator { - INSTANCE; - public enum Strategy { - INTEGER(StellarParser.IntLiteralContext.class, new IntLiteralEvaluator()) - , DOUBLE(StellarParser.DoubleLiteralContext.class, new DoubleLiteralEvaluator()) - , FLOAT(StellarParser.FloatLiteralContext.class, new FloatLiteralEvaluator()) - , LONG(StellarParser.LongLiteralContext.class, new LongLiteralEvaluator()); - Class context; - NumberEvaluator evaluator; - private static Map, NumberEvaluator> strategyMap; + INSTANCE; + + public enum Strategy { + INTEGER(StellarParser.IntLiteralContext.class, new IntLiteralEvaluator()), + DOUBLE(StellarParser.DoubleLiteralContext.class, new DoubleLiteralEvaluator()), + FLOAT(StellarParser.FloatLiteralContext.class, new FloatLiteralEvaluator()), + LONG(StellarParser.LongLiteralContext.class, new LongLiteralEvaluator()); + Class context; + NumberEvaluator evaluator; + private static final Map, NumberEvaluator> strategyMap; - static { - strategyMap = new HashMap<>(); - for (Strategy strat : Strategy.values()) { - strategyMap.put(strat.context, strat.evaluator); - } + static { + strategyMap = new HashMap<>(); + for (Strategy strat : Strategy.values()) { + strategyMap.put(strat.context, strat.evaluator); + } + } + + Strategy(Class context, + NumberEvaluator evaluator + ) { + this.context = context; + this.evaluator = evaluator; + } } - Strategy(Class context - , NumberEvaluator evaluator + @SuppressWarnings("unchecked") + Token evaluate(StellarParser.Arithmetic_operandsContext context, + Map, NumberEvaluator> instanceMap, + FrameContext.Context contextVariety ) { - this.context = context; - this.evaluator = evaluator; + NumberEvaluator evaluator = instanceMap.get(context.getClass()); + if (evaluator == null) { + throw new ParseException("Does not support evaluation for type " + context.getClass()); + } + return evaluator.evaluate(context, contextVariety); } - } - @SuppressWarnings("unchecked") - Token evaluate(StellarParser.Arithmetic_operandsContext context, - Map, NumberEvaluator> instanceMap, - FrameContext.Context contextVariety - ) { - NumberEvaluator evaluator = instanceMap.get(context.getClass()); - if (evaluator == null) { - throw new ParseException("Does not support evaluation for type " + context.getClass()); + public Token evaluate(StellarParser.Arithmetic_operandsContext context, + FrameContext.Context contextVariety) { + return evaluate(context, Strategy.strategyMap, contextVariety); } - return evaluator.evaluate(context, contextVariety); - } - - public Token evaluate(StellarParser.Arithmetic_operandsContext context, FrameContext.Context contextVariety) { - return evaluate(context, Strategy.strategyMap, contextVariety); - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java index cf52955d..9636c49d 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java @@ -17,20 +17,20 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; -import org.apache.commons.collections4.IterableUtils; -import org.apache.commons.collections4.trie.PatriciaTrie; -import org.apache.commons.lang.StringUtils; - import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.SortedMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.collections4.IterableUtils; +import org.apache.commons.collections4.trie.PatriciaTrie; +import org.apache.commons.lang.StringUtils; /** @@ -38,183 +38,191 @@ */ public class DefaultStellarAutoCompleter implements StellarAutoCompleter { - enum OperationType { - DOC, - MAGIC, - NORMAL - } + enum OperationType { + DOC, + MAGIC, + NORMAL + } + + enum AutoCompleteType implements AutoCompleteTransformation { + FUNCTION((type, key) -> { + if (OperationType.DOC == type) { + return "?" + key; + + } else if (OperationType.NORMAL == type) { + return key + "("; + } + + return key; + }), + VARIABLE((type, key) -> key), + TOKEN((type, key) -> key); - enum AutoCompleteType implements AutoCompleteTransformation { - FUNCTION((type, key) -> { - if(OperationType.DOC == type) { - return "?" + key; + AutoCompleteTransformation transform; - } else if(OperationType.NORMAL == type) { - return key + "("; - } + AutoCompleteType(AutoCompleteTransformation transform) { + this.transform = transform; + } - return key; - }), - VARIABLE((type, key) -> key), - TOKEN((type, key) -> key); + @Override + public String transform(OperationType type, String key) { + return transform.transform(type, key); + } + } + + /** + * Prefix tree index of auto-completes. + */ + private final PatriciaTrie autocompleteIndex; + + private final ReadWriteLock indexLock = new ReentrantReadWriteLock(); + + public interface AutoCompleteTransformation { + String transform(OperationType type, String key); + } + + public DefaultStellarAutoCompleter() { + this.autocompleteIndex = initializeIndex(); + } + + /** + * Is a given expression a built-in magic?. + * + * @param expression The expression. + */ + private boolean isMagic(String expression) { + return StringUtils.startsWith(expression, "%"); + } - AutoCompleteTransformation transform; + /** + * Is a given expression asking for function documentation?. + * + * @param expression The expression. + */ + private boolean isDoc(String expression) { + return StringUtils.startsWith(expression, "?"); + } + + /** + * Auto-completes a partial Stellar expression. + * + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable autoCompleteNormal(String buffer) { + return autoComplete(buffer, OperationType.NORMAL); + } - AutoCompleteType(AutoCompleteTransformation transform) { - this.transform = transform; + /** + * Auto-completes a partial doc command. + * + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable autoCompleteDoc(String buffer) { + return autoComplete(buffer, OperationType.DOC); } + /** + * Auto-completes a partial magic commands. + * + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable autoCompleteMagic(String buffer) { + return autoComplete(buffer, OperationType.MAGIC); + } + + @Override + public Iterable autoComplete(String buffer) { + Iterable candidates = IterableUtils.emptyIterable(); + + final String lastToken = getLastToken(buffer); + if (StringUtils.isNotEmpty(lastToken)) { + + if (isDoc(lastToken)) { + candidates = autoCompleteDoc(lastToken.substring(1)); + + } else if (isMagic(lastToken)) { + candidates = autoCompleteMagic(lastToken); + + } else { + candidates = autoCompleteNormal(lastToken); + } + } + + return candidates; + } + + /** + * Returns a list of viable candidates for auto-completion. + * + * @param buffer The current buffer. + * @param opType The type of operation needing auto-completion. + * @return Viable candidates for auto-completion. + */ + private Iterable autoComplete(String buffer, final OperationType opType) { + indexLock.readLock().lock(); + try { + SortedMap ret = autocompleteIndex.prefixMap(buffer); + if (ret.isEmpty()) { + return new ArrayList<>(); + } + return Iterables.transform(ret.entrySet(), kv -> kv.getValue().transform(opType, kv.getKey())); + } finally { + indexLock.readLock().unlock(); + } + } + + /** + * Adds a candidate for auto-completing function names. + * + * @param name The name of the function candidate. + */ @Override - public String transform(OperationType type, String key) { - return transform.transform(type, key); - } - } - - /** - * Prefix tree index of auto-completes. - */ - private PatriciaTrie autocompleteIndex; - - private ReadWriteLock indexLock = new ReentrantReadWriteLock(); - - public interface AutoCompleteTransformation { - String transform(OperationType type, String key); - } - - public DefaultStellarAutoCompleter() { - this.autocompleteIndex = initializeIndex(); - } - - @Override - public Iterable autoComplete(String buffer) { - Iterable candidates = IterableUtils.emptyIterable(); - - final String lastToken = getLastToken(buffer); - if(StringUtils.isNotEmpty(lastToken)) { - - if (isDoc(lastToken)) { - candidates = autoCompleteDoc(lastToken.substring(1)); - - } else if (isMagic(lastToken)) { - candidates = autoCompleteMagic(lastToken); - - } else { - candidates = autoCompleteNormal(lastToken); - } - } - - return candidates; - } - - /** - * Is a given expression a built-in magic? - * @param expression The expression. - */ - private boolean isMagic(String expression) { - return StringUtils.startsWith(expression, "%"); - } - - /** - * Is a given expression asking for function documentation? - * @param expression The expression. - */ - private boolean isDoc(String expression) { - return StringUtils.startsWith(expression, "?"); - } - - /** - * Auto-completes a partial Stellar expression - * @param buffer The partial buffer that needs auto-completed. - * @return Viable candidates for auto-completion. - */ - private Iterable autoCompleteNormal(String buffer) { - return autoComplete(buffer, OperationType.NORMAL); - } - - /** - * Auto-completes a partial doc command. - * @param buffer The partial buffer that needs auto-completed. - * @return Viable candidates for auto-completion. - */ - private Iterable autoCompleteDoc(String buffer) { - return autoComplete(buffer, OperationType.DOC); - } - - /** - * Auto-completes a partial magic commands. - * @param buffer The partial buffer that needs auto-completed. - * @return Viable candidates for auto-completion. - */ - private Iterable autoCompleteMagic(String buffer) { - return autoComplete(buffer, OperationType.MAGIC); - } - - /** - * Returns a list of viable candidates for auto-completion. - * @param buffer The current buffer. - * @param opType The type of operation needing auto-completion. - * @return Viable candidates for auto-completion. - */ - private Iterable autoComplete(String buffer, final OperationType opType) { - indexLock.readLock().lock(); - try { - SortedMap ret = autocompleteIndex.prefixMap(buffer); - if (ret.isEmpty()) { - return new ArrayList<>(); - } - return Iterables.transform(ret.entrySet(), kv -> kv.getValue().transform(opType, kv.getKey())); - } - finally { - indexLock.readLock().unlock(); - } - } - - /** - * Adds a candidate for auto-completing function names. - * @param name The name of the function candidate. - */ - @Override - public void addCandidateFunction(String name) { - add(name, AutoCompleteType.FUNCTION); - } - - /** - * Adds a candidate for auto-completing variable names. - * @param name The name of the function candidate. - */ - @Override - public void addCandidateVariable(String name) { - add(name, AutoCompleteType.VARIABLE); - } - - /** - * Add a candidate for auto-completion. - * @param name The name of the candidate. - * @param type The type of candidate. - */ - private void add(String name, AutoCompleteType type) { - if(StringUtils.isNotBlank(name)) { - // add the candidate to the auto-complete index - indexLock.writeLock().lock(); - try { - this.autocompleteIndex.put(name, type); - } finally { - indexLock.writeLock().unlock(); - } - } - } - - private PatriciaTrie initializeIndex() { - Map index = new HashMap<>(); - index.put("==", AutoCompleteType.TOKEN); - index.put(">=", AutoCompleteType.TOKEN); - index.put("<=", AutoCompleteType.TOKEN); - - return new PatriciaTrie<>(index); - } - - private static String getLastToken(String buffer) { - String lastToken = Iterables.getLast(Splitter.on(" ").split(buffer), null); - return lastToken.trim(); - } + public void addCandidateFunction(String name) { + add(name, AutoCompleteType.FUNCTION); + } + + /** + * Adds a candidate for auto-completing variable names. + * + * @param name The name of the function candidate. + */ + @Override + public void addCandidateVariable(String name) { + add(name, AutoCompleteType.VARIABLE); + } + + /** + * Add a candidate for auto-completion. + * + * @param name The name of the candidate. + * @param type The type of candidate. + */ + private void add(String name, AutoCompleteType type) { + if (StringUtils.isNotBlank(name)) { + // add the candidate to the auto-complete index + indexLock.writeLock().lock(); + try { + this.autocompleteIndex.put(name, type); + } finally { + indexLock.writeLock().unlock(); + } + } + } + + private PatriciaTrie initializeIndex() { + Map index = new HashMap<>(); + index.put("==", AutoCompleteType.TOKEN); + index.put(">=", AutoCompleteType.TOKEN); + index.put("<=", AutoCompleteType.TOKEN); + + return new PatriciaTrie<>(index); + } + + private static String getLastToken(String buffer) { + String lastToken = Iterables.getLast(Splitter.on(" ").split(buffer), null); + return lastToken.trim(); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java index 99c62560..8fdacc28 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java @@ -17,9 +17,27 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell; +import static org.apache.metron.stellar.common.configuration.ConfigurationsUtils.readGlobalConfigBytesFromZookeeper; +import static org.apache.metron.stellar.common.shell.StellarResult.error; +import static org.apache.metron.stellar.common.shell.StellarResult.noop; +import static org.apache.metron.stellar.common.shell.StellarResult.success; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.STELLAR_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.ZOOKEEPER_CLIENT; + import com.google.common.collect.Maps; +import java.io.ByteArrayInputStream; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; import org.apache.commons.collections.map.UnmodifiableMap; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; @@ -48,369 +66,363 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; - -import static org.apache.metron.stellar.common.configuration.ConfigurationsUtils.readGlobalConfigBytesFromZookeeper; -import static org.apache.metron.stellar.common.shell.StellarResult.noop; -import static org.apache.metron.stellar.common.shell.StellarResult.error; -import static org.apache.metron.stellar.common.shell.StellarResult.success; -import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; -import static org.apache.metron.stellar.dsl.Context.Capabilities.STELLAR_CONFIG; -import static org.apache.metron.stellar.dsl.Context.Capabilities.ZOOKEEPER_CLIENT; - /** * Default implementation of a StellarShellExecutor. */ public class DefaultStellarShellExecutor implements StellarShellExecutor { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String SHELL_VARIABLES = "shellVariables"; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String SHELL_VARIABLES = "shellVariables"; - /** - * The variables known by Stellar. - */ - private Map variables; + /** + * The variables known by Stellar. + */ + private final Map variables; - /** - * The function resolver. - */ - private FunctionResolver functionResolver; + /** + * The function resolver. + */ + private final FunctionResolver functionResolver; - /** - * A Zookeeper client. Only defined if given a valid Zookeeper URL. - */ - private Optional zkClient; + /** + * A Zookeeper client. Only defined if given a valid Zookeeper URL. + */ + private final Optional zkClient; - /** - * A registry of all special commands; like %magic, ?doc, and quit. - */ - private List specials; + /** + * A registry of all special commands; like %magic, ?doc, and quit. + */ + private final List specials; - /** - * The Stellar execution context. - */ - private Context context; + /** + * The Stellar execution context. + */ + private final Context context; - /** - * Listeners that are notified when a function is defined. - */ - private List functionListeners; + /** + * Listeners that are notified when a function is defined. + */ + private final List functionListeners; - /** - * Listeners that are notified when a variable is defined. - */ - private List variableListeners; + /** + * Listeners that are notified when a variable is defined. + */ + private final List variableListeners; - /** - * Listeners that are notified when a special command is defined. - */ - private List specialListeners; + /** + * Listeners that are notified when a special command is defined. + */ + private final List specialListeners; - public DefaultStellarShellExecutor( + public DefaultStellarShellExecutor( FunctionResolver functionResolver, Properties properties, Optional zookeeperUrl, List specials) throws Exception { - this.functionListeners = new ArrayList<>(); - this.variableListeners = new ArrayList<>(); - this.specialListeners = new ArrayList<>(); - this.variables = new HashMap<>(); - this.zkClient = createZookeeperClient(zookeeperUrl); - this.context = createContext(properties, this.zkClient); - this.functionResolver = functionResolver; - this.specials = specials; - } - - public DefaultStellarShellExecutor( + this.functionListeners = new ArrayList<>(); + this.variableListeners = new ArrayList<>(); + this.specialListeners = new ArrayList<>(); + this.variables = new HashMap<>(); + this.zkClient = createZookeeperClient(zookeeperUrl); + this.context = createContext(properties, this.zkClient); + this.functionResolver = functionResolver; + this.specials = specials; + } + + public DefaultStellarShellExecutor( FunctionResolver functionResolver, Properties properties, Optional zookeeperUrl) throws Exception { - this(functionResolver, properties, zookeeperUrl, defaultSpecials()); - } + this(functionResolver, properties, zookeeperUrl, defaultSpecials()); + } - public DefaultStellarShellExecutor( + public DefaultStellarShellExecutor( Properties properties, Optional zookeeperUrl) throws Exception { - this(StellarFunctions.FUNCTION_RESOLVER(), properties, zookeeperUrl); - } - - /** - * The default specials that will be made available, if none are specified otherwise. - * @return The default special commands. - */ - public static List defaultSpecials() { - return Arrays.asList( - new AssignmentCommand(), - new DocCommand(), - new QuitCommand(), - new Comment(), - new MagicListFunctions(), - new MagicListVariables(), - new MagicDefineGlobal(), - new MagicUndefineGlobal(), - new MagicListGlobals() - ); - } - - @Override - public void init() { - StellarFunctions.initialize(this.context); - - // notify listeners about the available specials - for(SpecialCommand command : specials) { - notifySpecialListeners(command); + this(StellarFunctions.FUNCTION_RESOLVER(), properties, zookeeperUrl); } - // notify listeners about the available functions - for(StellarFunctionInfo fn : functionResolver.getFunctionInfo()) { - notifyFunctionListeners(fn); + /** + * The default specials that will be made available, if none are specified otherwise. + * + * @return The default special commands. + */ + public static List defaultSpecials() { + return Arrays.asList( + new AssignmentCommand(), + new DocCommand(), + new QuitCommand(), + new Comment(), + new MagicListFunctions(), + new MagicListVariables(), + new MagicDefineGlobal(), + new MagicUndefineGlobal(), + new MagicListGlobals() + ); } - } - - /** - * Add a listener that will be notified when a function is defined. - * @param listener The listener to notify. - */ - @Override - public void addFunctionListener(FunctionDefinedListener listener) { - this.functionListeners.add(listener); - } - - /** - * Notify function listeners that a function has been defined. - * @param functionInfo The function that was defined. - */ - private void notifyFunctionListeners(StellarFunctionInfo functionInfo) { - for(FunctionDefinedListener listener : functionListeners) { - listener.whenFunctionDefined(functionInfo); + + @Override + public void init() { + StellarFunctions.initialize(this.context); + + // notify listeners about the available specials + for (SpecialCommand command : specials) { + notifySpecialListeners(command); + } + + // notify listeners about the available functions + for (StellarFunctionInfo fn : functionResolver.getFunctionInfo()) { + notifyFunctionListeners(fn); + } + } + + /** + * Add a listener that will be notified when a function is defined. + * + * @param listener The listener to notify. + */ + @Override + public void addFunctionListener(FunctionDefinedListener listener) { + this.functionListeners.add(listener); + } + + /** + * Notify function listeners that a function has been defined. + * + * @param functionInfo The function that was defined. + */ + private void notifyFunctionListeners(StellarFunctionInfo functionInfo) { + for (FunctionDefinedListener listener : functionListeners) { + listener.whenFunctionDefined(functionInfo); + } } - } - - /** - * Add a listener that will be notified when a variable is defined. - * @param listener The listener to notify. - */ - @Override - public void addVariableListener(VariableDefinedListener listener) { - this.variableListeners.add(listener); - } - - /** - * Notify variable listeners that a variable has been (re)defined. - * @param variableName The variable name. - * @param result The variable result. - */ - private void notifyVariableListeners(String variableName, VariableResult result) { - for(VariableDefinedListener listener : variableListeners) { - listener.whenVariableDefined(variableName, result); + + /** + * Add a listener that will be notified when a variable is defined. + * + * @param listener The listener to notify. + */ + @Override + public void addVariableListener(VariableDefinedListener listener) { + this.variableListeners.add(listener); } - } - - /** - * Add a listener that will be notified when a magic command is defined. - * @param listener The listener to notify. - */ - @Override - public void addSpecialListener(SpecialDefinedListener listener) { - this.specialListeners.add(listener); - } - - /** - * Notify listeners that a magic command has been defined. - * @param specialCommand The magic command. - */ - private void notifySpecialListeners(SpecialCommand specialCommand) { - for(SpecialDefinedListener listener : specialListeners) { - listener.whenSpecialDefined(specialCommand); + + /** + * Notify variable listeners that a variable has been (re)defined. + * + * @param variableName The variable name. + * @param result The variable result. + */ + private void notifyVariableListeners(String variableName, VariableResult result) { + for (VariableDefinedListener listener : variableListeners) { + listener.whenVariableDefined(variableName, result); + } } - } - @Override - public StellarResult execute(String expression) { + /** + * Add a listener that will be notified when a magic command is defined. + * + * @param listener The listener to notify. + */ + @Override + public void addSpecialListener(SpecialDefinedListener listener) { + this.specialListeners.add(listener); + } - // if only whitespace, there is nothing to do - expression = StringUtils.trimToEmpty(expression); - if(StringUtils.isBlank(expression)) { - return noop(); + /** + * Notify listeners that a magic command has been defined. + * + * @param specialCommand The magic command. + */ + private void notifySpecialListeners(SpecialCommand specialCommand) { + for (SpecialDefinedListener listener : specialListeners) { + listener.whenSpecialDefined(specialCommand); + } } - // is this a special command? - for(SpecialCommand command : specials) { - if(command.getMatcher().apply(expression)) { - return command.execute(expression, this); - } + @Override + public StellarResult execute(String expression) { + + // if only whitespace, there is nothing to do + expression = StringUtils.trimToEmpty(expression); + if (StringUtils.isBlank(expression)) { + return noop(); + } + + // is this a special command? + for (SpecialCommand command : specials) { + if (command.getMatcher().apply(expression)) { + return command.execute(expression, this); + } + } + + // otherwise, this must be a stellar expression + return executeStellar(expression); } - // otherwise, this must be a stellar expression - return executeStellar(expression); - } - - /** - * Retrieves the GLOBAL_CONFIG, if it exists. If it does not, it creates the GLOBAL_CONFIG - * and adds it to the Stellar execution context. - * - * @return The global configuration. - */ - @Override - @SuppressWarnings("unchecked") - public Map getGlobalConfig() { - Map globals; - Optional capability = getContext().getCapability(GLOBAL_CONFIG, false); - if (capability.isPresent()) { - globals = (Map) capability.get(); - - } else { - throw new IllegalStateException("'GLOBAL_CONFIG' is missing"); + /** + * Retrieves the GLOBAL_CONFIG, if it exists. If it does not, it creates the GLOBAL_CONFIG + * and adds it to the Stellar execution context. + * + * @return The global configuration. + */ + @Override + @SuppressWarnings("unchecked") + public Map getGlobalConfig() { + Map globals; + Optional capability = getContext().getCapability(GLOBAL_CONFIG, false); + if (capability.isPresent()) { + globals = (Map) capability.get(); + + } else { + throw new IllegalStateException("'GLOBAL_CONFIG' is missing"); + } + + return globals; } - return globals; - } - - @Override - public void assign(String variableName, Object value, Optional expression) { - - // perform the variable assignment - VariableResult varResult = VariableResult.withExpression(value, expression); - this.variables.put(variableName, varResult); - - // notify any listeners - notifyVariableListeners(variableName, varResult); - } - - @Override - public FunctionResolver getFunctionResolver() { - return functionResolver; - } - - @Override - @SuppressWarnings("unchecked") - public Map getState() { - return UnmodifiableMap.decorate(variables); - } - - /** - * Returns all variables that have been defined. Unlike 'getState' this unwraps - * the VariableResult so that we have the actual value. - * - * @return All variables that have been defined. - */ - public Map getVariables() { - return Maps.transformValues(variables, (v) -> v.getResult()); - } - - @Override - public Context getContext() { - return context; - } - - /** - * Creates a Zookeeper client. - * @param zookeeperUrl The Zookeeper URL. - */ - private Optional createZookeeperClient(Optional zookeeperUrl) { - Optional client = Optional.empty(); - - // can only create client, if have valid zookeeper URL - if(zookeeperUrl.isPresent()) { - String url = zookeeperUrl.get(); - if (StringUtils.isNotBlank(url)) { - - LOG.debug(String.format("Connecting to Zookeeper; url=%s", url)); - CuratorFramework c = ConfigurationsUtils.getClient(url); - c.start(); - client = Optional.of(c); - } + @Override + public void assign(String variableName, Object value, Optional expression) { + + // perform the variable assignment + VariableResult varResult = VariableResult.withExpression(value, expression); + this.variables.put(variableName, varResult); + + // notify any listeners + notifyVariableListeners(variableName, varResult); } - return client; - } + @Override + public FunctionResolver getFunctionResolver() { + return functionResolver; + } - /** - * Creates a Context initialized with configuration stored in Zookeeper. - * @param properties Properties to configure the context. - * @param zkClient An optional Zookeeper client. - */ - private Context createContext(Properties properties, Optional zkClient) throws Exception { + @Override + @SuppressWarnings("unchecked") + public Map getState() { + return UnmodifiableMap.decorate(variables); + } - Context.Builder contextBuilder = new Context.Builder(); - Map globals; - if (zkClient.isPresent()) { - LOG.debug("Zookeeper client present; fetching globals from Zookeeper."); + /** + * Returns all variables that have been defined. Unlike 'getState' this unwraps + * the VariableResult so that we have the actual value. + * + * @return All variables that have been defined. + */ + public Map getVariables() { + return Maps.transformValues(variables, (v) -> v.getResult()); + } - // fetch globals from zookeeper - globals = fetchGlobalConfig(zkClient.get()); - contextBuilder.with(ZOOKEEPER_CLIENT, () -> zkClient.get()); + @Override + public Context getContext() { + return context; + } - } else { - LOG.debug("No Zookeeper client; initializing empty globals."); + /** + * Creates a Zookeeper client. + * + * @param zookeeperUrl The Zookeeper URL. + */ + private Optional createZookeeperClient(Optional zookeeperUrl) { + Optional client = Optional.empty(); + + // can only create client, if have valid zookeeper URL + if (zookeeperUrl.isPresent()) { + String url = zookeeperUrl.get(); + if (StringUtils.isNotBlank(url)) { + + LOG.debug(String.format("Connecting to Zookeeper; url=%s", url)); + CuratorFramework c = ConfigurationsUtils.getClient(url); + c.start(); + client = Optional.of(c); + } + } + + return client; + } - // use empty globals to allow a user to '%define' their own - globals = new HashMap<>(); + /** + * Creates a Context initialized with configuration stored in Zookeeper. + * + * @param properties Properties to configure the context. + * @param zkClient An optional Zookeeper client. + */ + private Context createContext(Properties properties, Optional zkClient) throws Exception { + + Context.Builder contextBuilder = new Context.Builder(); + Map globals; + if (zkClient.isPresent()) { + LOG.debug("Zookeeper client present; fetching globals from Zookeeper."); + + // fetch globals from zookeeper + globals = fetchGlobalConfig(zkClient.get()); + contextBuilder.with(ZOOKEEPER_CLIENT, () -> zkClient.get()); + + } else { + LOG.debug("No Zookeeper client; initializing empty globals."); + + // use empty globals to allow a user to '%define' their own + globals = new HashMap<>(); + } + + return contextBuilder + .with(SHELL_VARIABLES, () -> variables) + .with(GLOBAL_CONFIG, () -> globals) + .with(STELLAR_CONFIG, () -> getStellarConfig(globals, properties)) + .build(); } - return contextBuilder - .with(SHELL_VARIABLES, () -> variables) - .with(GLOBAL_CONFIG, () -> globals) - .with(STELLAR_CONFIG, () -> getStellarConfig(globals, properties)) - .build(); - } - - /** - * Fetches the global configuration from Zookeeper. - * @param zkClient The Zookeeper client. - * @return The global configuration retrieved from Zookeeper. - * @throws Exception - */ - private Map fetchGlobalConfig(CuratorFramework zkClient) throws Exception { - byte[] raw = readGlobalConfigBytesFromZookeeper(zkClient); - return JSONUtils.INSTANCE.load( new ByteArrayInputStream(raw), JSONUtils.MAP_SUPPLIER); - } - - /** - * @param globalConfig The global configuration. - * @param props Property values - * @return The Stellar configuration. - */ - private Map getStellarConfig(Map globalConfig, Properties props) { - Map stellarConfig = new HashMap<>(); - stellarConfig.putAll(globalConfig); - if(props != null) { - for (Map.Entry kv : props.entrySet()) { - stellarConfig.put(kv.getKey().toString(), kv.getValue()); - } + /** + * Fetches the global configuration from Zookeeper. + * + * @param zkClient The Zookeeper client. + * @return The global configuration retrieved from Zookeeper. + */ + private Map fetchGlobalConfig(CuratorFramework zkClient) throws Exception { + byte[] raw = readGlobalConfigBytesFromZookeeper(zkClient); + return JSONUtils.INSTANCE.load(new ByteArrayInputStream(raw), JSONUtils.MAP_SUPPLIER); } - return stellarConfig; - } - - /** - * Executes Stellar expressions. - * @param expression The expression to execute. - */ - private StellarResult executeStellar(String expression) { - StellarResult result; - - try { - // execute the stellar expression - VariableResolver variableResolver = new MapVariableResolver(getVariables()); - Object exprResult = new StellarProcessor().parse(expression, variableResolver, functionResolver, context); - result = success(exprResult); - - } catch (Throwable t) { - result = error(t); + + /** + * Gets Stellar config from global config. + * + * @param globalConfig The global configuration. + * @param props Property values + * @return The Stellar configuration. + */ + private Map getStellarConfig(Map globalConfig, Properties props) { + Map stellarConfig = new HashMap<>(); + stellarConfig.putAll(globalConfig); + if (props != null) { + for (Map.Entry kv : props.entrySet()) { + stellarConfig.put(kv.getKey().toString(), kv.getValue()); + } + } + return stellarConfig; } - return result; - } + /** + * Executes Stellar expressions. + * + * @param expression The expression to execute. + */ + private StellarResult executeStellar(String expression) { + StellarResult result; + + try { + // execute the stellar expression + VariableResolver variableResolver = new MapVariableResolver(getVariables()); + Object exprResult = new StellarProcessor().parse(expression, variableResolver, functionResolver, context); + result = success(exprResult); + + } catch (Throwable t) { + result = error(t); + } + + return result; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java index 35ab5ffb..81c8154a 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java @@ -17,6 +17,7 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell; /** @@ -24,22 +25,25 @@ */ public interface StellarAutoCompleter { - /** - * Auto-completes based on the given buffer. - * @param buffer The partial buffer that needs auto-completed. - * @return Viable candidates for auto-completion. - */ - Iterable autoComplete(String buffer); + /** + * Auto-completes based on the given buffer. + * + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + Iterable autoComplete(String buffer); - /** - * Adds a candidate for auto-completing function names. - * @param name The name of the function candidate. - */ - void addCandidateFunction(String name); + /** + * Adds a candidate for auto-completing function names. + * + * @param name The name of the function candidate. + */ + void addCandidateFunction(String name); - /** - * Adds a candidate for auto-completing variable names. - * @param name The name of the function candidate. - */ - void addCandidateVariable(String name); + /** + * Adds a candidate for auto-completing variable names. + * + * @param name The name of the function candidate. + */ + void addCandidateVariable(String name); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java index 9734241b..11353d0b 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java @@ -17,6 +17,7 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell; import org.apache.metron.stellar.common.shell.specials.SpecialCommand; @@ -28,24 +29,24 @@ */ public class StellarExecutionListeners { - /** - * A listener that is notified when a function is defined. - */ - public interface FunctionDefinedListener { - void whenFunctionDefined(StellarFunctionInfo functionInfo); - } + /** + * A listener that is notified when a function is defined. + */ + public interface FunctionDefinedListener { + void whenFunctionDefined(StellarFunctionInfo functionInfo); + } - /** - * A listener that is notified when a variable is defined or redefined. - */ - public interface VariableDefinedListener { - void whenVariableDefined(String variableName, VariableResult result); - } + /** + * A listener that is notified when a variable is defined or redefined. + */ + public interface VariableDefinedListener { + void whenVariableDefined(String variableName, VariableResult result); + } - /** - * A listener that is notified when a special command is defined. - */ - public interface SpecialDefinedListener { - void whenSpecialDefined(SpecialCommand magic); - } + /** + * A listener that is notified when a special command is defined. + */ + public interface SpecialDefinedListener { + void whenSpecialDefined(SpecialCommand magic); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java index f4ebf2ef..ffc1a803 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java @@ -17,6 +17,7 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell; /** @@ -24,21 +25,24 @@ */ public interface StellarExecutionNotifier { - /** - * Add a listener that will be notified when a magic command is defined. - * @param listener The listener to notify. - */ - void addSpecialListener(StellarExecutionListeners.SpecialDefinedListener listener); + /** + * Add a listener that will be notified when a magic command is defined. + * + * @param listener The listener to notify. + */ + void addSpecialListener(StellarExecutionListeners.SpecialDefinedListener listener); - /** - * Add a listener that will be notified when a function is defined. - * @param listener The listener to notify. - */ - void addFunctionListener(StellarExecutionListeners.FunctionDefinedListener listener); + /** + * Add a listener that will be notified when a function is defined. + * + * @param listener The listener to notify. + */ + void addFunctionListener(StellarExecutionListeners.FunctionDefinedListener listener); - /** - * Add a listener that will be notified when a variable is defined. - * @param listener The listener to notify. - */ - void addVariableListener(StellarExecutionListeners.VariableDefinedListener listener); + /** + * Add a listener that will be notified when a variable is defined. + * + * @param listener The listener to notify. + */ + void addVariableListener(StellarExecutionListeners.VariableDefinedListener listener); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarResult.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarResult.java index 4e5c81ee..e7fe3b95 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarResult.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarResult.java @@ -17,6 +17,7 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell; import java.util.Optional; @@ -26,170 +27,185 @@ */ public class StellarResult { - /** - * Indicates that a Stellar expression resulted in either - * success or an error. - */ - enum Status { - SUCCESS, - ERROR, - TERMINATE - } - - /** - * Indicates either success or failure of executing the expression. - */ - private Status status; - - /** - * The result of executing the expression. Only valid when execution is successful. - */ - private Optional value; - - /** - * The error that occurred when executing the expression. Only valid when execution results in an error. - */ - private Optional exception; - - /** - * Indicates if the value is null; - * - * A null is a valid result, but cannot be unwrapped from an Optional. Because of this - * a boolean is used to indicate if the result is a success and the value is null. - */ - private boolean isValueNull; - - /** - * Private constructor to construct a result indicate success. Use the static methods; success. - * - * @param status Indicates success or failure. - * @param value The value of executing the expression. - */ - private StellarResult(Status status, Object value) { - this.status = status; - this.value = Optional.ofNullable(value); - this.exception = Optional.empty(); - this.isValueNull = (value == null) && (status == Status.SUCCESS); - } - - /** - * Private constructor to construct a result indicating an error occurred. Use the static method; error. - * - * @param status Indicates success or failure. - * @param exception The exception that occurred when executing the expression. - */ - private StellarResult(Status status, Throwable exception) { - this.status = status; - this.value = Optional.empty(); - this.exception = Optional.of(exception); - this.isValueNull = false; - } - - /** - * Create a result indicating the execution of an expression was successful. - * - * @param value The result of executing the expression. - * @return A Result indicating success. - */ - public static StellarResult success(Object value) { - return new StellarResult(Status.SUCCESS, value); - } - - /** - * Create a result indicating that the execution of an expression was not successful. - * - * @param exception The exception that occurred while executing the expression. - * @return A Result indicating that an error occurred. - */ - public static StellarResult error(Throwable exception) { - return new StellarResult(Status.ERROR, exception); - } - - /** - * Create a result indicating that the execution of an expression was not successful. - * - * @param errorMessage An error message. - * @return A Result indicating that an error occurred. - */ - public static StellarResult error(String errorMessage) { - return new StellarResult(Status.ERROR, new IllegalArgumentException(errorMessage)); - } - - /** - * Indicates an empty result; one that is successful yet has no result. For example, - * executing a comment. - * - * @return An empty result. - */ - public static StellarResult noop() { - return new StellarResult(Status.SUCCESS, ""); - } - - /** - * Indicates that the user would like to terminate the session. - * - * @return A result indicating that the session should be terminated. - */ - public static StellarResult terminate() { - return new StellarResult(Status.TERMINATE, ""); - } - - /** - * @return True, if the result indicates success. Otherwise, false. - */ - public boolean isSuccess() { - return status == Status.SUCCESS; - } - - /** - * @return True, if the result indicates an error. Otherwise, false. - */ - public boolean isError() { - return status == Status.ERROR; - } - - /** - * @return True, if status indicates terminate was requested. Otherwise, false. - */ - public boolean isTerminate() { - return status == Status.TERMINATE; - } - - /** - * @return True, if the value is null. Otherwise, false. - */ - public boolean isValueNull() { - return isValueNull; - } - - /** - * @return The status which indicates success or failure. - */ - public Status getStatus() { - return status; - } - - /** - * @return An optional value that only applies when status is success. - */ - public Optional getValue() { - return value; - } - - /** - * @return An optional exception that only applies when status is error. - */ - public Optional getException() { - return exception; - } - - @Override - public String toString() { - return "StellarResult{" + - "status=" + status + - ", value=" + value + - ", exception=" + exception + - ", isValueNull=" + isValueNull + - '}'; - } + /** + * Indicates that a Stellar expression resulted in either + * success or an error. + */ + enum Status { + SUCCESS, + ERROR, + TERMINATE + } + + /** + * Indicates either success or failure of executing the expression. + */ + private final Status status; + + /** + * The result of executing the expression. Only valid when execution is successful. + */ + private final Optional value; + + /** + * The error that occurred when executing the expression. Only valid when execution results in an error. + */ + private final Optional exception; + + /** + * Indicates if the value is null; + * + *

    + * A null is a valid result, but cannot be unwrapped from an Optional. Because of this + * a boolean is used to indicate if the result is a success and the value is null. + */ + private final boolean isValueNull; + + /** + * Private constructor to construct a result indicate success. Use the static methods; success. + * + * @param status Indicates success or failure. + * @param value The value of executing the expression. + */ + private StellarResult(Status status, Object value) { + this.status = status; + this.value = Optional.ofNullable(value); + this.exception = Optional.empty(); + this.isValueNull = (value == null) && (status == Status.SUCCESS); + } + + /** + * Private constructor to construct a result indicating an error occurred. Use the static method; error. + * + * @param status Indicates success or failure. + * @param exception The exception that occurred when executing the expression. + */ + private StellarResult(Status status, Throwable exception) { + this.status = status; + this.value = Optional.empty(); + this.exception = Optional.of(exception); + this.isValueNull = false; + } + + /** + * Create a result indicating the execution of an expression was successful. + * + * @param value The result of executing the expression. + * @return A Result indicating success. + */ + public static StellarResult success(Object value) { + return new StellarResult(Status.SUCCESS, value); + } + + /** + * Create a result indicating that the execution of an expression was not successful. + * + * @param exception The exception that occurred while executing the expression. + * @return A Result indicating that an error occurred. + */ + public static StellarResult error(Throwable exception) { + return new StellarResult(Status.ERROR, exception); + } + + /** + * Create a result indicating that the execution of an expression was not successful. + * + * @param errorMessage An error message. + * @return A Result indicating that an error occurred. + */ + public static StellarResult error(String errorMessage) { + return new StellarResult(Status.ERROR, new IllegalArgumentException(errorMessage)); + } + + /** + * Indicates an empty result; one that is successful yet has no result. For example, + * executing a comment. + * + * @return An empty result. + */ + public static StellarResult noop() { + return new StellarResult(Status.SUCCESS, ""); + } + + /** + * Indicates that the user would like to terminate the session. + * + * @return A result indicating that the session should be terminated. + */ + public static StellarResult terminate() { + return new StellarResult(Status.TERMINATE, ""); + } + + /** + * Success status getter. + * + * @return True, if the result indicates success. Otherwise, false. + */ + public boolean isSuccess() { + return status == Status.SUCCESS; + } + + /** + * Error status getter. + * + * @return True, if the result indicates an error. Otherwise, false. + */ + public boolean isError() { + return status == Status.ERROR; + } + + /** + * Terminate status getter. + * + * @return True, if status indicates terminate was requested. Otherwise, false. + */ + public boolean isTerminate() { + return status == Status.TERMINATE; + } + + /** + * isValueNull getter. + * + * @return True, if the value is null. Otherwise, false. + */ + public boolean isValueNull() { + return isValueNull; + } + + /** + * Status getter. + * + * @return The status which indicates success or failure. + */ + public Status getStatus() { + return status; + } + + /** + * Value getter. + * + * @return An optional value that only applies when status is success. + */ + public Optional getValue() { + return value; + } + + /** + * Exception getter. + * + * @return An optional exception that only applies when status is error. + */ + public Optional getException() { + return exception; + } + + @Override + public String toString() { + return "StellarResult{" + + "status=" + status + + ", value=" + value + + ", exception=" + exception + + ", isValueNull=" + isValueNull + + '}'; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarShellExecutor.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarShellExecutor.java index 7e2d4fc2..15776b91 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarShellExecutor.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/StellarShellExecutor.java @@ -17,63 +17,69 @@ * limitations under the License. * */ -package org.apache.metron.stellar.common.shell; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +package org.apache.metron.stellar.common.shell; import java.util.Map; import java.util.Optional; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; /** * Responsible for executing Stellar in a shell-like environment. * + *

    * Provides the additional capabilities expected of executing Stellar * in a shell-like environment including maintaining state, variable assignment, * magic commands, doc strings, and comments. */ public interface StellarShellExecutor extends StellarExecutionNotifier { - /** - * Initialize the Stellar executor. - */ - void init(); + /** + * Initialize the Stellar executor. + */ + void init(); - /** - * Execute the Stellar expression. - * @param expression The Stellar expression to execute. - * @return The result of executing the Stellar expression. - */ - StellarResult execute(String expression); + /** + * Execute the Stellar expression. + * + * @param expression The Stellar expression to execute. + * @return The result of executing the Stellar expression. + */ + StellarResult execute(String expression); - /** - * Update the state of the executor by assign a value to a variable. - * @param variable The name of the variable. - * @param value The value to assign. - * @param expression The expression that resulted in the given value. Optional. - */ - void assign(String variable, Object value, Optional expression); + /** + * Update the state of the executor by assign a value to a variable. + * + * @param variable The name of the variable. + * @param value The value to assign. + * @param expression The expression that resulted in the given value. Optional. + */ + void assign(String variable, Object value, Optional expression); - /** - * The current state of the Stellar execution environment. - */ - Map getState(); + /** + * The current state of the Stellar execution environment. + */ + Map getState(); - /** - * Returns the Context for the Stellar execution environment. - * @return The execution context. - */ - Context getContext(); + /** + * Returns the Context for the Stellar execution environment. + * + * @return The execution context. + */ + Context getContext(); - /** - * Returns the global configuration of the Stellar execution environment. - * @return A map of values defined in the global configuration. - */ - Map getGlobalConfig(); + /** + * Returns the global configuration of the Stellar execution environment. + * + * @return A map of values defined in the global configuration. + */ + Map getGlobalConfig(); - /** - * Returns the function resolver of the Stellar execution environment. - * @return The function resolver. - */ - FunctionResolver getFunctionResolver(); + /** + * Returns the function resolver of the Stellar execution environment. + * + * @return The function resolver. + */ + FunctionResolver getFunctionResolver(); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/VariableResult.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/VariableResult.java index 6cb1173b..4be89bfc 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/VariableResult.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/VariableResult.java @@ -17,6 +17,7 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell; import java.util.Optional; @@ -24,78 +25,79 @@ /** * The value assigned to a variable. * + *

    * Allows us to maintain not only the resulting value, but the * expression that resulted in that value. */ public class VariableResult { - /** - * The expression that resulted in the value. Not always available. - */ - private Optional expression; + /** + * The expression that resulted in the value. Not always available. + */ + private final Optional expression; - /** - * The value of the variable. - */ - private Object result; + /** + * The value of the variable. + */ + private final Object result; - /** - * Create a new VariableResult when the expression that resulted in a value is known. - * - * @param value The value. - * @param expression The expression that resulted in the given value. - * @return A VariableResult. - */ - public static VariableResult withExpression(Object value, String expression) { - return new VariableResult(Optional.of(expression), value); - } + /** + * Create a new VariableResult when the expression that resulted in a value is known. + * + * @param value The value. + * @param expression The expression that resulted in the given value. + * @return A VariableResult. + */ + public static VariableResult withExpression(Object value, String expression) { + return new VariableResult(Optional.of(expression), value); + } - /** - * Create a new VariableResult when the expression that resulted in a value is known. - * - * @param value The value. - * @param expression The expression that resulted in the given value. - * @return A VariableResult. - */ - public static VariableResult withExpression(Object value, Optional expression) { - return new VariableResult(expression, value); - } + /** + * Create a new VariableResult when the expression that resulted in a value is known. + * + * @param value The value. + * @param expression The expression that resulted in the given value. + * @return A VariableResult. + */ + public static VariableResult withExpression(Object value, Optional expression) { + return new VariableResult(expression, value); + } - /** - * Create a new VariableResult when only the value is known. - * - * @param value The value. - * @return A VariableResult. - */ - public static VariableResult withValue(Object value) { - return new VariableResult(Optional.empty(), value); - } + /** + * Create a new VariableResult when only the value is known. + * + * @param value The value. + * @return A VariableResult. + */ + public static VariableResult withValue(Object value) { + return new VariableResult(Optional.empty(), value); + } - /** - * Private constructor. Use the static method 'withExpression' and 'withValue'. - * - * @param expression The expression that resulted in the given value. - * @param result The value assigned to the variable. - */ - private VariableResult(Optional expression, Object result) { - this.expression = expression; - this.result = result; - } + /** + * Private constructor. Use the static method 'withExpression' and 'withValue'. + * + * @param expression The expression that resulted in the given value. + * @param result The value assigned to the variable. + */ + private VariableResult(Optional expression, Object result) { + this.expression = expression; + this.result = result; + } - public Optional getExpression() { - return expression; - } + public Optional getExpression() { + return expression; + } - public Object getResult() { - return result; - } + public Object getResult() { + return result; + } - @Override - public String toString() { - String ret = "" + result; - if(getExpression().isPresent()) { - ret += " via " + expression.get(); + @Override + public String toString() { + String ret = "" + result; + if (getExpression().isPresent()) { + ret += " via " + expression.get(); + } + return ret; } - return ret; - } } \ No newline at end of file diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java index c72d66f9..29c57c41 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java @@ -17,6 +17,7 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell.cli; import java.io.IOException; @@ -30,350 +31,351 @@ * and the spawned program both share a buffer. This causes contention and unpredictable * results (e.g. an input may be consumed by either the aeshell thread or the spawned program) * + *

    * Because you can inject an input stream into the console, we create this which can act as a * facade to System.in under normal 'unpaused' circumstances, and when paused, turn off the * access to System.in. This allows us to turn off access to aeshell while maintaining access * to the external program. - * */ public class PausableInput extends InputStream { - private InputStream in = System.in; - private AtomicBoolean paused = new AtomicBoolean(false); - private PausableInput() { - super(); - } + private final InputStream in = System.in; + private final AtomicBoolean paused = new AtomicBoolean(false); + + private PausableInput() { + super(); + } - /** - * Stop mirroring stdin - */ - public void pause() { - paused.set(true); - } + /** + * Stop mirroring stdin. + */ + public void pause() { + paused.set(true); + } - /** - * Resume mirroring stdin. - * @throws IOException - */ - public void unpause() throws IOException { - paused.set(false); - } + /** + * Resume mirroring stdin. + */ + public void unpause() throws IOException { + paused.set(false); + } - public final static PausableInput INSTANCE = new PausableInput(); + public static final PausableInput INSTANCE = new PausableInput(); - /** - * Reads the next byte of data from the input stream. The value byte is - * returned as an int in the range 0 to - * 255. If no byte is available because the end of the stream - * has been reached, the value -1 is returned. This method - * blocks until input data is available, the end of the stream is detected, - * or an exception is thrown. - *

    - *

    A subclass must provide an implementation of this method. - * - * @return the next byte of data, or -1 if the end of the - * stream is reached. - * @throws IOException if an I/O error occurs. - */ - @Override - public int read() throws IOException { - if(paused.get()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return 0; + /** + * Reads the next byte of data from the input stream. The value byte is + * returned as an int in the range 0 to + * 255. If no byte is available because the end of the stream + * has been reached, the value -1 is returned. This method + * blocks until input data is available, the end of the stream is detected, + * or an exception is thrown. + * + *

    A subclass must provide an implementation of this method. + * + * @return the next byte of data, or -1 if the end of the + * stream is reached. + * @throws IOException if an I/O error occurs. + */ + @Override + public int read() throws IOException { + if (paused.get()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return 0; + } + return in.read(); } - return in.read(); - } - /** - * Reads some number of bytes from the input stream and stores them into - * the buffer array b. The number of bytes actually read is - * returned as an integer. This method blocks until input data is - * available, end of file is detected, or an exception is thrown. - *

    - *

    If the length of b is zero, then no bytes are read and - * 0 is returned; otherwise, there is an attempt to read at - * least one byte. If no byte is available because the stream is at the - * end of the file, the value -1 is returned; otherwise, at - * least one byte is read and stored into b. - *

    - *

    The first byte read is stored into element b[0], the - * next one into b[1], and so on. The number of bytes read is, - * at most, equal to the length of b. Let k be the - * number of bytes actually read; these bytes will be stored in elements - * b[0] through b[k-1], - * leaving elements b[k] through - * b[b.length-1] unaffected. - *

    - *

    The read(b) method for class InputStream - * has the same effect as: 

     read(b, 0, b.length)
    - * - * @param b the buffer into which the data is read. - * @return the total number of bytes read into the buffer, or - * -1 if there is no more data because the end of - * the stream has been reached. - * @throws IOException If the first byte cannot be read for any reason - * other than the end of the file, if the input stream has been closed, or - * if some other I/O error occurs. - * @throws NullPointerException if b is null. - * @see InputStream#read(byte[], int, int) - */ - @Override - public int read(byte[] b) throws IOException { + /** + * Reads some number of bytes from the input stream and stores them into + * the buffer array b. The number of bytes actually read is + * returned as an integer. This method blocks until input data is + * available, end of file is detected, or an exception is thrown. + * + *

    If the length of b is zero, then no bytes are read and + * 0 is returned; otherwise, there is an attempt to read at + * least one byte. If no byte is available because the stream is at the + * end of the file, the value -1 is returned; otherwise, at + * least one byte is read and stored into b. + * + *

    The first byte read is stored into element b[0], the + * next one into b[1], and so on. The number of bytes read is, + * at most, equal to the length of b. Let k be the + * number of bytes actually read; these bytes will be stored in elements + * b[0] through b[k-1], + * leaving elements b[k] through + * b[b.length-1] unaffected. + * + *

    The read(b) method for class InputStream + * has the same effect as: 

     read(b, 0, b.length)
    + * + * @param b the buffer into which the data is read. + * @return the total number of bytes read into the buffer, or + * -1 if there is no more data because the end of + * the stream has been reached. + * @throws IOException If the first byte cannot be read for any reason + * other than the end of the file, if the input stream has been closed, or + * if some other I/O error occurs. + * @throws NullPointerException if b is null. + * @see InputStream#read(byte[], int, int) + */ + @Override + public int read(byte[] b) throws IOException { - if(paused.get()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return 0; + if (paused.get()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return 0; + } + int ret = in.read(b); + return ret; } - int ret = in.read(b); - return ret; - } - /** - * Reads up to len bytes of data from the input stream into - * an array of bytes. An attempt is made to read as many as - * len bytes, but a smaller number may be read. - * The number of bytes actually read is returned as an integer. - *

    - *

    This method blocks until input data is available, end of file is - * detected, or an exception is thrown. - *

    - *

    If len is zero, then no bytes are read and - * 0 is returned; otherwise, there is an attempt to read at - * least one byte. If no byte is available because the stream is at end of - * file, the value -1 is returned; otherwise, at least one - * byte is read and stored into b. - *

    - *

    The first byte read is stored into element b[off], the - * next one into b[off+1], and so on. The number of bytes read - * is, at most, equal to len. Let k be the number of - * bytes actually read; these bytes will be stored in elements - * b[off] through b[off+k-1], - * leaving elements b[off+k] through - * b[off+len-1] unaffected. - *

    - *

    In every case, elements b[0] through - * b[off] and elements b[off+len] through - * b[b.length-1] are unaffected. - *

    - *

    The read(b, off, len) method - * for class InputStream simply calls the method - * read() repeatedly. If the first such call results in an - * IOException, that exception is returned from the call to - * the read(b, off, len) method. If - * any subsequent call to read() results in a - * IOException, the exception is caught and treated as if it - * were end of file; the bytes read up to that point are stored into - * b and the number of bytes read before the exception - * occurred is returned. The default implementation of this method blocks - * until the requested amount of input data len has been read, - * end of file is detected, or an exception is thrown. Subclasses are encouraged - * to provide a more efficient implementation of this method. - * - * @param b the buffer into which the data is read. - * @param off the start offset in array b - * at which the data is written. - * @param len the maximum number of bytes to read. - * @return the total number of bytes read into the buffer, or - * -1 if there is no more data because the end of - * the stream has been reached. - * @throws IOException If the first byte cannot be read for any reason - * other than end of file, or if the input stream has been closed, or if - * some other I/O error occurs. - * @throws NullPointerException If b is null. - * @throws IndexOutOfBoundsException If off is negative, - * len is negative, or len is greater than - * b.length - off - * @see InputStream#read() - */ - @Override - public int read(byte[] b, int off, int len) throws IOException { - if(paused.get()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return 0; + /** + * Reads up to len bytes of data from the input stream into + * an array of bytes. An attempt is made to read as many as + * len bytes, but a smaller number may be read. + * The number of bytes actually read is returned as an integer. + * + *

    This method blocks until input data is available, end of file is + * detected, or an exception is thrown. + * + *

    If len is zero, then no bytes are read and + * 0 is returned; otherwise, there is an attempt to read at + * least one byte. If no byte is available because the stream is at end of + * file, the value -1 is returned; otherwise, at least one + * byte is read and stored into b. + * + *

    The first byte read is stored into element b[off], the + * next one into b[off+1], and so on. The number of bytes read + * is, at most, equal to len. Let k be the number of + * bytes actually read; these bytes will be stored in elements + * b[off] through b[off+k-1], + * leaving elements b[off+k] through + * b[off+len-1] unaffected. + * + *

    In every case, elements b[0] through + * b[off] and elements b[off+len] through + * b[b.length-1] are unaffected. + * + *

    The read(b, off, len) method + * for class InputStream simply calls the method + * read() repeatedly. If the first such call results in an + * IOException, that exception is returned from the call to + * the read(b, off, len) method. If + * any subsequent call to read() results in a + * IOException, the exception is caught and treated as if it + * were end of file; the bytes read up to that point are stored into + * b and the number of bytes read before the exception + * occurred is returned. The default implementation of this method blocks + * until the requested amount of input data len has been read, + * end of file is detected, or an exception is thrown. Subclasses are encouraged + * to provide a more efficient implementation of this method. + * + * @param b the buffer into which the data is read. + * @param off the start offset in array b + * at which the data is written. + * @param len the maximum number of bytes to read. + * @return the total number of bytes read into the buffer, or + * -1 if there is no more data because the end of + * the stream has been reached. + * @throws IOException If the first byte cannot be read for any reason + * other than end of file, or if the input stream has been closed, or if + * some other I/O error occurs. + * @throws NullPointerException If b is null. + * @throws IndexOutOfBoundsException If off is negative, + * len is negative, or len is greater than + * b.length - off + * @see InputStream#read() + */ + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (paused.get()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return 0; + } + int ret = in.read(b, off, len); + return ret; } - int ret = in.read(b, off, len); - return ret; - } - /** - * Skips over and discards n bytes of data from this input - * stream. The skip method may, for a variety of reasons, end - * up skipping over some smaller number of bytes, possibly 0. - * This may result from any of a number of conditions; reaching end of file - * before n bytes have been skipped is only one possibility. - * The actual number of bytes skipped is returned. If {@code n} is - * negative, the {@code skip} method for class {@code InputStream} always - * returns 0, and no bytes are skipped. Subclasses may handle the negative - * value differently. - *

    - *

    The skip method of this class creates a - * byte array and then repeatedly reads into it until n bytes - * have been read or the end of the stream has been reached. Subclasses are - * encouraged to provide a more efficient implementation of this method. - * For instance, the implementation may depend on the ability to seek. - * - * @param n the number of bytes to be skipped. - * @return the actual number of bytes skipped. - * @throws IOException if the stream does not support seek, - * or if some other I/O error occurs. - */ - @Override - public long skip(long n) throws IOException { + /** + * Skips over and discards n bytes of data from this input + * stream. The skip method may, for a variety of reasons, end + * up skipping over some smaller number of bytes, possibly 0. + * This may result from any of a number of conditions; reaching end of file + * before n bytes have been skipped is only one possibility. + * The actual number of bytes skipped is returned. If {@code n} is + * negative, the {@code skip} method for class {@code InputStream} always + * returns 0, and no bytes are skipped. Subclasses may handle the negative + * value differently. + * + *

    The skip method of this class creates a + * byte array and then repeatedly reads into it until n bytes + * have been read or the end of the stream has been reached. Subclasses are + * encouraged to provide a more efficient implementation of this method. + * For instance, the implementation may depend on the ability to seek. + * + * @param n the number of bytes to be skipped. + * @return the actual number of bytes skipped. + * @throws IOException if the stream does not support seek, + * or if some other I/O error occurs. + */ + @Override + public long skip(long n) throws IOException { - return in.skip(n); - } + return in.skip(n); + } - /** - * Returns an estimate of the number of bytes that can be read (or - * skipped over) from this input stream without blocking by the next - * invocation of a method for this input stream. The next invocation - * might be the same thread or another thread. A single read or skip of this - * many bytes will not block, but may read or skip fewer bytes. - *

    - *

    Note that while some implementations of {@code InputStream} will return - * the total number of bytes in the stream, many will not. It is - * never correct to use the return value of this method to allocate - * a buffer intended to hold all data in this stream. - *

    - *

    A subclass' implementation of this method may choose to throw an - * {@link IOException} if this input stream has been closed by - * invoking the {@link #close()} method. - *

    - *

    The {@code available} method for class {@code InputStream} always - * returns {@code 0}. - *

    - *

    This method should be overridden by subclasses. - * - * @return an estimate of the number of bytes that can be read (or skipped - * over) from this input stream without blocking or {@code 0} when - * it reaches the end of the input stream. - * @throws IOException if an I/O error occurs. - */ - @Override - public int available() throws IOException { + /** + * Returns an estimate of the number of bytes that can be read (or + * skipped over) from this input stream without blocking by the next + * invocation of a method for this input stream. The next invocation + * might be the same thread or another thread. A single read or skip of this + * many bytes will not block, but may read or skip fewer bytes. + * + *

    Note that while some implementations of {@code InputStream} will return + * the total number of bytes in the stream, many will not. It is + * never correct to use the return value of this method to allocate + * a buffer intended to hold all data in this stream. + * + *

    A subclass' implementation of this method may choose to throw an + * {@link IOException} if this input stream has been closed by + * invoking the {@link #close()} method. + * + *

    The {@code available} method for class {@code InputStream} always + * returns {@code 0}. + * + *

    This method should be overridden by subclasses. + * + * @return an estimate of the number of bytes that can be read (or skipped + * over) from this input stream without blocking or {@code 0} when + * it reaches the end of the input stream. + * @throws IOException if an I/O error occurs. + */ + @Override + public int available() throws IOException { - return in.available(); - } + return in.available(); + } - /** - * Closes this input stream and releases any system resources associated - * with the stream. - *

    - *

    The close method of InputStream does - * nothing. - * - * @throws IOException if an I/O error occurs. - */ - @Override - public void close() throws IOException { - in.close(); - } + /** + * Closes this input stream and releases any system resources associated + * with the stream. + * + *

    The close method of InputStream does + * nothing. + * + * @throws IOException if an I/O error occurs. + */ + @Override + public void close() throws IOException { + in.close(); + } - /** - * Marks the current position in this input stream. A subsequent call to - * the reset method repositions this stream at the last marked - * position so that subsequent reads re-read the same bytes. - *

    - *

    The readlimit arguments tells this input stream to - * allow that many bytes to be read before the mark position gets - * invalidated. - *

    - *

    The general contract of mark is that, if the method - * markSupported returns true, the stream somehow - * remembers all the bytes read after the call to mark and - * stands ready to supply those same bytes again if and whenever the method - * reset is called. However, the stream is not required to - * remember any data at all if more than readlimit bytes are - * read from the stream before reset is called. - *

    - *

    Marking a closed stream should not have any effect on the stream. - *

    - *

    The mark method of InputStream does - * nothing. - * - * @param readlimit the maximum limit of bytes that can be read before - * the mark position becomes invalid. - * @see InputStream#reset() - */ - @Override - public synchronized void mark(int readlimit) { - in.mark(readlimit); - } + /** + * Marks the current position in this input stream. A subsequent call to + * the reset method repositions this stream at the last marked + * position so that subsequent reads re-read the same bytes. + * + *

    The readlimit arguments tells this input stream to + * allow that many bytes to be read before the mark position gets + * invalidated. + * + *

    The general contract of mark is that, if the method + * markSupported returns true, the stream somehow + * remembers all the bytes read after the call to mark and + * stands ready to supply those same bytes again if and whenever the method + * reset is called. However, the stream is not required to + * remember any data at all if more than readlimit bytes are + * read from the stream before reset is called. + * + *

    Marking a closed stream should not have any effect on the stream. + * + *

    The mark method of InputStream does + * nothing. + * + * @param readlimit the maximum limit of bytes that can be read before + * the mark position becomes invalid. + * @see InputStream#reset() + */ + @Override + public synchronized void mark(int readlimit) { + in.mark(readlimit); + } - /** - * Repositions this stream to the position at the time the - * mark method was last called on this input stream. - *

    - *

    The general contract of reset is: - *

    - *

      - *
    • If the method markSupported returns - * true, then: - *

      - *

      • If the method mark has not been called since - * the stream was created, or the number of bytes read from the stream - * since mark was last called is larger than the argument - * to mark at that last call, then an - * IOException might be thrown. - *

        - *

      • If such an IOException is not thrown, then the - * stream is reset to a state such that all the bytes read since the - * most recent call to mark (or since the start of the - * file, if mark has not been called) will be resupplied - * to subsequent callers of the read method, followed by - * any bytes that otherwise would have been the next input data as of - * the time of the call to reset.
      - *

      - *

    • If the method markSupported returns - * false, then: - *

      - *

      • The call to reset may throw an - * IOException. - *

        - *

      • If an IOException is not thrown, then the stream - * is reset to a fixed state that depends on the particular type of the - * input stream and how it was created. The bytes that will be supplied - * to subsequent callers of the read method depend on the - * particular type of the input stream.
    - *

    - *

    The method reset for class InputStream - * does nothing except throw an IOException. - * - * @throws IOException if this stream has not been marked or if the - * mark has been invalidated. - * @see InputStream#mark(int) - * @see IOException - */ - @Override - public synchronized void reset() throws IOException { - in.reset(); - } + /** + * Repositions this stream to the position at the time the + * mark method was last called on this input stream. + * + *

    The general contract of reset is: + * + *

    + *

      + *
    • If the method markSupported returns + * true, then: + * + *
      • If the method mark has not been called since + * the stream was created, or the number of bytes read from the stream + * since mark was last called is larger than the argument + * to mark at that last call, then an + * IOException might be thrown. + * + *
      • If such an IOException is not thrown, then the + * stream is reset to a state such that all the bytes read since the + * most recent call to mark (or since the start of the + * file, if mark has not been called) will be resupplied + * to subsequent callers of the read method, followed by + * any bytes that otherwise would have been the next input data as of + * the time of the call to reset.
      + * + *
    • If the method markSupported returns + * false, then: + * + *
      • The call to reset may throw an + * IOException. + * + *
      • If an IOException is not thrown, then the stream + * is reset to a fixed state that depends on the particular type of the + * input stream and how it was created. The bytes that will be supplied + * to subsequent callers of the read method depend on the + * particular type of the input stream.
    + * + *

    The method reset for class InputStream + * does nothing except throw an IOException. + * + * @throws IOException if this stream has not been marked or if the + * mark has been invalidated. + * @see InputStream#mark(int) + * @see IOException + */ + @Override + public synchronized void reset() throws IOException { + in.reset(); + } - /** - * Tests if this input stream supports the mark and - * reset methods. Whether or not mark and - * reset are supported is an invariant property of a - * particular input stream instance. The markSupported method - * of InputStream returns false. - * - * @return true if this stream instance supports the mark - * and reset methods; false otherwise. - * @see InputStream#mark(int) - * @see InputStream#reset() - */ - @Override - public boolean markSupported() { - return in.markSupported(); - } + /** + * Tests if this input stream supports the mark and + * reset methods. Whether or not mark and + * reset are supported is an invariant property of a + * particular input stream instance. The markSupported method + * of InputStream returns false. + * + * @return true if this stream instance supports the mark + * and reset methods; false otherwise. + * @see InputStream#mark(int) + * @see InputStream#reset() + */ + @Override + public boolean markSupported() { + return in.markSupported(); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java index 083bad2c..d64076f4 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java @@ -20,8 +20,21 @@ package org.apache.metron.stellar.common.shell.cli; +import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.SHELL_VARIABLES; + import com.google.common.base.Splitter; import com.google.common.collect.Iterables; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; @@ -49,380 +62,380 @@ import org.jboss.aesh.terminal.TerminalCharacter; import org.jboss.aesh.terminal.TerminalColor; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; - -import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE; -import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; -import static org.apache.metron.stellar.dsl.Context.Capabilities.SHELL_VARIABLES; - /** * A REPL environment for Stellar. * + *

    * Useful for debugging Stellar expressions. */ public class StellarShell extends AeshConsoleCallback implements Completion { - static final String WELCOME = "Stellar, Go!\n" + - "Functions are loading lazily in the background and will be unavailable until loaded fully."; - - private List EXPRESSION_PROMPT = new ArrayList() - {{ - add(new TerminalCharacter('[', new TerminalColor(Color.RED, Color.DEFAULT))); - add(new TerminalCharacter('S', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('t', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('e', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('a', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter('r', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); - add(new TerminalCharacter(']', new TerminalColor(Color.RED, Color.DEFAULT))); - add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); - add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); - add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); - add(new TerminalCharacter(' ', new TerminalColor(Color.DEFAULT, Color.DEFAULT))); - }}; - - public static final String ERROR_PROMPT = "[!] "; - public static final String STELLAR_PROPERTIES_FILENAME = "stellar.properties"; - - /** - * Executes Stellar expressions for the shell environment. - */ - private StellarShellExecutor executor; - - /** - * The Aesh shell console. - */ - private Console console; - - /** - * Provides auto-complete functionality. - */ - private StellarAutoCompleter autoCompleter; - - /** - * Execute the Stellar REPL. - */ - public static void main(String[] args) throws Exception { - StellarShell shell = new StellarShell(args); - shell.run(); - } - - /** - * Create a Stellar REPL. - * @param args The commmand-line arguments. - */ - public StellarShell(String[] args) throws Exception { - - // define valid command-line options - CommandLineParser parser = new PosixParser(); - Options options = defineCommandLineOptions(); - CommandLine commandLine = parser.parse(options, args); - - // print help - if(commandLine.hasOption("h")) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("stellar", options); - System.exit(0); + static final String WELCOME = "Stellar, Go!\n" + + "Functions are loading lazily in the background and " + + "will be unavailable until loaded fully."; + + private static final List EXPRESSION_PROMPT = new ArrayList() { + { + add(new TerminalCharacter('[', new TerminalColor(Color.RED, Color.DEFAULT))); + add(new TerminalCharacter('S', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('t', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('e', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('a', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('r', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter(']', new TerminalColor(Color.RED, Color.DEFAULT))); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter(' ', new TerminalColor(Color.DEFAULT, Color.DEFAULT))); + } + }; + + public static final String ERROR_PROMPT = "[!] "; + public static final String STELLAR_PROPERTIES_FILENAME = "stellar.properties"; + + /** + * Executes Stellar expressions for the shell environment. + */ + private final StellarShellExecutor executor; + + /** + * The Aesh shell console. + */ + private final Console console; + + /** + * Provides auto-complete functionality. + */ + private final StellarAutoCompleter autoCompleter; + + /** + * Execute the Stellar REPL. + */ + public static void main(String[] args) throws Exception { + StellarShell shell = new StellarShell(args); + shell.run(); } - // validate the command line options - try { - StellarShellOptionsValidator.validateOptions(commandLine); + /** + * Create a Stellar REPL. + * + * @param args The commmand-line arguments. + */ + public StellarShell(String[] args) throws Exception { + + // define valid command-line options + CommandLineParser parser = new PosixParser(); + Options options = defineCommandLineOptions(); + CommandLine commandLine = parser.parse(options, args); + + // print help + if (commandLine.hasOption("h")) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("stellar", options); + System.exit(0); + } + + // validate the command line options + try { + StellarShellOptionsValidator.validateOptions(commandLine); + + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + System.exit(1); + } - } catch(IllegalArgumentException e){ - System.err.println(e.getMessage()); - System.exit(1); + // setup logging, if specified + if (commandLine.hasOption("l")) { + PropertyConfigurator.configure(commandLine.getOptionValue("l")); + } + + console = createConsole(commandLine); + autoCompleter = new DefaultStellarAutoCompleter(); + Properties props = getStellarProperties(commandLine); + executor = createExecutor(commandLine, console, props, autoCompleter); + loadVariables(commandLine, executor); + console.setPrompt(new Prompt(EXPRESSION_PROMPT)); + console.addCompletion(this); + console.setConsoleCallback(this); } - // setup logging, if specified - if(commandLine.hasOption("l")) { - PropertyConfigurator.configure(commandLine.getOptionValue("l")); + /** + * Getter for valid command line options. + * + * @return The valid command line options. + */ + private Options defineCommandLineOptions() { + Options options = new Options(); + options.addOption( + "z", + "zookeeper", + true, + "Zookeeper URL fragment in the form [HOSTNAME|IPADDRESS]:PORT"); + options.addOption( + "v", + "variables", + true, + "File containing a JSON Map of variables"); + options.addOption( + "irc", + "inputrc", + true, + "File containing the inputrc if not the default ~/.inputrc"); + options.addOption( + "na", + "no_ansi", + false, + "Make the input prompt not use ANSI colors."); + options.addOption( + "h", + "help", + false, + "Print help"); + options.addOption( + "p", + "properties", + true, + "File containing Stellar properties"); + Option log4j = new Option( + "l", + "log4j", + true, + "The log4j properties file to load"); + log4j.setArgName("FILE"); + log4j.setRequired(false); + options.addOption(log4j); + + return options; } - console = createConsole(commandLine); - autoCompleter = new DefaultStellarAutoCompleter(); - Properties props = getStellarProperties(commandLine); - executor = createExecutor(commandLine, console, props, autoCompleter); - loadVariables(commandLine, executor); - console.setPrompt(new Prompt(EXPRESSION_PROMPT)); - console.addCompletion(this); - console.setConsoleCallback(this); - } - - /** - * @return The valid command line options. - */ - private Options defineCommandLineOptions() { - Options options = new Options(); - options.addOption( - "z", - "zookeeper", - true, - "Zookeeper URL fragment in the form [HOSTNAME|IPADDRESS]:PORT"); - options.addOption( - "v", - "variables", - true, - "File containing a JSON Map of variables"); - options.addOption( - "irc", - "inputrc", - true, - "File containing the inputrc if not the default ~/.inputrc"); - options.addOption( - "na", - "no_ansi", - false, - "Make the input prompt not use ANSI colors."); - options.addOption( - "h", - "help", - false, - "Print help"); - options.addOption( - "p", - "properties", - true, - "File containing Stellar properties"); - Option log4j = new Option( - "l", - "log4j", - true, - "The log4j properties file to load"); - log4j.setArgName("FILE"); - log4j.setRequired(false); - options.addOption(log4j); - - return options; - } - - /** - * Loads any variables defined in an external file. - * @param commandLine The command line arguments. - * @param executor The stellar executor. - * @throws IOException - */ - private static void loadVariables( + /** + * Loads any variables defined in an external file. + * + * @param commandLine The command line arguments. + * @param executor The stellar executor. + */ + private static void loadVariables( CommandLine commandLine, StellarShellExecutor executor) throws IOException { - if(commandLine.hasOption("v")) { + if (commandLine.hasOption("v")) { - // load variables defined in a file - String variablePath = commandLine.getOptionValue("v"); - Map variables = JSONUtils.INSTANCE.load( - new File(variablePath), - JSONUtils.MAP_SUPPLIER); + // load variables defined in a file + String variablePath = commandLine.getOptionValue("v"); + Map variables = JSONUtils.INSTANCE.load( + new File(variablePath), + JSONUtils.MAP_SUPPLIER); - // for each variable... - for(Map.Entry kv : variables.entrySet()) { - String variable = kv.getKey(); - Object value = kv.getValue(); + // for each variable... + for (Map.Entry kv : variables.entrySet()) { + String variable = kv.getKey(); + Object value = kv.getValue(); - // define the variable - no expression available - executor.assign(variable, value, Optional.empty()); - } + // define the variable - no expression available + executor.assign(variable, value, Optional.empty()); + } + } } - } - - /** - * Creates the Stellar execution environment. - * @param commandLine The command line arguments. - * @param console The console which drives the REPL. - * @param properties Stellar properties. - */ - private StellarShellExecutor createExecutor( + + /** + * Creates the Stellar execution environment. + * + * @param commandLine The command line arguments. + * @param console The console which drives the REPL. + * @param properties Stellar properties. + */ + private StellarShellExecutor createExecutor( CommandLine commandLine, Console console, Properties properties, StellarAutoCompleter autoCompleter) throws Exception { - // setup zookeeper client - Optional zookeeperUrl = Optional.empty(); - if(commandLine.hasOption("z")) { - zookeeperUrl = Optional.of(commandLine.getOptionValue("z")); - } - - StellarShellExecutor executor = new DefaultStellarShellExecutor(properties, zookeeperUrl); + // setup zookeeper client + Optional zookeeperUrl = Optional.empty(); + if (commandLine.hasOption("z")) { + zookeeperUrl = Optional.of(commandLine.getOptionValue("z")); + } - // the 'CONSOLE' capability is only available with the CLI REPL - executor.getContext().addCapability(CONSOLE, () -> console); + StellarShellExecutor executor = new DefaultStellarShellExecutor(properties, zookeeperUrl); - // allows some Stellar functions to access Stellar internals; should probably use %magics instead - executor.getContext().addCapability(SHELL_VARIABLES, () -> executor.getState()); + // the 'CONSOLE' capability is only available with the CLI REPL + executor.getContext().addCapability(CONSOLE, () -> console); - // register the auto-completer to be notified when needed - executor.addSpecialListener( (special) -> autoCompleter.addCandidateFunction(special.getCommand())); - executor.addFunctionListener( (function) -> autoCompleter.addCandidateFunction(function.getName())); - executor.addVariableListener((name, val) -> autoCompleter.addCandidateVariable(name)); + // allows some Stellar functions to access Stellar internals; should probably use %magics instead + executor.getContext().addCapability(SHELL_VARIABLES, executor::getState); - executor.init(); - return executor; - } + // register the auto-completer to be notified when needed + executor.addSpecialListener((special) -> autoCompleter.addCandidateFunction(special.getCommand())); + executor.addFunctionListener((function) -> autoCompleter.addCandidateFunction(function.getName())); + executor.addVariableListener((name, val) -> autoCompleter.addCandidateVariable(name)); - /** - * Creates the REPL's console. - * @param commandLine The command line options. - */ - private Console createConsole(CommandLine commandLine) { + executor.init(); + return executor; + } - // console settings - boolean useAnsi = !commandLine.hasOption("na"); - SettingsBuilder settings = new SettingsBuilder().enableAlias(true) - .enableMan(true) - .ansi(useAnsi) - .parseOperators(false) - .inputStream(PausableInput.INSTANCE); + /** + * Creates the REPL's console. + * + * @param commandLine The command line options. + */ + private Console createConsole(CommandLine commandLine) { + + // console settings + boolean useAnsi = !commandLine.hasOption("na"); + SettingsBuilder settings = new SettingsBuilder().enableAlias(true) + .enableMan(true) + .ansi(useAnsi) + .parseOperators(false) + .inputStream(PausableInput.INSTANCE); + + if (commandLine.hasOption("irc")) { + settings = settings.inputrc(new File(commandLine.getOptionValue("irc"))); + } - if(commandLine.hasOption("irc")) { - settings = settings.inputrc(new File(commandLine.getOptionValue("irc"))); + return new Console(settings.create()); } - return new Console(settings.create()); - } - - /** - * Retrieves the Stellar properties. The properties are either loaded from a file in - * the classpath or a set of defaults are used. - */ - private Properties getStellarProperties(CommandLine commandLine) throws IOException { - Properties properties = new Properties(); - - if (commandLine.hasOption("p")) { - // attempt to load properties from a file specified on the command-line - try (InputStream in = new FileInputStream(commandLine.getOptionValue("p"))) { - if(in != null) { - properties.load(in); + /** + * Retrieves the Stellar properties. The properties are either loaded from a file in + * the classpath or a set of defaults are used. + */ + private Properties getStellarProperties(CommandLine commandLine) throws IOException { + Properties properties = new Properties(); + + if (commandLine.hasOption("p")) { + // attempt to load properties from a file specified on the command-line + try (InputStream in = new FileInputStream(commandLine.getOptionValue("p"))) { + if (in != null) { + properties.load(in); + } + } + + } else { + // otherwise attempt to load properties from the classpath + try (InputStream in = getClass().getClassLoader().getResourceAsStream(STELLAR_PROPERTIES_FILENAME)) { + if (in != null) { + properties.load(in); + } + } } - } - } else { - // otherwise attempt to load properties from the classpath - try (InputStream in = getClass().getClassLoader().getResourceAsStream(STELLAR_PROPERTIES_FILENAME)) { - if(in != null) { - properties.load(in); + return properties; + } + + /** + * Handles the main loop for the REPL. + */ + public void run() { + // welcome message + writeLine(WELCOME); + + // print the globals if we got 'em + executor.getContext() + .getCapability(GLOBAL_CONFIG, false) + .ifPresent(conf -> writeLine(conf.toString())); + + console.start(); + } + + /** + * Quits the console. + */ + private void handleQuit() { + try { + console.stop(); + StellarFunctions.close(); + } catch (Throwable e) { + e.printStackTrace(); } - } } - return properties; - } - - /** - * Handles the main loop for the REPL. - */ - public void run() { - // welcome message - writeLine(WELCOME); - - // print the globals if we got 'em - executor.getContext() - .getCapability(GLOBAL_CONFIG, false) - .ifPresent(conf -> writeLine(conf.toString())); - - console.start(); - } - - /** - * Quits the console. - */ - private void handleQuit() { - try { - console.stop(); - StellarFunctions.close(); - } catch (Throwable e) { - e.printStackTrace(); + private void writeLine(String out) { + console.getShell().out().println(out); } - } - private void writeLine(String out) { - console.getShell().out().println(out); - } + @Override + public int execute(ConsoleOperation output) throws InterruptedException { - @Override - public int execute(ConsoleOperation output) throws InterruptedException { + // grab the user the input + String expression = StringUtils.trimToEmpty(output.getBuffer()); + if (StringUtils.isNotBlank(expression)) { - // grab the user the input - String expression = StringUtils.trimToEmpty(output.getBuffer()); - if(StringUtils.isNotBlank(expression) ) { + // execute the expression + StellarResult result = executor.execute(expression); - // execute the expression - StellarResult result = executor.execute(expression); + if (result.isSuccess()) { + // on success + result.getValue().ifPresent(v -> writeLine(v.toString())); - if(result.isSuccess()) { - // on success - result.getValue().ifPresent(v -> writeLine(v.toString())); + } else if (result.isError()) { + // on error + result.getException().ifPresent(e -> writeLine(ERROR_PROMPT + e.getMessage())); + result.getException().ifPresent(e -> e.printStackTrace()); - } else if (result.isError()) { - // on error - result.getException().ifPresent(e -> writeLine(ERROR_PROMPT + e.getMessage())); - result.getException().ifPresent(e -> e.printStackTrace()); + } else if (result.isTerminate()) { + // on quit + handleQuit(); - } else if(result.isTerminate()) { - // on quit - handleQuit(); + } else { + // should never happen + throw new IllegalStateException( + "An execution result is neither a success nor a failure. Please file a bug report."); + } + } - } else { - // should never happen - throw new IllegalStateException("An execution result is neither a success nor a failure. Please file a bug report."); - } + return 0; } - return 0; - } - - /** - * Performs auto-completion for the shell. - * @param completeOperation The auto-complete operation. - */ - @Override - public void complete(CompleteOperation completeOperation) { - String buffer = completeOperation.getBuffer(); - final String lastToken = getLastToken(buffer); - Iterable candidates = autoCompleter.autoComplete(buffer); - - // transform the candidates into valid completions - if(candidates != null && !Iterables.isEmpty(candidates)) { - for(String candidate : candidates) { - String completion = stripOff(buffer, lastToken) + candidate; - completeOperation.addCompletionCandidate(completion); - } + /** + * Performs auto-completion for the shell. + * + * @param completeOperation The auto-complete operation. + */ + @Override + public void complete(CompleteOperation completeOperation) { + String buffer = completeOperation.getBuffer(); + final String lastToken = getLastToken(buffer); + Iterable candidates = autoCompleter.autoComplete(buffer); + + // transform the candidates into valid completions + if (candidates != null && !Iterables.isEmpty(candidates)) { + for (String candidate : candidates) { + String completion = stripOff(buffer, lastToken) + candidate; + completeOperation.addCompletionCandidate(completion); + } + } } - } - private static String getLastToken(String buffer) { - String lastToken = Iterables.getLast(Splitter.on(" ").split(buffer), null); - return lastToken.trim(); - } + private static String getLastToken(String buffer) { + String lastToken = Iterables.getLast(Splitter.on(" ").split(buffer), null); + return lastToken.trim(); + } + + private static String stripOff(String baseString, String lastBit) { + int index = baseString.lastIndexOf(lastBit); + if (index < 0) { + return baseString; + } + return baseString.substring(0, index); + } + + /** + * Executor getter. + * + * @return The executor of Stellar expressions. + */ + public StellarShellExecutor getExecutor() { + return executor; + } - private static String stripOff(String baseString, String lastBit) { - int index = baseString.lastIndexOf(lastBit); - if(index < 0) { - return baseString; + /** + * Console getter. + * + * @return The console. + */ + public Console getConsole() { + return console; } - return baseString.substring(0, index); - } - - /** - * @return The executor of Stellar expressions. - */ - public StellarShellExecutor getExecutor() { - return executor; - } - - /** - * @return The console. - */ - public Console getConsole() { - return console; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidator.java index 585add54..4d284cbe 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidator.java @@ -20,108 +20,106 @@ package org.apache.metron.stellar.common.shell.cli; +import com.google.common.base.Splitter; import java.io.File; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; - -import com.google.common.base.Splitter; import org.apache.commons.cli.CommandLine; import org.apache.commons.lang3.StringUtils; import org.apache.commons.validator.routines.InetAddressValidator; public class StellarShellOptionsValidator { - private static final Pattern validPortPattern = Pattern.compile("(^.*)[:](\\d+)$"); - private static final Predicate hostnameValidator = hostname -> { - if(StringUtils.isEmpty(hostname)) { - return false; - } - try { - InetAddress add = InetAddress.getByName(hostname); - return true; - } catch (UnknownHostException e) { - return false; - } - }; - - - - private static final InetAddressValidator inetAddressValidator = InetAddressValidator - .getInstance(); + private static final Pattern validPortPattern = Pattern.compile("(^.*)[:](\\d+)$"); + private static final Predicate hostnameValidator = hostname -> { + if (StringUtils.isEmpty(hostname)) { + return false; + } + try { + InetAddress add = InetAddress.getByName(hostname); + return true; + } catch (UnknownHostException e) { + return false; + } + }; + + + private static final InetAddressValidator inetAddressValidator = InetAddressValidator + .getInstance(); + + /** + * Validates Stellar CLI Options. + */ + public static void validateOptions(CommandLine commandLine) throws IllegalArgumentException { + if (commandLine.hasOption('z')) { + validateZookeeperOption(commandLine.getOptionValue('z')); + } + // v, irc, p are files + if (commandLine.hasOption('v')) { + validateFileOption("v", commandLine.getOptionValue('v')); + } + if (commandLine.hasOption("irc")) { + validateFileOption("irc", commandLine.getOptionValue("irc")); + } + if (commandLine.hasOption('p')) { + validateFileOption("p", commandLine.getOptionValue('p')); + } - /** - * Validates Stellar CLI Options. - */ - public static void validateOptions(CommandLine commandLine) throws IllegalArgumentException { - if (commandLine.hasOption('z')) { - validateZookeeperOption(commandLine.getOptionValue('z')); } - // v, irc, p are files - if (commandLine.hasOption('v')) { - validateFileOption("v", commandLine.getOptionValue('v')); - } - if (commandLine.hasOption("irc")) { - validateFileOption("irc", commandLine.getOptionValue("irc")); - } - if (commandLine.hasOption('p')) { - validateFileOption("p", commandLine.getOptionValue('p')); - } - - } - /** - * Zookeeper argument should be in the form [HOST|IP]:PORT. - * - * @param zMulti the zookeeper url fragment - */ - private static void validateZookeeperOption(String zMulti) throws IllegalArgumentException { - for(String z : Splitter.on(",").split(zMulti)) { - Matcher matcher = validPortPattern.matcher(z); - boolean hasPort = z.contains(":"); - if (hasPort && !matcher.matches()) { - throw new IllegalArgumentException(String.format("Zookeeper option must have valid port: %s", z)); - } - - if (hasPort && matcher.groupCount() != 2) { - throw new IllegalArgumentException( - String.format("Zookeeper Option must be in the form of [HOST|IP]:PORT %s", z)); - } - String name = hasPort?matcher.group(1):z; - Integer port = hasPort?Integer.parseInt(matcher.group(2)):null; - - if (!hostnameValidator.test(name) && !inetAddressValidator.isValid(name)) { - throw new IllegalArgumentException( - String.format("Zookeeper Option %s is not a valid host name or ip address %s", name, z)); - } - - if (hasPort && (port == 0 || port > 65535)) { - throw new IllegalArgumentException( - String.format("Zookeeper Option %s port is not valid", z)); - } + /** + * Zookeeper argument should be in the form [HOST|IP]:PORT. + * + * @param zooMulti the zookeeper url fragment + */ + private static void validateZookeeperOption(String zooMulti) throws IllegalArgumentException { + for (String z : Splitter.on(",").split(zooMulti)) { + Matcher matcher = validPortPattern.matcher(z); + boolean hasPort = z.contains(":"); + if (hasPort && !matcher.matches()) { + throw new IllegalArgumentException(String.format("Zookeeper option must have valid port: %s", z)); + } + + if (hasPort && matcher.groupCount() != 2) { + throw new IllegalArgumentException( + String.format("Zookeeper Option must be in the form of [HOST|IP]:PORT %s", z)); + } + String name = hasPort ? matcher.group(1) : z; + Integer port = hasPort ? Integer.parseInt(matcher.group(2)) : null; + + if (!hostnameValidator.test(name) && !inetAddressValidator.isValid(name)) { + throw new IllegalArgumentException( + String.format("Zookeeper Option %s is not a valid host name or ip address %s", name, z)); + } + + if (hasPort && (port == 0 || port > 65535)) { + throw new IllegalArgumentException( + String.format("Zookeeper Option %s port is not valid", z)); + } + } } - } - /** - * File options must exist and be readable. - * - * @param option name of the option - * @param fileName the file name - */ - private static void validateFileOption(String option, String fileName) - throws IllegalArgumentException { - File file = new File(fileName); - if (!file.exists()) { - throw new IllegalArgumentException( - String.format("%s: File %s doesn't exist", option, fileName)); - } - if (!file.canRead()) { - throw new IllegalArgumentException( - String.format("%s: File %s is not readable", option, fileName)); + /** + * File options must exist and be readable. + * + * @param option name of the option + * @param fileName the file name + */ + private static void validateFileOption(String option, String fileName) + throws IllegalArgumentException { + File file = new File(fileName); + if (!file.exists()) { + throw new IllegalArgumentException( + String.format("%s: File %s doesn't exist", option, fileName)); + } + if (!file.canRead()) { + throw new IllegalArgumentException( + String.format("%s: File %s is not readable", option, fileName)); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java index 664e01e6..da708baf 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java @@ -16,70 +16,66 @@ * limitations under the License. * */ -package org.apache.metron.stellar.common.shell.specials; -import org.apache.metron.stellar.common.StellarAssignment; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; +package org.apache.metron.stellar.common.shell.specials; import java.util.Optional; import java.util.function.Function; - -import static org.apache.metron.stellar.common.shell.StellarResult.error; +import org.apache.metron.stellar.common.StellarAssignment; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; /** * A special command that allows for variable assignment. Variable * assignment is not implemented directly within Stellar. * - * x := 2 + 2 + *

    + * x := 2 + 2 */ public class AssignmentCommand implements SpecialCommand { - public static final String ASSIGNMENT_OP = ":="; + public static final String ASSIGNMENT_OP = ":="; - @Override - public Function getMatcher() { - return (input) -> StellarAssignment.isAssignment(input); - } - - @Override - public String getCommand() { - return ASSIGNMENT_OP; - } + @Override + public Function getMatcher() { + return (input) -> StellarAssignment.isAssignment(input); + } - /** - * Handles variable assignment. - * @param input The assignment expression to execute. - * @param executor A stellar execution environment. - * @return - */ - @Override - public StellarResult execute(String input, StellarShellExecutor executor) { - assert StellarAssignment.isAssignment(input); + @Override + public String getCommand() { + return ASSIGNMENT_OP; + } - // extract the variable and assignment expression - StellarAssignment assignment = StellarAssignment.from(input); - String varName = assignment.getVariable(); - String varExpr = assignment.getStatement(); + /** + * Handles variable assignment. + * + * @param input The assignment expression to execute. + * @param executor A stellar execution environment. + */ + @Override + public StellarResult execute(String input, StellarShellExecutor executor) { + assert StellarAssignment.isAssignment(input); - // execute the stellar expression - StellarResult result = executor.execute(varExpr); - if(result.isSuccess()) { + // extract the variable and assignment expression + StellarAssignment assignment = StellarAssignment.from(input); + String varName = assignment.getVariable(); + String varExpr = assignment.getStatement(); - Object value = null; - if(result.getValue().isPresent()) { - value = result.getValue().get(); + // execute the stellar expression + StellarResult result = executor.execute(varExpr); + if (result.isSuccess()) { - } else if(result.isValueNull()) { - value = null; - } + Object value = null; + if (result.getValue().isPresent()) { + value = result.getValue().get(); + } - // variable assignment - executor.assign(varName, value, Optional.of(varExpr)); - return result; + // variable assignment + executor.assign(varName, value, Optional.of(varExpr)); + return result; - } else { - return result; + } else { + return result; + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/Comment.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/Comment.java index cea0c91a..e4ae102c 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/Comment.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/Comment.java @@ -16,38 +16,39 @@ * limitations under the License. * */ -package org.apache.metron.stellar.common.shell.specials; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; - -import java.util.function.Function; +package org.apache.metron.stellar.common.shell.specials; import static org.apache.commons.lang3.StringUtils.startsWith; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.apache.metron.stellar.common.shell.StellarResult.noop; +import java.util.function.Function; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; + /** * A special command that handles comments. * - * # this is a comment + *

    + * # this is a comment */ public class Comment implements SpecialCommand { - public static final String COMMENT_PREFIX = "#"; + public static final String COMMENT_PREFIX = "#"; - @Override - public String getCommand() { - return "#"; - } + @Override + public String getCommand() { + return "#"; + } - @Override - public Function getMatcher() { - return (input) -> startsWith(trimToEmpty(input), COMMENT_PREFIX); - } + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), COMMENT_PREFIX); + } - @Override - public StellarResult execute(String expression, StellarShellExecutor executor) { - return noop(); - } + @Override + public StellarResult execute(String expression, StellarShellExecutor executor) { + return noop(); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/DocCommand.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/DocCommand.java index f362f745..06e04ac1 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/DocCommand.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/DocCommand.java @@ -16,90 +16,92 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell.specials; -import org.apache.commons.lang3.StringUtils; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; -import org.apache.metron.stellar.dsl.StellarFunctionInfo; +import static org.apache.metron.stellar.common.shell.StellarResult.error; +import static org.apache.metron.stellar.common.shell.StellarResult.success; import java.util.Optional; import java.util.Spliterator; import java.util.function.Function; import java.util.stream.StreamSupport; - -import static org.apache.metron.stellar.common.shell.StellarResult.error; -import static org.apache.metron.stellar.common.shell.StellarResult.success; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.dsl.StellarFunctionInfo; /** * A special command that allows a user to request doc string * about a Stellar function. * + *

    * For example `?TO_STRING` will output the docs for the function `TO_STRING` */ public class DocCommand implements SpecialCommand { - public static final String DOC_PREFIX = "?"; + public static final String DOC_PREFIX = "?"; - @Override - public String getCommand() { - return DOC_PREFIX; - } + @Override + public String getCommand() { + return DOC_PREFIX; + } - @Override - public Function getMatcher() { - return (input) -> StringUtils.startsWith(input, DOC_PREFIX); - } + @Override + public Function getMatcher() { + return (input) -> StringUtils.startsWith(input, DOC_PREFIX); + } - @Override - public StellarResult execute(String command, StellarShellExecutor executor) { - StellarResult result; + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + StellarResult result; - // expect ?functionName - String functionName = StringUtils.substring(command, 1); + // expect ?functionName + String functionName = StringUtils.substring(command, 1); - // grab any docs for the given function - Spliterator fnIterator = executor.getFunctionResolver().getFunctionInfo().spliterator(); - Optional functionInfo = StreamSupport - .stream(fnIterator, false) - .filter(info -> StringUtils.equals(functionName, info.getName())) - .findFirst(); + // grab any docs for the given function + Spliterator fnIterator = executor.getFunctionResolver().getFunctionInfo().spliterator(); + Optional functionInfo = StreamSupport + .stream(fnIterator, false) + .filter(info -> StringUtils.equals(functionName, info.getName())) + .findFirst(); - if(functionInfo.isPresent()) { - result = success(docFormat(functionInfo.get())); - } else { - result = error(String.format("No docs available for function '%s'", functionName)); - } + if (functionInfo.isPresent()) { + result = success(docFormat(functionInfo.get())); + } else { + result = error(String.format("No docs available for function '%s'", functionName)); + } - return result; - } - - /** - * Formats the Stellar function info object into a readable string. - * @param info The stellar function info object. - * @return A readable string. - */ - private String docFormat(StellarFunctionInfo info) { - StringBuffer docString = new StringBuffer(); - - // name - docString.append(info.getName() + "\n"); - - // description - docString.append(String.format("Description: %-60s\n\n", info.getDescription())); - - // params - if(info.getParams().length > 0) { - docString.append("Arguments:\n"); - for(String param : info.getParams()) { - docString.append(String.format("\t%-60s\n", param)); - } - docString.append("\n"); + return result; } - // returns - docString.append(String.format("Returns: %-60s\n", info.getReturns())); - - return docString.toString(); - } + /** + * Formats the Stellar function info object into a readable string. + * + * @param info The stellar function info object. + * @return A readable string. + */ + private String docFormat(StellarFunctionInfo info) { + StringBuffer docString = new StringBuffer(); + + // name + docString.append(info.getName() + "\n"); + + // description + docString.append(String.format("Description: %-60s\n\n", info.getDescription())); + + // params + if (info.getParams().length > 0) { + docString.append("Arguments:\n"); + for (String param : info.getParams()) { + docString.append(String.format("\t%-60s\n", param)); + } + docString.append("\n"); + } + + // returns + docString.append(String.format("Returns: %-60s\n", info.getReturns())); + + return docString.toString(); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobal.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobal.java index 6e094f95..dc038c9d 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobal.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobal.java @@ -16,71 +16,72 @@ * limitations under the License. * */ -package org.apache.metron.stellar.common.shell.specials; - -import org.apache.commons.lang3.StringUtils; -import org.apache.metron.stellar.common.StellarAssignment; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; -import java.util.function.Function; +package org.apache.metron.stellar.common.shell.specials; import static org.apache.commons.lang3.StringUtils.startsWith; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.apache.metron.stellar.common.shell.StellarResult.error; import static org.apache.metron.stellar.common.shell.StellarResult.success; +import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.StellarAssignment; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; + /** * Allows a variable to be defined (or redefined) a within the global configuration. * - * %define newVar := newValue + *

    + * %define newVar := newValue */ public class MagicDefineGlobal implements SpecialCommand { - public static final String MAGIC_DEFINE = "%define"; + public static final String MAGIC_DEFINE = "%define"; - @Override - public String getCommand() { - return MAGIC_DEFINE; - } + @Override + public String getCommand() { + return MAGIC_DEFINE; + } - @Override - public Function getMatcher() { - return (input) -> startsWith(trimToEmpty(input), MAGIC_DEFINE); - } + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_DEFINE); + } - @Override - public StellarResult execute(String command, StellarShellExecutor executor) { + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { - // grab the expression in '%define ' - String assignExpr = StringUtils.trimToEmpty(command.substring(MAGIC_DEFINE.length())); - if(StringUtils.length(assignExpr) < 1) { - return error(MAGIC_DEFINE + " missing assignment expression"); - } + // grab the expression in '%define ' + String assignExpr = StringUtils.trimToEmpty(command.substring(MAGIC_DEFINE.length())); + if (StringUtils.length(assignExpr) < 1) { + return error(MAGIC_DEFINE + " missing assignment expression"); + } - // the expression must be an assignment - if(!StellarAssignment.isAssignment(assignExpr)) { - return error(MAGIC_DEFINE + " expected assignment expression"); - } + // the expression must be an assignment + if (!StellarAssignment.isAssignment(assignExpr)) { + return error(MAGIC_DEFINE + " expected assignment expression"); + } - // execute the expression - StellarAssignment expr = StellarAssignment.from(assignExpr); - StellarResult result = executor.execute(expr.getStatement()); + // execute the expression + StellarAssignment expr = StellarAssignment.from(assignExpr); + StellarResult result = executor.execute(expr.getStatement()); - // execution must be successful - if(!result.isSuccess()) { - return error(MAGIC_DEFINE + " expression execution failed"); - } + // execution must be successful + if (!result.isSuccess()) { + return error(MAGIC_DEFINE + " expression execution failed"); + } - // expression should have a result - if(!result.getValue().isPresent()) { - return error(MAGIC_DEFINE + " expression produced no result"); - } + // expression should have a result + if (!result.getValue().isPresent()) { + return error(MAGIC_DEFINE + " expression produced no result"); + } - // alter the global configuration - Object value = result.getValue().get(); - executor.getGlobalConfig().put(expr.getVariable(), value); + // alter the global configuration + Object value = result.getValue().get(); + executor.getGlobalConfig().put(expr.getVariable(), value); - return success(value); - } + return success(value); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctions.java index 22fc0021..24973864 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctions.java @@ -16,57 +16,59 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell.specials; -import org.apache.commons.lang3.StringUtils; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; +import static org.apache.commons.lang3.StringUtils.startsWith; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.StreamSupport; - -import static org.apache.commons.lang3.StringUtils.*; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; /** * A MagicCommand that lists the functions available within * a Stellar execution environment. * - * %functions + *

    + * %functions */ public class MagicListFunctions implements SpecialCommand { - public static final String MAGIC_FUNCTIONS = "%functions"; + public static final String MAGIC_FUNCTIONS = "%functions"; - @Override - public String getCommand() { - return MAGIC_FUNCTIONS; - } + @Override + public String getCommand() { + return MAGIC_FUNCTIONS; + } - @Override - public Function getMatcher() { - return (input) -> startsWith(trimToEmpty(input), MAGIC_FUNCTIONS); - } + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_FUNCTIONS); + } - @Override - public StellarResult execute(String command, StellarShellExecutor executor) { + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { - // if '%functions FOO' then show only functions that contain 'FOO' - String startsWith = StringUtils.trimToEmpty(command.substring(MAGIC_FUNCTIONS.length())); - Predicate nameFilter = (name -> true); - if (StringUtils.isNotBlank(startsWith)) { - nameFilter = (name -> name.contains(startsWith)); - } + // if '%functions FOO' then show only functions that contain 'FOO' + String startsWith = StringUtils.trimToEmpty(command.substring(MAGIC_FUNCTIONS.length())); + Predicate nameFilter = (name -> true); + if (StringUtils.isNotBlank(startsWith)) { + nameFilter = (name -> name.contains(startsWith)); + } - // '%functions' -> list all functions in scope - String functions = StreamSupport - .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false) - .map(info -> String.format("%s", info.getName())) - .filter(nameFilter) - .sorted() - .collect(Collectors.joining(", ")); + // '%functions' -> list all functions in scope + String functions = StreamSupport + .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false) + .map(info -> String.format("%s", info.getName())) + .filter(nameFilter) + .sorted() + .collect(Collectors.joining(", ")); - return StellarResult.success(functions); - } + return StellarResult.success(functions); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobals.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobals.java index 09a66796..ff640ce3 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobals.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobals.java @@ -16,39 +16,40 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell.specials; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; +import static org.apache.commons.lang3.StringUtils.startsWith; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; import java.util.Map; import java.util.function.Function; - -import static org.apache.commons.lang3.StringUtils.startsWith; -import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; /** * Displays all currently defined global configuration variables. * - * %globals + *

    + * %globals */ public class MagicListGlobals implements SpecialCommand { - public static final String MAGIC_GLOBALS = "%globals"; + public static final String MAGIC_GLOBALS = "%globals"; - @Override - public String getCommand() { - return MAGIC_GLOBALS; - } + @Override + public String getCommand() { + return MAGIC_GLOBALS; + } - @Override - public Function getMatcher() { - return (input) -> startsWith(trimToEmpty(input), MAGIC_GLOBALS); - } + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_GLOBALS); + } - @Override - public StellarResult execute(String command, StellarShellExecutor executor) { - Map globals = executor.getGlobalConfig(); - return StellarResult.success(globals.toString()); - } + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + Map globals = executor.getGlobalConfig(); + return StellarResult.success(globals.toString()); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListVariables.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListVariables.java index 5669c44a..812d4177 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListVariables.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicListVariables.java @@ -16,73 +16,74 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell.specials; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; -import org.apache.metron.stellar.common.shell.VariableResult; +import static org.apache.commons.lang3.StringUtils.startsWith; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.apache.metron.stellar.common.shell.StellarResult.success; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; - -import static org.apache.commons.lang3.StringUtils.startsWith; -import static org.apache.commons.lang3.StringUtils.trimToEmpty; -import static org.apache.metron.stellar.common.shell.StellarResult.success; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.VariableResult; /** * A MagicCommand that lists the variables available within * the Stellar execution environment. * - * %vars + *

    + * %vars */ public class MagicListVariables implements SpecialCommand { - public static final String MAGIC_VARS = "%vars"; + public static final String MAGIC_VARS = "%vars"; - @Override - public String getCommand() { - return MAGIC_VARS; - } + @Override + public String getCommand() { + return MAGIC_VARS; + } - @Override - public Function getMatcher() { - return (input) -> startsWith(trimToEmpty(input), MAGIC_VARS); - } + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_VARS); + } + + /** + * Lists each variable, its value, and if available, the expression that resulted in that value. + * + *

    + * x = 4 via `2 + 2` + * + * @param command The command to execute. + * @param executor A stellar execution environment. + */ + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { - /** - * Lists each variable, its value, and if available, the expression that resulted in that value. - * - * x = 4 via `2 + 2` - * - * @param command The command to execute. - * @param executor A stellar execution environment. - * @return - */ - @Override - public StellarResult execute(String command, StellarShellExecutor executor) { + // format a string containing each variable and it's value + String vars = executor + .getState() + .entrySet() + .stream() + .map(this::format) + .collect(Collectors.joining(", ")); - // format a string containing each variable and it's value - String vars = executor - .getState() - .entrySet() - .stream() - .map(e -> format(e)) - .collect(Collectors.joining(", ")); + return success(vars); + } - return success(vars); - } + private String format(Map.Entry var) { - private String format(Map.Entry var) { + // 'varName = varValue' + String out = String.format("%s = %s", var.getKey(), var.getValue().getResult()); - // 'varName = varValue' - String out = String.format("%s = %s", var.getKey(), var.getValue().getResult()); + // 'via varExpression', if the expression is known + if (var.getValue().getExpression().isPresent()) { + out += String.format(" via `%s`", var.getValue().getExpression().get()); + } - // 'via varExpression', if the expression is known - if(var.getValue().getExpression().isPresent()) { - out += String.format(" via `%s`", var.getValue().getExpression().get()); + return out; } - - return out; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobal.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobal.java index b7727e7d..adf98dfa 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobal.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobal.java @@ -16,55 +16,56 @@ * limitations under the License. * */ -package org.apache.metron.stellar.common.shell.specials; - -import org.apache.commons.lang3.StringUtils; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; -import java.util.Map; -import java.util.function.Function; +package org.apache.metron.stellar.common.shell.specials; import static org.apache.commons.lang3.StringUtils.startsWith; import static org.apache.commons.lang3.StringUtils.trimToEmpty; -import static org.apache.metron.stellar.common.shell.StellarResult.noop; import static org.apache.metron.stellar.common.shell.StellarResult.error; +import static org.apache.metron.stellar.common.shell.StellarResult.noop; + +import java.util.Map; +import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; /** * Allows a variable to be removed from the global configuration. * - * %undefine varName + *

    + * %undefine varName */ public class MagicUndefineGlobal implements SpecialCommand { - public static final String MAGIC_UNDEFINE = "%undefine"; + public static final String MAGIC_UNDEFINE = "%undefine"; - @Override - public String getCommand() { - return MAGIC_UNDEFINE; - } + @Override + public String getCommand() { + return MAGIC_UNDEFINE; + } - @Override - public Function getMatcher() { - return (input) -> startsWith(trimToEmpty(input), MAGIC_UNDEFINE); - } + @Override + public Function getMatcher() { + return (input) -> startsWith(trimToEmpty(input), MAGIC_UNDEFINE); + } - @Override - public StellarResult execute(String command, StellarShellExecutor executor) { - StellarResult result; + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + StellarResult result; - String variable = StringUtils.trimToEmpty(command.substring(MAGIC_UNDEFINE.length())); - if(StringUtils.isNotBlank(variable)) { + String variable = StringUtils.trimToEmpty(command.substring(MAGIC_UNDEFINE.length())); + if (StringUtils.isNotBlank(variable)) { - // remove the variable from the globals - Map globals = executor.getGlobalConfig(); - globals.remove(variable); - result = noop(); + // remove the variable from the globals + Map globals = executor.getGlobalConfig(); + globals.remove(variable); + result = noop(); - } else { - result = error(String.format("%s expected name of global, got '%s'", MAGIC_UNDEFINE, variable)); - } + } else { + result = error(String.format("%s expected name of global, got '%s'", MAGIC_UNDEFINE, variable)); + } - return result; - } + return result; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/QuitCommand.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/QuitCommand.java index 7786d4d2..ad777b18 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/QuitCommand.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/QuitCommand.java @@ -16,36 +16,37 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.shell.specials; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; +import static org.apache.metron.stellar.common.shell.StellarResult.terminate; import java.util.function.Function; - -import static org.apache.metron.stellar.common.shell.StellarResult.terminate; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; /** * A special command that allows the user to 'quit' their REPL session. * - * quit + *

    + * quit */ public class QuitCommand implements SpecialCommand { - public static final String QUIT_COMMAND = "quit"; + public static final String QUIT_COMMAND = "quit"; - @Override - public String getCommand() { - return QUIT_COMMAND; - } + @Override + public String getCommand() { + return QUIT_COMMAND; + } - @Override - public Function getMatcher() { - return (input) -> QUIT_COMMAND.equals(input); - } + @Override + public Function getMatcher() { + return QUIT_COMMAND::equals; + } - @Override - public StellarResult execute(String command, StellarShellExecutor executor) { - return terminate(); - } + @Override + public StellarResult execute(String command, StellarShellExecutor executor) { + return terminate(); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/SpecialCommand.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/SpecialCommand.java index a40e1e53..5e8d688d 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/SpecialCommand.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/shell/specials/SpecialCommand.java @@ -20,41 +20,47 @@ package org.apache.metron.stellar.common.shell.specials; -import org.apache.metron.stellar.common.shell.StellarShellExecutor; -import org.apache.metron.stellar.common.shell.StellarResult; - import java.util.function.Function; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; /** * A special command that can be run within a Stellar execution * environment. * + *

    * Most functionality expected of running Stellar in a shell-like * environment, that is not directly implemented in the Stellar * language itself, is implemented as a SpecialCommand. This * includes magics, doc strings, comments and quit. * + *

    * This is typically an action performed on the execution * environment, not something that could be executed within Stellar. */ public interface SpecialCommand { - /** - * @return A function that when applied to the input buffer returns - * true, if this special command should be applied. - */ - Function getMatcher(); + /** + * Matcher getter. + * + * @return A function that when applied to the input buffer returns + * true, if this special command should be applied. + */ + Function getMatcher(); - /** - * @return The name of the command, used for auto-completion. - */ - String getCommand(); + /** + * Command getter. + * + * @return The name of the command, used for auto-completion. + */ + String getCommand(); - /** - * Execute the magic command. - * @param expression The expression to execute. - * @param executor A stellar execution environment. - * @return The result of executing the magic command. - */ - StellarResult execute(String expression, StellarShellExecutor executor); + /** + * Execute the magic command. + * + * @param expression The expression to execute. + * @param executor A stellar execution environment. + * @return The result of executing the magic command. + */ + StellarResult execute(String expression, StellarShellExecutor executor); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/system/Clock.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/system/Clock.java index 66c9e9d6..557f3877 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/system/Clock.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/system/Clock.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.system; import java.text.SimpleDateFormat; @@ -22,16 +25,16 @@ import java.util.TimeZone; public class Clock { - private final static String UTC = "UTC"; + private static final String UTC = "UTC"; - public long currentTimeMillis() { - return System.currentTimeMillis(); - } + public long currentTimeMillis() { + return System.currentTimeMillis(); + } - public String currentTimeFormatted(String stdDateFormat) { - SimpleDateFormat format = new SimpleDateFormat(stdDateFormat); - format.setTimeZone(TimeZone.getTimeZone(UTC)); - return format.format(new Date(currentTimeMillis())); - } + public String currentTimeFormatted(String stdDateFormat) { + SimpleDateFormat format = new SimpleDateFormat(stdDateFormat); + format.setTimeZone(TimeZone.getTimeZone(UTC)); + return format.format(new Date(currentTimeMillis())); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/system/Environment.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/system/Environment.java index 2ad4b196..79432f3d 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/system/Environment.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/system/Environment.java @@ -6,22 +6,25 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.system; /** - * Useful so we can test mock dependency injection with environment variables + * Useful so we can test mock dependency injection with environment variables. */ public class Environment { - public String get(String variable) { - return System.getenv().get(variable); - } + public String get(String variable) { + return System.getenv().get(variable); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/BloomFilter.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/BloomFilter.java index f2057709..dbf93c60 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/BloomFilter.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/BloomFilter.java @@ -6,87 +6,95 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.utils; import com.google.common.hash.Funnel; import com.google.common.hash.PrimitiveSink; - import java.io.Serializable; +import java.util.Objects; import java.util.function.Function; public class BloomFilter implements Serializable { - private static class BloomFunnel implements Funnel, Serializable { + private static class BloomFunnel implements Funnel, Serializable { - Function serializer; + Function serializer; - public BloomFunnel(Function serializer) { - this.serializer = serializer; - } + public BloomFunnel(Function serializer) { + this.serializer = serializer; + } - @Override - public void funnel(T obj, PrimitiveSink primitiveSink) { - primitiveSink.putBytes(serializer.apply(obj)); - } + @Override + public void funnel(T obj, PrimitiveSink primitiveSink) { + primitiveSink.putBytes(serializer.apply(obj)); + } - @Override - public boolean equals(Object obj) { - return this.getClass().equals(obj.getClass()); - } + @Override + public boolean equals(Object obj) { + return this.getClass().equals(obj.getClass()); + } - @Override - public int hashCode() { - return super.hashCode() * 31; + @Override + public int hashCode() { + return super.hashCode() * 31; + } } - } - public static class DefaultSerializer implements Function, Serializable { - @Override - public byte[] apply(T t) { - return SerDeUtils.toBytes(t); + public static class DefaultSerializer implements Function, Serializable { + @Override + public byte[] apply(T t) { + return SerDeUtils.toBytes(t); + } } - } - private com.google.common.hash.BloomFilter filter; + private final com.google.common.hash.BloomFilter filter; - public BloomFilter(Function serializer, int expectedInsertions, double falsePositiveRate) { - filter = com.google.common.hash.BloomFilter.create(new BloomFunnel(serializer), expectedInsertions, falsePositiveRate); - } + public BloomFilter(Function serializer, int expectedInsertions, double falsePositiveRate) { + filter = com.google.common.hash.BloomFilter.create(new BloomFunnel(serializer), expectedInsertions, + falsePositiveRate); + } - public boolean mightContain(T key) { - return filter.mightContain(key); - } + public boolean mightContain(T key) { + return filter.mightContain(key); + } - public void add(T key) { - filter.put(key); - } + public void add(T key) { + filter.put(key); + } - public void merge(BloomFilter filter2) { - filter.putAll(filter2.filter); - } + public void merge(BloomFilter filter2) { + filter.putAll(filter2.filter); + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - BloomFilter that = (BloomFilter) o; + BloomFilter that = (BloomFilter) o; - return filter != null ? filter.equals(that.filter) : that.filter == null; + return Objects.equals(filter, that.filter); - } + } - @Override - public int hashCode() { - return filter != null ? filter.hashCode() : 0; - } + @Override + public int hashCode() { + return filter != null ? filter.hashCode() : 0; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java index 1f17bf4d..9246f2e0 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,12 +28,12 @@ import com.google.common.base.Joiner; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; - import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; /** @@ -39,221 +41,210 @@ * a union of all of the maps, but rather keeps the maps separate. Key/Value resolution is * done via a first-wins strategy (i.e. the first map which has a key will be used). * + *

    * Also, note, that this is an immutable map, so operations which require mutation will have * UnsupportedOperationException thrown. */ public class ConcatMap implements Map, Serializable, KryoSerializable { - List variableMappings = new ArrayList<>(); - public ConcatMap(List variableMappings) { - this.variableMappings = variableMappings; - } + List variableMappings = new ArrayList<>(); - @Override - public int size() { - int size = 0; - for(Map m : variableMappings) { - size += m.size(); + public ConcatMap(List variableMappings) { + this.variableMappings = variableMappings; } - return size; - } - @Override - public boolean isEmpty() { - boolean isEmpty = true; - for(Map m : variableMappings) { - isEmpty &= m.isEmpty(); + @Override + public int size() { + int size = 0; + for (Map m : variableMappings) { + size += m.size(); + } + return size; } - return isEmpty; - } - /** - * If any maps contains the key, then this will return true. - * @param key - * @return - */ - @Override - public boolean containsKey(Object key) { - for(Map m : variableMappings) { - if(m.containsKey(key)) { - return true; - } + @Override + public boolean isEmpty() { + boolean isEmpty = true; + for (Map m : variableMappings) { + isEmpty &= m.isEmpty(); + } + return isEmpty; } - return false; - } - /** - * - * If any maps contains the value, then this will return true. - * @param value - * @return - */ - @Override - public boolean containsValue(Object value) { - for(Map m : variableMappings) { - if(m.containsValue(value)) { - return true; - } + /** + * If any maps contains the key, then this will return true. + * + */ + @Override + public boolean containsKey(Object key) { + for (Map m : variableMappings) { + if (m.containsKey(key)) { + return true; + } + } + return false; } - return false; - } - /** - * The first map which contains the key will have the associated value returned. - * @param key - * @return - */ - @Override - public Object get(Object key) { - Object ret = null; - for(Map m : variableMappings) { - ret = m.get(key); - if(ret != null) { - break; - } + /** + * If any maps contains the value, then this will return true. + * + */ + @Override + public boolean containsValue(Object value) { + for (Map m : variableMappings) { + if (m.containsValue(value)) { + return true; + } + } + return false; } - return ret; - } - /** - * This is an immutable map and this operation is not supported. - * @param key - * @param value - * @return - */ - @Override - public Object put(String key, Object value) { - throw new UnsupportedOperationException("Merged map is immutable."); - } + /** + * The first map which contains the key will have the associated value returned. + */ + @Override + public Object get(Object key) { + Object ret = null; + for (Map m : variableMappings) { + ret = m.get(key); + if (ret != null) { + break; + } + } + return ret; + } + + /** + * This is an immutable map and this operation is not supported. + */ + @Override + public Object put(String key, Object value) { + throw new UnsupportedOperationException("Merged map is immutable."); + } - /** - * - * This is an immutable map and this operation is not supported. - * @param key - * @return - */ - @Override - public Object remove(Object key) { - throw new UnsupportedOperationException("Merged map is immutable."); - } + /** + * This is an immutable map and this operation is not supported. + */ + @Override + public Object remove(Object key) { + throw new UnsupportedOperationException("Merged map is immutable."); + } - /** - * - * This is an immutable map and this operation is not supported. - * @param m - */ - @Override - public void putAll(Map m) { - throw new UnsupportedOperationException("Merged map is immutable."); - } + /** + * This is an immutable map and this operation is not supported. + */ + @Override + public void putAll(Map m) { + throw new UnsupportedOperationException("Merged map is immutable."); + } - /** - * - * This is an immutable map and this operation is not supported. - */ - @Override - public void clear() { - throw new UnsupportedOperationException("Merged map is immutable."); - } + /** + * This is an immutable map and this operation is not supported. + */ + @Override + public void clear() { + throw new UnsupportedOperationException("Merged map is immutable."); + } - @Override - @SuppressWarnings("unchecked") - public Set keySet() { - Set ret = null; - for (Map m : variableMappings) { - if (ret == null) { - ret = m.keySet(); - } - else { - ret = Sets.union(ret, m.keySet()); - } + @Override + @SuppressWarnings("unchecked") + public Set keySet() { + Set ret = null; + for (Map m : variableMappings) { + if (ret == null) { + ret = m.keySet(); + } else { + ret = Sets.union(ret, m.keySet()); + } + } + return ret; } - return ret; - } - /** - * Note: this makes a copy of the values, so it is not fundamentally lazy. - * @return - */ - @Override - @SuppressWarnings("unchecked") - public Collection values() { - Collection ret = new ArrayList<>(size()); - for (Map m : variableMappings) { - ret.addAll(m.values()); + /** + * Note: this makes a copy of the values, so it is not fundamentally lazy. + */ + @Override + @SuppressWarnings("unchecked") + public Collection values() { + Collection ret = new ArrayList<>(size()); + for (Map m : variableMappings) { + ret.addAll(m.values()); + } + return ret; } - return ret; - } - /** - * This is a lazy entry collection of the associated maps. If there are duplicate keys, they will appear - * twice here, so be careful. - * @return - */ - @Override - @SuppressWarnings("unchecked") - public Set> entrySet() { - Set> ret = null; - for (Map m : variableMappings) { - if (ret == null) { - ret = m.entrySet(); - } else { - ret = Sets.union(ret, m.entrySet()); - } + /** + * This is a lazy entry collection of the associated maps. If there are duplicate keys, they will appear + * twice here, so be careful. + */ + @Override + @SuppressWarnings("unchecked") + public Set> entrySet() { + Set> ret = null; + for (Map m : variableMappings) { + if (ret == null) { + ret = m.entrySet(); + } else { + ret = Sets.union(ret, m.entrySet()); + } + } + return ret; } - return ret; - } - @Override - @SuppressWarnings("unchecked") - public String toString() { - Iterable>> transformed = - Iterables.transform(variableMappings, x -> x.entrySet()); - Iterable> it = Iterables.filter( Iterables.concat(transformed) - , x -> x.getValue() != null - ); - return "{" + Joiner.on(", ").join(it) + "}"; - } + @Override + @SuppressWarnings("unchecked") + public String toString() { + Iterable>> transformed = + Iterables.transform(variableMappings, x -> x.entrySet()); + Iterable> it = Iterables.filter(Iterables.concat(transformed), + x -> x.getValue() != null + ); + return "{" + Joiner.on(", ").join(it) + "}"; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - ConcatMap concatMap = (ConcatMap) o; + ConcatMap concatMap = (ConcatMap) o; - return variableMappings != null ? variableMappings.equals(concatMap.variableMappings) : concatMap.variableMappings == null; + return Objects.equals(variableMappings, concatMap.variableMappings); - } + } - @Override - public int hashCode() { - return variableMappings != null ? variableMappings.hashCode() : 0; - } + @Override + public int hashCode() { + return variableMappings != null ? variableMappings.hashCode() : 0; + } - @Override - public void write(Kryo kryo, Output output) { - int numVariableMappings = variableMappings.isEmpty()?0:variableMappings.size(); - output.writeShort(numVariableMappings); - for(Map m : variableMappings) { - byte[] b = m == null?new byte[]{}:SerDeUtils.toBytes(m); - output.writeInt(b.length); - if(b.length > 0) { - output.writeBytes(b); - } + @Override + public void write(Kryo kryo, Output output) { + int numVariableMappings = variableMappings.isEmpty() ? 0 : variableMappings.size(); + output.writeShort(numVariableMappings); + for (Map m : variableMappings) { + byte[] b = m == null ? new byte[] {} : SerDeUtils.toBytes(m); + output.writeInt(b.length); + if (b.length > 0) { + output.writeBytes(b); + } + } } - } - @Override - public void read(Kryo kryo, Input input) { - int numVariableMappings = input.readShort(); - variableMappings = new ArrayList<>(numVariableMappings); - for(int i = 0;i < numVariableMappings;++i) { - int size = input.readInt(); - if(size > 0) { - byte[] bytes = input.readBytes(size); - Map m = SerDeUtils.fromBytes(bytes, Map.class); - variableMappings.add(m); - } + @Override + public void read(Kryo kryo, Input input) { + int numVariableMappings = input.readShort(); + variableMappings = new ArrayList<>(numVariableMappings); + for (int i = 0; i < numVariableMappings; ++i) { + int size = input.readInt(); + if (size > 0) { + byte[] bytes = input.readBytes(size); + Map m = SerDeUtils.fromBytes(bytes, Map.class); + variableMappings.add(m); + } + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/ConversionUtils.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/ConversionUtils.java index 783afaeb..63fe1194 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/ConversionUtils.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/ConversionUtils.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,48 +28,48 @@ import org.apache.commons.beanutils.ConvertUtilsBean; public class ConversionUtils { - private static ThreadLocal UTILS_BEAN = new ThreadLocal() { - @Override - protected ConvertUtilsBean initialValue() { - ConvertUtilsBean ret = BeanUtilsBean2.getInstance().getConvertUtils(); - ret.deregister(); - ret.register(false, true, 1); - return ret; - } - }; + private static final ThreadLocal UTILS_BEAN = new ThreadLocal() { + @Override + protected ConvertUtilsBean initialValue() { + ConvertUtilsBean ret = BeanUtilsBean2.getInstance().getConvertUtils(); + ret.deregister(); + ret.register(false, true, 1); + return ret; + } + }; - public static T convert(Object o, Class clazz) { - if (o == null) { - return null; + public static T convert(Object o, Class clazz) { + if (o == null) { + return null; + } + return clazz.cast(UTILS_BEAN.get().convert(o, clazz)); } - return clazz.cast(UTILS_BEAN.get().convert(o, clazz)); - } - /** - * Performs naive List type conversion. - * - * @param from Source list - * @param clazz Class type to cast the List elements to - * @param Source element type - * @param Desired element type - * @return New List with the elements cast to the desired type - */ - public static List convertList(List from, Class clazz) { - return Lists.transform(from, s -> convert(s, clazz)); - } + /** + * Performs naive List type conversion. + * + * @param from Source list + * @param clazz Class type to cast the List elements to + * @param Source element type + * @param Desired element type + * @return New List with the elements cast to the desired type + */ + public static List convertList(List from, Class clazz) { + return Lists.transform(from, s -> convert(s, clazz)); + } - /** - * Performs naive Map type conversion on values. Key types remain unchanged. - * - * @param from Source map - * @param clazz Class type to cast the Map values to - * @param Map key type - * @param Source value type - * @param Desired value type - * @return New Map with the values cast to the desired type - */ - public static Map convertMap(Map from, Class clazz) { - return Maps.transformValues(from, s -> convert(s, clazz)); - } + /** + * Performs naive Map type conversion on values. Key types remain unchanged. + * + * @param from Source map + * @param clazz Class type to cast the Map values to + * @param Map key type + * @param Source value type + * @param Desired value type + * @return New Map with the values cast to the desired type + */ + public static Map convertMap(Map from, Class clazz) { + return Maps.transformValues(from, s -> convert(s, clazz)); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java index 90620a8a..e3cb8cfd 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,94 +34,98 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; - import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; public enum JSONUtils { - INSTANCE; - - public static class ReferenceSupplier implements Supplier> { - Type type; - protected ReferenceSupplier() { - Type superClass = this.getClass().getGenericSuperclass(); - if(superClass instanceof Class) { - throw new IllegalArgumentException("Internal error: ReferenceSupplier constructed without actual type information"); - } else { - this.type = ((ParameterizedType)superClass).getActualTypeArguments()[0]; - } - } + INSTANCE; + + public static class ReferenceSupplier implements Supplier> { + Type type; + + protected ReferenceSupplier() { + Type superClass = this.getClass().getGenericSuperclass(); + if (superClass instanceof Class) { + throw new IllegalArgumentException( + "Internal error: ReferenceSupplier constructed without actual type information"); + } else { + this.type = ((ParameterizedType) superClass).getActualTypeArguments()[0]; + } + } - @Override - public TypeReference get() { - return new TypeReference() { @Override - public Type getType() { - return type; + public TypeReference get() { + return new TypeReference() { + @Override + public Type getType() { + return type; + } + }; } - }; } - } - public final static ReferenceSupplier> MAP_SUPPLIER = new ReferenceSupplier>(){}; - public final static ReferenceSupplier> LIST_SUPPLIER = new ReferenceSupplier>(){}; + public static final ReferenceSupplier> MAP_SUPPLIER = + new ReferenceSupplier>() { + }; + public static final ReferenceSupplier> LIST_SUPPLIER = new ReferenceSupplier>() { + }; - private static ThreadLocal _parser = ThreadLocal.withInitial(() -> + private static final ThreadLocal _parser = ThreadLocal.withInitial(() -> new JSONParser()); - private static ThreadLocal _mapper = ThreadLocal.withInitial(() -> + private static final ThreadLocal _mapper = ThreadLocal.withInitial(() -> new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL)); - public T load(InputStream is, ReferenceSupplier ref) throws IOException { - return _mapper.get().readValue(is, (TypeReference)ref.get()); - } + public T load(InputStream is, ReferenceSupplier ref) throws IOException { + return _mapper.get().readValue(is, ref.get()); + } + + public T load(String is, ReferenceSupplier ref) throws IOException { + return _mapper.get().readValue(is, ref.get()); + } + + public T load(File f, ReferenceSupplier ref) throws IOException { + try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { + return _mapper.get().readValue(is, ref.get()); + } + } + + public T load(InputStream is, Class clazz) throws IOException { + return _mapper.get().readValue(is, clazz); + } - public T load(String is, ReferenceSupplier ref) throws IOException { - return _mapper.get().readValue(is, (TypeReference)ref.get()); - } + public T load(File f, Class clazz) throws IOException { + try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { + return _mapper.get().readValue(is, clazz); + } + } - public T load(File f, ReferenceSupplier ref) throws IOException { - try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { - return _mapper.get().readValue(is, (TypeReference)ref.get()); + public T load(String is, Class clazz) throws IOException { + return _mapper.get().readValue(is, clazz); } - } - public T load(InputStream is, Class clazz) throws IOException { - return _mapper.get().readValue(is, clazz); - } + public String toJSON(Object o, boolean pretty) throws JsonProcessingException { + if (pretty) { + return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsString(o); + } else { + return _mapper.get().writeValueAsString(o); + } + } - public T load(File f, Class clazz) throws IOException { - try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { - return _mapper.get().readValue(is, clazz); + public byte[] toJSON(Object config) throws JsonProcessingException { + return _mapper.get().writeValueAsBytes(config); } - } - public T load(String is, Class clazz) throws IOException { - return _mapper.get().readValue(is, clazz); - } + /** + * Transforms a bean (aka POJO) to a JSONObject. + */ + public JSONObject toJSONObject(Object o) throws JsonProcessingException, ParseException { + return toJSONObject(toJSON(o, false)); + } - public String toJSON(Object o, boolean pretty) throws JsonProcessingException { - if (pretty) { - return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsString(o); - } else { - return _mapper.get().writeValueAsString(o); + public JSONObject toJSONObject(String json) throws ParseException { + return (JSONObject) _parser.get().parse(json); } - } - - public byte[] toJSON(Object config) throws JsonProcessingException { - return _mapper.get().writeValueAsBytes(config); - } - - /** - * Transforms a bean (aka POJO) to a JSONObject. - */ - public JSONObject toJSONObject(Object o) throws JsonProcessingException, ParseException { - return toJSONObject(toJSON(o, false)); - } - - public JSONObject toJSONObject(String json) throws ParseException { - return (JSONObject) _parser.get().parse(json); - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/PatternCache.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/PatternCache.java index f0fcee98..69b53547 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/PatternCache.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/PatternCache.java @@ -15,23 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.utils; import java.util.HashMap; import java.util.regex.Pattern; public enum PatternCache { - INSTANCE; + INSTANCE; - private static final ThreadLocal> _cache = ThreadLocal.withInitial(() -> + private static final ThreadLocal> _cache = ThreadLocal.withInitial(() -> new HashMap<>()); - public Pattern getPattern(String patternString){ - Pattern pattern = _cache.get().get(patternString); - if(pattern == null){ - pattern = Pattern.compile(patternString); - _cache.get().put(patternString,pattern); + public Pattern getPattern(String patternString) { + Pattern pattern = _cache.get().get(patternString); + if (pattern == null) { + pattern = Pattern.compile(patternString); + _cache.get().put(patternString, pattern); + } + return pattern; } - return pattern; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/SerDeUtils.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/SerDeUtils.java index aeaa6eb7..982dfd81 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/SerDeUtils.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/SerDeUtils.java @@ -20,6 +20,8 @@ package org.apache.metron.stellar.common.utils; +import static com.esotericsoftware.kryo.util.Util.className; + import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.KryoException; import com.esotericsoftware.kryo.io.Input; @@ -51,7 +53,6 @@ import java.lang.reflect.Modifier; import java.time.LocalDate; import java.time.LocalDateTime; -import java.util.Arrays; import java.util.Collections; import java.util.GregorianCalendar; import java.util.function.Function; @@ -62,211 +63,210 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.esotericsoftware.kryo.util.Util.className; - /** * Provides basic functionality to serialize and deserialize the allowed * value types for a ProfileMeasurement. */ public class SerDeUtils { - protected static final Logger LOG = LoggerFactory.getLogger(SerDeUtils.class); - private static ThreadLocal kryo = new ThreadLocal() { - @Override - protected Kryo initialValue() { - Kryo ret = new Kryo(); - ret.setReferences(true); - ret.setInstantiatorStrategy(new DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); + protected static final Logger LOG = LoggerFactory.getLogger(SerDeUtils.class); + private static final ThreadLocal kryo = new ThreadLocal() { + @Override + protected Kryo initialValue() { + Kryo ret = new Kryo(); + ret.setReferences(true); + ret.setInstantiatorStrategy(new DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); - ret.register(Arrays.asList("").getClass(), new ArraysAsListSerializer()); - ret.register(Collections.EMPTY_LIST.getClass(), new CollectionsEmptyListSerializer()); - ret.register(Collections.EMPTY_MAP.getClass(), new CollectionsEmptyMapSerializer()); - ret.register(Collections.EMPTY_SET.getClass(), new CollectionsEmptySetSerializer()); - ret.register(Collections.singletonList("").getClass(), new CollectionsSingletonListSerializer()); - ret.register(Collections.singleton("").getClass(), new CollectionsSingletonSetSerializer()); - ret.register(Collections.singletonMap("", "").getClass(), new CollectionsSingletonMapSerializer()); - ret.register(GregorianCalendar.class, new GregorianCalendarSerializer()); - ret.register(InvocationHandler.class, new JdkProxySerializer()); - UnmodifiableCollectionsSerializer.registerSerializers(ret); - SynchronizedCollectionsSerializer.registerSerializers(ret); + ret.register(Collections.singletonList("").getClass(), new ArraysAsListSerializer()); + ret.register(Collections.EMPTY_LIST.getClass(), new CollectionsEmptyListSerializer()); + ret.register(Collections.EMPTY_MAP.getClass(), new CollectionsEmptyMapSerializer()); + ret.register(Collections.EMPTY_SET.getClass(), new CollectionsEmptySetSerializer()); + ret.register(Collections.singletonList("").getClass(), new CollectionsSingletonListSerializer()); + ret.register(Collections.singleton("").getClass(), new CollectionsSingletonSetSerializer()); + ret.register(Collections.singletonMap("", "").getClass(), new CollectionsSingletonMapSerializer()); + ret.register(GregorianCalendar.class, new GregorianCalendarSerializer()); + ret.register(InvocationHandler.class, new JdkProxySerializer()); + UnmodifiableCollectionsSerializer.registerSerializers(ret); + SynchronizedCollectionsSerializer.registerSerializers(ret); - // custom serializers for non-jdk libs + // custom serializers for non-jdk libs - // register CGLibProxySerializer, works in combination with the appropriate action in handleUnregisteredClass (see below) - ret.register(CGLibProxySerializer.CGLibProxyMarker.class, new CGLibProxySerializer()); + // register CGLibProxySerializer, works in combination with the appropriate action in handleUnregisteredClass (see below) + ret.register(CGLibProxySerializer.CGLibProxyMarker.class, new CGLibProxySerializer()); - // joda DateTime, LocalDate and LocalDateTime - ret.register(LocalDate.class, new JodaLocalDateSerializer()); - ret.register(LocalDateTime.class, new JodaLocalDateTimeSerializer()); + // joda DateTime, LocalDate and LocalDateTime + ret.register(LocalDate.class, new JodaLocalDateSerializer()); + ret.register(LocalDateTime.class, new JodaLocalDateTimeSerializer()); - // guava ImmutableList, ImmutableSet, ImmutableMap, ImmutableMultimap, UnmodifiableNavigableSet - ImmutableListSerializer.registerSerializers(ret); - ImmutableSetSerializer.registerSerializers(ret); - ImmutableMapSerializer.registerSerializers(ret); - ImmutableMultimapSerializer.registerSerializers(ret); - return ret; - } - }; + // guava ImmutableList, ImmutableSet, ImmutableMap, ImmutableMultimap, UnmodifiableNavigableSet + ImmutableListSerializer.registerSerializers(ret); + ImmutableSetSerializer.registerSerializers(ret); + ImmutableMapSerializer.registerSerializers(ret); + ImmutableMultimapSerializer.registerSerializers(ret); + return ret; + } + }; - /** - * This was backported from a more recent version of kryo than we currently run. The reason why it exists is - * that we want a strategy for instantiation of classes which attempts a no-arg constructor first and THEN falls - * back to reflection for performance reasons alone (this is, after all, in the critical path). - * - */ - static private class DefaultInstantiatorStrategy implements org.objenesis.strategy.InstantiatorStrategy { - private InstantiatorStrategy fallbackStrategy; + /** + * This was backported from a more recent version of kryo than we currently run. The reason why it exists is + * that we want a strategy for instantiation of classes which attempts a no-arg constructor first and THEN falls + * back to reflection for performance reasons alone (this is, after all, in the critical path). + */ + private static class DefaultInstantiatorStrategy implements org.objenesis.strategy.InstantiatorStrategy { + private InstantiatorStrategy fallbackStrategy; - public DefaultInstantiatorStrategy () { - } + public DefaultInstantiatorStrategy() { + } - public DefaultInstantiatorStrategy (InstantiatorStrategy fallbackStrategy) { - this.fallbackStrategy = fallbackStrategy; - } + public DefaultInstantiatorStrategy(InstantiatorStrategy fallbackStrategy) { + this.fallbackStrategy = fallbackStrategy; + } - public void setFallbackInstantiatorStrategy (final InstantiatorStrategy fallbackStrategy) { - this.fallbackStrategy = fallbackStrategy; - } + public void setFallbackInstantiatorStrategy(final InstantiatorStrategy fallbackStrategy) { + this.fallbackStrategy = fallbackStrategy; + } - public InstantiatorStrategy getFallbackInstantiatorStrategy () { - return fallbackStrategy; - } + public InstantiatorStrategy getFallbackInstantiatorStrategy() { + return fallbackStrategy; + } - @Override - @SuppressWarnings("unchecked") - public ObjectInstantiator newInstantiatorOf(final Class type) { - if (!Util.isAndroid) { - // Use ReflectASM if the class is not a non-static member class. - Class enclosingType = type.getEnclosingClass(); - boolean isNonStaticMemberClass = enclosingType != null && type.isMemberClass() - && !Modifier.isStatic(type.getModifiers()); - if (!isNonStaticMemberClass) { - try { - final ConstructorAccess access = ConstructorAccess.get(type); - return new ObjectInstantiator() { - @Override - public Object newInstance () { + @Override + @SuppressWarnings({"unchecked", "checkstyle:EmptyCatchBlock"}) + public ObjectInstantiator newInstantiatorOf(final Class type) { + if (!Util.isAndroid) { + // Use ReflectASM if the class is not a non-static member class. + Class enclosingType = type.getEnclosingClass(); + boolean isNonStaticMemberClass = enclosingType != null && type.isMemberClass() + && !Modifier.isStatic(type.getModifiers()); + if (!isNonStaticMemberClass) { + try { + final ConstructorAccess access = ConstructorAccess.get(type); + return new ObjectInstantiator() { + @Override + public Object newInstance() { + try { + return access.newInstance(); + } catch (Exception ex) { + throw new KryoException("Error constructing instance of class: " + className(type), + ex); + } + } + }; + } catch (Exception ignored) { + } + } + } + // Reflection. + try { + Constructor ctor; try { - return access.newInstance(); + ctor = type.getConstructor((Class[]) null); } catch (Exception ex) { - throw new KryoException("Error constructing instance of class: " + className(type), ex); + ctor = type.getDeclaredConstructor((Class[]) null); + ctor.setAccessible(true); } - } - }; - } catch (Exception ignored) { - } - } - } - // Reflection. - try { - Constructor ctor; - try { - ctor = type.getConstructor((Class[])null); - } catch (Exception ex) { - ctor = type.getDeclaredConstructor((Class[])null); - ctor.setAccessible(true); - } - final Constructor constructor = ctor; - return new ObjectInstantiator() { - @Override - public Object newInstance () { - try { - return constructor.newInstance(); - } catch (Exception ex) { - throw new KryoException("Error constructing instance of class: " + className(type), ex); + final Constructor constructor = ctor; + return new ObjectInstantiator() { + @Override + public Object newInstance() { + try { + return constructor.newInstance(); + } catch (Exception ex) { + throw new KryoException("Error constructing instance of class: " + className(type), ex); + } + } + }; + } catch (Exception ignored) { } - } - }; - } catch (Exception ignored) { - } - if (fallbackStrategy == null) { - if (type.isMemberClass() && !Modifier.isStatic(type.getModifiers())) - throw new KryoException("Class cannot be created (non-static member class): " + className(type)); - else - throw new KryoException("Class cannot be created (missing no-arg constructor): " + className(type)); - } - // InstantiatorStrategy. - return fallbackStrategy.newInstantiatorOf(type); + if (fallbackStrategy == null) { + if (type.isMemberClass() && !Modifier.isStatic(type.getModifiers())) { + throw new KryoException("Class cannot be created (non-static member class): " + className(type)); + } else { + throw new KryoException("Class cannot be created (missing no-arg constructor): " + className(type)); + } + } + // InstantiatorStrategy. + return fallbackStrategy.newInstantiatorOf(type); + } } - } - public static Serializer SERIALIZER = new Serializer(); + public static Serializer SERIALIZER = new Serializer(); - private static class Serializer implements Function, Serializable { - /** - * Serializes the given Object into bytes. - * - */ - @Override - public byte[] apply(Object o) { - return toBytes(o); + private static class Serializer implements Function, Serializable { + /** + * Serializes the given Object into bytes. + */ + @Override + public byte[] apply(Object o) { + return toBytes(o); + } } - } - public static class Deserializer implements Function, Serializable { + public static class Deserializer implements Function, Serializable { - private Class clazz; + private final Class clazz; - public Deserializer(Class clazz) { - this.clazz = clazz; - } - /** - * Deserializes the given bytes. - * - * @param bytes the function argument - * @return the function result - */ - @Override - public T apply(byte[] bytes) { - return fromBytes(bytes, clazz); - } - } + public Deserializer(Class clazz) { + this.clazz = clazz; + } + /** + * Deserializes the given bytes. + * + * @param bytes the function argument + * @return the function result + */ + @Override + public T apply(byte[] bytes) { + return fromBytes(bytes, clazz); + } + } - private SerDeUtils() { - // do not instantiate - } - /** - * Serialize a profile measurement's value. - * - * The value produced by a Profile definition can be any numeric data type. The data - * type depends on how the profile is defined by the user. The user should be able to - * choose the data type that is most suitable for their use case. - * - * @param value The value to serialize. - */ - public static byte[] toBytes(Object value) { - try { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - Output output = new Output(bos); - kryo.get().writeClassAndObject(output, value); - output.flush(); - bos.flush(); - return bos.toByteArray(); - } - catch(Throwable t) { - LOG.error("Unable to serialize: " + value + " because " + t.getMessage(), t); - throw new IllegalStateException("Unable to serialize " + value + " because " + t.getMessage(), t); + private SerDeUtils() { + // do not instantiate } - } - /** - * Deserialize a profile measurement's value. - * - * The value produced by a Profile definition can be any numeric data type. The data - * type depends on how the profile is defined by the user. The user should be able to - * choose the data type that is most suitable for their use case. - * - * @param value The value to deserialize. - */ - public static T fromBytes(byte[] value, Class clazz) { - try { - Input input = new Input(new ByteArrayInputStream(value)); - return clazz.cast(kryo.get().readClassAndObject(input)); + /** + * Serialize a profile measurement's value. + * + *

    + * The value produced by a Profile definition can be any numeric data type. The data + * type depends on how the profile is defined by the user. The user should be able to + * choose the data type that is most suitable for their use case. + * + * @param value The value to serialize. + */ + public static byte[] toBytes(Object value) { + try { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + Output output = new Output(bos); + kryo.get().writeClassAndObject(output, value); + output.flush(); + bos.flush(); + return bos.toByteArray(); + } catch (Throwable t) { + LOG.error("Unable to serialize: " + value + " because " + t.getMessage(), t); + throw new IllegalStateException("Unable to serialize " + value + " because " + t.getMessage(), t); + } } - catch(Throwable t) { - LOG.error("Unable to deserialize because " + t.getMessage(), t); - throw t; + + /** + * Deserialize a profile measurement's value. + * + *

    + * The value produced by a Profile definition can be any numeric data type. The data + * type depends on how the profile is defined by the user. The user should be able to + * choose the data type that is most suitable for their use case. + * + * @param value The value to deserialize. + */ + public static T fromBytes(byte[] value, Class clazz) { + try { + Input input = new Input(new ByteArrayInputStream(value)); + return clazz.cast(kryo.get().readClassAndObject(input)); + } catch (Throwable t) { + LOG.error("Unable to deserialize because " + t.getMessage(), t); + throw t; + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java index d6ba189b..c7cc1586 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,282 +51,286 @@ */ public class StellarProcessorUtils { - /** - * Execute and validate a Stellar expression. - * - *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression - * validates successfully and produces a result that can be serialized correctly. - * - * @param expression The expression to execute. - * @param varResolver The variable resolver to use - * @param context The execution context. - * @return The result of executing the expression. - */ - public static Object run(String expression, VariableResolver varResolver, Context context) { - - validate(expression, context); - Object result = execute(expression, varResolver, context); - ensureKryoSerializable(result, expression); - ensureJavaSerializable(result, expression); - - return result; - } - - /** - * Execute and validate a Stellar expression. - * - *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression - * validates successfully and produces a result that can be serialized correctly. - * - * @param expression The expression to execute. - * @param variables The variables to expose to the expression. - * @param context The execution context. - * @return The result of executing the expression. - */ - public static Object run(String expression, Map variables, Context context) { - VariableResolver varResolver = new DefaultVariableResolver( - x -> { - if(x.equals(MapVariableResolver.ALL_FIELDS)) { - return variables; - } - return variables.get(x); - } - ,x-> x.equals(MapVariableResolver.ALL_FIELDS) || variables.containsKey(x) - ); - return run(expression, varResolver, context); - } - - /** - * Execute a Stellar expression. - * - * @param expression The expression to execute. - * @param variableResolver Variable Resolver to use - * @param context The execution context. - * @return The result of executing the expression. - */ - private static Object execute(String expression, VariableResolver variableResolver, Context context) { - - StellarProcessor processor = new StellarProcessor(); - - Object result = processor.parse( - expression, - variableResolver, - StellarFunctions.FUNCTION_RESOLVER(), - context); - - return result; - } - - /** - * Ensure that a value can be serialized and deserialized using Kryo. - * - *

    When a Stellar function is used in a Storm topology there are cases when the result - * needs to be serializable, like when using the Profiler. Storm can use either Kryo or - * basic Java serialization. It is highly recommended that all Stellar functions return a - * result that is Kryo serializable to allow for the broadest possible use of the function. - * - * @param value The value to validate. - */ - private static void ensureKryoSerializable(Object value, String expression) { - - String msg = String.format("Expression result is not Kryo serializable. It is highly recommended for all " + - "functions to return a result that is Kryo serializable to allow for their broadest possible use. " + - "expr=%s, value=%s", expression, value); - - byte[] raw = SerDeUtils.toBytes(value); - Object actual = SerDeUtils.fromBytes(raw, Object.class); - if((value == null && actual != null) || (value != null && !value.equals(actual))) { - throw new AssertionError(msg); + /** + * Execute and validate a Stellar expression. + * + *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. + * @param varResolver The variable resolver to use + * @param context The execution context. + * @return The result of executing the expression. + */ + public static Object run(String expression, VariableResolver varResolver, Context context) { + + validate(expression, context); + Object result = execute(expression, varResolver, context); + ensureKryoSerializable(result, expression); + ensureJavaSerializable(result, expression); + + return result; + } + + /** + * Execute and validate a Stellar expression. + * + *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. + * @param variables The variables to expose to the expression. + * @param context The execution context. + * @return The result of executing the expression. + */ + public static Object run(String expression, Map variables, Context context) { + VariableResolver varResolver = new DefaultVariableResolver( + x -> { + if (x.equals(MapVariableResolver.ALL_FIELDS)) { + return variables; + } + return variables.get(x); + }, + x -> x.equals(MapVariableResolver.ALL_FIELDS) || variables.containsKey(x) + ); + return run(expression, varResolver, context); + } + + /** + * Execute and validate a Stellar expression. + * + *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. + * @param variables The variables to expose to the expression. + * @return The result of executing the expression. + */ + public static Object run(String expression, Map variables) { + return run(expression, variables, Context.EMPTY_CONTEXT()); } - } - - /** - * Ensure a value can be serialized and deserialized using Java serialization. - * - *

    When a Stellar function is used in a Storm topology there are cases when the result - * needs to be serializable, like when using the Profiler. Storm can use either Kryo or - * basic Java serialization. It is highly recommended that all Stellar functions return a - * result that is Java serializable to allow for the broadest possible use of the function. - * - * @param value The value to serialize - */ - private static void ensureJavaSerializable(Object value, String expression) { - - String msg = String.format("Expression result is not Java serializable. It is highly recommended for all " + - "functions to return a result that is Java serializable to allow for their broadest possible use. " + - "expr=%s, value=%s", expression, value); - - try { - // serialize using java - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(bytes); - out.writeObject(value); - - // the serialized bits - byte[] raw = bytes.toByteArray(); - if(!(raw.length > 0)) { - throw new AssertionError("Serialized byte length not greater than 0"); - } - - // deserialize using java - ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(raw)); - Object actual = in.readObject(); - - // ensure that the round-trip was successful - if((value == null && actual != null) || (value != null && !value.equals(actual))) { - throw new AssertionError(msg); - } - - } catch(IOException | ClassNotFoundException e) { - - String error = String.format("Expression result is not Java serializable. It is highly recommended for all " + - "functions to return a result that is Java serializable to allow for their broadest possible use. " + - "expr=%s, value=%s, error=%s", expression, value, ExceptionUtils.getRootCauseMessage(e)); - throw new AssertionError(error); + + /** + * Execute and validate a Stellar expression. + * + *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. + * @param variables The variables to expose to the expression. + * @return The result of executing the expression. + */ + public static Object run(String expression, VariableResolver variables) { + return run(expression, variables, Context.EMPTY_CONTEXT()); + } + + /** + * Execute and validate a Stellar expression. + * + *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. + * @param context The execution context. + * @return The result of executing the expression. + */ + public static Object run(String expression, Context context) { + return run(expression, Collections.emptyMap(), context); + } + + /** + * Execute a Stellar expression. + * + * @param expression The expression to execute. + * @param variableResolver Variable Resolver to use + * @param context The execution context. + * @return The result of executing the expression. + */ + private static Object execute(String expression, VariableResolver variableResolver, Context context) { + + StellarProcessor processor = new StellarProcessor(); + + Object result = processor.parse( + expression, + variableResolver, + StellarFunctions.FUNCTION_RESOLVER(), + context); + + return result; } - } - - /** - * Execute and validate a Stellar expression. - * - *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression - * validates successfully and produces a result that can be serialized correctly. - * - * @param expression The expression to execute. - * @param variables The variables to expose to the expression. - * @return The result of executing the expression. - */ - public static Object run(String expression, Map variables) { - return run(expression, variables, Context.EMPTY_CONTEXT()); - } - - /** - * Execute and validate a Stellar expression. - * - *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression - * validates successfully and produces a result that can be serialized correctly. - * - * @param expression The expression to execute. - * @param variables The variables to expose to the expression. - * @return The result of executing the expression. - */ - public static Object run(String expression, VariableResolver variables) { - return run(expression, variables, Context.EMPTY_CONTEXT()); - } - - /** - * Execute and validate a Stellar expression. - * - *

    This is intended for use while unit testing Stellar expressions. This ensures that the expression - * validates successfully and produces a result that can be serialized correctly. - * - * @param expression The expression to execute. - * @param context The execution context. - * @return The result of executing the expression. - */ - public static Object run(String expression, Context context) { - return run(expression, Collections.emptyMap(), context); - } - - public static void validate(String expression, Context context) { - StellarProcessor processor = new StellarProcessor(); - if(!processor.validate(expression, context)) { - throw new AssertionError("Invalid expression; expr=" + expression); + + /** + * Ensure that a value can be serialized and deserialized using Kryo. + * + *

    When a Stellar function is used in a Storm topology there are cases when the result + * needs to be serializable, like when using the Profiler. Storm can use either Kryo or + * basic Java serialization. It is highly recommended that all Stellar functions return a + * result that is Kryo serializable to allow for the broadest possible use of the function. + * + * @param value The value to validate. + */ + private static void ensureKryoSerializable(Object value, String expression) { + + String msg = String.format("Expression result is not Kryo serializable. It is highly recommended for all " + + "functions to return a result that is Kryo serializable to allow for their broadest possible use. " + + "expr=%s, value=%s", expression, value); + + byte[] raw = SerDeUtils.toBytes(value); + Object actual = SerDeUtils.fromBytes(raw, Object.class); + if ((value == null && actual != null) || (value != null && !value.equals(actual))) { + throw new AssertionError(msg); + } } - } - public static void validate(String rule) { - validate(rule, Context.EMPTY_CONTEXT()); - } + /** + * Ensure a value can be serialized and deserialized using Java serialization. + * + *

    When a Stellar function is used in a Storm topology there are cases when the result + * needs to be serializable, like when using the Profiler. Storm can use either Kryo or + * basic Java serialization. It is highly recommended that all Stellar functions return a + * result that is Java serializable to allow for the broadest possible use of the function. + * + * @param value The value to serialize + */ + private static void ensureJavaSerializable(Object value, String expression) { + + String msg = String.format("Expression result is not Java serializable. It is highly recommended for all " + + "functions to return a result that is Java serializable to allow for their broadest possible use. " + + "expr=%s, value=%s", expression, value); + + try { + // serialize using java + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ObjectOutputStream out = new ObjectOutputStream(bytes); + out.writeObject(value); + + // the serialized bits + byte[] raw = bytes.toByteArray(); + if (!(raw.length > 0)) { + throw new AssertionError("Serialized byte length not greater than 0"); + } + + // deserialize using java + ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(raw)); + Object actual = in.readObject(); - public static boolean runPredicate(String rule, Map resolver) { - return runPredicate(rule, resolver, Context.EMPTY_CONTEXT()); - } + // ensure that the round-trip was successful + if ((value == null && actual != null) || (value != null && !value.equals(actual))) { + throw new AssertionError(msg); + } - public static boolean runPredicate(String rule, Map resolver, Context context) { - return runPredicate(rule, new MapVariableResolver(resolver), context); - } + } catch (IOException | ClassNotFoundException e) { - public static boolean runPredicate(String rule, VariableResolver resolver) { - return runPredicate(rule, resolver, Context.EMPTY_CONTEXT()); - } + String error = + String.format("Expression result is not Java serializable. It is highly recommended for all " + + "functions to return a result that is Java serializable to allow for their broadest possible use. " + + "expr=%s, value=%s, error=%s", expression, value, + ExceptionUtils.getRootCauseMessage(e)); + throw new AssertionError(error); + } + } - public static boolean runPredicate(String rule, VariableResolver resolver, Context context) { - StellarPredicateProcessor processor = new StellarPredicateProcessor(); - if(!processor.validate(rule)) { - throw new AssertionError(rule + " not valid."); + public static void validate(String expression, Context context) { + StellarProcessor processor = new StellarProcessor(); + if (!processor.validate(expression, context)) { + throw new AssertionError("Invalid expression; expr=" + expression); + } } - return processor.parse(rule, resolver, StellarFunctions.FUNCTION_RESOLVER(), context); - } - - public static void runWithArguments(String function, Object argument, Object expected) { - runWithArguments(function, ImmutableList.of(argument), expected); - } - - public static void runWithArguments(String function, List arguments, Object expected) { - Supplier>> kvStream = () -> StreamSupport - .stream(new XRange(arguments.size()), false) - .map(i -> new AbstractMap.SimpleImmutableEntry<>("var" + i, arguments.get(i))); - - String args = kvStream.get().map(kv -> kv.getKey()).collect(Collectors.joining(",")); - Map variables = kvStream.get().collect(Collectors.toMap(kv -> kv.getKey(), kv -> kv.getValue())); - String stellarStatement = function + "(" + args + ")"; - String reason = stellarStatement + " != " + expected + " with variables: " + variables; - - if (expected instanceof Double) { - if(!(Math.abs((Double) expected - (Double) run(stellarStatement, variables)) < 1e-6)) { - throw new AssertionError(reason); - } - } else { - if(!expected.equals(run(stellarStatement, variables))) { - throw new AssertionError(reason); - } + + public static void validate(String rule) { + validate(rule, Context.EMPTY_CONTEXT()); } - } - public static class XRange extends Spliterators.AbstractIntSpliterator { - int end; - int i = 0; + public static boolean runPredicate(String rule, Map resolver) { + return runPredicate(rule, resolver, Context.EMPTY_CONTEXT()); + } - public XRange(int start, int end) { - super(end - start, 0); - i = start; - this.end = end; + public static boolean runPredicate(String rule, Map resolver, Context context) { + return runPredicate(rule, new MapVariableResolver(resolver), context); } - public XRange(int end) { - this(0, end); + public static boolean runPredicate(String rule, VariableResolver resolver) { + return runPredicate(rule, resolver, Context.EMPTY_CONTEXT()); } - @Override - public boolean tryAdvance(IntConsumer action) { - boolean isDone = i >= end; - if (isDone) { - return false; - } else { - action.accept(i); - i++; - return true; - } + public static boolean runPredicate(String rule, VariableResolver resolver, Context context) { + StellarPredicateProcessor processor = new StellarPredicateProcessor(); + if (!processor.validate(rule)) { + throw new AssertionError(rule + " not valid."); + } + return processor.parse(rule, resolver, StellarFunctions.FUNCTION_RESOLVER(), context); } - /** - * {@inheritDoc} - * - * @param action to {@code IntConsumer} and passed to {@link #tryAdvance(IntConsumer)}; - * otherwise the action is adapted to an instance of {@code IntConsumer}, by boxing the - * argument of {@code IntConsumer}, and then passed to {@link #tryAdvance(IntConsumer)}. - */ - @Override - public boolean tryAdvance(Consumer action) { - boolean isDone = i >= end; - if (isDone) { - return false; - } else { - action.accept(i); - i++; - return true; - } + public static void runWithArguments(String function, Object argument, Object expected) { + runWithArguments(function, ImmutableList.of(argument), expected); + } + + public static void runWithArguments(String function, List arguments, Object expected) { + Supplier>> kvStream = () -> StreamSupport + .stream(new XRange(arguments.size()), false) + .map(i -> new AbstractMap.SimpleImmutableEntry<>("var" + i, arguments.get(i))); + + String args = kvStream.get().map(kv -> kv.getKey()).collect(Collectors.joining(",")); + Map variables = + kvStream.get().collect(Collectors.toMap(kv -> kv.getKey(), kv -> kv.getValue())); + String stellarStatement = function + "(" + args + ")"; + String reason = stellarStatement + " != " + expected + " with variables: " + variables; + + if (expected instanceof Double) { + if (!(Math.abs((Double) expected - (Double) run(stellarStatement, variables)) < 1e-6)) { + throw new AssertionError(reason); + } + } else { + if (!expected.equals(run(stellarStatement, variables))) { + throw new AssertionError(reason); + } + } + } + + public static class XRange extends Spliterators.AbstractIntSpliterator { + int end; + @SuppressWarnings("checkstyle:MemberName") + int i = 0; + + public XRange(int start, int end) { + super(end - start, 0); + i = start; + this.end = end; + } + + public XRange(int end) { + this(0, end); + } + + @Override + public boolean tryAdvance(IntConsumer action) { + boolean isDone = i >= end; + if (isDone) { + return false; + } else { + action.accept(i); + i++; + return true; + } + } + + /** + * {@inheritDoc} + * + * @param action to {@code IntConsumer} and passed to {@link #tryAdvance(IntConsumer)}; + * otherwise the action is adapted to an instance of {@code IntConsumer}, by boxing the + * argument of {@code IntConsumer}, and then passed to {@link #tryAdvance(IntConsumer)}. + */ + @Override + public boolean tryAdvance(Consumer action) { + boolean isDone = i >= end; + if (isDone) { + return false; + } else { + action.accept(i); + i++; + return true; + } + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/UniqueFileReplicator.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/UniqueFileReplicator.java index 5bdcf257..2d94e140 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/UniqueFileReplicator.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/UniqueFileReplicator.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.FileSelector; @@ -46,22 +45,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * - */ public class UniqueFileReplicator implements VfsComponent, FileReplicator { - private static final char[] TMP_RESERVED_CHARS = new char[] {'?', '/', '\\', ' ', '&', '"', '\'', '*', '#', ';', ':', '<', '>', '|'}; + private static final char[] TMP_RESERVED_CHARS = + new char[] {'?', '/', '\\', ' ', '&', '"', '\'', '*', '#', ';', ':', '<', '>', '|'}; private static final Logger log = LoggerFactory.getLogger(UniqueFileReplicator.class); - private File tempDir; + private final File tempDir; private VfsComponentContext context; - private List tmpFiles = Collections.synchronizedList(new ArrayList()); + private final List tmpFiles = Collections.synchronizedList(new ArrayList()); public UniqueFileReplicator(File tempDir) { this.tempDir = tempDir; - if (!tempDir.exists() && !tempDir.mkdirs()) + if (!tempDir.exists() && !tempDir.mkdirs()) { log.warn("Unexpected error creating directory " + tempDir); + } } @Override @@ -102,15 +100,17 @@ public void init() throws FileSystemException { public void close() { synchronized (tmpFiles) { for (File tmpFile : tmpFiles) { - if (!tmpFile.delete()) + if (!tmpFile.delete()) { log.warn("File does not exist: " + tmpFile); + } } } if (tempDir.exists()) { int numChildren = tempDir.list().length; - if (0 == numChildren && !tempDir.delete()) + if (0 == numChildren && !tempDir.delete()) { log.warn("Cannot delete empty directory: " + tempDir); + } } } } \ No newline at end of file diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/VFSClassloaderUtil.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/VFSClassloaderUtil.java index 825be7b0..0c53a373 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/VFSClassloaderUtil.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/VFSClassloaderUtil.java @@ -6,18 +6,29 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.utils; -import org.apache.commons.vfs2.*; +import java.io.File; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Optional; +import org.apache.commons.vfs2.CacheStrategy; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemManager; +import org.apache.commons.vfs2.FileType; import org.apache.commons.vfs2.cache.SoftRefFilesCache; import org.apache.commons.vfs2.impl.DefaultFileSystemManager; import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; @@ -26,149 +37,141 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Optional; - public class VFSClassloaderUtil { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** - * Create a FileSystem manager suitable for our purposes. - * This manager supports files of the following types: - * * res - resource files - * * jar - * * tar - * * bz2 - * * tgz - * * zip - * * HDFS - * * FTP - * * HTTP/S - * * file - * @return vfs - * @throws FileSystemException - */ - public static FileSystemManager generateVfs() throws FileSystemException { - DefaultFileSystemManager vfs = new DefaultFileSystemManager(); - vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider()); - vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider()); - vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider()); - vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider()); - vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider()); - vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); - vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider()); - vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider()); - vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider()); - vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider()); - vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); - vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); - vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); - vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); - vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); - vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider()); - vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); - vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); - vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); - vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider()); - vfs.addProvider("hdfs", new HdfsFileProvider()); - vfs.addExtensionMap("jar", "jar"); - vfs.addExtensionMap("zip", "zip"); - vfs.addExtensionMap("gz", "gz"); - vfs.addExtensionMap("tar", "tar"); - vfs.addExtensionMap("tbz2", "tar"); - vfs.addExtensionMap("tgz", "tar"); - vfs.addExtensionMap("bz2", "bz2"); - vfs.addMimeTypeMap("application/x-tar", "tar"); - vfs.addMimeTypeMap("application/x-gzip", "gz"); - vfs.addMimeTypeMap("application/zip", "zip"); - vfs.addMimeTypeMap("application/java-archive", "jar"); - vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory()); - vfs.setFilesCache(new SoftRefFilesCache()); - vfs.setReplicator(new UniqueFileReplicator(new File(System.getProperty("java.io.tmpdir")))); - vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE); - vfs.init(); - return vfs; - } + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - /** - * Create a classloader backed by a virtual filesystem which can handle the following URI types: - * * res - resource files - * * jar - * * tar - * * bz2 - * * tgz - * * zip - * * HDFS - * * FTP - * * HTTP/S - * * file - * @param paths A set of comma separated paths. The paths are URIs or URIs with a regex pattern at the end. - * @return A classloader object if it can create it - * @throws FileSystemException - */ - public static Optional configureClassloader(String paths) throws FileSystemException { - LOG.debug("Configuring class loader with paths = {}", paths); - if(paths.trim().isEmpty()) { - LOG.debug("No paths provided. Not returning a ClassLoader."); - return Optional.empty(); + /** + * Create a FileSystem manager suitable for our purposes. + * This manager supports files of the following types: + * * res - resource files + * * jar + * * tar + * * bz2 + * * tgz + * * zip + * * HDFS + * * FTP + * * HTTP/S + * * file + */ + public static FileSystemManager generateVfs() throws FileSystemException { + DefaultFileSystemManager vfs = new DefaultFileSystemManager(); + vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider()); + vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider()); + vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider()); + vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider()); + vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider()); + vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider()); + vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider()); + vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider()); + vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider()); + vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider()); + vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider()); + vfs.addProvider("hdfs", new HdfsFileProvider()); + vfs.addExtensionMap("jar", "jar"); + vfs.addExtensionMap("zip", "zip"); + vfs.addExtensionMap("gz", "gz"); + vfs.addExtensionMap("tar", "tar"); + vfs.addExtensionMap("tbz2", "tar"); + vfs.addExtensionMap("tgz", "tar"); + vfs.addExtensionMap("bz2", "bz2"); + vfs.addMimeTypeMap("application/x-tar", "tar"); + vfs.addMimeTypeMap("application/x-gzip", "gz"); + vfs.addMimeTypeMap("application/zip", "zip"); + vfs.addMimeTypeMap("application/java-archive", "jar"); + vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory()); + vfs.setFilesCache(new SoftRefFilesCache()); + vfs.setReplicator(new UniqueFileReplicator(new File(System.getProperty("java.io.tmpdir")))); + vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE); + vfs.init(); + return vfs; } - FileSystemManager vfs = generateVfs(); - FileObject[] objects = resolve(vfs, paths); - if(objects == null || objects.length == 0) { - LOG.debug("No Classloader able to be resolved from provided paths. Not returning a ClassLoader."); - return Optional.empty(); - } - LOG.debug("vfs = {}, objects = {}", vfs, objects); - return Optional.of(new VFSClassLoader(objects, vfs, vfs.getClass().getClassLoader())); - } - /** - * Resolve a set of URIs into FileObject objects. - * This is not recursive. The URIs can refer directly to a file or directory or an optional regex at the end. - * (NOTE: This is NOT a glob). - * @param vfs The file system manager to use to resolve URIs - * @param uris comma separated URIs and URI + globs - * @return - * @throws FileSystemException - */ - static FileObject[] resolve(FileSystemManager vfs, String uris) throws FileSystemException { - if (uris == null) { - return new FileObject[0]; + /** + * Create a classloader backed by a virtual filesystem which can handle the following URI types: + * * res - resource files + * * jar + * * tar + * * bz2 + * * tgz + * * zip + * * HDFS + * * FTP + * * HTTP/S + * * file. + * + * @param paths A set of comma separated paths. The paths are URIs or URIs with a regex pattern at the end. + * @return A classloader object if it can create it + */ + public static Optional configureClassloader(String paths) throws FileSystemException { + LOG.debug("Configuring class loader with paths = {}", paths); + if (paths.trim().isEmpty()) { + LOG.debug("No paths provided. Not returning a ClassLoader."); + return Optional.empty(); + } + FileSystemManager vfs = generateVfs(); + FileObject[] objects = resolve(vfs, paths); + if (objects == null || objects.length == 0) { + LOG.debug("No Classloader able to be resolved from provided paths. Not returning a ClassLoader."); + return Optional.empty(); + } + LOG.debug("vfs = {}, objects = {}", vfs, objects); + return Optional.of(new VFSClassLoader(objects, vfs, vfs.getClass().getClassLoader())); } - ArrayList classpath = new ArrayList<>(); - for (String path : uris.split(",")) { - path = path.trim(); - if (path.equals("")) { - continue; - } - FileObject fo = vfs.resolveFile(path); - switch (fo.getType()) { - case FILE: - case FOLDER: - classpath.add(fo); - break; - case IMAGINARY: - // assume its a pattern - String pattern = fo.getName().getBaseName(); - if (fo.getParent() != null && fo.getParent().getType() == FileType.FOLDER) { - FileObject[] children = fo.getParent().getChildren(); - for (FileObject child : children) { - if (child.getType() == FileType.FILE && child.getName().getBaseName().matches(pattern)) { - classpath.add(child); - } + /** + * Resolve a set of URIs into FileObject objects. + * This is not recursive. The URIs can refer directly to a file or directory or an optional regex at the end. + * (NOTE: This is NOT a glob). + * + * @param vfs The file system manager to use to resolve URIs + * @param uris comma separated URIs and URI + globs + */ + static FileObject[] resolve(FileSystemManager vfs, String uris) throws FileSystemException { + if (uris == null) { + return new FileObject[0]; + } + + ArrayList classpath = new ArrayList<>(); + for (String path : uris.split(",")) { + path = path.trim(); + if (path.equals("")) { + continue; + } + FileObject fo = vfs.resolveFile(path); + switch (fo.getType()) { + case FILE: + case FOLDER: + classpath.add(fo); + break; + case IMAGINARY: + // assume its a pattern + String pattern = fo.getName().getBaseName(); + if (fo.getParent() != null && fo.getParent().getType() == FileType.FOLDER) { + FileObject[] children = fo.getParent().getChildren(); + for (FileObject child : children) { + if (child.getType() == FileType.FILE && child.getName().getBaseName().matches(pattern)) { + classpath.add(child); + } + } + } else { + LOG.warn("ignoring classpath entry {}", fo); + } + break; + default: + LOG.warn("ignoring classpath entry {}", fo); + break; } - } else { - LOG.warn("ignoring classpath entry {}", fo); - } - break; - default: - LOG.warn("ignoring classpath entry {}", fo); - break; - } + } + return classpath.toArray(new FileObject[classpath.size()]); } - return classpath.toArray(new FileObject[classpath.size()]); - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/cli/OptionHandler.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/cli/OptionHandler.java index 074158aa..4febd86f 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/cli/OptionHandler.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/cli/OptionHandler.java @@ -6,26 +6,27 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.utils.cli; import com.google.common.base.Function; +import java.util.Optional; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; -import java.util.Optional; - -public abstract class OptionHandler> implements Function -{ - public Optional getValue(OPT_T option, CommandLine cli) { - return Optional.empty(); - } +public abstract class OptionHandler> implements Function { + public Optional getValue(OPT_T option, CommandLine cli) { + return Optional.empty(); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/DefaultHasher.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/DefaultHasher.java index b2eeca56..cac6fb9d 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/DefaultHasher.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/DefaultHasher.java @@ -15,137 +15,142 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.stellar.common.utils.hashing; -import com.google.common.base.Joiner; -import org.apache.commons.codec.BinaryEncoder; -import org.apache.commons.codec.Charsets; -import org.apache.commons.codec.EncoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.lang3.SerializationUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.metron.stellar.common.utils.ConversionUtils; +package org.apache.metron.stellar.common.utils.hashing; import java.io.Serializable; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.nio.charset.UnsupportedCharsetException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.Security; -import java.util.*; -import java.util.function.Function; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.codec.BinaryEncoder; +import org.apache.commons.codec.EncoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; public class DefaultHasher implements Hasher { - public enum Config implements EnumConfigurable { - CHARSET("charset"), - ; - private String key; - Config(String key) { - this.key = key; + public enum Config implements EnumConfigurable { + CHARSET("charset"), + ; + private final String key; + + Config(String key) { + this.key = key; + } + + @Override + public String getKey() { + return key; + } + } + private final String algorithm; + private final BinaryEncoder encoder; + private Charset charset; + + /** + * Builds a utility to hash values based on a given algorithm. + * + * @param algorithm The algorithm used when hashing a value. + * @param encoder The encoder to use to encode the hashed value. + * @param charset The charset that will be used during hashing and encoding. + * @see java.security.Security + * @see java.security.MessageDigest + */ + public DefaultHasher(final String algorithm, final BinaryEncoder encoder, final Charset charset) { + this.algorithm = algorithm; + this.encoder = encoder; + this.charset = charset; + } + + /** + * Builds a utility to hash values based on a given algorithm. Uses {@link StandardCharsets#UTF_8} for encoding. + * + * @param algorithm The algorithm used when hashing a value. + * @param encoder The encoder to use to encode the hashed value. + * @see java.security.Security + * @see java.security.MessageDigest + */ + public DefaultHasher(final String algorithm, final BinaryEncoder encoder) { + this.algorithm = algorithm; + this.encoder = encoder; + this.charset = StandardCharsets.UTF_8; + } + + /** + * Builds a utility to hash values based on a given algorithm. Uses {@link StandardCharsets#UTF_8} for encoding. + * + * @param algorithm The algorithm used when hashing a value. + * @see java.security.Security + * @see java.security.MessageDigest + */ + public DefaultHasher(final String algorithm) { + this(algorithm, new Hex(StandardCharsets.UTF_8)); + } + + /** + * {@inheritDoc} + * + *

    + * Returns a hash which has been encoded using the supplied encoder. If input is null then a string + * containing all '0' will be returned. The length of the string is determined by the hashing algorithm + * used. + * + * @param toHash The value to hash. + * @return A hash of {@code toHash} that has been encoded. + * @throws EncoderException If unable to encode the hash then this exception occurs. + * @throws NoSuchAlgorithmException If the supplied algorithm is not known. + */ @Override - public String getKey() { - return key; + public String getHash(final Object toHash) throws EncoderException, NoSuchAlgorithmException { + final MessageDigest messageDigest = MessageDigest.getInstance(algorithm); + + if (toHash == null) { + return StringUtils.repeat("00", messageDigest.getDigestLength()); + } else if (toHash instanceof String) { + return getHash(messageDigest, toHash.toString().getBytes(charset)); + } else if (toHash instanceof Serializable) { + final byte[] serialized = SerializationUtils.serialize((Serializable) toHash); + return getHash(messageDigest, serialized); + } + + return null; } - } - - private String algorithm; - private BinaryEncoder encoder; - private Charset charset; - - /** - * Builds a utility to hash values based on a given algorithm. - * @param algorithm The algorithm used when hashing a value. - * @param encoder The encoder to use to encode the hashed value. - * @param charset The charset that will be used during hashing and encoding. - * @see java.security.Security - * @see java.security.MessageDigest - */ - public DefaultHasher(final String algorithm, final BinaryEncoder encoder, final Charset charset) { - this.algorithm = algorithm; - this.encoder = encoder; - this.charset = charset; - } - - /** - * Builds a utility to hash values based on a given algorithm. Uses {@link StandardCharsets#UTF_8} for encoding. - * @param algorithm The algorithm used when hashing a value. - * @param encoder The encoder to use to encode the hashed value. - * @see java.security.Security - * @see java.security.MessageDigest - */ - public DefaultHasher(final String algorithm, final BinaryEncoder encoder) { - this.algorithm = algorithm; - this.encoder = encoder; - this.charset = StandardCharsets.UTF_8; - } - - /** - * Builds a utility to hash values based on a given algorithm. Uses {@link StandardCharsets#UTF_8} for encoding. - * @param algorithm The algorithm used when hashing a value. - * @see java.security.Security - * @see java.security.MessageDigest - */ - public DefaultHasher(final String algorithm) { - this(algorithm, new Hex(StandardCharsets.UTF_8)); - } - - /** - * {@inheritDoc} - * - * Returns a hash which has been encoded using the supplied encoder. If input is null then a string - * containing all '0' will be returned. The length of the string is determined by the hashing algorithm - * used. - * @param toHash The value to hash. - * @return A hash of {@code toHash} that has been encoded. - * @throws EncoderException If unable to encode the hash then this exception occurs. - * @throws NoSuchAlgorithmException If the supplied algorithm is not known. - */ - @Override - public String getHash(final Object toHash) throws EncoderException, NoSuchAlgorithmException { - final MessageDigest messageDigest = MessageDigest.getInstance(algorithm); - - if (toHash == null) { - return StringUtils.repeat("00", messageDigest.getDigestLength()); - } else if (toHash instanceof String) { - return getHash(messageDigest, toHash.toString().getBytes(charset)); - } else if (toHash instanceof Serializable) { - final byte[] serialized = SerializationUtils.serialize((Serializable) toHash); - return getHash(messageDigest, serialized); + private String getHash(final MessageDigest messageDigest, final byte[] toHash) throws EncoderException { + messageDigest.update(toHash); + final byte[] encode = encoder.encode(messageDigest.digest()); + + return new String(encode, charset); } - return null; - } - - private String getHash(final MessageDigest messageDigest, final byte[] toHash) throws EncoderException { - messageDigest.update(toHash); - final byte[] encode = encoder.encode(messageDigest.digest()); - - return new String(encode, charset); - } - - @Override - public void configure(Optional> config) { - if(config.isPresent() && !config.get().isEmpty()) { - charset = Config.CHARSET.get(config.get() - , o -> { - String charset = ConversionUtils.convert(o, String.class); - if(charset != null) { - Charset set = Charset.forName(charset); - return set; - } - return null; - } - ).orElse(charset); + @Override + public void configure(Optional> config) { + if (config.isPresent() && !config.get().isEmpty()) { + charset = Config.CHARSET.get(config.get(), + o -> { + String charset = ConversionUtils.convert(o, String.class); + if (charset != null) { + Charset set = Charset.forName(charset); + return set; + } + return null; + } + ).orElse(charset); + } } - } - public static final Set supportedHashes() { - return new HashSet<>(Security.getAlgorithms("MessageDigest")); - } + public static final Set supportedHashes() { + return new HashSet<>(Security.getAlgorithms("MessageDigest")); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/EnumConfigurable.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/EnumConfigurable.java index 42923a07..b987fce8 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/EnumConfigurable.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/EnumConfigurable.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.utils.hashing; import java.util.Map; @@ -22,10 +23,10 @@ import java.util.function.Function; public interface EnumConfigurable { - String getKey(); + String getKey(); - default Optional get(Map config, Function converter) { - Object o = config.get(getKey()); - return o == null?Optional.empty():Optional.ofNullable(converter.apply(o)); - } + default Optional get(Map config, Function converter) { + Object o = config.get(getKey()); + return o == null ? Optional.empty() : Optional.ofNullable(converter.apply(o)); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/HashStrategy.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/HashStrategy.java index a26832b8..a72ec65b 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/HashStrategy.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/HashStrategy.java @@ -15,70 +15,76 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.common.utils.hashing; import com.google.common.base.Joiner; -import org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHHasher; - -import java.util.*; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.function.Function; +import org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHHasher; /** * This is an enum implementing a hashing strategy pattern. Because hash types may * be quite different, but have a similar interface we have laid on top this abstraction * to allow new types of hashes to be added easily. * + *

    * In order to add a new family of hashes, simply implement the Hasher interface and register it with this * enum. The search order for algorithms to their respective Hasher is in the order of the entries * in this enum. */ public enum HashStrategy { - TLSH(a -> new TLSHHasher(), TLSHHasher.supportedHashes()), - DEFAULT(a -> new DefaultHasher(a), DefaultHasher.supportedHashes()) - ; + TLSH(a -> new TLSHHasher(), TLSHHasher.supportedHashes()), + DEFAULT(a -> new DefaultHasher(a), DefaultHasher.supportedHashes()); - /** - * An accumulated list of all the supported hash algorithms. - */ - public static final Set ALL_SUPPORTED_HASHES = new HashSet<>(); - static { - for(HashStrategy factory : HashStrategy.values()) { - ALL_SUPPORTED_HASHES.addAll(factory.supportedHashes); - } - } + /** + * An accumulated list of all the supported hash algorithms. + */ + public static final Set ALL_SUPPORTED_HASHES = new HashSet<>(); - Function hasherCreator; - Set supportedHashes; + static { + for (HashStrategy factory : HashStrategy.values()) { + ALL_SUPPORTED_HASHES.addAll(factory.supportedHashes); + } + } - HashStrategy(Function hasherCreator, Set supportedHashes) { - this.hasherCreator = hasherCreator; - this.supportedHashes = supportedHashes; - } + Function hasherCreator; + Set supportedHashes; - /** - * Return the appropriate hasher given the algorithm. - * @param algorithm The algorithm to find a hasher handler for. Note: this is upper-cased prior to search - * @param config The config for the hasher - * @return The hasher which will handle the algorithm. If the algorithm is not supported by any registered - * hashers, an IllegalArgumentException is thrown. - */ - public static Hasher getHasher(String algorithm, Optional> config) { - Hasher h = null; - for(HashStrategy factory : HashStrategy.values()) { - if(factory.getSupportedHashes().contains(algorithm.toUpperCase())) { - h = factory.hasherCreator.apply(algorithm); - break; - } + HashStrategy(Function hasherCreator, Set supportedHashes) { + this.hasherCreator = hasherCreator; + this.supportedHashes = supportedHashes; } - if(h == null) { - throw new IllegalArgumentException("Unsupported hash function: " + algorithm - + ". Supported algorithms are " + Joiner.on(",").join(ALL_SUPPORTED_HASHES)); + + /** + * Return the appropriate hasher given the algorithm. + * + * @param algorithm The algorithm to find a hasher handler for. Note: this is upper-cased prior to search + * @param config The config for the hasher + * @return The hasher which will handle the algorithm. If the algorithm is not supported by any registered + * hashers, an IllegalArgumentException is thrown. + */ + public static Hasher getHasher(String algorithm, Optional> config) { + Hasher h = null; + for (HashStrategy factory : HashStrategy.values()) { + if (factory.getSupportedHashes().contains(algorithm.toUpperCase())) { + h = factory.hasherCreator.apply(algorithm); + break; + } + } + if (h == null) { + throw new IllegalArgumentException("Unsupported hash function: " + algorithm + + ". Supported algorithms are " + + Joiner.on(",").join(ALL_SUPPORTED_HASHES)); + } + h.configure(config); + return h; } - h.configure(config); - return h; - } - public Set getSupportedHashes() { - return supportedHashes; - } + public Set getSupportedHashes() { + return supportedHashes; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/Hasher.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/Hasher.java index a0598422..d3af16a1 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/Hasher.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/Hasher.java @@ -15,32 +15,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.stellar.common.utils.hashing; -import org.apache.commons.codec.EncoderException; +package org.apache.metron.stellar.common.utils.hashing; import java.security.NoSuchAlgorithmException; -import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; +import org.apache.commons.codec.EncoderException; public interface Hasher { - /** - * Returns an encoded string representation of the hash value of the input. It is expected that - * this implementation does throw exceptions when the input is null. - * @param toHash The value to hash. - * @return A hash of {@code toHash} that has been encoded. - * @throws EncoderException If unable to encode the hash then this exception occurs. - * @throws NoSuchAlgorithmException If the supplied algorithm is not known. - */ - Object getHash(final Object toHash) throws EncoderException, NoSuchAlgorithmException; + /** + * Returns an encoded string representation of the hash value of the input. It is expected that + * this implementation does throw exceptions when the input is null. + * + * @param toHash The value to hash. + * @return A hash of {@code toHash} that has been encoded. + * @throws EncoderException If unable to encode the hash then this exception occurs. + * @throws NoSuchAlgorithmException If the supplied algorithm is not known. + */ + Object getHash(final Object toHash) throws EncoderException, NoSuchAlgorithmException; - /** - * Configure the hasher with a string to object map. - * @param config - */ - void configure(final Optional> config); + /** + * Configure the hasher with a string to object map. + */ + void configure(final Optional> config); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/SlidingWindow.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/SlidingWindow.java index bdf7786e..0d5d4a6a 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/SlidingWindow.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/SlidingWindow.java @@ -24,9 +24,9 @@ public int[] getWindow() { final IntUnaryOperator reverseIterate = i -> i == 0 ? window.length - 1 : i - 1; final IntUnaryOperator mapper = i -> window[i] & 0xFF; return IntStream.iterate(startPosition, reverseIterate) - .limit(window.length) - .map(mapper) - .toArray(); + .limit(window.length) + .map(mapper) + .toArray(); } public int getByteCount() { diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSH.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSH.java index 4d6aa645..23ca0637 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSH.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSH.java @@ -15,12 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.stellar.common.utils.hashing.tlsh; -import java.nio.ByteBuffer; +package org.apache.metron.stellar.common.utils.hashing.tlsh; import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHUtil.swapNibble; +import java.nio.ByteBuffer; + /** * The abstraction around interacting with TLSH. */ @@ -37,6 +38,7 @@ public class TLSH { /** * The encoded length value. */ + @SuppressWarnings("checkstyle:MemberName") private final int lValue; /** * The q1 ratio. @@ -48,6 +50,7 @@ public class TLSH { private final int q2Ratio; + @SuppressWarnings("checkstyle:ParameterName") public TLSH(int[] checksum, int[] codes, int lValue, int q1, int q2) { this.checksum = checksum; this.codes = codes; diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHBuilder.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHBuilder.java index f9f88a85..fb729abb 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHBuilder.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHBuilder.java @@ -1,9 +1,6 @@ package org.apache.metron.stellar.common.utils.hashing.tlsh; -import java.nio.ByteBuffer; -import java.util.Arrays; - import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.T0; import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.T11; import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.T13; @@ -17,11 +14,15 @@ import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHUtil.lCapturing; import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHUtil.swapNibble; +import java.nio.ByteBuffer; +import java.util.Arrays; + public class TLSHBuilder { private SlidingWindow slidingWindow; private final CHECKSUM_OPTION checksumOption; private final BUCKET_OPTION bucketOption; private final int[] checksum; + @SuppressWarnings("checkstyle:MemberName") private final int[] aBucket; @@ -86,7 +87,7 @@ public TLSH fromHex(byte[] hash) { break; default: throw new IllegalArgumentException( - String.format("Illegal hash buffer length: %d, must be one of 35,37,67,69", hash.length)); + String.format("Illegal hash buffer length: %d, must be one of 35,37,67,69", hash.length)); } final ByteBuffer buf = ByteBuffer.wrap(hash); final int[] checksum = new int[checksumLength]; @@ -168,6 +169,7 @@ public void clean() { } + @SuppressWarnings("checkstyle:TypeName") public enum CHECKSUM_OPTION { CHECKSUM_1(1), CHECKSUM_3(3); @@ -186,6 +188,7 @@ public static CHECKSUM_OPTION fromVal(int val) { } } + @SuppressWarnings("checkstyle:TypeName") public enum BUCKET_OPTION { BUCKET_128(128), BUCKET_256(256); diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHConstants.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHConstants.java index 0957ab17..c2f24ac8 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHConstants.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHConstants.java @@ -7,18 +7,28 @@ public final class TLSHConstants { /** * Pearson's table. */ - public static final int[] PEARSON_TABLE = {1, 87, 49, 12, 176, 178, 102, 166, 121, 193, 6, 84, 249, 230, 44, 163, 14, 197, 213, - 181, 161, 85, 218, 80, 64, 239, 24, 226, 236, 142, 38, 200, 110, 177, 104, 103, 141, 253, 255, 50, 77, 101, - 81, 18, 45, 96, 31, 222, 25, 107, 190, 70, 86, 237, 240, 34, 72, 242, 20, 214, 244, 227, 149, 235, 97, 234, - 57, 22, 60, 250, 82, 175, 208, 5, 127, 199, 111, 62, 135, 248, 174, 169, 211, 58, 66, 154, 106, 195, 245, - 171, 17, 187, 182, 179, 0, 243, 132, 56, 148, 75, 128, 133, 158, 100, 130, 126, 91, 13, 153, 246, 216, 219, - 119, 68, 223, 78, 83, 88, 201, 99, 122, 11, 92, 32, 136, 114, 52, 10, 138, 30, 48, 183, 156, 35, 61, 26, - 143, 74, 251, 94, 129, 162, 63, 152, 170, 7, 115, 167, 241, 206, 3, 150, 55, 59, 151, 220, 90, 53, 23, 131, - 125, 173, 15, 238, 79, 95, 89, 16, 105, 137, 225, 224, 217, 160, 37, 123, 118, 73, 2, 157, 46, 116, 9, 145, - 134, 228, 207, 212, 202, 215, 69, 229, 27, 188, 67, 124, 168, 252, 42, 4, 29, 108, 21, 247, 19, 205, 39, - 203, 233, 40, 186, 147, 198, 192, 155, 33, 164, 191, 98, 204, 165, 180, 117, 76, 140, 36, 210, 172, 41, 54, - 159, 8, 185, 232, 113, 196, 231, 47, 146, 120, 51, 65, 28, 144, 254, 221, 93, 189, 194, 139, 112, 43, 71, - 109, 184, 209}; + public static final int[] PEARSON_TABLE = { + 1, 87, 49, 12, 176, 178, 102, 166, 121, 193, 6, 84, 249, 230, 44, 163, 14, 197, 213, + 181, 161, 85, 218, 80, 64, 239, 24, 226, 236, 142, 38, 200, 110, 177, 104, 103, 141, 253, 255, 50, 77, + 101, + 81, 18, 45, 96, 31, 222, 25, 107, 190, 70, 86, 237, 240, 34, 72, 242, 20, 214, 244, 227, 149, 235, 97, + 234, + 57, 22, 60, 250, 82, 175, 208, 5, 127, 199, 111, 62, 135, 248, 174, 169, 211, 58, 66, 154, 106, 195, + 245, + 171, 17, 187, 182, 179, 0, 243, 132, 56, 148, 75, 128, 133, 158, 100, 130, 126, 91, 13, 153, 246, 216, + 219, + 119, 68, 223, 78, 83, 88, 201, 99, 122, 11, 92, 32, 136, 114, 52, 10, 138, 30, 48, 183, 156, 35, 61, 26, + 143, 74, 251, 94, 129, 162, 63, 152, 170, 7, 115, 167, 241, 206, 3, 150, 55, 59, 151, 220, 90, 53, 23, + 131, + 125, 173, 15, 238, 79, 95, 89, 16, 105, 137, 225, 224, 217, 160, 37, 123, 118, 73, 2, 157, 46, 116, 9, + 145, + 134, 228, 207, 212, 202, 215, 69, 229, 27, 188, 67, 124, 168, 252, 42, 4, 29, 108, 21, 247, 19, 205, 39, + 203, 233, 40, 186, 147, 198, 192, 155, 33, 164, 191, 98, 204, 165, 180, 117, 76, 140, 36, 210, 172, 41, + 54, + 159, 8, 185, 232, 113, 196, 231, 47, 146, 120, 51, 65, 28, 144, 254, 221, 93, 189, 194, 139, 112, 43, + 71, + 109, 184, 209 + }; /** * Lookup table for the logs of the length value. The last entry saturates the @@ -30,79 +40,93 @@ public final class TLSHConstants { * logLength is n. */ static final long[] TOPVAL = {/* 0 */ 1, /* 1 */ 2, /* 2 */ 3, /* 3 */ 5, /* 4 */ 7, /* 5 */ 11, /* 6 */ 17, - /* 7 */ 25, /* 8 */ 38, /* 9 */ 57, /* 10 */ 86, /* 11 */ 129, /* 12 */ 194, /* 13 */ 291, /* 14 */ 437, - /* 15 */ 656, /* 16 */ 854, /* 17 */ 1_110, /* 18 */ 1443, /* 19 */ 1876, /* 20 */ 2439, /* 21 */ 3171, - /* 22 */ 3475, /* 23 */ 3823, /* 24 */ 4205, /* 25 */ 4626, /* 26 */ 5088, /* 27 */ 5597, /* 28 */ 6157, - /* 29 */ 6772, /* 30 */ 7450, /* 31 */ 8195, /* 32 */ 9014, /* 33 */ 9916, /* 34 */ 10_907, /* 35 */ 11_998, - /* 36 */ 13_198, /* 37 */ 14_518, /* 38 */ 15_970, /* 39 */ 17_567, /* 40 */ 19_323, /* 41 */ 21_256, - /* 42 */ 23_382, /* 43 */ 25_720, /* 44 */ 28_292, /* 45 */ 31_121, /* 46 */ 34_233, /* 47 */ 37_656, - /* 48 */ 41_422, /* 49 */ 45_564, /* 50 */ 50_121, /* 51 */ 55_133, /* 52 */ 60_646, /* 53 */ 66_711, - /* 54 */ 73_382, /* 55 */ 80_721, /* 56 */ 88_793, /* 57 */ 97_672, /* 58 */ 107_439, /* 59 */ 118_183, - /* 60 */ 130_002, /* 61 */ 143_002, /* 62 */ 157_302, /* 63 */ 173_032, /* 64 */ 190_335, /* 65 */ 209_369, - /* 66 */ 230_306, /* 67 */ 253_337, /* 68 */ 278_670, /* 69 */ 306_538, /* 70 */ 337_191, /* 71 */ 370_911, - /* 72 */ 408_002, /* 73 */ 448_802, /* 74 */ 493_682, /* 75 */ 543_050, /* 76 */ 597_356, /* 77 */ 657_091, - /* 78 */ 722_800, /* 79 */ 795_081, /* 80 */ 874_589, /* 81 */ 962_048, /* 82 */ 1_058_252, - /* 83 */ 1_164_078, /* 84 */ 1_280_486, /* 85 */ 1_408_534, /* 86 */ 1_549_388, /* 87 */ 1_704_327, - /* 88 */ 1_874_759, /* 89 */ 2_062_236, /* 90 */ 2_268_459, /* 91 */ 2_495_305, /* 92 */ 2_744_836, - /* 93 */ 3_019_320, /* 94 */ 3_321_252, /* 95 */ 3_653_374, /* 96 */ 4_018_711, /* 97 */ 4_420_582, - /* 98 */ 4_862_641, /* 99 */ 5_348_905, /* 100 */ 5_883_796, /* 101 */ 6_472_176, /* 102 */ 7_119_394, - /* 103 */ 7_831_333, /* 104 */ 8_614_467, /* 105 */ 9_475_909, /* 106 */ 10_423_501, /* 107 */ 11_465_851, - /* 108 */ 12_612_437, /* 109 */ 13_873_681, /* 110 */ 15_261_050, /* 111 */ 16_787_154, - /* 112 */ 18_465_870, /* 113 */ 20_312_458, /* 114 */ 22_343_706, /* 115 */ 24_578_077, - /* 116 */ 27_035_886, /* 117 */ 29_739_474, /* 118 */ 32_713_425, /* 119 */ 35_984_770, - /* 120 */ 39_583_245, /* 121 */ 43_541_573, /* 122 */ 47_895_730, /* 123 */ 52_685_306, - /* 124 */ 57_953_837, /* 125 */ 63_749_221, /* 126 */ 70_124_148, /* 127 */ 77_136_564, - /* 128 */ 84_850_228, /* 129 */ 93_335_252, /* 130 */ 102_668_779, /* 131 */ 112_935_659, - /* 132 */ 124_229_227, /* 133 */ 136_652_151, /* 134 */ 150_317_384, /* 135 */ 165_349_128, - /* 136 */ 181_884_040, /* 137 */ 200_072_456, /* 138 */ 220_079_703, /* 139 */ 242_087_671, - /* 140 */ 266_296_456, /* 141 */ 292_926_096, /* 142 */ 322_218_735, /* 143 */ 354_440_623, - /* 144 */ 389_884_688, /* 145 */ 428_873_168, /* 146 */ 471_760_495, /* 147 */ 518_936_559, - /* 148 */ 570_830_240, /* 149 */ 627_913_311, /* 150 */ 690_704_607, /* 151 */ 759_775_136, - /* 152 */ 835_752_671, /* 153 */ 919_327_967, /* 154 */ 1_011_260_767, /* 155 */ 1_112_386_880, - /* 156 */ 1_223_623_232, /* 157 */ 1_345_985_727, /* 158 */ 1_480_584_256, /* 159 */ 1_628_642_751, - /* 160 */ 1_791_507_135, /* 161 */ 1_970_657_856, /* 162 */ 2_167_723_648L, /* 163 */ 2_384_496_256L, - /* 164 */ 2_622_945_920L, /* 165 */ 2_885_240_448L, /* 166 */ 3_173_764_736L, /* 167 */ 3_491_141_248L, - /* 168 */ 3_840_255_616L, /* 169 */ 4_224_281_216L, /* 170 */ 4_646_709_504L, /* 171 */ 5_111_380_735L, - /* 172 */ 5_622_519_040L, /* 173 */ 6_184_770_816L, /* 174 */ 6_803_248_384L, /* 175 */ 7_483_572_991L, - /* 176 */ 8_231_930_623L, /* 177 */ 9_055_123_968L, /* 178 */ 9_960_636_928L, /* 179 */ 10_956_701_183L, - /* 180 */ 12_052_370_943L, /* 181 */ 13_257_608_703L, /* 182 */ 14_583_370_240L, /* 183 */ 16_041_708_032L, - /* 184 */ 17_645_878_271L, /* 185 */ 19_410_467_839L, /* 186 */ 21_351_515_136L, /* 187 */ 23_486_667_775L, - /* 188 */ 25_835_334_655L, /* 189 */ 28_418_870_271L, /* 190 */ 31_260_756_991L, /* 191 */ 34_386_835_455L, - /* 192 */ 37_825_517_567L, /* 193 */ 41_608_071_168L, /* 194 */ 45_768_882_175L, /* 195 */ 50_345_768_959L, - /* 196 */ 55_380_346_880L, /* 197 */ 60_918_384_640L, /* 198 */ 67_010_226_176L, /* 199 */ 73_711_251_455L, - /* 200 */ 81_082_380_287L, /* 201 */ 89_190_617_088L, /* 202 */ 98_109_681_663L, /* 203 */ 107_920_658_432L, - /* 204 */ 118_712_725_503L, /* 205 */ 130_584_006_656L, /* 206 */ 143_642_402_816L, - /* 207 */ 158_006_648_832L, /* 208 */ 173_807_329_279L, /* 209 */ 191_188_066_303L, - /* 210 */ 210_306_867_200L, /* 211 */ 231_337_566_208L, /* 212 */ 254_471_331_839L, - /* 213 */ 279_918_460_927L, /* 214 */ 307_910_328_319L, /* 215 */ 338_701_369_343L, - /* 216 */ 372_571_521_024L, /* 217 */ 409_827_917_823L, /* 218 */ 450_810_724_351L, - /* 219 */ 495_891_791_872L, /* 220 */ 545_481_015_295L, /* 221 */ 600_029_102_079L, - /* 222 */ 660_032_028_671L, /* 223 */ 726_035_300_351L, /* 224 */ 798_638_833_663L, - /* 225 */ 878_502_772_736L, /* 226 */ 966_353_059_839L, /* 227 */ 1_062_988_382_207L, - /* 228 */ 1_169_287_217_151L, /* 229 */ 1_286_216_024_063L, /* 230 */ 1_414_837_633_024L, - /* 231 */ 1_556_321_468_416L, /* 232 */ 1_711_953_739_776L, /* 233 */ 1_883_149_107_199L, - /* 234 */ 2_071_464_050_688L, /* 235 */ 2_278_610_567_167L, /* 236 */ 2_506_471_636_992L, - /* 237 */ 2_757_119_049_728L, /* 238 */ 3_032_831_098_880L, /* 239 */ 3_336_114_143_231L, - /* 240 */ 3_669_725_675_520L, /* 241 */ 4_036_698_439_680L, /* 242 */ 4_440_368_349_184L, - /* 243 */ 4_884_405_157_887L, /* 244 */ 5_372_846_014_464L, /* 245 */ 5_910_131_113_984L, - /* 246 */ 6_501_144_199_168L, /* 247 */ 7_151_258_697_727L, /* 248 */ 7_866_384_908_288L, - /* 249 */ 8_653_023_477_760L, /* 250 */ 9_518_326_480_895L, /* 251 */ 10_470_159_810_560L, - /* 252 */ 11_517_175_529_472L, /* 253 */ 12_668_893_659_136L, /* 254 */ 13_935_783_182_336L, - /* 255 */ /* 15329425519609L */ Long.MAX_VALUE}; + /* 7 */ 25, /* 8 */ 38, /* 9 */ 57, /* 10 */ 86, /* 11 */ 129, /* 12 */ 194, /* 13 */ 291, /* 14 */ 437, + /* 15 */ 656, /* 16 */ 854, /* 17 */ 1_110, /* 18 */ 1443, /* 19 */ 1876, /* 20 */ 2439, /* 21 */ 3171, + /* 22 */ 3475, /* 23 */ 3823, /* 24 */ 4205, /* 25 */ 4626, /* 26 */ 5088, /* 27 */ 5597, /* 28 */ 6157, + /* 29 */ 6772, /* 30 */ 7450, /* 31 */ 8195, /* 32 */ 9014, /* 33 */ 9916, /* 34 */ 10_907, /* 35 */ 11_998, + /* 36 */ 13_198, /* 37 */ 14_518, /* 38 */ 15_970, /* 39 */ 17_567, /* 40 */ 19_323, /* 41 */ 21_256, + /* 42 */ 23_382, /* 43 */ 25_720, /* 44 */ 28_292, /* 45 */ 31_121, /* 46 */ 34_233, /* 47 */ 37_656, + /* 48 */ 41_422, /* 49 */ 45_564, /* 50 */ 50_121, /* 51 */ 55_133, /* 52 */ 60_646, /* 53 */ 66_711, + /* 54 */ 73_382, /* 55 */ 80_721, /* 56 */ 88_793, /* 57 */ 97_672, /* 58 */ 107_439, /* 59 */ 118_183, + /* 60 */ 130_002, /* 61 */ 143_002, /* 62 */ 157_302, /* 63 */ 173_032, /* 64 */ 190_335, /* 65 */ 209_369, + /* 66 */ 230_306, /* 67 */ 253_337, /* 68 */ 278_670, /* 69 */ 306_538, /* 70 */ 337_191, /* 71 */ 370_911, + /* 72 */ 408_002, /* 73 */ 448_802, /* 74 */ 493_682, /* 75 */ 543_050, /* 76 */ 597_356, /* 77 */ 657_091, + /* 78 */ 722_800, /* 79 */ 795_081, /* 80 */ 874_589, /* 81 */ 962_048, /* 82 */ 1_058_252, + /* 83 */ 1_164_078, /* 84 */ 1_280_486, /* 85 */ 1_408_534, /* 86 */ 1_549_388, /* 87 */ 1_704_327, + /* 88 */ 1_874_759, /* 89 */ 2_062_236, /* 90 */ 2_268_459, /* 91 */ 2_495_305, /* 92 */ 2_744_836, + /* 93 */ 3_019_320, /* 94 */ 3_321_252, /* 95 */ 3_653_374, /* 96 */ 4_018_711, /* 97 */ 4_420_582, + /* 98 */ 4_862_641, /* 99 */ 5_348_905, /* 100 */ 5_883_796, /* 101 */ 6_472_176, /* 102 */ 7_119_394, + /* 103 */ 7_831_333, /* 104 */ 8_614_467, /* 105 */ 9_475_909, /* 106 */ 10_423_501, /* 107 */ 11_465_851, + /* 108 */ 12_612_437, /* 109 */ 13_873_681, /* 110 */ 15_261_050, /* 111 */ 16_787_154, + /* 112 */ 18_465_870, /* 113 */ 20_312_458, /* 114 */ 22_343_706, /* 115 */ 24_578_077, + /* 116 */ 27_035_886, /* 117 */ 29_739_474, /* 118 */ 32_713_425, /* 119 */ 35_984_770, + /* 120 */ 39_583_245, /* 121 */ 43_541_573, /* 122 */ 47_895_730, /* 123 */ 52_685_306, + /* 124 */ 57_953_837, /* 125 */ 63_749_221, /* 126 */ 70_124_148, /* 127 */ 77_136_564, + /* 128 */ 84_850_228, /* 129 */ 93_335_252, /* 130 */ 102_668_779, /* 131 */ 112_935_659, + /* 132 */ 124_229_227, /* 133 */ 136_652_151, /* 134 */ 150_317_384, /* 135 */ 165_349_128, + /* 136 */ 181_884_040, /* 137 */ 200_072_456, /* 138 */ 220_079_703, /* 139 */ 242_087_671, + /* 140 */ 266_296_456, /* 141 */ 292_926_096, /* 142 */ 322_218_735, /* 143 */ 354_440_623, + /* 144 */ 389_884_688, /* 145 */ 428_873_168, /* 146 */ 471_760_495, /* 147 */ 518_936_559, + /* 148 */ 570_830_240, /* 149 */ 627_913_311, /* 150 */ 690_704_607, /* 151 */ 759_775_136, + /* 152 */ 835_752_671, /* 153 */ 919_327_967, /* 154 */ 1_011_260_767, /* 155 */ 1_112_386_880, + /* 156 */ 1_223_623_232, /* 157 */ 1_345_985_727, /* 158 */ 1_480_584_256, /* 159 */ 1_628_642_751, + /* 160 */ 1_791_507_135, /* 161 */ 1_970_657_856, /* 162 */ 2_167_723_648L, /* 163 */ 2_384_496_256L, + /* 164 */ 2_622_945_920L, /* 165 */ 2_885_240_448L, /* 166 */ 3_173_764_736L, /* 167 */ 3_491_141_248L, + /* 168 */ 3_840_255_616L, /* 169 */ 4_224_281_216L, /* 170 */ 4_646_709_504L, /* 171 */ 5_111_380_735L, + /* 172 */ 5_622_519_040L, /* 173 */ 6_184_770_816L, /* 174 */ 6_803_248_384L, /* 175 */ 7_483_572_991L, + /* 176 */ 8_231_930_623L, /* 177 */ 9_055_123_968L, /* 178 */ 9_960_636_928L, /* 179 */ 10_956_701_183L, + /* 180 */ 12_052_370_943L, /* 181 */ 13_257_608_703L, /* 182 */ 14_583_370_240L, /* 183 */ 16_041_708_032L, + /* 184 */ 17_645_878_271L, /* 185 */ 19_410_467_839L, /* 186 */ 21_351_515_136L, /* 187 */ 23_486_667_775L, + /* 188 */ 25_835_334_655L, /* 189 */ 28_418_870_271L, /* 190 */ 31_260_756_991L, /* 191 */ 34_386_835_455L, + /* 192 */ 37_825_517_567L, /* 193 */ 41_608_071_168L, /* 194 */ 45_768_882_175L, /* 195 */ 50_345_768_959L, + /* 196 */ 55_380_346_880L, /* 197 */ 60_918_384_640L, /* 198 */ 67_010_226_176L, /* 199 */ 73_711_251_455L, + /* 200 */ 81_082_380_287L, /* 201 */ 89_190_617_088L, /* 202 */ 98_109_681_663L, /* 203 */ 107_920_658_432L, + /* 204 */ 118_712_725_503L, /* 205 */ 130_584_006_656L, /* 206 */ 143_642_402_816L, + /* 207 */ 158_006_648_832L, /* 208 */ 173_807_329_279L, /* 209 */ 191_188_066_303L, + /* 210 */ 210_306_867_200L, /* 211 */ 231_337_566_208L, /* 212 */ 254_471_331_839L, + /* 213 */ 279_918_460_927L, /* 214 */ 307_910_328_319L, /* 215 */ 338_701_369_343L, + /* 216 */ 372_571_521_024L, /* 217 */ 409_827_917_823L, /* 218 */ 450_810_724_351L, + /* 219 */ 495_891_791_872L, /* 220 */ 545_481_015_295L, /* 221 */ 600_029_102_079L, + /* 222 */ 660_032_028_671L, /* 223 */ 726_035_300_351L, /* 224 */ 798_638_833_663L, + /* 225 */ 878_502_772_736L, /* 226 */ 966_353_059_839L, /* 227 */ 1_062_988_382_207L, + /* 228 */ 1_169_287_217_151L, /* 229 */ 1_286_216_024_063L, /* 230 */ 1_414_837_633_024L, + /* 231 */ 1_556_321_468_416L, /* 232 */ 1_711_953_739_776L, /* 233 */ 1_883_149_107_199L, + /* 234 */ 2_071_464_050_688L, /* 235 */ 2_278_610_567_167L, /* 236 */ 2_506_471_636_992L, + /* 237 */ 2_757_119_049_728L, /* 238 */ 3_032_831_098_880L, /* 239 */ 3_336_114_143_231L, + /* 240 */ 3_669_725_675_520L, /* 241 */ 4_036_698_439_680L, /* 242 */ 4_440_368_349_184L, + /* 243 */ 4_884_405_157_887L, /* 244 */ 5_372_846_014_464L, /* 245 */ 5_910_131_113_984L, + /* 246 */ 6_501_144_199_168L, /* 247 */ 7_151_258_697_727L, /* 248 */ 7_866_384_908_288L, + /* 249 */ 8_653_023_477_760L, /* 250 */ 9_518_326_480_895L, /* 251 */ 10_470_159_810_560L, + /* 252 */ 11_517_175_529_472L, /* 253 */ 12_668_893_659_136L, /* 254 */ 13_935_783_182_336L, + /* 255 */ /* 15329425519609L */ Long.MAX_VALUE}; - /** The Pearson default hash of 0. */ + /** + * The Pearson default hash of 0. + */ public static final int T0 = 1 /* T[0] */; - /** The Pearson default hash of 2. */ + /** + * The Pearson default hash of 2. + */ public static final int T2 = 49 /* T[2] */; - /** The Pearson default hash of 3. */ + /** + * The Pearson default hash of 3. + */ public static final int T3 = 12 /* T[3] */; - /** The Pearson default hash of 5. */ + /** + * The Pearson default hash of 5. + */ public static final int T5 = 178 /* T[5] */; - /** The Pearson default hash of 7. */ + /** + * The Pearson default hash of 7. + */ public static final int T7 = 166 /* T[7] */; - /** The Pearson default hash of 11. */ + /** + * The Pearson default hash of 11. + */ public static final int T11 = 84 /* T[11] */; - /** The Pearson default hash of 13. */ + /** + * The Pearson default hash of 13. + */ public static final int T13 = 230 /* T[13] */; /** @@ -111,25 +135,39 @@ public final class TLSHConstants { public static final int DIFF_SCALE = 12; - /** The length threshold for step 1. */ + /** + * The length threshold for step 1. + */ public static final int LEN_STEP_1 = 656; - /** The log(1.5) constant used in CPP reference implementation for step 1. */ + /** + * The log(1.5) constant used in CPP reference implementation for step 1. + */ public static final double LOG_1_5 = 0.405_465_100D; - /** The length threshold for step 2. */ + /** + * The length threshold for step 2. + */ public static final int LEN_STEP_2 = 3199; - /** The adjustment for step 2. */ + /** + * The adjustment for step 2. + */ public static final double LEN_ADJ_2 = 8.727_770D; - /** The log(1.3) constant used in CPP reference implementation for step 2. */ + /** + * The log(1.3) constant used in CPP reference implementation for step 2. + */ public static final double LOG_1_3 = 0.262_364_260D; - /** The adjustment for step 3. */ + /** + * The adjustment for step 3. + */ public static final double LEN_ADJ_3 = 62.547_200D; - /** The log(1.1) constant used in CPP reference implementation for step 3. */ + /** + * The log(1.1) constant used in CPP reference implementation for step 3. + */ public static final double LOG_1_1 = 0.095_310_180D; public static final byte[] HEX_ARRAY = "0123456789ABCDEF".getBytes(StandardCharsets.US_ASCII); - private TLSHConstants(){ + private TLSHConstants() { } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHHasher.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHHasher.java index 14468856..cb5827c7 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHHasher.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHHasher.java @@ -15,14 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.stellar.common.utils.hashing.tlsh; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.metron.stellar.common.utils.ConversionUtils; -import org.apache.metron.stellar.common.utils.SerDeUtils; -import org.apache.metron.stellar.common.utils.hashing.EnumConfigurable; -import org.apache.metron.stellar.common.utils.hashing.Hasher; +package org.apache.metron.stellar.common.utils.hashing.tlsh; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -34,6 +28,12 @@ import java.util.Optional; import java.util.Random; import java.util.Set; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.SerDeUtils; +import org.apache.metron.stellar.common.utils.hashing.EnumConfigurable; +import org.apache.metron.stellar.common.utils.hashing.Hasher; public class TLSHHasher implements Hasher { public static final String TLSH_KEY = "tlsh"; @@ -44,7 +44,7 @@ public enum Config implements EnumConfigurable { CHECKSUM("checksumBytes"), HASHES("hashes"), FORCE("force"); - final public String key; + public final String key; Config(String key) { this.key = key; @@ -67,11 +67,11 @@ public String getKey() { * * @param o The value to hash. * @return A hash of {@code toHash} that has been encoded. - * */ @Override public Object getHash(Object o) { - TLSHBuilder builder = new TLSHBuilder(TLSHBuilder.CHECKSUM_OPTION.fromVal(checksumOption), TLSHBuilder.BUCKET_OPTION.fromVal(bucketOption)); + TLSHBuilder builder = new TLSHBuilder(TLSHBuilder.CHECKSUM_OPTION.fromVal(checksumOption), + TLSHBuilder.BUCKET_OPTION.fromVal(bucketOption)); byte[] data; if (o instanceof String) { data = ((String) o).getBytes(StandardCharsets.UTF_8); @@ -120,48 +120,51 @@ public Map bin(String hash) throws DecoderException { return ret; } + @SuppressWarnings("checkstyle:LocalVariableName") @Override public void configure(Optional> config) { if (config.isPresent() && !config.get().isEmpty()) { - bucketOption = Config.BUCKET_SIZE.get(config.get() - , o -> { - Integer bucketSize = ConversionUtils.convert(o, Integer.class); - return bucketSize.equals(256) ? 256 : 128; - } + bucketOption = Config.BUCKET_SIZE.get(config.get(), + o -> { + Integer bucketSize = ConversionUtils.convert(o, Integer.class); + return bucketSize.equals(256) ? 256 : 128; + } ).orElse(bucketOption); - checksumOption = Config.CHECKSUM.get(config.get() - , o -> { - Integer checksumBytes = ConversionUtils.convert(o, Integer.class); - return checksumBytes.equals(3) ? 3 : 1; - } + checksumOption = Config.CHECKSUM.get(config.get(), + o -> { + Integer checksumBytes = ConversionUtils.convert(o, Integer.class); + return checksumBytes.equals(3) ? 3 : 1; + } ).orElse(checksumOption); - force = Config.FORCE.get(config.get() - , o -> ConversionUtils.convert(o, Boolean.class) + force = Config.FORCE.get(config.get(), + o -> ConversionUtils.convert(o, Boolean.class) ).orElse(force); - hashes = Config.HASHES.get(config.get() - , o -> { - List ret = new ArrayList<>(); - if (o instanceof List) { - List vals = (List) o; - for (Object oVal : vals) { - ret.add(ConversionUtils.convert(oVal, Integer.class)); - } - } else { - ret.add(ConversionUtils.convert(o, Integer.class)); - } - return ret; - } + hashes = Config.HASHES.get(config.get(), + o -> { + List ret = new ArrayList<>(); + if (o instanceof List) { + List vals = (List) o; + for (Object oVal : vals) { + ret.add(ConversionUtils.convert(oVal, Integer.class)); + } + } else { + ret.add(ConversionUtils.convert(o, Integer.class)); + } + return ret; + } ).orElse(hashes); } } public static Set supportedHashes() { - return new HashSet() {{ - add("TLSH"); - }}; + return new HashSet() { + { + add("TLSH"); + } + }; } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHScorer.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHScorer.java index 8d465e2e..65e89d90 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHScorer.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHScorer.java @@ -1,9 +1,9 @@ package org.apache.metron.stellar.common.utils.hashing.tlsh; -import java.util.Arrays; - import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.DIFF_SCALE; +import java.util.Arrays; + public class TLSHScorer { private final BitPairsTable diffTable; @@ -29,7 +29,7 @@ public int score(TLSH tlsh1, TLSH tlsh2, boolean lenDiff) { private int scoreBuckets(final int[] buckets1, final int[] buckets2) { if (buckets1.length != buckets2.length) { throw new IllegalArgumentException( - String.format("Number of body bytes differ %d != %d", buckets1.length, buckets2.length)); + String.format("Number of body bytes differ %d != %d", buckets1.length, buckets2.length)); } int diff = 0; @@ -42,7 +42,7 @@ private int scoreBuckets(final int[] buckets1, final int[] buckets2) { private int scoreChecksum(final int[] checksumA, final int[] checksumB) { if (checksumA.length != checksumB.length) { throw new IllegalArgumentException( - String.format("Number of checksum bytes differ %d != %d", checksumA.length, checksumB.length)); + String.format("Number of checksum bytes differ %d != %d", checksumA.length, checksumB.length)); } return Arrays.equals(checksumA, checksumB) ? 0 : 1; } @@ -53,6 +53,7 @@ private int scoreQ(final int q2, final int q3) { return q1diff <= 1 ? q1diff : (q1diff - 1) * DIFF_SCALE; } + @SuppressWarnings("checkstyle:ParameterName") private int scoreLValue(final int lValue2, final int lValue3) { final int ldiff = modDiff(lValue2, lValue3, 256); switch (ldiff) { diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHUtil.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHUtil.java index 56616202..258a1fda 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHUtil.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSHUtil.java @@ -1,10 +1,6 @@ package org.apache.metron.stellar.common.utils.hashing.tlsh; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.stream.IntStream; - import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.HEX_ARRAY; import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.LEN_ADJ_2; import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.LEN_ADJ_3; @@ -15,7 +11,9 @@ import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.LOG_1_5; import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.PEARSON_TABLE; import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHConstants.TOPVAL; -import java.io.ByteArrayOutputStream; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; public final class TLSHUtil { @@ -36,6 +34,7 @@ public static int bucketMapping(final int salt, final int i, final int j, final return PEARSON_TABLE[PEARSON_TABLE[PEARSON_TABLE[PEARSON_TABLE[salt] ^ i] ^ j] ^ k]; } + @SuppressWarnings("checkstyle:ParameterName") public static int fastBucketMapping(final int mod_salt, final int i, final int j, final int k) { return PEARSON_TABLE[PEARSON_TABLE[PEARSON_TABLE[mod_salt ^ i] ^ j] ^ k]; } @@ -46,6 +45,7 @@ public static int fastBucketMapping(final int mod_salt, final int i, final int j * @param len the length * @return the byte value */ + @SuppressWarnings("checkstyle:MethodName") public static int lCapturing(final int len) { final int x = Arrays.binarySearch(TOPVAL, len); return x >= 0 ? x : -x - 1; @@ -60,6 +60,7 @@ public static int lCapturing(final int len) { * @param len the length * @return the byte value */ + @SuppressWarnings("checkstyle:MethodName") public static int lCapturingLog(final int len) { if (len <= 0) { return 0; diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/MathOperation.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/MathOperation.java index 803d1300..bc697334 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/MathOperation.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/MathOperation.java @@ -17,29 +17,31 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.utils.math; import java.util.function.Function; public class MathOperation { - private int maxArgs; - private int minArgs; - private Function operation; - public MathOperation(Function operation, int minArgs, int maxArgs) { - this.operation = operation; - this.maxArgs = maxArgs; - this.minArgs = minArgs; - } + private final int maxArgs; + private final int minArgs; + private final Function operation; + + public MathOperation(Function operation, int minArgs, int maxArgs) { + this.operation = operation; + this.maxArgs = maxArgs; + this.minArgs = minArgs; + } - public int getMaxArgs() { - return maxArgs; - } + public int getMaxArgs() { + return maxArgs; + } - public int getMinArgs() { - return minArgs; - } + public int getMinArgs() { + return minArgs; + } - public Function getOperation() { - return operation; - } + public Function getOperation() { + return operation; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/MathOperations.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/MathOperations.java index bccdc001..20e02c42 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/MathOperations.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/MathOperations.java @@ -17,67 +17,72 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.utils.math; import java.util.function.BiFunction; import java.util.function.Function; public enum MathOperations implements Function { - ABS(d -> Math.abs(d)), - CEIL(d -> Math.ceil(d)), - COS(d -> Math.cos(d)), - FLOOR(d -> Math.floor(d)), - LOG10(d -> Math.log10(d)), - LOG2(d -> Math.log(d)/Math.log(2)), - LN(d -> Math.log(d)), - SIN(d -> Math.sin(d)), - SQRT(d -> Math.sqrt(d)), - TAN(d -> Math.tan(d)), - EXP(d -> Math.exp(d)), - ROUND(new MathOperation(d -> { - Double val = d[0].doubleValue(); - return Double.isNaN(val)?Double.NaN:Math.round(d[0].doubleValue()); - }, 1, 1)), - ; + ABS(d -> Math.abs(d)), + CEIL(d -> Math.ceil(d)), + COS(d -> Math.cos(d)), + FLOOR(d -> Math.floor(d)), + LOG10(d -> Math.log10(d)), + LOG2(d -> Math.log(d) / Math.log(2)), + LN(d -> Math.log(d)), + SIN(d -> Math.sin(d)), + SQRT(d -> Math.sqrt(d)), + TAN(d -> Math.tan(d)), + EXP(d -> Math.exp(d)), + ROUND(new MathOperation(d -> { + Double val = d[0].doubleValue(); + return Double.isNaN(val) ? Double.NaN : Math.round(d[0].doubleValue()); + }, 1, 1)), + ; - private static class SingleOpFunc implements Function { - Function f; - public SingleOpFunc(Function f) { - this.f = f; - } - @Override - public Number apply(Number[] numbers) { - return f.apply(numbers[0].doubleValue()); - } - } + private static class SingleOpFunc implements Function { + Function func; - private static class BinaryOpFunc implements Function { - BiFunction f; - public BinaryOpFunc(BiFunction f) { - this.f = f; + public SingleOpFunc(Function f) { + this.func = f; + } + + @Override + public Number apply(Number[] numbers) { + return func.apply(numbers[0].doubleValue()); + } } - @Override - public Number apply(Number[] numbers) { - return f.apply(numbers[0].doubleValue(), numbers[1].doubleValue()); + + private static class BinaryOpFunc implements Function { + BiFunction func; + + public BinaryOpFunc(BiFunction f) { + this.func = f; + } + + @Override + public Number apply(Number[] numbers) { + return func.apply(numbers[0].doubleValue(), numbers[1].doubleValue()); + } } - } - MathOperation op; - MathOperations(Function singleArg) { - op = new MathOperation(new SingleOpFunc(singleArg), 1, 1); - } + MathOperation op; - MathOperations(BiFunction binaryArg) { - op = new MathOperation(new BinaryOpFunc(binaryArg), 2, 2); - } + MathOperations(Function singleArg) { + op = new MathOperation(new SingleOpFunc(singleArg), 1, 1); + } - MathOperations(MathOperation op) - { - this.op = op; - } + MathOperations(BiFunction binaryArg) { + op = new MathOperation(new BinaryOpFunc(binaryArg), 2, 2); + } - @Override - public Number apply(Number[] in) { - return op.getOperation().apply(in); - } + MathOperations(MathOperation op) { + this.op = op; + } + + @Override + public Number apply(Number[] in) { + return op.getOperation().apply(in); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/StellarMathFunction.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/StellarMathFunction.java index 0f9eb169..f48457db 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/StellarMathFunction.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/math/StellarMathFunction.java @@ -17,48 +17,52 @@ * limitations under the License. * */ + package org.apache.metron.stellar.common.utils.math; +import java.util.List; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.StellarFunction; -import java.util.List; - public class StellarMathFunction implements StellarFunction { - MathOperation _func; - public StellarMathFunction(MathOperations _func) { - this._func = _func.op; - } - - public StellarMathFunction(MathOperation _func) { - this._func = _func; - } + @SuppressWarnings("checkstyle:MemberName") + MathOperation _func; - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() < _func.getMinArgs()) { - return Double.NaN; + @SuppressWarnings("checkstyle:ParameterName") + public StellarMathFunction(MathOperations _func) { + this._func = _func.op; } - Number[] nums = new Number[_func.getMaxArgs()]; - for(int i = 0; i < _func.getMaxArgs();++i) { - nums[i] = (Number)args.get(i); - if(nums[i] == null) { - return Double.NaN; - } + + @SuppressWarnings("checkstyle:ParameterName") + public StellarMathFunction(MathOperation _func) { + this._func = _func; } - Object ret = _func.getOperation().apply(nums); - return ret; - } + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() < _func.getMinArgs()) { + return Double.NaN; + } + Number[] nums = new Number[_func.getMaxArgs()]; + for (int i = 0; i < _func.getMaxArgs(); ++i) { + nums[i] = (Number) args.get(i); + if (nums[i] == null) { + return Double.NaN; + } + } - @Override - public void initialize(Context context) { + Object ret = _func.getOperation().apply(nums); + return ret; + } - } + @Override + public void initialize(Context context) { - @Override - public boolean isInitialized() { - return true; - } + } + + @Override + public boolean isInitialized() { + return true; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/BaseStellarFunction.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/BaseStellarFunction.java index 92a41297..58a7f4f6 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/BaseStellarFunction.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/BaseStellarFunction.java @@ -7,36 +7,39 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl; import java.util.List; /** - * Functions that do not require initialization can extend this class rather than directly implement StellarFunction + * Functions that do not require initialization can extend this class rather than directly implement StellarFunction. */ public abstract class BaseStellarFunction implements StellarFunction { - public abstract Object apply(List args); + public abstract Object apply(List args); - @Override - public Object apply(List args, Context context) throws ParseException { - return apply(args); - } + @Override + public Object apply(List args, Context context) throws ParseException { + return apply(args); + } - @Override - public void initialize(Context context) { + @Override + public void initialize(Context context) { - } + } - @Override - public boolean isInitialized() { - return true; - } + @Override + public boolean isInitialized() { + return true; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Context.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Context.java index d31580b3..b738eee0 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Context.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Context.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl; import java.io.Serializable; @@ -24,137 +27,131 @@ public class Context implements Serializable { - public interface Capability { - Object get(); - } - - public enum Capabilities { - HBASE_PROVIDER - , - /** - * This capability indicates that the global config is available. - */ - GLOBAL_CONFIG - , - /** - * This capability indicates that a zookeeper client (i.e. a Curator client, specifically) is available. - */ - ZOOKEEPER_CLIENT - , - /** - * This capability indicates that a MaaS service discoverer is available. - */ - SERVICE_DISCOVERER - , - /** - * This capability indicates that a map configuring stellar is available. Generally this is done within the global config - * inside of storm, but may be sourced elsewhere (e.g. the CLI when running the REPL). - */ - STELLAR_CONFIG - , - /** - * This capability indicates that the Console object is available. This is available when run via the CLI (e.g. from the REPL). - */ - CONSOLE - , - /** - * This capability indicates that shell variables are available. This is available when run via the CLI (e.g. from the REPL). - */ - SHELL_VARIABLES - , - /** - * This capability indicates that the StellarProcessor should use a Caffeine cache to cache expression -{@literal >} results. If an expression - * is in the cache, then the cached result will be returned instead of recomputing. - */ - CACHE - , - /** - * This capability indicates that a http client (i.e. a CloseableHttpClient, specifically) is available. - */ - HTTP_CLIENT - } - - public enum ActivityType { - VALIDATION_ACTIVITY, - PARSE_ACTIVITY - } - - private static ThreadLocal _activityType = ThreadLocal.withInitial(() -> - null); - - public static class Builder { - - private Map capabilityMap = new HashMap<>(); - - public Builder with(String s, Capability capability) { - capabilityMap.put(s, capability); - return this; + public interface Capability { + Object get(); } - public Builder with(Enum s, Capability capability) { - capabilityMap.put(s.toString(), capability); - return this; + public enum Capabilities { + HBASE_PROVIDER, + /** + * This capability indicates that the global config is available. + */ + GLOBAL_CONFIG, + /** + * This capability indicates that a zookeeper client (i.e. a Curator client, specifically) is available. + */ + ZOOKEEPER_CLIENT, + /** + * This capability indicates that a MaaS service discoverer is available. + */ + SERVICE_DISCOVERER, + /** + * This capability indicates that a map configuring stellar is available. Generally this is done within the global config + * inside of storm, but may be sourced elsewhere (e.g. the CLI when running the REPL). + */ + STELLAR_CONFIG, + /** + * This capability indicates that the Console object is available. This is available when run via the CLI (e.g. from the REPL). + */ + CONSOLE, + /** + * This capability indicates that shell variables are available. This is available when run via the CLI (e.g. from the REPL). + */ + SHELL_VARIABLES, + /** + * This capability indicates that the StellarProcessor should use a Caffeine cache to cache expression -{@literal >} results. + * If an expression is in the cache, then the cached result will be returned instead of recomputing. + */ + CACHE, + /** + * This capability indicates that a http client (i.e. a CloseableHttpClient, specifically) is available. + */ + HTTP_CLIENT } - - public Builder withAll(Map externalConfig) { - for(Map.Entry entry : externalConfig.entrySet()) { - capabilityMap.put(entry.getKey(), () -> entry.getValue()); - } - return this; + public enum ActivityType { + VALIDATION_ACTIVITY, + PARSE_ACTIVITY } - public Context build() { - return new Context(capabilityMap); - } - } + private static final ThreadLocal _activityType = ThreadLocal.withInitial(() -> + null); + + public static class Builder { + + private final Map capabilityMap = new HashMap<>(); + + public Builder with(String s, Capability capability) { + capabilityMap.put(s, capability); + return this; + } - public static Context EMPTY_CONTEXT() { - return new Context(new HashMap<>()){}; - } + public Builder with(Enum s, Capability capability) { + capabilityMap.put(s.toString(), capability); + return this; + } + + public Builder withAll(Map externalConfig) { + for (Map.Entry entry : externalConfig.entrySet()) { + + capabilityMap.put(entry.getKey(), () -> entry.getValue()); + } + return this; + } + + public Context build() { + return new Context(capabilityMap); + } + } - private Map capabilities; + @SuppressWarnings("checkstyle:MethodName") + public static Context EMPTY_CONTEXT() { + return new Context(new HashMap<>()) { + }; + } - private Context( Map capabilities) { - this.capabilities = capabilities; - } + private final Map capabilities; - public Optional getCapability(Enum capability) { - return getCapability(capability, true); - } + private Context(Map capabilities) { + this.capabilities = capabilities; + } - public Optional getCapability(Enum capability, boolean errorIfNotThere) { - return getCapability(capability.toString(), errorIfNotThere); - } + public Optional getCapability(Enum capability) { + return getCapability(capability, true); + } - public Optional getCapability(String capability) { - return getCapability(capability, true); - } + public Optional getCapability(Enum capability, boolean errorIfNotThere) { + return getCapability(capability.toString(), errorIfNotThere); + } - public Optional getCapability(String capability, boolean errorIfNotThere) { - Capability c = capabilities.get(capability); - if(c == null && errorIfNotThere) { - throw new IllegalStateException("Unable to find capability " + capability + "; it may not be available in your context."); + public Optional getCapability(String capability) { + return getCapability(capability, true); } - else if(c == null) { - return Optional.empty(); + + public Optional getCapability(String capability, boolean errorIfNotThere) { + Capability c = capabilities.get(capability); + if (c == null && errorIfNotThere) { + throw new IllegalStateException( + "Unable to find capability " + capability + "; it may not be available in your context."); + } else if (c == null) { + return Optional.empty(); + } + return Optional.ofNullable(c.get()); } - return Optional.ofNullable(c.get()); - } - public void addCapability(String s, Capability capability) { - this.capabilities.put(s, capability); - } + public void addCapability(String s, Capability capability) { + this.capabilities.put(s, capability); + } - public void addCapability(Enum s, Capability capability) { - this.capabilities.put(s.toString(), capability); - } + public void addCapability(Enum s, Capability capability) { + this.capabilities.put(s.toString(), capability); + } - public ActivityType getActivityType() { - return _activityType.get(); - } + public ActivityType getActivityType() { + return _activityType.get(); + } - public void setActivityType(ActivityType activityType) { - _activityType.set(activityType); - } + public void setActivityType(ActivityType activityType) { + _activityType.set(activityType); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java index fc2c2b7d..212a6a1a 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,25 +22,27 @@ import java.util.function.Function; -public class DefaultVariableResolver implements VariableResolver{ - Function resolveFunc; - Function existsFunc; +public class DefaultVariableResolver implements VariableResolver { + Function resolveFunc; + Function existsFunc; - public DefaultVariableResolver(Function resolveFunc, Function existsFunc){ - this.resolveFunc = resolveFunc; - this.existsFunc = existsFunc; - } - @Override - public Object resolve(String variable) { - return resolveFunc.apply(variable); - } + public DefaultVariableResolver(Function resolveFunc, Function existsFunc) { + this.resolveFunc = resolveFunc; + this.existsFunc = existsFunc; + } - @Override - public boolean exists(String variable) { - return existsFunc.apply(variable); - } + @Override + public Object resolve(String variable) { + return resolveFunc.apply(variable); + } - public static DefaultVariableResolver NULL_RESOLVER() { - return new DefaultVariableResolver(x -> null, x -> false); - } + @Override + public boolean exists(String variable) { + return existsFunc.apply(variable); + } + + @SuppressWarnings("checkstyle:MethodName") + public static DefaultVariableResolver NULL_RESOLVER() { + return new DefaultVariableResolver(x -> null, x -> false); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/ErrorListener.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/ErrorListener.java index 3e30775a..ee5bf15f 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/ErrorListener.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/ErrorListener.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +20,7 @@ package org.apache.metron.stellar.dsl; +import java.util.BitSet; import org.antlr.v4.runtime.ANTLRErrorListener; import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.RecognitionException; @@ -25,26 +28,28 @@ import org.antlr.v4.runtime.atn.ATNConfigSet; import org.antlr.v4.runtime.dfa.DFA; -import java.util.BitSet; - public class ErrorListener implements ANTLRErrorListener { - @Override - public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { - throw new ParseException("Syntax error @ " + line + ":" + charPositionInLine+ " " + msg, e); - } + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, + String msg, RecognitionException e) { + throw new ParseException("Syntax error @ " + line + ":" + charPositionInLine + " " + msg, e); + } - @Override - public void reportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, boolean exact, BitSet ambigAlts, ATNConfigSet configs) { - } + @Override + public void reportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, boolean exact, + BitSet ambigAlts, ATNConfigSet configs) { + } - @Override - public void reportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet conflictingAlts, ATNConfigSet configs) { - } + @Override + public void reportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, int stopIndex, + BitSet conflictingAlts, ATNConfigSet configs) { + } - @Override - public void reportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, int prediction, ATNConfigSet configs) { - } + @Override + public void reportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, int prediction, + ATNConfigSet configs) { + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/FunctionMarker.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/FunctionMarker.java index 96baeab1..9f1914e7 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/FunctionMarker.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/FunctionMarker.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,4 +20,5 @@ package org.apache.metron.stellar.dsl; -public class FunctionMarker { } +public class FunctionMarker { +} diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/GrammarUtils.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/GrammarUtils.java index 03f5c111..c1983530 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/GrammarUtils.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/GrammarUtils.java @@ -17,117 +17,111 @@ * limitations under the License. * */ -package org.apache.metron.stellar.dsl; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.Token; +package org.apache.metron.stellar.dsl; import java.util.ArrayList; import java.util.List; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.tree.ParseTree; public class GrammarUtils { - public static String toSyntaxTree(ParseTree tree) { - return new AST(tree).toString(); - } + public static String toSyntaxTree(ParseTree tree) { + return new AST(tree).toString(); + } - /** - * This is a utility class to walk the parse tree for an antlr ParseTree - */ - private static class AST { + /** + * This is a utility class to walk the parse tree for an antlr ParseTree. + */ + private static class AST { - private final Object payload; - private final List children; + private final Object payload; + private final List children; - public AST(ParseTree tree) { - this(null, tree); - } - - private AST(AST ast, ParseTree tree) { - this(ast, tree, new ArrayList<>()); - } + public AST(ParseTree tree) { + this(null, tree); + } - private AST(AST parent, ParseTree tree, List children) { - this.payload = getPayload(tree); - this.children = children; - if (parent == null) { - walk(tree, this); - } - else { - parent.children.add(this); - } - } + private AST(AST ast, ParseTree tree) { + this(ast, tree, new ArrayList<>()); + } - private Object getPayload(ParseTree tree) { - if (tree.getChildCount() == 0) { - return tree.getPayload(); - } - else { - String ruleName = tree.getClass().getSimpleName().replace("Context", ""); - return Character.toLowerCase(ruleName.charAt(0)) + ruleName.substring(1); - } - } + private AST(AST parent, ParseTree tree, List children) { + this.payload = getPayload(tree); + this.children = children; + if (parent == null) { + walk(tree, this); + } else { + parent.children.add(this); + } + } - private static void walk(ParseTree tree, AST ast) { - if (tree.getChildCount() == 0) { - new AST(ast, tree); - } - else if (tree.getChildCount() == 1) { - walk(tree.getChild(0), ast); - } - else if (tree.getChildCount() > 1) { - for (int i = 0; i < tree.getChildCount(); i++) { - AST temp = new AST(ast, tree.getChild(i)); - if (!(temp.payload instanceof Token)) { - walk(tree.getChild(i), temp); - } + private Object getPayload(ParseTree tree) { + if (tree.getChildCount() == 0) { + return tree.getPayload(); + } else { + String ruleName = tree.getClass().getSimpleName().replace("Context", ""); + return Character.toLowerCase(ruleName.charAt(0)) + ruleName.substring(1); + } } - } - } - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - AST ast = this; - List firstStack = new ArrayList<>(); - firstStack.add(ast); - List> childListStack = new ArrayList<>(); - childListStack.add(firstStack); - while (!childListStack.isEmpty()) { - List childStack = childListStack.get(childListStack.size() - 1); - if (childStack.isEmpty()) { - childListStack.remove(childListStack.size() - 1); + private static void walk(ParseTree tree, AST ast) { + if (tree.getChildCount() == 0) { + new AST(ast, tree); + } else if (tree.getChildCount() == 1) { + walk(tree.getChild(0), ast); + } else if (tree.getChildCount() > 1) { + for (int i = 0; i < tree.getChildCount(); i++) { + AST temp = new AST(ast, tree.getChild(i)); + if (!(temp.payload instanceof Token)) { + walk(tree.getChild(i), temp); + } + } + } } - else { - ast = childStack.remove(0); - String caption; - if (ast.payload instanceof Token) { - Token token = (Token) ast.payload; - caption = String.format("TOKEN[type: %s, text: %s]", - token.getType(), token.getText().replace("\n", "\\n")); - } - else { - caption = String.valueOf(ast.payload); - } - String indent = ""; - for (int i = 0; i < childListStack.size() - 1; i++) { - indent += (childListStack.get(i).size() > 0) ? "| " : " "; - } - builder.append(indent) - .append(childStack.isEmpty() ? "'- " : "|- ") - .append(caption) - .append("\n"); - if (ast.children.size() > 0) { - List children = new ArrayList<>(); - for (int i = 0; i < ast.children.size(); i++) { - children.add(ast.children.get(i)); + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + AST ast = this; + List firstStack = new ArrayList<>(); + firstStack.add(ast); + List> childListStack = new ArrayList<>(); + childListStack.add(firstStack); + while (!childListStack.isEmpty()) { + List childStack = childListStack.get(childListStack.size() - 1); + if (childStack.isEmpty()) { + childListStack.remove(childListStack.size() - 1); + } else { + ast = childStack.remove(0); + String caption; + if (ast.payload instanceof Token) { + Token token = (Token) ast.payload; + caption = String.format("TOKEN[type: %s, text: %s]", + token.getType(), token.getText().replace("\n", "\\n")); + } else { + caption = String.valueOf(ast.payload); + } + String indent = ""; + for (int i = 0; i < childListStack.size() - 1; i++) { + indent += (childListStack.get(i).size() > 0) ? "| " : " "; + } + builder.append(indent) + .append(childStack.isEmpty() ? "'- " : "|- ") + .append(caption) + .append("\n"); + + if (ast.children.size() > 0) { + List children = new ArrayList<>(); + for (int i = 0; i < ast.children.size(); i++) { + children.add(ast.children.get(i)); + } + childListStack.add(children); + } + } } - childListStack.add(children); - } + return builder.toString(); } - } - return builder.toString(); } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java index 872211d7..249f1c46 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,51 +21,50 @@ package org.apache.metron.stellar.dsl; -import org.apache.metron.stellar.common.utils.ConcatMap; - import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.common.utils.ConcatMap; public class MapVariableResolver implements VariableResolver { - List variableMappings = new ArrayList<>(); + List variableMappings = new ArrayList<>(); - public MapVariableResolver(Map variableMappingOne, Map... variableMapping) { - if (variableMappingOne != null) { - variableMappings.add(variableMappingOne); + public MapVariableResolver(Map variableMappingOne, Map... variableMapping) { + if (variableMappingOne != null) { + variableMappings.add(variableMappingOne); + } + add(variableMapping); } - add(variableMapping); - } - public void add(Map... ms) { - if (ms != null) { - for (Map m : ms) { - if (m != null) { - this.variableMappings.add(m); + public void add(Map... ms) { + if (ms != null) { + for (Map m : ms) { + if (m != null) { + this.variableMappings.add(m); + } + } } - } } - } - @Override - public Object resolve(String variable) { - if(variable != null && variable.equals(VariableResolver.ALL_FIELDS)) { - return new ConcatMap(variableMappings); - } + @Override + public Object resolve(String variable) { + if (variable != null && variable.equals(VariableResolver.ALL_FIELDS)) { + return new ConcatMap(variableMappings); + } - for (Map variableMapping : variableMappings) { - Object o = variableMapping.get(variable); - if (o != null) { - return o; - } + for (Map variableMapping : variableMappings) { + Object o = variableMapping.get(variable); + if (o != null) { + return o; + } + } + return null; } - return null; - } - @Override - public boolean exists(String variable) { - return true; - } + @Override + public boolean exists(String variable) { + return true; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/ParseException.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/ParseException.java index 0fd8af0f..7f911fca 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/ParseException.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/ParseException.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,10 +21,11 @@ package org.apache.metron.stellar.dsl; public class ParseException extends RuntimeException { - public ParseException(String reason) { - super(reason); - } - public ParseException(String reason, Throwable t) { - super(reason, t); - } + public ParseException(String reason) { + super(reason); + } + + public ParseException(String reason, Throwable t) { + super(reason, t); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Predicate2StellarFunction.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Predicate2StellarFunction.java index eee50533..b0f663c4 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Predicate2StellarFunction.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Predicate2StellarFunction.java @@ -7,27 +7,31 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl; import java.util.List; import java.util.function.Predicate; public class Predicate2StellarFunction extends BaseStellarFunction { - Predicate> pred; - public Predicate2StellarFunction(Predicate> pred) { - this.pred = pred; - } + Predicate> pred; + + public Predicate2StellarFunction(Predicate> pred) { + this.pred = pred; + } - @Override - public Object apply(List objects) { - return pred.test(objects); - } + @Override + public Object apply(List objects) { + return pred.test(objects); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Stellar.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Stellar.java index 8afcad31..3324c27e 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Stellar.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Stellar.java @@ -7,27 +7,33 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.stellar.dsl; -import org.atteo.classindex.IndexAnnotated; +package org.apache.metron.stellar.dsl; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import org.atteo.classindex.IndexAnnotated; @Retention(RetentionPolicy.RUNTIME) @IndexAnnotated public @interface Stellar { - String namespace() default ""; - String name(); - String description() default ""; - String returns() default ""; - String[] params() default {}; + String namespace() default ""; + + String name(); + + String description() default ""; + + String returns() default ""; + + String[] params() default {}; } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java index 4fabfafd..0e329f9b 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl; import java.io.Closeable; @@ -22,12 +25,14 @@ import java.util.List; public interface StellarFunction extends Closeable { - Object apply(List args, Context context) throws ParseException; - void initialize(Context context); - boolean isInitialized(); + Object apply(List args, Context context) throws ParseException; + + void initialize(Context context); + + boolean isInitialized(); - @Override - default void close() throws IOException { + @Override + default void close() throws IOException { - } + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunctionInfo.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunctionInfo.java index 86067230..1f870e08 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunctionInfo.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunctionInfo.java @@ -7,110 +7,127 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl; import java.util.Arrays; +import java.util.Objects; /** * Describes a Stellar function. */ public class StellarFunctionInfo { - /** - * The name of the function. - */ - String name; - - /** - * A description of the function. Used for documentation purposes. - */ - String description; - - /** - * A description of what the function returns. Used for documentation purposes. - */ - String returns; - - /** - * The function parameters. Used for documentation purposes. - */ - String[] params; - - /** - * The actual function that can be executed. - */ - StellarFunction function; - - public StellarFunctionInfo(String description, String name, String[] params, String returns, StellarFunction function) { - this.description = description; - this.name = name; - this.params = params; - this.function = function; - this.returns = returns; - } - - public String getReturns() { - return returns; - } - - public String getDescription() { - return description; - } - - public String getName() { - return name; - } - - public String[] getParams() { - return params; - } - - public StellarFunction getFunction() { - return function; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - StellarFunctionInfo that = (StellarFunctionInfo) o; - - if (name != null ? !name.equals(that.name) : that.name != null) return false; - if (description != null ? !description.equals(that.description) : that.description != null) return false; - if (returns != null ? !returns.equals(that.returns) : that.returns != null) return false; - // Probably incorrect - comparing Object[] arrays with Arrays.equals - if (!Arrays.equals(params, that.params)) return false; - return function != null ? function.equals(that.function) : that.function == null; - - } - - @Override - public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + (description != null ? description.hashCode() : 0); - result = 31 * result + (returns != null ? returns.hashCode() : 0); - result = 31 * result + Arrays.hashCode(params); - result = 31 * result + (function != null ? function.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "StellarFunctionInfo{" + - "name='" + name + '\'' + - ", description='" + description + '\'' + - ", returns='" + returns + '\'' + - ", params=" + Arrays.toString(params) + - ", function=" + function + - '}'; - } + /** + * The name of the function. + */ + String name; + + /** + * A description of the function. Used for documentation purposes. + */ + String description; + + /** + * A description of what the function returns. Used for documentation purposes. + */ + String returns; + + /** + * The function parameters. Used for documentation purposes. + */ + String[] params; + + /** + * The actual function that can be executed. + */ + StellarFunction function; + + public StellarFunctionInfo(String description, String name, String[] params, String returns, + StellarFunction function) { + this.description = description; + this.name = name; + this.params = params; + this.function = function; + this.returns = returns; + } + + public String getReturns() { + return returns; + } + + public String getDescription() { + return description; + } + + public String getName() { + return name; + } + + public String[] getParams() { + return params; + } + + public StellarFunction getFunction() { + return function; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + StellarFunctionInfo that = (StellarFunctionInfo) o; + + if (!Objects.equals(name, that.name)) { + return false; + } + if (!Objects.equals(description, that.description)) { + return false; + } + if (!Objects.equals(returns, that.returns)) { + return false; + } + // Probably incorrect - comparing Object[] arrays with Arrays.equals + if (!Arrays.equals(params, that.params)) { + return false; + } + return Objects.equals(function, that.function); + + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (description != null ? description.hashCode() : 0); + result = 31 * result + (returns != null ? returns.hashCode() : 0); + result = 31 * result + Arrays.hashCode(params); + result = 31 * result + (function != null ? function.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "StellarFunctionInfo{" + + "name='" + name + '\'' + + ", description='" + description + '\'' + + ", returns='" + returns + '\'' + + ", params=" + Arrays.toString(params) + + ", function=" + function + + '}'; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java index 73df82f7..b9ae79bb 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,15 +26,16 @@ public class StellarFunctions { - public static FunctionResolver FUNCTION_RESOLVER() { - return SingletonFunctionResolver.getInstance(); - } + @SuppressWarnings("checkstyle:MethodName") + public static FunctionResolver FUNCTION_RESOLVER() { + return SingletonFunctionResolver.getInstance(); + } - public static void initialize(Context context) { - SingletonFunctionResolver.getInstance().initialize(context); - } + public static void initialize(Context context) { + SingletonFunctionResolver.getInstance().initialize(context); + } - public static void close() throws IOException { - SingletonFunctionResolver.getInstance().close(); - } + public static void close() throws IOException { + SingletonFunctionResolver.getInstance().close(); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Token.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Token.java index d6c8458a..37c88678 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Token.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/Token.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,53 +23,60 @@ import org.apache.metron.stellar.common.FrameContext; public class Token { - T value; - Class underlyingType; - FrameContext.Context multiArgContext; + T value; + Class underlyingType; + FrameContext.Context multiArgContext; - public Token(T value, Class clazz) { - this(value, clazz, null); - } + public Token(T value, Class clazz) { + this(value, clazz, null); + } - public Token(T value, Class clazz, FrameContext.Context multiArgContext) { - this.value = value; - this.underlyingType = clazz; - this.multiArgContext = multiArgContext; - } + public Token(T value, Class clazz, FrameContext.Context multiArgContext) { + this.value = value; + this.underlyingType = clazz; + this.multiArgContext = multiArgContext; + } - public FrameContext.Context getMultiArgContext() { - return multiArgContext; - } + public FrameContext.Context getMultiArgContext() { + return multiArgContext; + } - public T getValue() { - return value; - } + public T getValue() { + return value; + } - public Class getUnderlyingType() { - return underlyingType; - } + public Class getUnderlyingType() { + return underlyingType; + } - @Override - public String toString() { - return "" + value; - } + @Override + public String toString() { + return "" + value; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - Token token = (Token) o; + Token token = (Token) o; - if (getValue() != null ? !getValue().equals(token.getValue()) : token.getValue() != null) return false; - return getUnderlyingType() != null ? getUnderlyingType().equals(token.getUnderlyingType()) : token.getUnderlyingType() == null; + if (getValue() != null ? !getValue().equals(token.getValue()) : token.getValue() != null) { + return false; + } + return getUnderlyingType() != null ? getUnderlyingType().equals(token.getUnderlyingType()) + : token.getUnderlyingType() == null; - } + } - @Override - public int hashCode() { - int result = getValue() != null ? getValue().hashCode() : 0; - result = 31 * result + (getUnderlyingType() != null ? getUnderlyingType().hashCode() : 0); - return result; - } + @Override + public int hashCode() { + int result = getValue() != null ? getValue().hashCode() : 0; + result = 31 * result + (getUnderlyingType() != null ? getUnderlyingType().hashCode() : 0); + return result; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java index fb95d27d..57f3b564 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +22,9 @@ public interface VariableResolver { - public static final String ALL_FIELDS = "_"; - Object resolve(String variable); - boolean exists(String variable); + String ALL_FIELDS = "_"; + + Object resolve(String variable); + + boolean exists(String variable); } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/ConversionFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/ConversionFunctions.java index 6a63fc67..2440b174 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/ConversionFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/ConversionFunctions.java @@ -7,77 +7,85 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions; +import java.util.List; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.BaseStellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.common.utils.ConversionUtils; - -import java.util.List; +@SuppressWarnings("checkstyle:TypeName") public class ConversionFunctions { - public static class Cast extends BaseStellarFunction { - Class clazz; - public Cast(Class clazz) { - this.clazz = clazz; - } + public static class Cast extends BaseStellarFunction { + Class clazz; + + public Cast(Class clazz) { + this.clazz = clazz; + } - @Override - public Object apply(List strings ) { - return strings.get(0) == null?null: ConversionUtils.convert(strings.get(0), clazz); + @Override + public Object apply(List strings) { + return strings.get(0) == null ? null : ConversionUtils.convert(strings.get(0), clazz); + } } - } - @Stellar(name="TO_INTEGER" - , description="Transforms the first argument to an integer" - , params = { "input - Object of string or numeric type"} - , returns = "Integer version of the first argument" - ) - public static class TO_INTEGER extends Cast { + @Stellar(name = "TO_INTEGER", + description = "Transforms the first argument to an integer", + params = {"input - Object of string or numeric type"}, + returns = "Integer version of the first argument" + ) + public static class TO_INTEGER extends Cast { - public TO_INTEGER() { - super(Integer.class); + public TO_INTEGER() { + super(Integer.class); + } } - } - @Stellar(name="TO_DOUBLE" - , description="Transforms the first argument to a double precision number" - , params = { "input - Object of string or numeric type"} - , returns = "Double version of the first argument" - ) - public static class TO_DOUBLE extends Cast { + @Stellar(name = "TO_DOUBLE", + description = "Transforms the first argument to a double precision number", + params = {"input - Object of string or numeric type"}, + returns = "Double version of the first argument" + ) + public static class TO_DOUBLE extends Cast { - public TO_DOUBLE() { - super(Double.class); + public TO_DOUBLE() { + super(Double.class); + } } - } - @Stellar(name="TO_LONG" - , description="Transforms the first argument to a long integer" - , params = { "input - Object of string or numeric type"} - , returns = "Long version of the first argument" - ) - public static class TO_LONG extends Cast { + @Stellar(name = "TO_LONG", + description = "Transforms the first argument to a long integer", + params = {"input - Object of string or numeric type"}, + returns = "Long version of the first argument" + ) + public static class TO_LONG extends Cast { - public TO_LONG() { super(Long.class); } - } + public TO_LONG() { + super(Long.class); + } + } - @Stellar(name="TO_FLOAT" - , description="Transforms the first argument to a float" - , params = { "input - Object of string or numeric type"} - , returns = "Float version of the first argument" - ) - public static class TO_FLOAT extends Cast { + @Stellar(name = "TO_FLOAT", + description = "Transforms the first argument to a float", + params = {"input - Object of string or numeric type"}, + returns = "Float version of the first argument" + ) + public static class TO_FLOAT extends Cast { - public TO_FLOAT() { super(Float.class); } - } + public TO_FLOAT() { + super(Float.class); + } + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java index 9a8ec1d5..fa09b9aa 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java @@ -7,221 +7,220 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions; +import java.util.Collection; +import java.util.List; +import java.util.Map; import org.apache.metron.stellar.common.utils.BloomFilter; import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.common.utils.SerDeUtils; import org.apache.metron.stellar.dsl.BaseStellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import java.util.Collection; -import java.util.List; -import java.util.Map; - public class DataStructureFunctions { - @Stellar(name="ADD" - , namespace="BLOOM" - , description="Adds an element to the bloom filter passed in" - , params = { "bloom - The bloom filter" - , "value(s) - The value(s) to add" - } - , returns = "Bloom Filter" - ) - public static class BloomAdd extends BaseStellarFunction { - - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - BloomFilter filter = (BloomFilter)args.get(0); - for (int i = 1;i < args.size();++i) { - Object arg = args.get(i); - if (arg != null) { - filter.add(args.get(i)); + @Stellar(name = "ADD", + namespace = "BLOOM", + description = "Adds an element to the bloom filter passed in", + params = {"bloom - The bloom filter", + "value(s) - The value(s) to add" + }, + returns = "Bloom Filter" + ) + public static class BloomAdd extends BaseStellarFunction { + + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + BloomFilter filter = (BloomFilter) args.get(0); + for (int i = 1; i < args.size(); ++i) { + Object arg = args.get(i); + if (arg != null) { + filter.add(args.get(i)); + } + } + return filter; } - } - return filter; } - } - - @Stellar(name = "EXISTS", - namespace = "BLOOM", - description = "If the bloom filter contains the value", - params = { "bloom - The bloom filter", - "value - The value to check" - }, - returns = "True if the filter might contain the value and false otherwise" - ) - public static class BloomExists extends BaseStellarFunction { - - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - if (args.size() == 0) { - return false; - } - BloomFilter filter = (BloomFilter)args.get(0); - if (args.size() > 1) { - Object arg = args.get(1); - if (arg == null) { - return false; + + @Stellar(name = "EXISTS", + namespace = "BLOOM", + description = "If the bloom filter contains the value", + params = {"bloom - The bloom filter", + "value - The value to check" + }, + returns = "True if the filter might contain the value and false otherwise" + ) + public static class BloomExists extends BaseStellarFunction { + + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + if (args.size() == 0) { + return false; + } + BloomFilter filter = (BloomFilter) args.get(0); + if (args.size() > 1) { + Object arg = args.get(1); + if (arg == null) { + return false; + } + return filter.mightContain(arg); + } + return false; } - return filter.mightContain(arg); - } - return false; - } - } - - @Stellar(name = "INIT", - namespace = "BLOOM", - description = "Returns an empty bloom filter", - params = { "expectedInsertions - The expected insertions", - "falsePositiveRate - The false positive rate you are willing to tolerate" - }, - returns = "Bloom Filter" - ) - public static class BloomInit extends BaseStellarFunction { - - @Override - public Object apply(List args) { - int expectedInsertions = 100000; - float falsePositiveRate = 0.01f; - if (args.size() > 1) { - expectedInsertions = ConversionUtils.convert(args.get(0), Integer.class); - } - if (args.size() > 2) { - falsePositiveRate = ConversionUtils.convert(args.get(1), Float.class); - } - return new BloomFilter<>(SerDeUtils.SERIALIZER, expectedInsertions, falsePositiveRate); } - } - - @Stellar( name="MERGE" - , namespace="BLOOM" - , description="Returns a merged bloom filter" - , params = { "bloomfilters - A list of bloom filters to merge" - } - , returns = "Bloom Filter or null if the list is empty" - ) - public static class BloomMerge extends BaseStellarFunction { - - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - if (args.size() > 0) { - Object firstArg = args.get(0); - if (firstArg instanceof List) { - BloomFilter ret = null; - for (Object bf : (List)firstArg) { - if (bf instanceof BloomFilter) { - if (ret == null) { - ret = (BloomFilter)bf; - } else { - ret.merge((BloomFilter)bf); - } + + @Stellar(name = "INIT", + namespace = "BLOOM", + description = "Returns an empty bloom filter", + params = {"expectedInsertions - The expected insertions", + "falsePositiveRate - The false positive rate you are willing to tolerate" + }, + returns = "Bloom Filter" + ) + public static class BloomInit extends BaseStellarFunction { + + @Override + public Object apply(List args) { + int expectedInsertions = 100000; + float falsePositiveRate = 0.01f; + if (args.size() > 1) { + expectedInsertions = ConversionUtils.convert(args.get(0), Integer.class); } - } - return ret; + if (args.size() > 2) { + falsePositiveRate = ConversionUtils.convert(args.get(1), Float.class); + } + return new BloomFilter<>(SerDeUtils.SERIALIZER, expectedInsertions, falsePositiveRate); } - else { - return null; + } + + @Stellar(name = "MERGE", + namespace = "BLOOM", + description = "Returns a merged bloom filter", + params = {"bloomfilters - A list of bloom filters to merge" + }, + returns = "Bloom Filter or null if the list is empty" + ) + public static class BloomMerge extends BaseStellarFunction { + + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + if (args.size() > 0) { + Object firstArg = args.get(0); + if (firstArg instanceof List) { + BloomFilter ret = null; + for (Object bf : (List) firstArg) { + if (bf instanceof BloomFilter) { + if (ret == null) { + ret = (BloomFilter) bf; + } else { + ret.merge((BloomFilter) bf); + } + } + } + return ret; + } else { + return null; + } + } + return null; } - } - return null; } - } - - @Stellar(name="IS_EMPTY" - , description="Returns true if string or collection is empty or null and false if otherwise." - , params = { "input - Object of string or collection type (for example, list)"} - , returns = "True if the string or collection is empty or null and false if otherwise." - ) - public static class IsEmpty extends BaseStellarFunction { - - @Override - public Object apply(List list) { - if(null == list || list.size() == 0) { - return true; - } - Object o = list.get(0); - if(o instanceof Collection) { - return ((Collection)o).isEmpty(); - } - else if(o instanceof String) { - String val = (String) list.get(0); - return val == null || val.isEmpty() ? true : false; - } - else if(o instanceof Map) { - return (((Map)o).isEmpty()); - } - else { - return o == null; - } + + @Stellar(name = "IS_EMPTY", + description = "Returns true if string or collection is empty or null and false if otherwise.", + params = {"input - Object of string or collection type (for example, list)"}, + returns = "True if the string or collection is empty or null and false if otherwise." + ) + public static class IsEmpty extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if (null == list || list.size() == 0) { + return true; + } + Object o = list.get(0); + if (o instanceof Collection) { + return ((Collection) o).isEmpty(); + } else if (o instanceof String) { + String val = (String) list.get(0); + return val == null || val.isEmpty(); + } else if (o instanceof Map) { + return (((Map) o).isEmpty()); + } else { + return o == null; + } + } } - } - @Stellar(name="ADD" - ,namespace="LIST" - , description="Adds an element to a list." - , params = { "list - List to add element to." - , "element - Element to add to list" - } - , returns = "Resulting list with the item added at the end." - ) - - public static class ListAdd extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() == 0) { - return null; - } - Object o = list.get(0); - if (list.size() == 1) { - return o; - } - if (o instanceof List) { - List l = (List)o; - Object arg = list.get(1); - l.add(arg); - return l; - } else { - return o; - } + + @Stellar(name = "ADD", + namespace = "LIST", + description = "Adds an element to a list.", + params = {"list - List to add element to.", + "element - Element to add to list" + }, + returns = "Resulting list with the item added at the end." + ) + + public static class ListAdd extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() == 0) { + return null; + } + Object o = list.get(0); + if (list.size() == 1) { + return o; + } + if (o instanceof List) { + List l = (List) o; + Object arg = list.get(1); + l.add(arg); + return l; + } else { + return o; + } + } } - } - - @Stellar(name = "LENGTH", - description = "Returns the length of a string or size of a collection. Returns 0 for empty or null Strings", - params = { "input - Object of string or collection type (e.g. list)"}, - returns = "Integer" - ) - public static class Length extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() == 0) { - return 0; - } - Object o = list.get(0); - if (o instanceof Collection) { - return ((Collection)o).size(); - } else if (o instanceof Map) { - return ((Map)o).size(); - } else if (o instanceof String) { - String val = (String) list.get(0); - return val == null || val.isEmpty() ? 0 : val.length(); - } else { - return 0; - } + + @Stellar(name = "LENGTH", + description = "Returns the length of a string or size of a collection. Returns 0 for empty or null Strings", + params = {"input - Object of string or collection type (e.g. list)"}, + returns = "Integer" + ) + public static class Length extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() == 0) { + return 0; + } + Object o = list.get(0); + if (o instanceof Collection) { + return ((Collection) o).size(); + } else if (o instanceof Map) { + return ((Map) o).size(); + } else if (o instanceof String) { + String val = (String) list.get(0); + return val == null || val.isEmpty() ? 0 : val.length(); + } else { + return 0; + } + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java index 7ad91b7d..6cef0f09 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,391 +23,409 @@ import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; -import org.apache.metron.stellar.dsl.BaseStellarFunction; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.common.utils.ConversionUtils; - import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.TimeZone; import java.util.concurrent.ExecutionException; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; /** * Stellar data functions. */ public class DateFunctions { - private static class TimezonedFormat { - - private String format; - private Optional timezone; - - public TimezonedFormat(String format, String timezone) { - this.format = format; - this.timezone = Optional.of(timezone); - } + private static class TimezonedFormat { - public TimezonedFormat(String format) { - this.format = format; - this.timezone = Optional.empty(); - } + private final String format; + private final Optional timezone; - public SimpleDateFormat toDateFormat() { - return createFormat(format, timezone); - } + public TimezonedFormat(String format, String timezone) { + this.format = format; + this.timezone = Optional.of(timezone); + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + public TimezonedFormat(String format) { + this.format = format; + this.timezone = Optional.empty(); + } - TimezonedFormat that = (TimezonedFormat) o; + public SimpleDateFormat toDateFormat() { + return createFormat(format, timezone); + } - if (format != null ? !format.equals(that.format) : that.format != null) return false; - return timezone != null ? timezone.equals(that.timezone) : that.timezone == null; - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + TimezonedFormat that = (TimezonedFormat) o; + + if (!Objects.equals(format, that.format)) { + return false; + } + return Objects.equals(timezone, that.timezone); + } - @Override - public int hashCode() { - int result = format != null ? format.hashCode() : 0; - result = 31 * result + (timezone != null ? timezone.hashCode() : 0); - return result; + @Override + public int hashCode() { + int result = format != null ? format.hashCode() : 0; + result = 31 * result + (timezone != null ? timezone.hashCode() : 0); + return result; + } } - } - private static LoadingCache> formatCache = + private static final LoadingCache> formatCache = Caffeine.newBuilder().build( - new CacheLoader>() { + new CacheLoader>() { @Override public ThreadLocal load(final TimezonedFormat format) throws Exception { - return new ThreadLocal() { - @Override - public SimpleDateFormat initialValue() { - return format.toDateFormat(); - } - }; + return new ThreadLocal() { + @Override + public SimpleDateFormat initialValue() { + return format.toDateFormat(); + } + }; } - }); + }); - public static SimpleDateFormat createFormat(String format, Optional timezone) { - SimpleDateFormat sdf = new SimpleDateFormat(format); - if(timezone.isPresent()) { - sdf.setTimeZone(TimeZone.getTimeZone(timezone.get())); + public static SimpleDateFormat createFormat(String format, Optional timezone) { + SimpleDateFormat sdf = new SimpleDateFormat(format); + if (timezone.isPresent()) { + sdf.setTimeZone(TimeZone.getTimeZone(timezone.get())); + } + return sdf; + } + + public static long getEpochTime(String date, String format, Optional timezone) + throws ExecutionException, ParseException { + TimezonedFormat fmt; + if (timezone.isPresent()) { + fmt = new TimezonedFormat(format, timezone.get()); + } else { + fmt = new TimezonedFormat(format); + } + SimpleDateFormat sdf = formatCache.get(fmt).get(); + return sdf.parse(date).getTime(); } - return sdf; - } - - public static long getEpochTime(String date, String format, Optional timezone) throws ExecutionException, ParseException { - TimezonedFormat fmt; - if(timezone.isPresent()) { - fmt = new TimezonedFormat(format, timezone.get()); - } else { - fmt = new TimezonedFormat(format); + + public static String getDateFormat(String format, Optional epochTime, Optional timezone) { + Long time = epochTime.orElseGet(System::currentTimeMillis); + TimezonedFormat fmt = + timezone.map(s -> new TimezonedFormat(format, s)).orElseGet(() -> new TimezonedFormat(format)); + SimpleDateFormat sdf = formatCache.get(fmt).get(); + return sdf.format(new Date(time)); } - SimpleDateFormat sdf = formatCache.get(fmt).get(); - return sdf.parse(date).getTime(); - } - - public static String getDateFormat(String format, Optional epochTime, Optional timezone) { - Long time = epochTime.orElseGet(System::currentTimeMillis); - TimezonedFormat fmt = timezone.map(s -> new TimezonedFormat(format, s)).orElseGet(() -> new TimezonedFormat(format)); - SimpleDateFormat sdf = formatCache.get(fmt).get(); - return sdf.format(new Date(time)); - } - - - /** - * Stellar Function: TO_EPOCH_TIMESTAMP - */ - @Stellar( name="TO_EPOCH_TIMESTAMP" - , description="Returns the epoch timestamp of the dateTime in the specified format. " + - "If the format does not have a timestamp and you wish to assume a " + - "given timestamp, you may specify the timezone optionally." - , params = { "dateTime - DateTime in String format" - , "format - DateTime format as a String" - , "timezone - Optional timezone in String format" - } - , returns = "Epoch timestamp") - public static class ToTimestamp extends BaseStellarFunction { - @Override - public Object apply(List objects) { - Object dateObj = objects.get(0); - Object formatObj = objects.get(1); - Object tzObj = null; - if(objects.size() >= 3) { - tzObj = objects.get(2); - } - if(dateObj != null && formatObj != null) { - try { - Optional tz = (tzObj == null) ? Optional.empty() : Optional.of(tzObj.toString()); - return getEpochTime(dateObj.toString(), formatObj.toString(), tz); - - } catch (ExecutionException | ParseException e) { - return null; + + + /** + * Stellar Function: TO_EPOCH_TIMESTAMP. + */ + @Stellar(name = "TO_EPOCH_TIMESTAMP", + description = "Returns the epoch timestamp of the dateTime in the specified format. " + + "If the format does not have a timestamp and you wish to assume a " + + "given timestamp, you may specify the timezone optionally.", + params = { + "dateTime - DateTime in String format", + "format - DateTime format as a String", + "timezone - Optional timezone in String format" + }, + returns = "Epoch timestamp") + public static class ToTimestamp extends BaseStellarFunction { + @Override + public Object apply(List objects) { + Object dateObj = objects.get(0); + Object formatObj = objects.get(1); + Object tzObj = null; + if (objects.size() >= 3) { + tzObj = objects.get(2); + } + if (dateObj != null && formatObj != null) { + try { + Optional tz = (tzObj == null) ? Optional.empty() : Optional.of(tzObj.toString()); + return getEpochTime(dateObj.toString(), formatObj.toString(), tz); + + } catch (ExecutionException | ParseException e) { + return null; + } + } + return null; } - } - return null; } - } - @Stellar(name = "DATE_FORMAT", + @Stellar(name = "DATE_FORMAT", description = "Takes an epoch timestamp and converts it to a date format.", params = {"format - DateTime format as a String.", - "timestampField - Optional epoch time in Long format. Defaults to now.", - "timezone - Optional timezone in String format."}, + "timestampField - Optional epoch time in Long format. Defaults to now.", + "timezone - Optional timezone in String format."}, returns = "Formatted date." - ) - public static class DateFormat extends BaseStellarFunction { - - @Override - public Object apply(List objects) { - int size = objects.size(); - Optional formatObj = Optional.ofNullable(objects.get(0)); - Optional epochObj = Optional.empty(); - Optional tzObj = Optional.empty(); - if (size > 1) { - if (size == 2) { - if (objects.get(1) == null) { - return null; - } - epochObj = objects.get(1) instanceof Long ? Optional.of((Long) objects.get(1)) : Optional.empty(); - tzObj = objects.get(1) instanceof String ? Optional.of((String) objects.get(1)) : Optional.empty(); - } else { - epochObj = Optional.ofNullable((Long) objects.get(1)); - tzObj = Optional.ofNullable((String) objects.get(2)); + ) + public static class DateFormat extends BaseStellarFunction { + + @Override + public Object apply(List objects) { + int size = objects.size(); + Optional formatObj = Optional.ofNullable(objects.get(0)); + Optional epochObj = Optional.empty(); + Optional tzObj = Optional.empty(); + if (size > 1) { + if (size == 2) { + if (objects.get(1) == null) { + return null; + } + epochObj = objects.get(1) instanceof Long ? Optional.of((Long) objects.get(1)) : Optional.empty(); + tzObj = objects.get(1) instanceof String ? Optional.of((String) objects.get(1)) : Optional.empty(); + } else { + epochObj = Optional.ofNullable((Long) objects.get(1)); + tzObj = Optional.ofNullable((String) objects.get(2)); + } + } + if (formatObj.isPresent()) { + return getDateFormat(formatObj.get().toString(), epochObj, tzObj); + } else { + return null; + } } - } - if(formatObj.isPresent()) { - return getDateFormat(formatObj.get().toString(), epochObj, tzObj); - } else { - return null; - } } - } - - /** - * Gets the value from a list of arguments. - * - * If the argument at the specified position does not exist, a default value will be returned. - * If the argument at the specified position exists, but cannot be coerced to the right type, null is returned. - * Otherwise, the argument value is returned. - * - * @param args A list of arguments. - * @param position The position of the argument to get. - * @param clazz The type of class expected. - * @param defaultValue The default value. - * @param The expected type of the argument. - */ - private static T getOrDefault(List args, int position, Class clazz, T defaultValue) { - T result = defaultValue; - if(args.size() > position) { - result = ConversionUtils.convert(args.get(position), clazz); - } - return result; - } - - /** - * Stellar Function: DAY_OF_WEEK - * - * The numbered day within the week. The first day of the week, Sunday, has a value of 1. - * - * If no argument is supplied, returns the current day of week. - */ - @Stellar( name="DAY_OF_WEEK" - , description="The numbered day within the week. The first day of the week, Sunday, has a value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The numbered day within the week.") - public static class DayOfWeek extends BaseStellarFunction { - @Override - public Object apply(List args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.DAY_OF_WEEK); + + /** + * Gets the value from a list of arguments. + * + *

    + * If the argument at the specified position does not exist, a default value will be returned. + * If the argument at the specified position exists, but cannot be coerced to the right type, null is returned. + * Otherwise, the argument value is returned. + * + * @param args A list of arguments. + * @param position The position of the argument to get. + * @param clazz The type of class expected. + * @param defaultValue The default value. + * @param The expected type of the argument. + */ + private static T getOrDefault(List args, int position, Class clazz, T defaultValue) { + T result = defaultValue; + if (args.size() > position) { + result = ConversionUtils.convert(args.get(position), clazz); + } + return result; } - } - - /** - * Stellar Function: DAY_OF_MONTH - * - * The day within the month. The first day within the month has a value of 1. - */ - @Stellar( name="DAY_OF_MONTH" - , description="The numbered day within the month. The first day within the month has a value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The numbered day within the month.") - public static class DayOfMonth extends BaseStellarFunction { - @Override - public Object apply(List args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.DAY_OF_MONTH); + + /** + * Stellar Function: DAY_OF_WEEK + * + *

    + * The numbered day within the week. The first day of the week, Sunday, has a value of 1. + * + *

    + * If no argument is supplied, returns the current day of week. + */ + @Stellar(name = "DAY_OF_WEEK", + description = "The numbered day within the week. The first day of the week, Sunday, has a value of 1.", + params = {"dateTime - The datetime as a long representing the milliseconds since unix epoch" + }, + returns = "The numbered day within the week.") + public static class DayOfWeek extends BaseStellarFunction { + @Override + public Object apply(List args) { + + // expects epoch millis, otherwise defaults to current time + Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); + if (epochMillis == null) { + return null; // invalid argument + } + + // create a calendar + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(epochMillis); + + return calendar.get(Calendar.DAY_OF_WEEK); + } } - } - - /** - * Stellar Function: WEEK_OF_MONTH - * - * The numbered week within the month. The first week has a value of 1. - */ - @Stellar( name="WEEK_OF_MONTH" - , description="The numbered week within the month. The first week within the month has a value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The numbered week within the month.") - public static class WeekOfMonth extends BaseStellarFunction { - @Override - public Object apply(List args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.WEEK_OF_MONTH); + + /** + * Stellar Function: DAY_OF_MONTH + * + *

    + * The day within the month. The first day within the month has a value of 1. + */ + @Stellar(name = "DAY_OF_MONTH", + description = "The numbered day within the month. The first day within the month has a value of 1.", + params = {"dateTime - The datetime as a long representing the milliseconds since unix epoch" + }, + returns = "The numbered day within the month.") + public static class DayOfMonth extends BaseStellarFunction { + @Override + public Object apply(List args) { + + // expects epoch millis, otherwise defaults to current time + Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); + if (epochMillis == null) { + return null; // invalid argument + } + + // create a calendar + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(epochMillis); + + return calendar.get(Calendar.DAY_OF_MONTH); + } } - } - - /** - * Stellar Function: WEEK_OF_YEAR - * - * The numbered week within the year. The first week in the year has a value of 1. - */ - @Stellar( name="WEEK_OF_YEAR" - , description="The numbered week within the year. The first week in the year has a value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The numbered week within the year.") - public static class WeekOfYear extends BaseStellarFunction { - @Override - public Object apply(List args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.WEEK_OF_YEAR); + + /** + * Stellar Function: WEEK_OF_MONTH + * + *

    + * The numbered week within the month. The first week has a value of 1. + */ + @Stellar(name = "WEEK_OF_MONTH", + description = "The numbered week within the month. The first week within the month has a value of 1.", + params = {"dateTime - The datetime as a long representing the milliseconds since unix epoch" + }, + returns = "The numbered week within the month.") + public static class WeekOfMonth extends BaseStellarFunction { + @Override + public Object apply(List args) { + + // expects epoch millis, otherwise defaults to current time + Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); + if (epochMillis == null) { + return null; // invalid argument + } + + // create a calendar + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(epochMillis); + + return calendar.get(Calendar.WEEK_OF_MONTH); + } } - } - - /** - * Stellar Function: MONTH - * - * A number representing the month. The first month, January, has a value of 0. - */ - @Stellar( name="MONTH" - , description="The number representing the month. The first month, January, has a value of 0." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The current month (0-based).") - public static class MonthOfYear extends BaseStellarFunction { - @Override - public Object apply(List args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.MONTH); + + /** + * Stellar Function: WEEK_OF_YEAR + * + *

    + * The numbered week within the year. The first week in the year has a value of 1. + */ + @Stellar(name = "WEEK_OF_YEAR", + description = "The numbered week within the year. The first week in the year has a value of 1.", + params = {"dateTime - The datetime as a long representing the milliseconds since unix epoch" + }, + returns = "The numbered week within the year.") + public static class WeekOfYear extends BaseStellarFunction { + @Override + public Object apply(List args) { + + // expects epoch millis, otherwise defaults to current time + Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); + if (epochMillis == null) { + return null; // invalid argument + } + + // create a calendar + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(epochMillis); + + return calendar.get(Calendar.WEEK_OF_YEAR); + } } - } - - /** - * Stellar Function: YEAR - * - * The calendar year. - */ - @Stellar( name="YEAR" - , description="The number representing the year. " - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The current year" - ) - public static class Year extends BaseStellarFunction { - @Override - public Object apply(List args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.YEAR); + + /** + * Stellar Function: MONTH + * + *

    + * A number representing the month. The first month, January, has a value of 0. + */ + @Stellar(name = "MONTH", + description = "The number representing the month. The first month, January, has a value of 0.", + params = {"dateTime - The datetime as a long representing the milliseconds since unix epoch" + }, + returns = "The current month (0-based).") + public static class MonthOfYear extends BaseStellarFunction { + @Override + public Object apply(List args) { + + // expects epoch millis, otherwise defaults to current time + Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); + if (epochMillis == null) { + return null; // invalid argument + } + + // create a calendar + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(epochMillis); + + return calendar.get(Calendar.MONTH); + } } - } - - /** - * Stellar Function: DAY_OF_YEAR - * - * The day number within the year. The first day of the year has value of 1. - */ - @Stellar( name="DAY_OF_YEAR" - , description="The day number within the year. The first day of the year has value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The day number within the year." - ) - public static class DayOfYear extends BaseStellarFunction { - @Override - public Object apply(List args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.DAY_OF_YEAR); + + /** + * Stellar Function: YEAR + * + *

    + * The calendar year. + */ + @Stellar(name = "YEAR", + description = "The number representing the year. ", + params = {"dateTime - The datetime as a long representing the milliseconds since unix epoch" + }, + returns = "The current year" + ) + public static class Year extends BaseStellarFunction { + @Override + public Object apply(List args) { + + // expects epoch millis, otherwise defaults to current time + Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); + if (epochMillis == null) { + return null; // invalid argument + } + + // create a calendar + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(epochMillis); + + return calendar.get(Calendar.YEAR); + } + } + + /** + * Stellar Function: DAY_OF_YEAR + * + *

    + * The day number within the year. The first day of the year has value of 1. + */ + @Stellar(name = "DAY_OF_YEAR", + description = "The day number within the year. The first day of the year has value of 1.", + params = {"dateTime - The datetime as a long representing the milliseconds since unix epoch" + }, + returns = "The day number within the year." + ) + public static class DayOfYear extends BaseStellarFunction { + @Override + public Object apply(List args) { + + // expects epoch millis, otherwise defaults to current time + Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); + if (epochMillis == null) { + return null; // invalid argument + } + + // create a calendar + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(epochMillis); + + return calendar.get(Calendar.DAY_OF_YEAR); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java index 41783c2a..eecdddbb 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * + *

    * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,128 +32,128 @@ */ public class EncodingFunctions { - @Stellar(name = "GET_SUPPORTED_ENCODINGS", - description = "Returns a list of the encodings that are currently supported as a list", - params = {}, - returns = "A list of supported encodings" - ) - public static class GetSupportedEncodings extends BaseStellarFunction { - - @Override - public Object apply(List list) { - return Encodings.SUPPORTED_LIST; + @Stellar(name = "GET_SUPPORTED_ENCODINGS", + description = "Returns a list of the encodings that are currently supported as a list", + params = {}, + returns = "A list of supported encodings" + ) + public static class GetSupportedEncodings extends BaseStellarFunction { + + @Override + public Object apply(List list) { + return Encodings.SUPPORTED_LIST; + } } - } - - @Stellar(name = "IS_ENCODING", - description = "Returns if the passed string is encoded in one of the supported encodings", - params = {"string - the string to test", - "encoding - the encoding to test, must be one of encodings returned from " - + "LIST_SUPPORTED_ENCODINGS" - }, - returns = "true if it is encoded, false if not" - ) - public static class IsEncoding extends BaseStellarFunction { - - @Override - public Object apply(List list) { - if (list.size() < 2) { - throw new IllegalStateException( - "IS_ENCODING expects two args: [string, encoding] where encoding is one from " - + "the supported list"); - } - String str = (String) list.get(0); - String encoding = (String) list.get(1); - if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { - return false; - } - - Encodings enc = null; - try { - enc = Encodings.valueOf(encoding.toUpperCase()); - } catch (IllegalArgumentException iae) { - throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); - } - return enc.is(str); + + @Stellar(name = "IS_ENCODING", + description = "Returns if the passed string is encoded in one of the supported encodings", + params = {"string - the string to test", + "encoding - the encoding to test, must be one of encodings returned from " + + "LIST_SUPPORTED_ENCODINGS" + }, + returns = "true if it is encoded, false if not" + ) + public static class IsEncoding extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if (list.size() < 2) { + throw new IllegalStateException( + "IS_ENCODING expects two args: [string, encoding] where encoding is one from " + + "the supported list"); + } + String str = (String) list.get(0); + String encoding = (String) list.get(1); + if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { + return false; + } + + Encodings enc = null; + try { + enc = Encodings.valueOf(encoding.toUpperCase()); + } catch (IllegalArgumentException iae) { + throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); + } + return enc.is(str); + } } - } - - @Stellar(name = "DECODE", - description = "Decodes the passed string with the provided encoding, " - + " must be one of the encodings returned from LIST_SUPPORTED_ENCODINGS", - params = {"string - the string to decode", - "encoding - the encoding to use, must be one of encodings returned from " - + "LIST_SUPPORTED_ENCODINGS", - "verify - (optional), true or false to determine if string should be verified as being " - + "encoded with the passed encoding" - }, - returns = "The decoded string on success\n" - + "The original string the string cannot be decoded\n" - + "null on usage error" - ) - public static class Decode extends BaseStellarFunction { - - @Override - public Object apply(List list) { - if (list.size() != 2 && list.size() != 3) { - throw new IllegalStateException( - "DECODE expects two or three args: [string, encoding] or " - + "[string, encoding, verify] where encoding is one from " - + "the supported list"); - } - Boolean verify = false; - String str = (String) list.get(0); - String encoding = (String) list.get(1); - - if (list.size() == 3) { - verify = (Boolean)list.get(2); - } - if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { - return null; - } - - Encodings enc = null; - try { - enc = Encodings.valueOf(encoding.toUpperCase()); - } catch (IllegalArgumentException iae) { - throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); - } - return enc.decode(str, verify); + + @Stellar(name = "DECODE", + description = "Decodes the passed string with the provided encoding, " + + " must be one of the encodings returned from LIST_SUPPORTED_ENCODINGS", + params = {"string - the string to decode", + "encoding - the encoding to use, must be one of encodings returned from " + + "LIST_SUPPORTED_ENCODINGS", + "verify - (optional), true or false to determine if string should be verified as being " + + "encoded with the passed encoding" + }, + returns = "The decoded string on success\n" + + "The original string the string cannot be decoded\n" + + "null on usage error" + ) + public static class Decode extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if (list.size() != 2 && list.size() != 3) { + throw new IllegalStateException( + "DECODE expects two or three args: [string, encoding] or " + + "[string, encoding, verify] where encoding is one from " + + "the supported list"); + } + Boolean verify = false; + String str = (String) list.get(0); + String encoding = (String) list.get(1); + + if (list.size() == 3) { + verify = (Boolean) list.get(2); + } + if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { + return null; + } + + Encodings enc = null; + try { + enc = Encodings.valueOf(encoding.toUpperCase()); + } catch (IllegalArgumentException iae) { + throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); + } + return enc.decode(str, verify); + } } - } - - @Stellar(name = "ENCODE", - description = "Encodes the passed string with the provided encoding, " - + " must be one of the encodings returned from LIST_SUPPORTED_ENCODINGS", - params = {"string - the string to encode", - "encoding - the encoding to use, must be one of encodings returned from " - + "LIST_SUPPORTED_ENCODINGS" - }, - returns = "The encoded string or null on error" - ) - public static class Encode extends BaseStellarFunction { - - @Override - public Object apply(List list) { - if (list.size() != 2 && list.size() != 3) { - throw new IllegalStateException( - "ENCODE expects two or three args: [string, encoding] where encoding is one from " - + "the supported list"); - } - String str = (String) list.get(0); - String encoding = (String) list.get(1); - - if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { - return null; - } - - Encodings enc = null; - try { - enc = Encodings.valueOf(encoding.toUpperCase()); - } catch (IllegalArgumentException iae) { - throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); - } - return enc.encode(str); + + @Stellar(name = "ENCODE", + description = "Encodes the passed string with the provided encoding, " + + " must be one of the encodings returned from LIST_SUPPORTED_ENCODINGS", + params = {"string - the string to encode", + "encoding - the encoding to use, must be one of encodings returned from " + + "LIST_SUPPORTED_ENCODINGS" + }, + returns = "The encoded string or null on error" + ) + public static class Encode extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if (list.size() != 2 && list.size() != 3) { + throw new IllegalStateException( + "ENCODE expects two or three args: [string, encoding] where encoding is one from " + + "the supported list"); + } + String str = (String) list.get(0); + String encoding = (String) list.get(1); + + if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { + return null; + } + + Encodings enc = null; + try { + enc = Encodings.valueOf(encoding.toUpperCase()); + } catch (IllegalArgumentException iae) { + throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); + } + return enc.encode(str); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java index c11fa609..58b1e1a6 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,198 +21,201 @@ package org.apache.metron.stellar.dsl.functions; import com.google.common.collect.Lists; -import org.apache.metron.stellar.dsl.BaseStellarFunction; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.common.LambdaExpression; - import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.metron.stellar.common.LambdaExpression; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; public class FunctionalFunctions { - @Stellar(name="MAP" - , description="Applies lambda expression to a list of arguments. e.g. MAP( [ 'foo', 'bar' ] , ( x ) -> TO_UPPER(x) ) would yield [ 'FOO', 'BAR' ]" - , params = { - "list - List of arguments." - ,"transform_expression - The lambda expression to apply. This expression is assumed to take one argument." - } - , returns = "The input list transformed item-wise by the lambda expression." - ) - public static class Map extends BaseStellarFunction { - - @Override - public Object apply(List args) { - Iterable input = getIterable(args.get(0)); - LambdaExpression expression = (LambdaExpression)args.get(1); - if(input == null || expression == null) { - return input; - } - List ret = new ArrayList<>(); - for(Object o : input) { - ret.add(expression.apply(listOf(o))); - } - return ret; - } - } - - @Stellar(name="FILTER" - , description="Applies a filter in the form of a lambda expression to a list. e.g. FILTER( [ 'foo', 'bar' ] , (x) -> x == 'foo') would yield [ 'foo']" - , params = { - "list - List of arguments." - ,"predicate - The lambda expression to apply. This expression is assumed to take one argument and return a boolean." - } - , returns = "The input list filtered by the predicate." - ) - public static class Filter extends BaseStellarFunction { - - @Override - public Object apply(List args) { - Iterable input = getIterable(args.get(0)); - LambdaExpression expression = (LambdaExpression) args.get(1); - if(input == null || expression == null) { - return input; - } - List ret = new ArrayList<>(); - for(Object o : input) { - Object result = expression.apply(listOf(o)); - if(result != null && result instanceof Boolean && (Boolean)result) { - ret.add(o); + @Stellar(name = "MAP", + description = "Applies lambda expression to a list of arguments. " + + "e.g. MAP( [ 'foo', 'bar' ] , ( x ) -> TO_UPPER(x) ) would yield [ 'FOO', 'BAR' ]", + params = { + "list - List of arguments.", + "transform_expression - The lambda expression to apply. This expression is assumed to take one argument." + }, + returns = "The input list transformed item-wise by the lambda expression." + ) + public static class Map extends BaseStellarFunction { + + @Override + public Object apply(List args) { + Iterable input = getIterable(args.get(0)); + LambdaExpression expression = (LambdaExpression) args.get(1); + if (input == null || expression == null) { + return input; + } + List ret = new ArrayList<>(); + for (Object o : input) { + ret.add(expression.apply(listOf(o))); + } + return ret; } - } - return ret; - } - } - - @Stellar(name="REDUCE" - , description="Reduces a list by a binary lambda expression. That is, the expression takes two arguments. Usage example: REDUCE( [ 1, 2, 3 ] , (x, y) -> x + y) would sum the input list, yielding 6." - , params = { - "list - List of arguments." - ,"binary_operation - The lambda expression function to apply to reduce the list. It is assumed that this takes two arguments, the first being the running total and the second being an item from the list." - ,"initial_value - The initial value to use." - } - , returns = "The reduction of the list." - ) - public static class Reduce extends BaseStellarFunction { - - @Override - public Object apply(List args) { - Iterable input = getIterable(args.get(0)); - if(input == null || args.size() < 3) { - return null; - } - LambdaExpression expression = (LambdaExpression) args.get(1); - - Object runningResult = args.get(2); - if(expression == null || runningResult == null) { - return null; - } - for(Object rhs : input) { - runningResult = expression.apply(listOf(runningResult, rhs)); - } - return runningResult; } - } - @SuppressWarnings("unchecked") - private static Iterable getIterable(Object o) { - if (o == null) { - return null; - } - if (o instanceof String) { - return Lists.charactersOf((String)o); - } - else if (o instanceof Iterable) { - return (Iterable)o; + @Stellar(name = "FILTER", + description = "Applies a filter in the form of a lambda expression to a list. " + + "e.g. FILTER( [ 'foo', 'bar' ] , (x) -> x == 'foo') would yield [ 'foo']", + params = { + "list - List of arguments.", + + "predicate - The lambda expression to apply. This expression is assumed to take one argument and return a boolean." + }, + returns = "The input list filtered by the predicate." + ) + public static class Filter extends BaseStellarFunction { + + @Override + public Object apply(List args) { + Iterable input = getIterable(args.get(0)); + LambdaExpression expression = (LambdaExpression) args.get(1); + if (input == null || expression == null) { + return input; + } + List ret = new ArrayList<>(); + for (Object o : input) { + Object result = expression.apply(listOf(o)); + if (result != null && result instanceof Boolean && (Boolean) result) { + ret.add(o); + } + } + return ret; + } } - else { - throw new IllegalArgumentException(o.getClass() + " is not an iterable, and therefore cannot be used."); + + @Stellar(name = "REDUCE", + description = "Reduces a list by a binary lambda expression. " + + "That is, the expression takes two arguments. " + + "Usage example: REDUCE( [ 1, 2, 3 ] , (x, y) -> x + y) would sum the input list, yielding 6.", + params = { + "list - List of arguments.", + + "binary_operation - The lambda expression function to apply to reduce the list. " + + "It is assumed that this takes two arguments, " + + "the first being the running total and the second being an item from the list.", + "initial_value - The initial value to use." + }, + returns = "The reduction of the list." + ) + public static class Reduce extends BaseStellarFunction { + + @Override + public Object apply(List args) { + Iterable input = getIterable(args.get(0)); + if (input == null || args.size() < 3) { + return null; + } + LambdaExpression expression = (LambdaExpression) args.get(1); + + Object runningResult = args.get(2); + if (expression == null || runningResult == null) { + return null; + } + for (Object rhs : input) { + runningResult = expression.apply(listOf(runningResult, rhs)); + } + return runningResult; + } } - } - - @Stellar(name="ZIP_LONGEST" - , description="Zips lists into a single list where the ith element is an list " + - "containing the ith items from the constituent lists. " + - "See [python](https://docs.python.org/3/library/itertools.html#itertools.zip_longest) " + - "and [wikipedia](https://en.wikipedia.org/wiki/Convolution_(computer_science)) for more context." - , params = { - "list(s) - List(s) to zip." - } - , returns = "The zip of the lists. The returned list is the max size of all the lists. " + - "Empty elements are null " + - "e.g. ZIP_LONGEST( [ 1, 2 ], [ 3, 4, 5] ) == [ [1, 3], [2, 4], [null, 5] ]" - ) - public static class LongestZip extends BaseStellarFunction { - - @Override - public Object apply(List args) { - if(args == null || args.size() == 0) { - return new ArrayList<>(); - } - return zip(args, true); + + @SuppressWarnings("unchecked") + private static Iterable getIterable(Object o) { + if (o == null) { + return null; + } + if (o instanceof String) { + return Lists.charactersOf((String) o); + } else if (o instanceof Iterable) { + return (Iterable) o; + } else { + throw new IllegalArgumentException(o.getClass() + " is not an iterable, and therefore cannot be used."); + } } - } - - @Stellar(name="ZIP" - , description="Zips lists into a single list where the ith element is an list containing the ith items from the constituent lists. " + - "See [python](https://docs.python.org/3/library/functions.html#zip) and [wikipedia](https://en.wikipedia.org/wiki/Convolution_(computer_science)) for more context." - , params = { - "list(s) - List(s) to zip." - } - ,returns = "The zip of the lists. The returned list is the min size of all the lists. " + - "e.g. ZIP( [ 1, 2 ], [ 3, 4, 5] ) == [ [1, 3], [2, 4] ]" - ) - public static class Zip extends BaseStellarFunction { - - @Override - public Object apply(List args) { - if(args == null || args.size() == 0) { - return new ArrayList<>(); - } - return zip(args, false); + + @Stellar(name = "ZIP_LONGEST", + description = "Zips lists into a single list where the ith element is an list " + + "containing the ith items from the constituent lists. " + + "See [python](https://docs.python.org/3/library/itertools.html#itertools.zip_longest) " + + "and [wikipedia](https://en.wikipedia.org/wiki/Convolution_(computer_science)) for more context.", + params = { + "list(s) - List(s) to zip." + }, + returns = "The zip of the lists. The returned list is the max size of all the lists. " + + "Empty elements are null " + + "e.g. ZIP_LONGEST( [ 1, 2 ], [ 3, 4, 5] ) == [ [1, 3], [2, 4], [null, 5] ]" + ) + public static class LongestZip extends BaseStellarFunction { + + @Override + public Object apply(List args) { + if (args == null || args.size() == 0) { + return new ArrayList<>(); + } + return zip(args, true); + } } - } - - @SuppressWarnings("unchecked") - private static List> zip(List args, boolean jagged) { - List> lists = new ArrayList<>(); - Integer resultSize = null; - for (Object o : args) { - if (o instanceof List) { - List l = (List)o; - if (resultSize == null) { - resultSize = l.size(); + + @Stellar(name = "ZIP", + description = + "Zips lists into a single list where the ith element is an list containing the ith items from the constituent lists. " + + "See [python](https://docs.python.org/3/library/functions.html#zip) and [wikipedia](https://en.wikipedia.org/wiki/Convolution_(computer_science)) for more context.", + params = { + "list(s) - List(s) to zip." + }, + returns = "The zip of the lists. The returned list is the min size of all the lists. " + + "e.g. ZIP( [ 1, 2 ], [ 3, 4, 5] ) == [ [1, 3], [2, 4] ]" + ) + public static class Zip extends BaseStellarFunction { + + @Override + public Object apply(List args) { + if (args == null || args.size() == 0) { + return new ArrayList<>(); + } + return zip(args, false); } - else if (jagged) { - resultSize = Math.max(l.size(), resultSize); + } + + @SuppressWarnings("unchecked") + private static List> zip(List args, boolean jagged) { + List> lists = new ArrayList<>(); + Integer resultSize = null; + for (Object o : args) { + if (o instanceof List) { + List l = (List) o; + if (resultSize == null) { + resultSize = l.size(); + } else if (jagged) { + resultSize = Math.max(l.size(), resultSize); + } else { + resultSize = Math.min(l.size(), resultSize); + } + lists.add(l); + } } - else { - resultSize = Math.min(l.size(), resultSize); + if (resultSize == null) { + return new ArrayList<>(); } - lists.add(l); - } - } - if (resultSize == null) { - return new ArrayList<>(); + + return IntStream.range(0, resultSize) + .mapToObj(i -> { + List o = new ArrayList<>(); + for (List list : lists) { + o.add(i < list.size() ? list.get(i) : null); + } + return o; + }) + .collect(Collectors.toList()); } - return IntStream.range(0, resultSize) - .mapToObj(i -> { - List o = new ArrayList<>(); - for (List list : lists) { - o.add(i < list.size() ? list.get(i) : null); - } - return o; - }) - .collect(Collectors.toList()); - } - - private static List listOf(Object... vals) { - List ret = new ArrayList<>(vals.length); - for(int i = 0;i < vals.length;++i) { - ret.add(vals[i]); + private static List listOf(Object... vals) { + List ret = new ArrayList<>(vals.length); + Collections.addAll(ret, vals); + return ret; } - return ret; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/HashFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/HashFunctions.java index d8110c94..54ff148c 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/HashFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/HashFunctions.java @@ -15,8 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.commons.codec.EncoderException; import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.common.utils.hashing.HashStrategy; @@ -26,18 +32,12 @@ import org.apache.metron.stellar.dsl.BaseStellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - public class HashFunctions { @Stellar( - name = "GET_HASHES_AVAILABLE", - description = "Will return all available hashing algorithms available to 'HASH'.", - returns = "A list containing all supported hashing algorithms." + name = "GET_HASHES_AVAILABLE", + description = "Will return all available hashing algorithms available to 'HASH'.", + returns = "A list containing all supported hashing algorithms." ) public static class ListSupportedHashTypes extends BaseStellarFunction { @@ -54,24 +54,25 @@ public List apply(final List args) { } + @SuppressWarnings("checkstyle:LineLength") @Stellar( - name = "HASH", - description = "Hashes a given value using the given hashing algorithm and returns a hex encoded string.", - params = { - "toHash - value to hash.", - "hashType - A valid string representation of a hashing algorithm. See 'GET_HASHES_AVAILABLE'.", - "config? - Configuration for the hash function in the form of a String to object map.\n" - + " For forensic hash TLSH (see https://github.com/trendmicro/tlsh and Jonathan Oliver, Chun Cheng, and Yanggui Chen, TLSH - A Locality Sensitive Hash. 4th Cybercrime and Trustworthy Computing Workshop, Sydney, November 2013):\n" - + " - bucketSize : This defines the size of the hash created. Valid values are 128 (default) or 256 (the former results in a 70 character hash and latter results in 134 characters) \n" - + " - checksumBytes : This defines how many bytes are used to capture the checksum. Valid values are 1 (default) and 3\n" - + " - force : If true (the default) then a hash can be generated from as few as 50 bytes. If false, then at least 256 bytes are required. Insufficient variation or size in the bytes result in a null being returned.\n" - + " - hashes : You can compute a second hash for use in fuzzy clustering TLSH signatures. The number of hashes is the lever to adjust the size of those clusters and \"fuzzy\" the clusters are. If this is specified, then one or more bins are created based on the specified size and the function will return a Map containing the bins.\n" - + " For all other hashes:\n" - + " - charset : The character set to use (UTF8 is default). \n" - }, - returns = "A hex encoded string of a hashed value using the given algorithm. If 'hashType' is null " + - "then '00', padded to the necessary length, will be returned. If 'toHash' is not able to be hashed or " + - "'hashType' is null then null is returned." + name = "HASH", + description = "Hashes a given value using the given hashing algorithm and returns a hex encoded string.", + params = { + "toHash - value to hash.", + "hashType - A valid string representation of a hashing algorithm. See 'GET_HASHES_AVAILABLE'.", + "config? - Configuration for the hash function in the form of a String to object map.\n" + + " For forensic hash TLSH (see https://github.com/trendmicro/tlsh and Jonathan Oliver, Chun Cheng, and Yanggui Chen, TLSH - A Locality Sensitive Hash. 4th Cybercrime and Trustworthy Computing Workshop, Sydney, November 2013):\n" + + " - bucketSize : This defines the size of the hash created. Valid values are 128 (default) or 256 (the former results in a 70 character hash and latter results in 134 characters) \n" + + " - checksumBytes : This defines how many bytes are used to capture the checksum. Valid values are 1 (default) and 3\n" + + " - force : If true (the default) then a hash can be generated from as few as 50 bytes. If false, then at least 256 bytes are required. Insufficient variation or size in the bytes result in a null being returned.\n" + + " - hashes : You can compute a second hash for use in fuzzy clustering TLSH signatures. The number of hashes is the lever to adjust the size of those clusters and \"fuzzy\" the clusters are. If this is specified, then one or more bins are created based on the specified size and the function will return a Map containing the bins.\n" + + " For all other hashes:\n" + + " - charset : The character set to use (UTF8 is default). \n" + }, + returns = "A hex encoded string of a hashed value using the given algorithm. If 'hashType' is null " + + "then '00', padded to the necessary length, will be returned. If 'toHash' is not able to be hashed or " + + "'hashType' is null then null is returned." ) public static class Hash extends BaseStellarFunction { @@ -100,30 +101,33 @@ public Object apply(final List args) { } catch (final EncoderException e) { return null; } catch (final NoSuchAlgorithmException e) { - throw new IllegalArgumentException("Invalid hash type: " + hashType.toString()); + throw new IllegalArgumentException("Invalid hash type: " + hashType); } } } @Stellar( - name = "DIST", - namespace = "TLSH", - params = { - "hash1 - The first TLSH hash", - "hash2 - The first TLSH hash", - "includeLength? - Include the length in the distance calculation or not?", - }, - description = "Will return the hamming distance between two TLSH hashes (note: must be computed with the same params). " + - "For more information, see https://github.com/trendmicro/tlsh and Jonathan Oliver, Chun Cheng, and Yanggui Chen, TLSH - A Locality Sensitive Hash. 4th Cybercrime and Trustworthy Computing Workshop, Sydney, November 2013. " + - "For a discussion of tradeoffs, see Table II on page 5 of https://github.com/trendmicro/tlsh/blob/master/TLSH_CTC_final.pdf", - returns = "An integer representing the distance between hash1 and hash2. The distance is roughly hamming distance, so 0 is very similar." + name = "DIST", + namespace = "TLSH", + params = { + "hash1 - The first TLSH hash", + "hash2 - The first TLSH hash", + "includeLength? - Include the length in the distance calculation or not?", + }, + description = + "Will return the hamming distance between two TLSH hashes (note: must be computed with the same params). " + + "For more information, see https://github.com/trendmicro/tlsh and Jonathan Oliver, Chun Cheng, and Yanggui Chen, TLSH - A Locality Sensitive Hash. 4th Cybercrime and Trustworthy Computing Workshop, Sydney, November 2013. " + + "For a discussion of tradeoffs, see Table II on page 5 of https://github.com/trendmicro/tlsh/blob/master/TLSH_CTC_final.pdf", + returns = "An integer representing the distance between hash1 and hash2." + + "The distance is roughly hamming distance, so 0 is very similar." ) public static class TlshDist extends BaseStellarFunction { @Override public Integer apply(final List args) { if (args == null || args.size() < 2) { - throw new IllegalArgumentException("Invalid call. This function requires at least 2 arguments: the two TLSH hashes."); + throw new IllegalArgumentException( + "Invalid call. This function requires at least 2 arguments: the two TLSH hashes."); } Object h1Obj = args.get(0); Object h2Obj = args.get(1); diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java index 5bf109ac..922a26c3 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,127 +30,133 @@ public class MapFunctions { - @Stellar(name="EXISTS" - ,namespace="MAP" - , description="Checks for existence of a key in a map." - , params = { - "key - The key to check for existence" - ,"map - The map to check for existence of the key" - } - , returns = "True if the key is found in the map and false if otherwise." - ) - public static class MapExists extends BaseStellarFunction { + @Stellar(name = "EXISTS", + namespace = "MAP", + description = "Checks for existence of a key in a map.", + params = { + "key - The key to check for existence", + "map - The map to check for existence of the key" + }, + returns = "True if the key is found in the map and false if otherwise." + ) + public static class MapExists extends BaseStellarFunction { - @Override - public Object apply(List list) { - if(list.size() < 2) { - return false; - } - Object key = list.get(0); - Object mapObj = list.get(1); - if(key != null && mapObj != null && mapObj instanceof Map) { - return ((Map)mapObj).containsKey(key); - } - return false; + @Override + public Object apply(List list) { + if (list.size() < 2) { + return false; + } + Object key = list.get(0); + Object mapObj = list.get(1); + if (key != null && mapObj != null && mapObj instanceof Map) { + return ((Map) mapObj).containsKey(key); + } + return false; + } } - } - @Stellar(name="GET" - ,namespace="MAP" - , description="Gets the value associated with a key from a map" - , params = { - "key - The key" - ,"map - The map" - ,"default - Optionally the default value to return if the key is not in the map." - } - , returns = "The object associated with the key in the map. If no value is associated with the key and default is specified, then default is returned. If no value is associated with the key or default, then null is returned." - ) - public static class MapGet extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List objects) { - Object keyObj = objects.get(0); - Object mapObj = objects.get(1); - Object defaultObj = null; - if(objects.size() >= 3) { - defaultObj = objects.get(2); - } - if(keyObj == null || mapObj == null) { - return defaultObj; - } - Map map = (Map)mapObj; - Object ret = map.get(keyObj); - if(ret == null && defaultObj != null) { - return defaultObj; - } - return ret; + @Stellar(name = "GET", + namespace = "MAP", + description = "Gets the value associated with a key from a map", + params = { + "key - The key", + "map - The map", + "default - Optionally the default value to return if the key is not in the map." + }, + returns = "The object associated with the key in the map. " + + "If no value is associated with the key and default is specified, then default is returned. " + + "If no value is associated with the key or default, then null is returned." + ) + public static class MapGet extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List objects) { + Object keyObj = objects.get(0); + Object mapObj = objects.get(1); + Object defaultObj = null; + if (objects.size() >= 3) { + defaultObj = objects.get(2); + } + if (keyObj == null || mapObj == null) { + return defaultObj; + } + Map map = (Map) mapObj; + Object ret = map.get(keyObj); + if (ret == null && defaultObj != null) { + return defaultObj; + } + return ret; + } } - } - @Stellar(name = "PUT", - namespace = "MAP", - description = "Adds a key/value pair to a map", - params = { - "key - The key", - "value - The value", - "map - The map to perform the put on" - }, - returns = "The original map modified with the key/value. If the map argument is null, a new map will be created and returned that contains the provided key and value - note: if the 'map' argument is null, only the returned map will be non-null and contain the key/value." - ) - public static class MapPut extends BaseStellarFunction { + @Stellar(name = "PUT", + namespace = "MAP", + description = "Adds a key/value pair to a map", + params = { + "key - The key", + "value - The value", + "map - The map to perform the put on" + }, + returns = "The original map modified with the key/value. " + + "If the map argument is null, a new map will be created " + + "and returned that contains the provided key and value - note: " + + "if the 'map' argument is null, only the returned map will be non-null and contain the key/value." + ) + public static class MapPut extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List objects) { - if (objects.size() < 3) { - throw new IllegalArgumentException("Must pass a key, value, and map"); - } else { - Object keyObj = objects.get(0); - Object valueObj = objects.get(1); - Object mapObj = objects.get(2); - if (mapObj == null) { - mapObj = new HashMap<>(); + @Override + @SuppressWarnings("unchecked") + public Object apply(List objects) { + if (objects.size() < 3) { + throw new IllegalArgumentException("Must pass a key, value, and map"); + } else { + Object keyObj = objects.get(0); + Object valueObj = objects.get(1); + Object mapObj = objects.get(2); + if (mapObj == null) { + mapObj = new HashMap<>(); + } + Map map = (Map) mapObj; + map.put(keyObj, valueObj); + return map; + } } - Map map = (Map) mapObj; - map.put(keyObj, valueObj); - return map; - } } - } - @Stellar(name = "MERGE", - namespace = "MAP", - description = "Merges a list of maps", - params = {"maps - A collection of maps to merge. Last entry wins for overlapping keys."}, - returns = "A Map. null if the list of maps is empty." - ) - public static class MapMerge extends BaseStellarFunction { + @Stellar(name = "MERGE", + namespace = "MAP", + description = "Merges a list of maps", + params = {"maps - A collection of maps to merge. Last entry wins for overlapping keys."}, + returns = "A Map. null if the list of maps is empty." + ) + public static class MapMerge extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() < 1) { - return null; - } - LinkedHashMap ret = new LinkedHashMap<>(); - Object o = list.get(0); - if (o != null) { - if (!(o instanceof Iterable)) { - throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass()); - } - Iterable maps = (Iterable) o; + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() < 1) { + return null; + } + LinkedHashMap ret = new LinkedHashMap<>(); + Object o = list.get(0); + if (o != null) { + if (!(o instanceof Iterable)) { + throw new IllegalArgumentException( + "Expected an Iterable, but " + o + " is of type " + o.getClass()); + } + Iterable maps = (Iterable) o; - if (Iterables.size(maps) == 1) { - return Iterables.getFirst(maps, null); - } + if (Iterables.size(maps) == 1) { + return Iterables.getFirst(maps, null); + } - for (Map m : maps) { - if (m != null) { - ret.putAll(m); - } + for (Map m : maps) { + if (m != null) { + ret.putAll(m); + } + } + } + return ret; } - } - return ret; } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java index 04116d24..3e026ec3 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java @@ -17,207 +17,204 @@ * limitations under the License. * */ + package org.apache.metron.stellar.dsl.functions; +import java.util.List; import org.apache.metron.stellar.common.utils.math.MathOperations; import org.apache.metron.stellar.common.utils.math.StellarMathFunction; import org.apache.metron.stellar.dsl.BaseStellarFunction; -import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.dsl.StellarFunction; - -import java.util.List; -import java.util.function.Function; public class MathFunctions { - @Stellar(name="ABS" - ,description="Returns the absolute value of a number." - ,params = { + @Stellar(name = "ABS", + description = "Returns the absolute value of a number.", + params = { "number - The number to take the absolute value of" - } - , returns="The absolute value of the number passed in." - ) - public static class Abs extends StellarMathFunction{ + }, + returns = "The absolute value of the number passed in." + ) + public static class Abs extends StellarMathFunction { - public Abs() { - super(MathOperations.ABS); + public Abs() { + super(MathOperations.ABS); + } } - } - @Stellar(name="LOG10" - ,description="Returns the log (base 10) of a number." - ,params = { + @Stellar(name = "LOG10", + description = "Returns the log (base 10) of a number.", + params = { "number - The number to take the log (base 10) value of" - } - , returns="The log (base 10) of the number passed in." - ) - public static class Log10 extends StellarMathFunction { - public Log10() { - super(MathOperations.LOG10); - } + }, + returns = "The log (base 10) of the number passed in." + ) + public static class Log10 extends StellarMathFunction { + public Log10() { + super(MathOperations.LOG10); + } - } + } - @Stellar(name="LOG2" - ,description="Returns the log (base 2) of a number." - ,params = { + @Stellar(name = "LOG2", + description = "Returns the log (base 2) of a number.", + params = { "number - The number to take the log (base 2) value of" - } - , returns="The log (base 2) of the number passed in." - ) - public static class Log2 extends StellarMathFunction { - public Log2() { - super(MathOperations.LOG2); - } + }, + returns = "The log (base 2) of the number passed in." + ) + public static class Log2 extends StellarMathFunction { + public Log2() { + super(MathOperations.LOG2); + } - } + } - @Stellar(name="LN" - ,description="Returns the natural log of a number." - ,params = { + @Stellar(name = "LN", + description = "Returns the natural log of a number.", + params = { "number - The number to take the natural log value of" - } - , returns="The natural log of the number passed in." - ) - public static class Ln extends StellarMathFunction { - public Ln() { - super(MathOperations.LN); - } + }, + returns = "The natural log of the number passed in." + ) + public static class Ln extends StellarMathFunction { + public Ln() { + super(MathOperations.LN); + } - } + } - @Stellar(name="SQRT" - ,description="Returns the square root of a number." - ,params = { + @Stellar(name = "SQRT", + description = "Returns the square root of a number.", + params = { "number - The number to take the square root of" - } - , returns="The square root of the number passed in." - ) - public static class Sqrt extends StellarMathFunction { - public Sqrt() { - super(MathOperations.SQRT); - } + }, + returns = "The square root of the number passed in." + ) + public static class Sqrt extends StellarMathFunction { + public Sqrt() { + super(MathOperations.SQRT); + } - } + } - @Stellar(name="CEILING" - ,description="Returns the ceiling of a number." - ,params = { + @Stellar(name = "CEILING", + description = "Returns the ceiling of a number.", + params = { "number - The number to take the ceiling of" - } - , returns="The ceiling of the number passed in." - ) - public static class Ceil extends StellarMathFunction { - public Ceil() { - super(MathOperations.CEIL); - } + }, + returns = "The ceiling of the number passed in." + ) + public static class Ceil extends StellarMathFunction { + public Ceil() { + super(MathOperations.CEIL); + } - } + } - @Stellar(name="FLOOR" - ,description="Returns the floor of a number." - ,params = { + @Stellar(name = "FLOOR", + description = "Returns the floor of a number.", + params = { "number - The number to take the floor of" - } - , returns="The floor of the number passed in." - ) - public static class Floor extends StellarMathFunction { - public Floor() { - super(MathOperations.FLOOR); + }, + returns = "The floor of the number passed in." + ) + public static class Floor extends StellarMathFunction { + public Floor() { + super(MathOperations.FLOOR); + } } - } - @Stellar(name="SIN" - ,description="Returns the sine of a number." - ,params = { + @Stellar(name = "SIN", + description = "Returns the sine of a number.", + params = { "number - The number to take the sine of" - } - , returns="The sine of the number passed in." - ) - public static class Sin extends StellarMathFunction { - public Sin() { - super(MathOperations.SIN); + }, + returns = "The sine of the number passed in." + ) + public static class Sin extends StellarMathFunction { + public Sin() { + super(MathOperations.SIN); + } } - } - @Stellar(name="COS" - ,description="Returns the cosine of a number." - ,params = { + @Stellar(name = "COS", + description = "Returns the cosine of a number.", + params = { "number - The number to take the cosine of" - } - , returns="The cosine of the number passed in." - ) - public static class Cos extends StellarMathFunction { - public Cos() { - super(MathOperations.COS); + }, + returns = "The cosine of the number passed in." + ) + public static class Cos extends StellarMathFunction { + public Cos() { + super(MathOperations.COS); + } } - } - @Stellar(name="TAN" - ,description="Returns the tangent of a number." - ,params = { + @Stellar(name = "TAN", + description = "Returns the tangent of a number.", + params = { "number - The number to take the tangent of" - } - , returns="The tangent of the number passed in." - ) - public static class Tan extends StellarMathFunction { - public Tan() { - super(MathOperations.TAN); + }, + returns = "The tangent of the number passed in." + ) + public static class Tan extends StellarMathFunction { + public Tan() { + super(MathOperations.TAN); + } } - } - @Stellar(name="EXP" - ,description="Returns Euler's number raised to the power of the argument" - ,params = { + @Stellar(name = "EXP", + description = "Returns Euler's number raised to the power of the argument", + params = { "number - The power to which e is raised." - } - , returns="Euler's number raised to the power of the argument." - ) - public static class Exp extends StellarMathFunction { - public Exp() { - super(MathOperations.EXP); + }, + returns = "Euler's number raised to the power of the argument." + ) + public static class Exp extends StellarMathFunction { + public Exp() { + super(MathOperations.EXP); + } } - } - @Stellar(name="ROUND" - ,description="Rounds a number to the nearest integer. This is half-up rounding." - ,params = { + @Stellar(name = "ROUND", + description = "Rounds a number to the nearest integer. This is half-up rounding.", + params = { "number - The number to round" - } - , returns="The nearest integer (based on half-up rounding)." - ) - public static class Round extends StellarMathFunction { - public Round() { - super(MathOperations.ROUND); + }, + returns = "The nearest integer (based on half-up rounding)." + ) + public static class Round extends StellarMathFunction { + public Round() { + super(MathOperations.ROUND); + } } - } - - @Stellar(name = "IS_NAN", - description = "Evaluates if the passed number is NaN. The number is evaluated as a double", - params = { - "number - number to evaluate" - }, - returns = "True if the value is NaN, false if it is not") - public static class IsNaN extends BaseStellarFunction { - - @Override - public Object apply(List args) { - - if (args == null || args.size() != 1) { - throw new IllegalStateException( - "IS_NAN expects one: [number] "); - } - - Object obj = args.get(0); - if (obj instanceof Number) { - return Double.isNaN(((Number) obj).doubleValue()); - } else { - throw new ParseException("IS_NAN() expects a number argument"); - } + + @Stellar(name = "IS_NAN", + description = "Evaluates if the passed number is NaN. The number is evaluated as a double", + params = { + "number - number to evaluate" + }, + returns = "True if the value is NaN, false if it is not") + public static class IsNaN extends BaseStellarFunction { + + @Override + public Object apply(List args) { + + if (args == null || args.size() != 1) { + throw new IllegalStateException( + "IS_NAN expects one: [number] "); + } + + Object obj = args.get(0); + if (obj instanceof Number) { + return Double.isNaN(((Number) obj).doubleValue()); + } else { + throw new ParseException("IS_NAN() expects a number argument"); + } + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/NetworkFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/NetworkFunctions.java index 1d2655dd..ed7f464e 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/NetworkFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/NetworkFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,279 +24,270 @@ import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.net.InternetDomainName; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; import org.apache.commons.lang3.StringUtils; import org.apache.commons.net.util.SubnetUtils; import org.apache.metron.stellar.dsl.BaseStellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.List; - public class NetworkFunctions { - @Stellar(name="IN_SUBNET" - ,description = "Returns true if an IP is within a subnet range." - ,params = { - "ip - The IP address in string form" - ,"cidr+ - One or more IP ranges specified in CIDR notation (for example 192.168.0.0/24)" - } - ,returns = "True if the IP address is within at least one of the network ranges and false if otherwise" - ) - public static class InSubnet extends BaseStellarFunction { + @Stellar(name = "IN_SUBNET", + description = "Returns true if an IP is within a subnet range.", + params = { + "ip - The IP address in string form", + "cidr+ - One or more IP ranges specified in CIDR notation (for example 192.168.0.0/24)" + }, + returns = "True if the IP address is within at least one of the network ranges and false if otherwise" + ) + public static class InSubnet extends BaseStellarFunction { - @Override - public Object apply(List list) { - if(list.size() < 2) { - throw new IllegalStateException("IN_SUBNET expects at least two args: [ip, cidr1, cidr2, ...]" - + " where cidr is the subnet mask in cidr form" - ); - } - String ip = (String) list.get(0); - if(ip == null) { - return false; - } - boolean inSubnet = false; - for(int i = 1;i < list.size() && !inSubnet;++i) { - String cidr = (String) list.get(i); - if(cidr == null) { - continue; - } - inSubnet |= new SubnetUtils(cidr).getInfo().isInRange(ip); - } + @Override + public Object apply(List list) { + if (list.size() < 2) { + throw new IllegalStateException("IN_SUBNET expects at least two args: [ip, cidr1, cidr2, ...]" + + " where cidr is the subnet mask in cidr form" + ); + } + String ip = (String) list.get(0); + if (ip == null) { + return false; + } + boolean inSubnet = false; + for (int i = 1; i < list.size() && !inSubnet; ++i) { + String cidr = (String) list.get(i); + if (cidr == null) { + continue; + } + inSubnet |= new SubnetUtils(cidr).getInfo().isInRange(ip); + } - return inSubnet; + return inSubnet; + } } - } - @Stellar(name="REMOVE_SUBDOMAINS" - ,namespace = "DOMAIN" - ,description = "Removes the subdomains from a domain." - , params = { - "domain - Fully qualified domain name" - } - , returns = "The domain without the subdomains. " + - "(for example, DOMAIN_REMOVE_SUBDOMAINS('mail.yahoo.com') yields 'yahoo.com')" - ) - public static class RemoveSubdomains extends BaseStellarFunction { + @Stellar(name = "REMOVE_SUBDOMAINS", + namespace = "DOMAIN", + description = "Removes the subdomains from a domain.", + params = { + "domain - Fully qualified domain name" + }, + returns = "The domain without the subdomains. " + + "(for example, DOMAIN_REMOVE_SUBDOMAINS('mail.yahoo.com') yields 'yahoo.com')" + ) + public static class RemoveSubdomains extends BaseStellarFunction { - @Override - public Object apply(List objects) { - if(objects.isEmpty()) { - return null; - } - Object dnObj = objects.get(0); - InternetDomainName idn = toDomainName(dnObj); - if(idn != null) { - String dn = dnObj.toString(); - String tld = extractTld(idn, dn); - if(!StringUtils.isEmpty(dn)) { - String suffix = safeSubstring(dn, 0, dn.length() - tld.length()); - String hostnameWithoutTLD = safeSubstring(suffix, 0, suffix.length() - 1); - if(hostnameWithoutTLD == null) { - return dn; - } - String hostnameWithoutSubsAndTLD = Iterables.getLast(Splitter.on(".").split(hostnameWithoutTLD), null); - if(hostnameWithoutSubsAndTLD == null) { + @Override + public Object apply(List objects) { + if (objects.isEmpty()) { + return null; + } + Object dnObj = objects.get(0); + InternetDomainName idn = toDomainName(dnObj); + if (idn != null) { + String dn = dnObj.toString(); + String tld = extractTld(idn, dn); + if (!StringUtils.isEmpty(dn)) { + String suffix = safeSubstring(dn, 0, dn.length() - tld.length()); + String hostnameWithoutTLD = safeSubstring(suffix, 0, suffix.length() - 1); + if (hostnameWithoutTLD == null) { + return dn; + } + String hostnameWithoutSubsAndTLD = + Iterables.getLast(Splitter.on(".").split(hostnameWithoutTLD), null); + if (hostnameWithoutSubsAndTLD == null) { + return null; + } + return hostnameWithoutSubsAndTLD + "." + tld; + } + } return null; - } - return hostnameWithoutSubsAndTLD + "." + tld; } - } - return null; } - } - @Stellar(name="REMOVE_TLD" - ,namespace = "DOMAIN" - ,description = "Removes the top level domain (TLD) suffix from a domain." - , params = { - "domain - Fully qualified domain name" - } - , returns = "The domain without the TLD. " + - "(for example, DOMAIN_REMOVE_TLD('mail.yahoo.co.uk') yields 'mail.yahoo')" - ) - public static class RemoveTLD extends BaseStellarFunction { - @Override - public Object apply(List objects) { - Object dnObj = objects.get(0); - InternetDomainName idn = toDomainName(dnObj); - if(idn != null) { - String dn = dnObj.toString(); - String tld = extractTld(idn, dn); - String suffix = safeSubstring(dn, 0, dn.length() - tld.length()); - if(StringUtils.isEmpty(suffix)) { - return suffix; - } - else { - return suffix.substring(0, suffix.length() - 1); + @Stellar(name = "REMOVE_TLD", + namespace = "DOMAIN", + description = "Removes the top level domain (TLD) suffix from a domain.", + params = { + "domain - Fully qualified domain name" + }, + returns = "The domain without the TLD. " + + "(for example, DOMAIN_REMOVE_TLD('mail.yahoo.co.uk') yields 'mail.yahoo')" + ) + public static class RemoveTLD extends BaseStellarFunction { + @Override + public Object apply(List objects) { + Object dnObj = objects.get(0); + InternetDomainName idn = toDomainName(dnObj); + if (idn != null) { + String dn = dnObj.toString(); + String tld = extractTld(idn, dn); + String suffix = safeSubstring(dn, 0, dn.length() - tld.length()); + if (StringUtils.isEmpty(suffix)) { + return suffix; + } else { + return suffix.substring(0, suffix.length() - 1); + } + } + return null; } - } - return null; } - } - @Stellar(name="TO_TLD" - ,namespace = "DOMAIN" - ,description = "Extracts the top level domain from a domain" - , params = { - "domain - Fully qualified domain name" - } - , returns = "The TLD of the domain. " + - "(for example, DOMAIN_TO_TLD('mail.yahoo.co.uk') yields 'co.uk')" - ) - public static class ExtractTLD extends BaseStellarFunction { - @Override - public Object apply(List objects) { - Object dnObj = objects.get(0); - InternetDomainName idn = toDomainName(dnObj); - return extractTld(idn, dnObj + ""); + @Stellar(name = "TO_TLD", + namespace = "DOMAIN", + description = "Extracts the top level domain from a domain", + params = { + "domain - Fully qualified domain name" + }, + returns = "The TLD of the domain. " + + "(for example, DOMAIN_TO_TLD('mail.yahoo.co.uk') yields 'co.uk')" + ) + public static class ExtractTLD extends BaseStellarFunction { + @Override + public Object apply(List objects) { + Object dnObj = objects.get(0); + InternetDomainName idn = toDomainName(dnObj); + return extractTld(idn, dnObj + ""); + } } - } - @Stellar(name="TO_PORT" - ,namespace="URL" - ,description = "Extract the port from a URL. " + - "If the port is not explicitly stated in the URL, then an implicit port is inferred based on the protocol." - , params = { - "url - URL in string form" - } - , returns = "The port used in the URL as an integer (for example, URL_TO_PORT('http://www.yahoo.com/foo') would yield 80)" - ) - public static class URLToPort extends BaseStellarFunction { - @Override - public Object apply(List objects) { - URL url = toUrl(objects.get(0)); - if(url == null) { - return null; - } - int port = url.getPort(); - return port >= 0?port:url.getDefaultPort(); + @Stellar(name = "TO_PORT", + namespace = "URL", + description = "Extract the port from a URL. " + + "If the port is not explicitly stated in the URL, then an implicit port is inferred based on the protocol.", + params = { + "url - URL in string form" + }, + returns = "The port used in the URL as an integer (for example, URL_TO_PORT('http://www.yahoo.com/foo') would yield 80)" + ) + public static class URLToPort extends BaseStellarFunction { + @Override + public Object apply(List objects) { + URL url = toUrl(objects.get(0)); + if (url == null) { + return null; + } + int port = url.getPort(); + return port >= 0 ? port : url.getDefaultPort(); + } } - } - @Stellar(name="TO_PATH" - ,namespace="URL" - ,description = "Extract the path from a URL." - , params = { - "url - URL in String form" - } - , returns = "The path from the URL as a String. e.g. URL_TO_PATH('http://www.yahoo.com/foo') would yield 'foo'") - public static class URLToPath extends BaseStellarFunction { - @Override - public Object apply(List objects) { - URL url = toUrl(objects.get(0)); - return url == null?null:url.getPath(); + @Stellar(name = "TO_PATH", + namespace = "URL", + description = "Extract the path from a URL.", + params = { + "url - URL in String form" + }, + returns = "The path from the URL as a String. e.g. URL_TO_PATH('http://www.yahoo.com/foo') would yield 'foo'") + public static class URLToPath extends BaseStellarFunction { + @Override + public Object apply(List objects) { + URL url = toUrl(objects.get(0)); + return url == null ? null : url.getPath(); + } } - } - @Stellar(name="TO_HOST" - ,namespace="URL" - ,description = "Extract the hostname from a URL." - , params = { - "url - URL in String form" - } - , returns = "The hostname from the URL as a String. e.g. URL_TO_HOST('http://www.yahoo.com/foo') would yield 'www.yahoo.com'" - ) - public static class URLToHost extends BaseStellarFunction { + @Stellar(name = "TO_HOST", + namespace = "URL", + description = "Extract the hostname from a URL.", + params = { + "url - URL in String form" + }, + returns = "The hostname from the URL as a String. e.g. URL_TO_HOST('http://www.yahoo.com/foo') would yield 'www.yahoo.com'" + ) + public static class URLToHost extends BaseStellarFunction { - @Override - public Object apply(List objects) { - URL url = toUrl(objects.get(0)); - return url == null?null:url.getHost(); + @Override + public Object apply(List objects) { + URL url = toUrl(objects.get(0)); + return url == null ? null : url.getHost(); + } } - } - @Stellar(name="TO_PROTOCOL" - ,namespace="URL" - ,description = "Extract the protocol from a URL." - , params = { - "url - URL in String form" - } - , returns = "The protocol from the URL as a String. e.g. URL_TO_PROTOCOL('http://www.yahoo.com/foo') would yield 'http'") - public static class URLToProtocol extends BaseStellarFunction { + @Stellar(name = "TO_PROTOCOL", + namespace = "URL", + description = "Extract the protocol from a URL.", + params = { + "url - URL in String form" + }, + returns = "The protocol from the URL as a String. e.g. URL_TO_PROTOCOL('http://www.yahoo.com/foo') would yield 'http'") + public static class URLToProtocol extends BaseStellarFunction { - @Override - public Object apply(List objects) { - URL url = toUrl(objects.get(0)); - return url == null?null:url.getProtocol(); + @Override + public Object apply(List objects) { + URL url = toUrl(objects.get(0)); + return url == null ? null : url.getProtocol(); + } } - } - /** - * Extract the TLD. If the domain is a normal domain, then we can handle the TLD via the InternetDomainName object. - * If it is not, then we default to returning the last segment after the final '.' - * @param idn - * @param dn - * @return The TLD of the domain - */ - private static String extractTld(InternetDomainName idn, String dn) { + /** + * Extract the TLD. If the domain is a normal domain, then we can handle the TLD via the InternetDomainName object. + * If it is not, then we default to returning the last segment after the final '.' + * + * @return The TLD of the domain + */ + private static String extractTld(InternetDomainName idn, String dn) { - if(idn != null && idn.hasPublicSuffix()) { - String ret = idn.publicSuffix().toString(); - if(ret.startsWith("InternetDomainName")) { - return Joiner.on(".").join(idn.publicSuffix().parts()); - } - else { - return ret; - } - } - else if(dn != null) { - StringBuffer tld = new StringBuffer(""); - for(int idx = dn.length() -1;idx >= 0;idx--) { - char c = dn.charAt(idx); - if(c == '.') { - break; - } - else { - tld.append(dn.charAt(idx)); + if (idn != null && idn.hasPublicSuffix()) { + String ret = idn.publicSuffix().toString(); + if (ret.startsWith("InternetDomainName")) { + return Joiner.on(".").join(idn.publicSuffix().parts()); + } else { + return ret; + } + } else if (dn != null) { + StringBuffer tld = new StringBuffer(); + for (int idx = dn.length() - 1; idx >= 0; idx--) { + char c = dn.charAt(idx); + if (c == '.') { + break; + } else { + tld.append(dn.charAt(idx)); + } + } + return tld.reverse().toString(); + } else { + return null; } - } - return tld.reverse().toString(); - } - else { - return null; - } - } - - private static String safeSubstring(String val, int start, int end) { - if(!StringUtils.isEmpty(val)) { - return val.substring(start, end); } - return null; - } - private static InternetDomainName toDomainName(Object dnObj) { - if(dnObj != null) { - if(dnObj instanceof String) { - String dn = dnObj.toString(); - try { - return InternetDomainName.from(dn); + private static String safeSubstring(String val, int start, int end) { + if (!StringUtils.isEmpty(val)) { + return val.substring(start, end); } - catch(IllegalArgumentException iae) { - return null; - } - } - else { - throw new IllegalArgumentException(dnObj + " is not a string and therefore also not a domain."); - } + return null; } - return null; - } - private static URL toUrl(Object urlObj) { - if(urlObj == null) { - return null; - } - if(urlObj instanceof String) { - String url = urlObj.toString(); - try { - return new URL(url); - } catch (MalformedURLException e) { + private static InternetDomainName toDomainName(Object dnObj) { + if (dnObj != null) { + if (dnObj instanceof String) { + String dn = dnObj.toString(); + try { + return InternetDomainName.from(dn); + } catch (IllegalArgumentException iae) { + return null; + } + } else { + throw new IllegalArgumentException(dnObj + " is not a string and therefore also not a domain."); + } + } return null; - } } - else { - throw new IllegalArgumentException(urlObj + " is not a string and therefore also not a URL."); + + private static URL toUrl(Object urlObj) { + if (urlObj == null) { + return null; + } + if (urlObj instanceof String) { + String url = urlObj.toString(); + try { + return new URL(url); + } catch (MalformedURLException e) { + return null; + } + } else { + throw new IllegalArgumentException(urlObj + " is not a string and therefore also not a URL."); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java index d3bd9ce1..3459be57 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions; /** @@ -23,13 +26,15 @@ public interface Ordinal { /** - * get the min value + * get the min value. + * * @return min value */ double getMin(); /** - * get the max value + * get the max value. + * * @return max value */ double getMax(); diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java index 12ac3f81..f157ac5a 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,122 +20,128 @@ package org.apache.metron.stellar.dsl.functions; +import static org.apache.metron.stellar.common.utils.ConversionUtils.convert; + import com.google.common.collect.Iterables; +import java.util.List; +import java.util.function.BiFunction; import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.BaseStellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import java.util.List; -import java.util.function.BiFunction; - -import static org.apache.metron.stellar.common.utils.ConversionUtils.convert; - public class OrdinalFunctions { - /** - * Stellar Function: MAX - *

    - * Return the maximum value of a list of input values in a Stellar list - */ - @Stellar(name = "MAX" - , description = "Returns the maximum value of a list of input values or from a statistics object" - , params = {"stats - The Stellar statistics object" - ,"list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface)" + - " Multi type numeric comparisons are supported: MAX([10,15L,15.3]) would return 15.3, but MAX(['23',25]) will fail and return null as strings and numbers can't be compared."} - , returns = "The maximum value in the list or from stats, or null if the list is empty or the input values were not comparable.") - public static class Max extends BaseStellarFunction { + /** + * Stellar Function: MAX. + * + *

    + * Return the maximum value of a list of input values in a Stellar list + */ + @Stellar(name = "MAX", + description = "Returns the maximum value of a list of input values or from a statistics object", + params = { + "stats - The Stellar statistics object", + "list - List of arguments. The list may only contain objects that are " + + "mutually comparable / ordinal (implement java.lang.Comparable interface)" + + " Multi type numeric comparisons are supported: MAX([10,15L,15.3]) would return 15.3, " + + "but MAX(['23',25]) will fail and return null as strings and numbers can't be compared." + }, + returns = "The maximum value in the list or from stats, or null if the list is empty or the input values were not comparable.") + public static class Max extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - if (args.size() < 1 || args.get(0) == null) { - throw new IllegalStateException("MAX function requires at least one argument"); - } - Object firstArg = args.get(0); - if (firstArg instanceof Ordinal) { - Ordinal stats = convert(firstArg, Ordinal.class); - return stats.getMax(); - } else if (firstArg instanceof Iterable) { - Iterable list = (Iterable) args.get(0); - return orderList(list, (ret, val) -> ret.compareTo(val) < 0, "MAX"); - } else { - throw new IllegalStateException("MAX function expects either 'a StatisticsProvider object' or 'Stellar list of values'"); - } + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + if (args.size() < 1 || args.get(0) == null) { + throw new IllegalStateException("MAX function requires at least one argument"); + } + Object firstArg = args.get(0); + if (firstArg instanceof Ordinal) { + Ordinal stats = convert(firstArg, Ordinal.class); + return stats.getMax(); + } else if (firstArg instanceof Iterable) { + Iterable list = (Iterable) args.get(0); + return orderList(list, (ret, val) -> ret.compareTo(val) < 0, "MAX"); + } else { + throw new IllegalStateException( + "MAX function expects either 'a StatisticsProvider object' or 'Stellar list of values'"); + } + } } - } - /** - * Stellar Function: MIN. - * - *

    - * Return the minimum value of a list of input values in a Stellar list - *

    - */ - @Stellar(name = "MIN", + /** + * Stellar Function: MIN. + * + *

    + * Return the minimum value of a list of input values in a Stellar list + *

    + */ + @Stellar(name = "MIN", description = "Returns the minimum value of a list of input values", params = {"stats - The Stellar statistics object", - "list - List of arguments. The list may only contain objects that are mutually comparable " - + "/ ordinal (implement java.lang.Comparable interface)" - + " Multi type numeric comparisons are supported: MIN([10,15L,15.3]) would return 10," - + "but MIN(['23',25]) will fail and return null as strings and numbers can't be compared."}, + "list - List of arguments. The list may only contain objects that are mutually comparable " + + "/ ordinal (implement java.lang.Comparable interface)" + + " Multi type numeric comparisons are supported: MIN([10,15L,15.3]) would return 10," + + "but MIN(['23',25]) will fail and return null as strings and numbers can't be compared."}, returns = "The minimum value in the list or from stats, or null if the list is empty or the input values" - + " were not comparable.") - public static class Min extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - if (args.size() < 1 || args.get(0) == null) { - throw new IllegalStateException("MIN function requires at least one argument"); - } - Object firstArg = args.get(0); - if (firstArg instanceof Ordinal) { - Ordinal stats = convert(firstArg, Ordinal.class); - return stats.getMin(); - } else if (firstArg instanceof Iterable) { - Iterable list = (Iterable) args.get(0); - return orderList(list, (ret, val) -> ret.compareTo(val) > 0, "MIN"); - } else { - throw new IllegalStateException("MIN function expects either 'a StatisticsProvider object' or 'Stellar list of values' "); - } + + " were not comparable.") + public static class Min extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + if (args.size() < 1 || args.get(0) == null) { + throw new IllegalStateException("MIN function requires at least one argument"); + } + Object firstArg = args.get(0); + if (firstArg instanceof Ordinal) { + Ordinal stats = convert(firstArg, Ordinal.class); + return stats.getMin(); + } else if (firstArg instanceof Iterable) { + Iterable list = (Iterable) args.get(0); + return orderList(list, (ret, val) -> ret.compareTo(val) > 0, "MIN"); + } else { + throw new IllegalStateException( + "MIN function expects either 'a StatisticsProvider object' or 'Stellar list of values' "); + } + } } - } - private static Comparable orderList(Iterable list, BiFunction eval, String funcName) { - if (Iterables.isEmpty(list)) { - return null; - } - Object o = Iterables.getFirst(list, null); - Comparable ret = null; - for(Object valueVal : list) { - if(valueVal == null) { - continue; - } - Comparable value = null; - if(!(valueVal instanceof Comparable)) { - throw new IllegalStateException("Noncomparable object type " + valueVal.getClass().getName() - + " submitted to " + funcName); - } - else { - value = (Comparable)valueVal; - } - try { - Comparable convertedRet = ConversionUtils.convert(ret, value.getClass()); - if(convertedRet == null && ret != null) { - throw new IllegalStateException("Incomparable objects were submitted to " + funcName - + ": " + ret.getClass() + " is incomparable to " + value.getClass()); + private static Comparable orderList(Iterable list, BiFunction eval, + String funcName) { + if (Iterables.isEmpty(list)) { + return null; } - if(ret == null || eval.apply(convertedRet, value) ) { - ret = value; + Object o = Iterables.getFirst(list, null); + Comparable ret = null; + for (Object valueVal : list) { + if (valueVal == null) { + continue; + } + Comparable value = null; + if (!(valueVal instanceof Comparable)) { + throw new IllegalStateException("Noncomparable object type " + valueVal.getClass().getName() + + " submitted to " + funcName); + } else { + value = (Comparable) valueVal; + } + try { + Comparable convertedRet = ConversionUtils.convert(ret, value.getClass()); + if (convertedRet == null && ret != null) { + throw new IllegalStateException("Incomparable objects were submitted to " + funcName + + ": " + ret.getClass() + " is incomparable to " + + value.getClass()); + } + if (ret == null || eval.apply(convertedRet, value)) { + ret = value; + } + } catch (ClassCastException cce) { + throw new IllegalStateException("Incomparable objects were submitted to " + funcName + + ": " + cce.getMessage(), cce); + } } - } - catch(ClassCastException cce) { - throw new IllegalStateException("Incomparable objects were submitted to " + funcName - + ": " + cce.getMessage(), cce); - } + return ret; } - return ret; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java index 48e82a36..d214957c 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * + *

    * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,126 +31,127 @@ public class RegExFunctions { - @Stellar(name = "REGEXP_MATCH", - description = "Determines whether a regex matches a string, if a list of patterns is passed, then the matching is an OR operation", - params = { - "string - The string to test", - "pattern - The proposed regex pattern or a list of proposed regex patterns" - }, - returns = "True if the regex pattern matches the string and false if otherwise.") - public static class RegexpMatch extends BaseStellarFunction { - - @Override - public Object apply(List list) { - if (list.size() < 2) { - throw new IllegalStateException( - "REGEXP_MATCH expects two args: [string, pattern] where pattern is a regexp pattern or a list of regexp patterns"); - } - Object patternObject = list.get(1); - String str = (String) list.get(0); - if (str == null || patternObject == null) { - return false; - } - if (patternObject instanceof String) { - return PatternCache.INSTANCE.getPattern((String)patternObject).matcher(str).matches(); - } else if (patternObject instanceof Iterable) { - boolean matches = false; - for (Object thisPatternObject : (Iterable)patternObject) { - if (thisPatternObject == null) { - continue; - } - if (PatternCache.INSTANCE.getPattern(thisPatternObject.toString()).matcher(str).matches()) { - matches = true; - break; - } + @Stellar(name = "REGEXP_MATCH", + description = "Determines whether a regex matches a string, " + + "if a list of patterns is passed, then the matching is an OR operation", + params = { + "string - The string to test", + "pattern - The proposed regex pattern or a list of proposed regex patterns" + }, + returns = "True if the regex pattern matches the string and false if otherwise.") + public static class RegexpMatch extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if (list.size() < 2) { + throw new IllegalStateException( + "REGEXP_MATCH expects two args: [string, pattern] where pattern is a regexp pattern or a list of regexp patterns"); + } + Object patternObject = list.get(1); + String str = (String) list.get(0); + if (str == null || patternObject == null) { + return false; + } + if (patternObject instanceof String) { + return PatternCache.INSTANCE.getPattern((String) patternObject).matcher(str).matches(); + } else if (patternObject instanceof Iterable) { + boolean matches = false; + for (Object thisPatternObject : (Iterable) patternObject) { + if (thisPatternObject == null) { + continue; + } + if (PatternCache.INSTANCE.getPattern(thisPatternObject.toString()).matcher(str).matches()) { + matches = true; + break; + } + } + return matches; + } + return false; } - return matches; - } - return false; } - } - - @Stellar(name = "REGEXP_GROUP_VAL", - description = "Returns the value of a group in a regex against a string", - params = { - "string - The string to test", - "pattern - The proposed regex pattern", - "group - integer that selects what group to select, starting at 1" - }, - returns = "The value of the group, or null if not matched or no group at index") - public static class RegexpGroupValue extends BaseStellarFunction { - - @Override - public Object apply(List list) { - if (list.size() != 3) { - throw new IllegalStateException( - "REGEXP_GROUP_VAL expects three args: [string, pattern, int]" + "" - + "where pattern is a regexp pattern"); - } - String stringPattern = (String) list.get(1); - String str = (String) list.get(0); - Integer groupNumber = ConversionUtils.convert(list.get(2), Integer.class); - - if (groupNumber == null) { - // group number was not a valid integer - return null; - } - - if (groupNumber == 0) { - // 0, by default is the entire input - // default to returning a non-null - return str; - } - - if (str == null || stringPattern == null) { - return null; - } - Pattern pattern = PatternCache.INSTANCE.getPattern(stringPattern); - Matcher matcher = pattern.matcher(str); - if (!matcher.matches()) { - return null; - } - - int groupCount = matcher.groupCount(); - if (groupCount == 0 || groupCount < groupNumber) { - return null; - } - return matcher.group(groupNumber); + + @Stellar(name = "REGEXP_GROUP_VAL", + description = "Returns the value of a group in a regex against a string", + params = { + "string - The string to test", + "pattern - The proposed regex pattern", + "group - integer that selects what group to select, starting at 1" + }, + returns = "The value of the group, or null if not matched or no group at index") + public static class RegexpGroupValue extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if (list.size() != 3) { + throw new IllegalStateException( + "REGEXP_GROUP_VAL expects three args: [string, pattern, int]" + + "where pattern is a regexp pattern"); + } + String stringPattern = (String) list.get(1); + String str = (String) list.get(0); + Integer groupNumber = ConversionUtils.convert(list.get(2), Integer.class); + + if (groupNumber == null) { + // group number was not a valid integer + return null; + } + + if (groupNumber == 0) { + // 0, by default is the entire input + // default to returning a non-null + return str; + } + + if (str == null || stringPattern == null) { + return null; + } + Pattern pattern = PatternCache.INSTANCE.getPattern(stringPattern); + Matcher matcher = pattern.matcher(str); + if (!matcher.matches()) { + return null; + } + + int groupCount = matcher.groupCount(); + if (groupCount == 0 || groupCount < groupNumber) { + return null; + } + return matcher.group(groupNumber); + } } - } - - @Stellar(name = "REGEXP_REPLACE", - description = "Replace all occurences of the regex pattern within the string by value", - params = { - "string - The input string", - "pattern - The regex pattern to be replaced. Special characters must be escaped (e.g. \\\\d)", - "value - The value to replace the regex pattern" - }, - returns = "The modified input string with replaced values") - public static class RegexpReplace extends BaseStellarFunction { - - @Override - public Object apply(List list) { - if (list.size() != 3) { - throw new IllegalStateException( - "REGEXP_REPLACE expects three args: [string, pattern, value]" - + " where pattern is a regexp pattern"); - } - String str = (String) list.get(0); - String stringPattern = (String) list.get(1); - String value = (String) list.get(2); - - if (StringUtils.isEmpty(str)) { - return null; - } - - if (StringUtils.isEmpty(stringPattern) || StringUtils.isEmpty(value)) { - return str; - } - - Pattern pattern = PatternCache.INSTANCE.getPattern(stringPattern); - Matcher matcher = pattern.matcher(str); - return matcher.replaceAll(value); + + @Stellar(name = "REGEXP_REPLACE", + description = "Replace all occurences of the regex pattern within the string by value", + params = { + "string - The input string", + "pattern - The regex pattern to be replaced. Special characters must be escaped (e.g. \\\\d)", + "value - The value to replace the regex pattern" + }, + returns = "The modified input string with replaced values") + public static class RegexpReplace extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if (list.size() != 3) { + throw new IllegalStateException( + "REGEXP_REPLACE expects three args: [string, pattern, value]" + + " where pattern is a regexp pattern"); + } + String str = (String) list.get(0); + String stringPattern = (String) list.get(1); + String value = (String) list.get(2); + + if (StringUtils.isEmpty(str)) { + return null; + } + + if (StringUtils.isEmpty(stringPattern) || StringUtils.isEmpty(value)) { + return str; + } + + Pattern pattern = PatternCache.INSTANCE.getPattern(stringPattern); + Matcher matcher = pattern.matcher(str); + return matcher.replaceAll(value); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java index a32faec7..cdf6c858 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java @@ -7,191 +7,193 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions; import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Map; /** * A Map containing the Stellar REST settings. */ public class RestConfig extends HashMap { - /** - * A global config prefix used for storing Stellar REST settings. - */ - public final static String STELLAR_REST_SETTINGS = "stellar.rest.settings"; - - /** - * A global config prefix used for storing Stellar REST GET settings. - */ - public final static String STELLAR_REST_GET_SETTINGS = "stellar.rest.get.settings"; - - /** - * A global config prefix used for storing Stellar REST POST settings. - */ - public final static String STELLAR_REST_POST_SETTINGS = "stellar.rest.post.settings"; - - /** - * User name for basic authentication. - */ - public final static String BASIC_AUTH_USER = "basic.auth.user"; - - /** - * Path to the basic authentication password file stored in HDFS. - */ - public final static String BASIC_AUTH_PASSWORD_PATH = "basic.auth.password.path"; - - /** - * Proxy host. - */ - public final static String PROXY_HOST = "proxy.host"; - - /** - * Proxy port. - */ - public final static String PROXY_PORT = "proxy.port"; - - /** - * User name for proxy basic authentication. - */ - public final static String PROXY_BASIC_AUTH_USER = "proxy.basic.auth.user"; - - /** - * Path to the proxy basic authentication password file stored in HDFS. - */ - public final static String PROXY_BASIC_AUTH_PASSWORD_PATH = "proxy.basic.auth.password.path"; - - /** - * Hard timeout for the total request time. - */ - public final static String TIMEOUT = "timeout"; - - /** - * Timeouts exposed by the HttpClient object. - */ - public final static String CONNECT_TIMEOUT = "connect.timeout"; - public final static String CONNECTION_REQUEST_TIMEOUT = "connection.request.timeout"; - public final static String SOCKET_TIMEOUT = "socket.timeout"; - - /** - * A list of response codes that are allowed. All others will be treated as errors. - */ - public final static String RESPONSE_CODES_ALLOWED = "response.codes.allowed"; - - /** - * The default value that will be returned on a successful request with empty content. Default is null. - */ - public final static String EMPTY_CONTENT_OVERRIDE = "empty.content.override"; - - /** - * The default value that will be returned on an error. Default is null. - */ - public final static String ERROR_VALUE_OVERRIDE = "error.value.override"; - - /** - * The maximum number of connections in the connection pool. - */ - public final static String POOLING_MAX_TOTAL = "pooling.max.total"; - - /** - * The default maximum number of connections per route in the connection pool. - */ - public final static String POOLING_DEFAULT_MAX_PER_RUOTE = "pooling.default.max.per.route"; - - /** - * Setting this to true will verify the actual body content length equals the content length header. - */ - public final static String VERIFY_CONTENT_LENGTH = "verify.content.length"; - - /** - * Setting this to true will verify POST data is well-formed JSON. - */ - public final static String ENFORCE_JSON = "enforce.json"; - - public RestConfig() { - put(TIMEOUT, 1000); - put(RESPONSE_CODES_ALLOWED, Collections.singletonList(200)); - put(VERIFY_CONTENT_LENGTH, false); - put(ENFORCE_JSON, true); - } - - public String getBasicAuthUser() { - return (String) get(BASIC_AUTH_USER); - } - - public String getBasicAuthPasswordPath() { - return (String) get(BASIC_AUTH_PASSWORD_PATH); - } - - public String getProxyHost() { - return (String) get(PROXY_HOST); - } - - public Integer getProxyPort() { - return (Integer) get(PROXY_PORT); - } - - public String getProxyBasicAuthUser() { - return (String) get(PROXY_BASIC_AUTH_USER); - } - - public String getProxyBasicAuthPasswordPath() { - return (String) get(PROXY_BASIC_AUTH_PASSWORD_PATH); - } - - public Integer getTimeout() { - return (Integer) get(TIMEOUT); - } - - public Integer getConnectTimeout() { - return (Integer) get(CONNECT_TIMEOUT); - } - - public Integer getConnectionRequestTimeout() { - return (Integer) get(CONNECTION_REQUEST_TIMEOUT); - } - - public Integer getSocketTimeout() { - return (Integer) get(SOCKET_TIMEOUT); - } - - @SuppressWarnings("unchecked") - public List getResponseCodesAllowed() { - return (List) get(RESPONSE_CODES_ALLOWED); - } - - public Object getEmptyContentOverride() { - return get(EMPTY_CONTENT_OVERRIDE); - } - - public Object getErrorValueOverride() { - return get(ERROR_VALUE_OVERRIDE); - } - - public Integer getPoolingMaxTotal() { - return (Integer) get(POOLING_MAX_TOTAL); - } - - public Integer getPoolingDefaultMaxPerRoute() { - return (Integer) get(POOLING_DEFAULT_MAX_PER_RUOTE); - } - - public Boolean verifyContentLength() { - return (Boolean) get(VERIFY_CONTENT_LENGTH); - } - - public Boolean enforceJson() { - return (Boolean) get(ENFORCE_JSON); - } + /** + * A global config prefix used for storing Stellar REST settings. + */ + public static final String STELLAR_REST_SETTINGS = "stellar.rest.settings"; + + /** + * A global config prefix used for storing Stellar REST GET settings. + */ + public static final String STELLAR_REST_GET_SETTINGS = "stellar.rest.get.settings"; + + /** + * A global config prefix used for storing Stellar REST POST settings. + */ + public static final String STELLAR_REST_POST_SETTINGS = "stellar.rest.post.settings"; + + /** + * User name for basic authentication. + */ + public static final String BASIC_AUTH_USER = "basic.auth.user"; + + /** + * Path to the basic authentication password file stored in HDFS. + */ + public static final String BASIC_AUTH_PASSWORD_PATH = "basic.auth.password.path"; + + /** + * Proxy host. + */ + public static final String PROXY_HOST = "proxy.host"; + + /** + * Proxy port. + */ + public static final String PROXY_PORT = "proxy.port"; + + /** + * User name for proxy basic authentication. + */ + public static final String PROXY_BASIC_AUTH_USER = "proxy.basic.auth.user"; + + /** + * Path to the proxy basic authentication password file stored in HDFS. + */ + public static final String PROXY_BASIC_AUTH_PASSWORD_PATH = "proxy.basic.auth.password.path"; + + /** + * Hard timeout for the total request time. + */ + public static final String TIMEOUT = "timeout"; + + /** + * Timeouts exposed by the HttpClient object. + */ + public static final String CONNECT_TIMEOUT = "connect.timeout"; + public static final String CONNECTION_REQUEST_TIMEOUT = "connection.request.timeout"; + public static final String SOCKET_TIMEOUT = "socket.timeout"; + + /** + * A list of response codes that are allowed. All others will be treated as errors. + */ + public static final String RESPONSE_CODES_ALLOWED = "response.codes.allowed"; + + /** + * The default value that will be returned on a successful request with empty content. Default is null. + */ + public static final String EMPTY_CONTENT_OVERRIDE = "empty.content.override"; + + /** + * The default value that will be returned on an error. Default is null. + */ + public static final String ERROR_VALUE_OVERRIDE = "error.value.override"; + + /** + * The maximum number of connections in the connection pool. + */ + public static final String POOLING_MAX_TOTAL = "pooling.max.total"; + + /** + * The default maximum number of connections per route in the connection pool. + */ + public static final String POOLING_DEFAULT_MAX_PER_RUOTE = "pooling.default.max.per.route"; + + /** + * Setting this to true will verify the actual body content length equals the content length header. + */ + public static final String VERIFY_CONTENT_LENGTH = "verify.content.length"; + + /** + * Setting this to true will verify POST data is well-formed JSON. + */ + public static final String ENFORCE_JSON = "enforce.json"; + + public RestConfig() { + put(TIMEOUT, 1000); + put(RESPONSE_CODES_ALLOWED, Collections.singletonList(200)); + put(VERIFY_CONTENT_LENGTH, false); + put(ENFORCE_JSON, true); + } + + public String getBasicAuthUser() { + return (String) get(BASIC_AUTH_USER); + } + + public String getBasicAuthPasswordPath() { + return (String) get(BASIC_AUTH_PASSWORD_PATH); + } + + public String getProxyHost() { + return (String) get(PROXY_HOST); + } + + public Integer getProxyPort() { + return (Integer) get(PROXY_PORT); + } + + public String getProxyBasicAuthUser() { + return (String) get(PROXY_BASIC_AUTH_USER); + } + + public String getProxyBasicAuthPasswordPath() { + return (String) get(PROXY_BASIC_AUTH_PASSWORD_PATH); + } + + public Integer getTimeout() { + return (Integer) get(TIMEOUT); + } + + public Integer getConnectTimeout() { + return (Integer) get(CONNECT_TIMEOUT); + } + + public Integer getConnectionRequestTimeout() { + return (Integer) get(CONNECTION_REQUEST_TIMEOUT); + } + + public Integer getSocketTimeout() { + return (Integer) get(SOCKET_TIMEOUT); + } + + @SuppressWarnings("unchecked") + public List getResponseCodesAllowed() { + return (List) get(RESPONSE_CODES_ALLOWED); + } + + public Object getEmptyContentOverride() { + return get(EMPTY_CONTENT_OVERRIDE); + } + + public Object getErrorValueOverride() { + return get(ERROR_VALUE_OVERRIDE); + } + + public Integer getPoolingMaxTotal() { + return (Integer) get(POOLING_MAX_TOTAL); + } + + public Integer getPoolingDefaultMaxPerRoute() { + return (Integer) get(POOLING_DEFAULT_MAX_PER_RUOTE); + } + + public Boolean verifyContentLength() { + return (Boolean) get(VERIFY_CONTENT_LENGTH); + } + + public Boolean enforceJson() { + return (Boolean) get(ENFORCE_JSON); + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java index 50585279..0943c826 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java @@ -7,17 +7,42 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions; +import static java.lang.String.format; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; +import static org.apache.metron.stellar.dsl.functions.RestConfig.POOLING_DEFAULT_MAX_PER_RUOTE; +import static org.apache.metron.stellar.dsl.functions.RestConfig.POOLING_MAX_TOTAL; +import static org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_GET_SETTINGS; +import static org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_POST_SETTINGS; +import static org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_SETTINGS; + import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -29,7 +54,11 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.*; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.StringEntity; @@ -47,25 +76,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.lang.invoke.MethodHandles; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import static java.lang.String.format; -import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; -import static org.apache.metron.stellar.dsl.functions.RestConfig.*; - /** * Defines functions that enable REST requests with proper result and error handling. Depends on an * Apache HttpComponents client being supplied as a Stellar HTTP_CLIENT capability. Exposes various Http settings @@ -74,520 +84,515 @@ */ public class RestFunctions { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** - * The CloseableHttpClient. - */ - private static CloseableHttpClient closeableHttpClient; - - /** - * Executor used to impose a hard request timeout. - */ - private static ScheduledExecutorService scheduledExecutorService; - - /** - * Initialize a single HttpClient to be shared by REST functions. - * @param context - */ - private static synchronized void initializeHttpClient(Context context) { - if (closeableHttpClient == null) { - closeableHttpClient = getHttpClient(context); - } - } - - /** - * Close the shared HttpClient. - */ - private static synchronized void closeHttpClient() throws IOException { - if (closeableHttpClient != null) { - closeableHttpClient.close(); - closeableHttpClient = null; - } - } - - /** - * Initialize a single ExecutorService to be shared by REST functions. - */ - private static synchronized void initializeExecutorService() { - if (scheduledExecutorService == null) { - scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - } - } - - /** - * Shutdown the shared ExecutorService. - */ - private static synchronized void closeExecutorService() { - if (scheduledExecutorService != null) { - scheduledExecutorService.shutdown(); - scheduledExecutorService = null; - } - } + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @Stellar( - namespace = "REST", - name = "GET", - description = "Performs a REST GET request and parses the JSON results into a map.", - params = { - "url - URI to the REST service", - "rest_config - Optional - Map (in curly braces) of name:value pairs, each overriding the global config parameter " + - "of the same name. Default is the empty Map, meaning no overrides.", - "query_parameters - Optional - Map (in curly braces) of name:value pairs that will be added to the request as query parameters" - }, - returns = "JSON results as a Map") - public static class RestGet implements StellarFunction { + /** + * The CloseableHttpClient. + */ + private static CloseableHttpClient closeableHttpClient; /** - * Whether the function has been initialized. + * Executor used to impose a hard request timeout. */ - private boolean initialized = false; + private static ScheduledExecutorService scheduledExecutorService; /** - * Initialize the function by creating a ScheduledExecutorService and looking up the CloseableHttpClient from the - * Stellar context. - * @param context + * Initialize a single HttpClient to be shared by REST functions. */ - @Override - public void initialize(Context context) { - initializeExecutorService(); - initializeHttpClient(context); - initialized = true; + private static synchronized void initializeHttpClient(Context context) { + if (closeableHttpClient == null) { + closeableHttpClient = getHttpClient(context); + } } - @Override - public boolean isInitialized() { - return initialized; + /** + * Close the shared HttpClient. + */ + private static synchronized void closeHttpClient() throws IOException { + if (closeableHttpClient != null) { + closeableHttpClient.close(); + closeableHttpClient = null; + } } /** - * Apply the function. - * @param args The function arguments including uri and rest config. - * @param context Stellar context + * Initialize a single ExecutorService to be shared by REST functions. */ - @Override - public Object apply(List args, Context context) throws ParseException { - String uriString = getArg(0, String.class, args); - Map functionRestConfig = null; - Map queryParameters = new HashMap<>(); - if (args.size() > 1) { - functionRestConfig = getArg(1, Map.class, args); - if (args.size() == 3) { - queryParameters = getArg(2, Map.class, args); - } - } - - // Build the RestConfig by applying settins in order of precedence - Map globalRestConfig = (Map) getGlobalConfig(context).get(STELLAR_REST_SETTINGS); - Map getRestConfig = (Map) getGlobalConfig(context).get(STELLAR_REST_GET_SETTINGS); - RestConfig restConfig = buildRestConfig(globalRestConfig, getRestConfig, functionRestConfig); - - try { - HttpGet httpGet = buildGetRequest(uriString, queryParameters); - return executeRequest(restConfig, httpGet); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e.getMessage(), e); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - return restConfig.getErrorValueOverride(); - } + private static synchronized void initializeExecutorService() { + if (scheduledExecutorService == null) { + scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + } } - @Override - public void close() throws IOException { - closeHttpClient(); - closeExecutorService(); + /** + * Shutdown the shared ExecutorService. + */ + private static synchronized void closeExecutorService() { + if (scheduledExecutorService != null) { + scheduledExecutorService.shutdown(); + scheduledExecutorService = null; + } } - private HttpGet buildGetRequest(String uri, Map queryParameters) throws URISyntaxException { - HttpGet httpGet = new HttpGet(getURI(uri, queryParameters)); - httpGet.addHeader("Accept", "application/json"); + @SuppressWarnings("checkstyle:LineLength") + @Stellar( + namespace = "REST", + name = "GET", + description = "Performs a REST GET request and parses the JSON results into a map.", + params = { + "url - URI to the REST service", + "rest_config - Optional - Map (in curly braces) of name:value pairs, each overriding the global config parameter " + + "of the same name. Default is the empty Map, meaning no overrides.", + "query_parameters - Optional - Map (in curly braces) of name:value pairs that will be added to the request as query parameters" + }, + returns = "JSON results as a Map") + public static class RestGet implements StellarFunction { + + /** + * Whether the function has been initialized. + */ + private boolean initialized = false; + + /** + * Initialize the function by creating a ScheduledExecutorService and looking up the CloseableHttpClient from the + * Stellar context. + */ + @Override + public void initialize(Context context) { + initializeExecutorService(); + initializeHttpClient(context); + initialized = true; + } + + @Override + public boolean isInitialized() { + return initialized; + } - return httpGet; + /** + * Apply the function. + * + * @param args The function arguments including uri and rest config. + * @param context Stellar context + */ + @Override + public Object apply(List args, Context context) throws ParseException { + String uriString = getArg(0, String.class, args); + Map functionRestConfig = null; + Map queryParameters = new HashMap<>(); + if (args.size() > 1) { + functionRestConfig = getArg(1, Map.class, args); + if (args.size() == 3) { + queryParameters = getArg(2, Map.class, args); + } + } + + // Build the RestConfig by applying settins in order of precedence + Map globalRestConfig = + (Map) getGlobalConfig(context).get(STELLAR_REST_SETTINGS); + Map getRestConfig = + (Map) getGlobalConfig(context).get(STELLAR_REST_GET_SETTINGS); + RestConfig restConfig = buildRestConfig(globalRestConfig, getRestConfig, functionRestConfig); + + try { + HttpGet httpGet = buildGetRequest(uriString, queryParameters); + return executeRequest(restConfig, httpGet); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + return restConfig.getErrorValueOverride(); + } + } + + @Override + public void close() throws IOException { + closeHttpClient(); + closeExecutorService(); + } + + private HttpGet buildGetRequest(String uri, Map queryParameters) throws URISyntaxException { + HttpGet httpGet = new HttpGet(getURI(uri, queryParameters)); + httpGet.addHeader("Accept", "application/json"); + + return httpGet; + } } - } - @Stellar( + @SuppressWarnings("checkstyle:LineLength") + @Stellar( namespace = "REST", name = "POST", description = "Performs a REST POST request and parses the JSON results into a map.", params = { - "url - URI to the REST service", - "post_data - POST data that will be sent in the POST request. Must be well-formed JSON unless the 'enforce.json' property is set to false.", - "rest_config - Optional - Map (in curly braces) of name:value pairs, each overriding the global config parameter " + - "of the same name. Default is the empty Map, meaning no overrides.", - "query_parameters - Optional - Map (in curly braces) of name:value pairs that will be added to the request as query parameters" + "url - URI to the REST service", + "post_data - POST data that will be sent in the POST request. Must be well-formed JSON unless the 'enforce.json' property is set to false.", + "rest_config - Optional - Map (in curly braces) of name:value pairs, each overriding the global config parameter " + + "of the same name. Default is the empty Map, meaning no overrides.", + "query_parameters - Optional - Map (in curly braces) of name:value pairs that will be added to the request as query parameters" }, returns = "JSON results as a Map") - public static class RestPost implements StellarFunction { + public static class RestPost implements StellarFunction { + + /** + * Whether the function has been initialized. + */ + private boolean initialized = false; + + /** + * Initialize the function by creating a ScheduledExecutorService and looking up the CloseableHttpClient from the + * Stellar context. + */ + @Override + public void initialize(Context context) { + initializeExecutorService(); + initializeHttpClient(context); + initialized = true; + } - /** - * Whether the function has been initialized. - */ - private boolean initialized = false; + @Override + public boolean isInitialized() { + return initialized; + } - /** - * Initialize the function by creating a ScheduledExecutorService and looking up the CloseableHttpClient from the - * Stellar context. - * @param context - */ - @Override - public void initialize(Context context) { - initializeExecutorService(); - initializeHttpClient(context); - initialized = true; - } + @Override + public Object apply(List args, Context context) throws ParseException { + String uriString = getArg(0, String.class, args); + Object dataObject = getArg(1, Object.class, args); + Map functionRestConfig = null; + Map queryParameters = new HashMap<>(); + if (args.size() > 2) { + functionRestConfig = getArg(2, Map.class, args); + if (args.size() == 4) { + queryParameters = getArg(3, Map.class, args); + } + } + + // Build the RestConfig by applying settins in order of precedence + Map globalRestConfig = + (Map) getGlobalConfig(context).get(STELLAR_REST_SETTINGS); + Map postRestConfig = + (Map) getGlobalConfig(context).get(STELLAR_REST_POST_SETTINGS); + RestConfig restConfig = buildRestConfig(globalRestConfig, postRestConfig, functionRestConfig); + + try { + HttpPost httpPost = buildPostRequest(restConfig, uriString, dataObject, queryParameters); + return executeRequest(restConfig, httpPost); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + return restConfig.getErrorValueOverride(); + } + } - @Override - public boolean isInitialized() { - return initialized; - } + @Override + public void close() throws IOException { + closeHttpClient(); + closeExecutorService(); + } - @Override - public Object apply(List args, Context context) throws ParseException { - String uriString = getArg(0, String.class, args); - Object dataObject = getArg(1, Object.class, args); - Map functionRestConfig = null; - Map queryParameters = new HashMap<>(); - if (args.size() > 2) { - functionRestConfig = getArg(2, Map.class, args); - if (args.size() == 4) { - queryParameters = getArg(3, Map.class, args); - } - } - - // Build the RestConfig by applying settins in order of precedence - Map globalRestConfig = (Map) getGlobalConfig(context).get(STELLAR_REST_SETTINGS); - Map postRestConfig = (Map) getGlobalConfig(context).get(STELLAR_REST_POST_SETTINGS); - RestConfig restConfig = buildRestConfig(globalRestConfig, postRestConfig, functionRestConfig); - - try { - HttpPost httpPost = buildPostRequest(restConfig, uriString, dataObject, queryParameters); - return executeRequest(restConfig, httpPost); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e.getMessage(), e); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - return restConfig.getErrorValueOverride(); - } - } + private HttpPost buildPostRequest(RestConfig restConfig, String uriString, Object dataObject, + Map queryParameters) + throws JsonProcessingException, URISyntaxException, UnsupportedEncodingException { + String body = getPostData(restConfig, dataObject); + + URI uri = getURI(uriString, queryParameters); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(body)); + httpPost.addHeader("Accept", "application/json"); + httpPost.addHeader("Content-type", "application/json"); - @Override - public void close() throws IOException { - closeHttpClient(); - closeExecutorService(); + return httpPost; + } + + /** + * Serializes the supplied POST data to be sent in the POST request. Checks for well-formed JSON by default unless 'enforce.json' is set to false. + * + * @param restConfig RestConfig + * @param arg POST data + * @return Serialized POST data + */ + private String getPostData(RestConfig restConfig, Object arg) throws JsonProcessingException { + String data = ""; + if (arg == null) { + return data; + } + if (arg instanceof Map) { + data = JSONUtils.INSTANCE.toJSON(arg, false); + } else { + data = arg.toString(); + if (restConfig.enforceJson()) { + try { + JSONUtils.INSTANCE.toJSONObject(data); + } catch (org.json.simple.parser.ParseException e) { + throw new IllegalArgumentException( + String.format("POST data '%s' must be properly formatted JSON. " + + "Set the '%s' property to false to disable this check.", data, + RestConfig.ENFORCE_JSON)); + } + } + } + return data; + } } - private HttpPost buildPostRequest(RestConfig restConfig, String uriString, Object dataObject, Map queryParameters) throws JsonProcessingException, URISyntaxException, UnsupportedEncodingException { - String body = getPostData(restConfig, dataObject); + /** + * Get an argument from a list of arguments. + * + * @param index The index within the list of arguments. + * @param clazz The type expected. + * @param args All of the arguments. + * @param The type of the argument expected. + */ + public static T getArg(int index, Class clazz, List args) { - URI uri = getURI(uriString, queryParameters); - HttpPost httpPost = new HttpPost(uri); - httpPost.setEntity(new StringEntity(body)); - httpPost.addHeader("Accept", "application/json"); - httpPost.addHeader("Content-type", "application/json"); + if (index >= args.size()) { + throw new IllegalArgumentException( + format("Expected at least %d argument(s), found %d", index + 1, args.size())); + } - return httpPost; + return ConversionUtils.convert(args.get(index), clazz); } /** - * Serializes the supplied POST data to be sent in the POST request. Checks for well-formed JSON by default unless 'enforce.json' is set to false. - * @param restConfig RestConfig - * @param arg POST data - * @return Serialized POST data - * @throws JsonProcessingException + * Retrieves the ClosableHttpClient from a pooling connection manager. + * + * @param context The execution context. + * @return A ClosableHttpClient. */ - private String getPostData(RestConfig restConfig, Object arg) throws JsonProcessingException { - String data = ""; - if (arg == null) { - return data; - } - if (arg instanceof Map) { - data = JSONUtils.INSTANCE.toJSON(arg, false); - } else { - data = arg.toString(); - if (restConfig.enforceJson()) { - try { - JSONUtils.INSTANCE.toJSONObject(data); - } catch (org.json.simple.parser.ParseException e) { - throw new IllegalArgumentException(String.format("POST data '%s' must be properly formatted JSON. " + - "Set the '%s' property to false to disable this check.", data, RestConfig.ENFORCE_JSON)); - } - } - } - return data; - } - } - - /** - * Get an argument from a list of arguments. - * - * @param index The index within the list of arguments. - * @param clazz The type expected. - * @param args All of the arguments. - * @param The type of the argument expected. - */ - public static T getArg(int index, Class clazz, List args) { - - if(index >= args.size()) { - throw new IllegalArgumentException(format("Expected at least %d argument(s), found %d", index+1, args.size())); - } + protected static CloseableHttpClient getHttpClient(Context context) { + RestConfig restConfig = buildRestConfig(getGlobalConfig(context)); - return ConversionUtils.convert(args.get(index), clazz); - } - - /** - * Retrieves the ClosableHttpClient from a pooling connection manager. - * - * @param context The execution context. - * @return A ClosableHttpClient. - */ - protected static CloseableHttpClient getHttpClient(Context context) { - RestConfig restConfig = buildRestConfig(getGlobalConfig(context)); - - PoolingHttpClientConnectionManager cm = getConnectionManager(restConfig); - - return HttpClients.custom() - .setConnectionManager(cm) - .build(); - } - - protected static PoolingHttpClientConnectionManager getConnectionManager(RestConfig restConfig) { - PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); - if (restConfig.containsKey(POOLING_MAX_TOTAL)) { - cm.setMaxTotal(restConfig.getPoolingMaxTotal()); - } - if (restConfig.containsKey(POOLING_DEFAULT_MAX_PER_RUOTE)) { - cm.setDefaultMaxPerRoute(restConfig.getPoolingDefaultMaxPerRoute()); - } - return cm; - } - - @SuppressWarnings("unchecked") - private static Map getGlobalConfig(Context context) { - Optional globalCapability = context.getCapability(GLOBAL_CONFIG, false); - return globalCapability.map(o -> (Map) o).orElseGet(HashMap::new); - } - - /** - * Build the RestConfig by applying settings in order of precedence (last item in the input list has highest priority). - * Only settings specified in the rest config will override lower priority config settings. - * @param configs - * @return - * @throws IOException - */ - @SuppressWarnings("unchecked") - protected static RestConfig buildRestConfig(Map... configs) { - RestConfig restConfig = new RestConfig(); - - // Add settings in order of precedence - for(Map config: configs) { - if (config != null) { - restConfig.putAll(config); - } - } - return restConfig; - } - - /** - * Builds a URI from the supplied URI string and adds query parameters. - * @param uriString - * @param queryParameters - * @return - * @throws URISyntaxException - */ - private static URI getURI(String uriString, Map queryParameters) throws URISyntaxException { - URIBuilder uriBuilder = new URIBuilder(uriString); - if (queryParameters != null) { - for(Map.Entry entry: queryParameters.entrySet()) { - uriBuilder.setParameter(entry.getKey(), (String) entry.getValue()); - } + PoolingHttpClientConnectionManager cm = getConnectionManager(restConfig); + + return HttpClients.custom() + .setConnectionManager(cm) + .build(); } - return uriBuilder.build(); - } - - /** - * Returns the proxy HttpHost object if the proxy rest config settings are set. - * @param restConfig - * @return - */ - protected static Optional getProxy(RestConfig restConfig) { - Optional proxy = Optional.empty(); - if (restConfig.getProxyHost() != null && restConfig.getProxyPort() != null) { - proxy = Optional.of(new HttpHost(restConfig.getProxyHost(), restConfig.getProxyPort(), "http")); + + protected static PoolingHttpClientConnectionManager getConnectionManager(RestConfig restConfig) { + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + if (restConfig.containsKey(POOLING_MAX_TOTAL)) { + cm.setMaxTotal(restConfig.getPoolingMaxTotal()); + } + if (restConfig.containsKey(POOLING_DEFAULT_MAX_PER_RUOTE)) { + cm.setDefaultMaxPerRoute(restConfig.getPoolingDefaultMaxPerRoute()); + } + return cm; } - return proxy; - } - - /** - * Builds the RequestConfig object by setting HttpClient settings defined in the rest config. - * @param restConfig - * @param proxy - * @return - */ - protected static RequestConfig getRequestConfig(RestConfig restConfig, Optional proxy) { - RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); - if (restConfig.getConnectTimeout() != null) { - requestConfigBuilder.setConnectTimeout(restConfig.getConnectTimeout()); + + @SuppressWarnings("unchecked") + private static Map getGlobalConfig(Context context) { + Optional globalCapability = context.getCapability(GLOBAL_CONFIG, false); + return globalCapability.map(o -> (Map) o).orElseGet(HashMap::new); } - if (restConfig.getConnectionRequestTimeout() != null) { - requestConfigBuilder.setConnectionRequestTimeout(restConfig.getConnectionRequestTimeout()); + + /** + * Build the RestConfig by applying settings in order of precedence (last item in the input list has highest priority). + * Only settings specified in the rest config will override lower priority config settings. + * + */ + @SuppressWarnings("unchecked") + protected static RestConfig buildRestConfig(Map... configs) { + RestConfig restConfig = new RestConfig(); + + // Add settings in order of precedence + for (Map config : configs) { + if (config != null) { + restConfig.putAll(config); + } + } + return restConfig; } - if (restConfig.getSocketTimeout() != null) { - requestConfigBuilder.setSocketTimeout(restConfig.getSocketTimeout()); + + /** + * Builds a URI from the supplied URI string and adds query parameters. + * + */ + private static URI getURI(String uriString, Map queryParameters) throws URISyntaxException { + URIBuilder uriBuilder = new URIBuilder(uriString); + if (queryParameters != null) { + for (Map.Entry entry : queryParameters.entrySet()) { + uriBuilder.setParameter(entry.getKey(), (String) entry.getValue()); + } + } + return uriBuilder.build(); } - proxy.ifPresent(requestConfigBuilder::setProxy); - return requestConfigBuilder.build(); - } - - /** - * Builds the HttpClientContext object by setting the basic auth and/or proxy basic auth credentials when the - * necessary rest config settings are configured. Passwords are stored in HDFS. - * @param restConfig - * @param target - * @param proxy - * @return - * @throws IOException - */ - protected static HttpClientContext getHttpClientContext(RestConfig restConfig, HttpHost target, Optional proxy) throws IOException { - HttpClientContext httpClientContext = HttpClientContext.create(); - boolean credentialsAdded = false; - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - - // Add the basic auth credentials if the rest config settings are present - if (restConfig.getBasicAuthUser() != null && restConfig.getBasicAuthPasswordPath() != null) { - String password = new String(readBytes(new Path(restConfig.getBasicAuthPasswordPath())), StandardCharsets.UTF_8); - credentialsProvider.setCredentials( - new AuthScope(target), - new UsernamePasswordCredentials(restConfig.getBasicAuthUser(), password)); - credentialsAdded = true; + /** + * Returns the proxy HttpHost object if the proxy rest config settings are set. + * + */ + protected static Optional getProxy(RestConfig restConfig) { + Optional proxy = Optional.empty(); + if (restConfig.getProxyHost() != null && restConfig.getProxyPort() != null) { + proxy = Optional.of(new HttpHost(restConfig.getProxyHost(), restConfig.getProxyPort(), "http")); + } + return proxy; } - // Add the proxy basic auth credentials if the rest config settings are present - if (proxy.isPresent() && restConfig.getProxyBasicAuthUser() != null && - restConfig.getProxyBasicAuthPasswordPath() != null) { - String password = new String(readBytes(new Path(restConfig.getProxyBasicAuthPasswordPath())), StandardCharsets.UTF_8); - credentialsProvider.setCredentials( - new AuthScope(proxy.get()), - new UsernamePasswordCredentials(restConfig.getProxyBasicAuthUser(), password)); - credentialsAdded = true; + /** + * Builds the RequestConfig object by setting HttpClient settings defined in the rest config. + * + */ + protected static RequestConfig getRequestConfig(RestConfig restConfig, Optional proxy) { + RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); + if (restConfig.getConnectTimeout() != null) { + requestConfigBuilder.setConnectTimeout(restConfig.getConnectTimeout()); + } + if (restConfig.getConnectionRequestTimeout() != null) { + requestConfigBuilder.setConnectionRequestTimeout(restConfig.getConnectionRequestTimeout()); + } + if (restConfig.getSocketTimeout() != null) { + requestConfigBuilder.setSocketTimeout(restConfig.getSocketTimeout()); + } + + proxy.ifPresent(requestConfigBuilder::setProxy); + return requestConfigBuilder.build(); } - if (credentialsAdded) { - httpClientContext.setCredentialsProvider(credentialsProvider); + + /** + * Builds the HttpClientContext object by setting the basic auth and/or proxy basic auth credentials when the + * necessary rest config settings are configured. Passwords are stored in HDFS. + * + */ + protected static HttpClientContext getHttpClientContext(RestConfig restConfig, HttpHost target, + Optional proxy) throws IOException { + HttpClientContext httpClientContext = HttpClientContext.create(); + boolean credentialsAdded = false; + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + + // Add the basic auth credentials if the rest config settings are present + if (restConfig.getBasicAuthUser() != null && restConfig.getBasicAuthPasswordPath() != null) { + String password = + new String(readBytes(new Path(restConfig.getBasicAuthPasswordPath())), StandardCharsets.UTF_8); + credentialsProvider.setCredentials( + new AuthScope(target), + new UsernamePasswordCredentials(restConfig.getBasicAuthUser(), password)); + credentialsAdded = true; + } + + // Add the proxy basic auth credentials if the rest config settings are present + if (proxy.isPresent() && restConfig.getProxyBasicAuthUser() != null + && restConfig.getProxyBasicAuthPasswordPath() != null) { + String password = + new String(readBytes(new Path(restConfig.getProxyBasicAuthPasswordPath())), StandardCharsets.UTF_8); + credentialsProvider.setCredentials( + new AuthScope(proxy.get()), + new UsernamePasswordCredentials(restConfig.getProxyBasicAuthUser(), password)); + credentialsAdded = true; + } + if (credentialsAdded) { + httpClientContext.setCredentialsProvider(credentialsProvider); + } + return httpClientContext; } - return httpClientContext; - } - - /** - * Read bytes from a HDFS path. - * @param inPath - * @return - * @throws IOException - */ - private static byte[] readBytes(Path inPath) throws IOException { - FileSystem fs = FileSystem.get(inPath.toUri(), new Configuration()); - try (FSDataInputStream inputStream = fs.open(inPath)) { - return IOUtils.toByteArray(inputStream); + + /** + * Read bytes from a HDFS path. + * + */ + private static byte[] readBytes(Path inPath) throws IOException { + FileSystem fs = FileSystem.get(inPath.toUri(), new Configuration()); + try (FSDataInputStream inputStream = fs.open(inPath)) { + return IOUtils.toByteArray(inputStream); + } } - } - - /** - * Perform the HttpClient request and handle the results. A configurable list of status codes are accepted and the - * response content (expected to be json) is parsed into a Map. Values returned on errors and when response content - * is also configurable. The rest config "timeout" setting is imposed in this method and will abort the get request - * if exceeded. - * - * @param restConfig - * @param httpRequestBase - * @return - * @throws IOException - */ - protected static Object executeRequest(RestConfig restConfig, HttpRequestBase httpRequestBase) throws IOException { - URI uri = httpRequestBase.getURI(); - HttpHost target = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme()); - Optional proxy = getProxy(restConfig); - HttpClientContext httpClientContext = getHttpClientContext(restConfig, target, proxy); - httpRequestBase.setConfig(getRequestConfig(restConfig, proxy)); - - // Schedule a command to abort the request if the timeout is exceeded - ScheduledFuture scheduledFuture = scheduledExecutorService.schedule(httpRequestBase::abort, restConfig.getTimeout(), TimeUnit.MILLISECONDS); - CloseableHttpResponse response; - try { - response = closeableHttpClient.execute(httpRequestBase, httpClientContext); - } catch(Exception e) { - // Report a timeout if the httpGet request was aborted. Otherwise rethrow exception. - if (httpRequestBase.isAborted()) { - throw new IOException(String.format("Total Stellar REST request time to %s exceeded the configured timeout of %d ms.", httpRequestBase.getURI().toString(), restConfig.getTimeout())); - } else { - throw e; - } + + /** + * Perform the HttpClient request and handle the results. A configurable list of status codes are accepted and the + * response content (expected to be json) is parsed into a Map. Values returned on errors and when response content + * is also configurable. The rest config "timeout" setting is imposed in this method and will abort the get request + * if exceeded. + * + */ + protected static Object executeRequest(RestConfig restConfig, HttpRequestBase httpRequestBase) throws IOException { + URI uri = httpRequestBase.getURI(); + HttpHost target = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme()); + Optional proxy = getProxy(restConfig); + HttpClientContext httpClientContext = getHttpClientContext(restConfig, target, proxy); + httpRequestBase.setConfig(getRequestConfig(restConfig, proxy)); + + // Schedule a command to abort the request if the timeout is exceeded + ScheduledFuture scheduledFuture = + scheduledExecutorService.schedule(httpRequestBase::abort, restConfig.getTimeout(), TimeUnit.MILLISECONDS); + CloseableHttpResponse response; + try { + response = closeableHttpClient.execute(httpRequestBase, httpClientContext); + } catch (Exception e) { + // Report a timeout if the httpGet request was aborted. Otherwise rethrow exception. + if (httpRequestBase.isAborted()) { + throw new IOException( + String.format("Total Stellar REST request time to %s exceeded the configured timeout of %d ms.", + httpRequestBase.getURI().toString(), restConfig.getTimeout())); + } else { + throw e; + } + } + + // Cancel the future if the request finished within the timeout + if (!scheduledFuture.isDone()) { + scheduledFuture.cancel(true); + } + int statusCode = response.getStatusLine().getStatusCode(); + LOG.debug("request = {}; response = {}", httpRequestBase, response); + if (restConfig.getResponseCodesAllowed().contains(statusCode)) { + HttpEntity httpEntity = response.getEntity(); + + // Parse the response if present, return the empty value override if not + Optional parsedResponse = parseResponse(restConfig, httpRequestBase, httpEntity); + return parsedResponse.orElseGet(restConfig::getEmptyContentOverride); + } else { + throw new IOException(String.format("Stellar REST request to %s expected status code to be one of %s but " + + "failed with http status code %d: %s", + httpRequestBase.getURI().toString(), + restConfig.getResponseCodesAllowed().toString(), + statusCode, + EntityUtils.toString(response.getEntity()))); + } } - // Cancel the future if the request finished within the timeout - if (!scheduledFuture.isDone()) { - scheduledFuture.cancel(true); + /** + * Parses the Http response into a Map and checks for content length. + * + */ + protected static Optional parseResponse(RestConfig restConfig, HttpUriRequest httpUriRequest, + HttpEntity httpEntity) throws IOException { + Optional parsedResponse = Optional.empty(); + if (httpEntity != null) { + int actualContentLength = 0; + String json = EntityUtils.toString(httpEntity); + if (json != null && !json.isEmpty()) { + actualContentLength = json.length(); + parsedResponse = Optional.of(JSONUtils.INSTANCE.load(json, JSONUtils.MAP_SUPPLIER)); + } + if (restConfig.verifyContentLength() && actualContentLength != httpEntity.getContentLength()) { + throw new IOException( + String.format("Stellar REST request to %s returned incorrect or missing content length. " + + "Content length in the response was %d but the actual body content length was %d.", + httpUriRequest.getURI().toString(), + httpEntity.getContentLength(), + actualContentLength)); + } + } + return parsedResponse; } - int statusCode = response.getStatusLine().getStatusCode(); - LOG.debug("request = {}; response = {}", httpRequestBase, response); - if (restConfig.getResponseCodesAllowed().contains(statusCode)) { - HttpEntity httpEntity = response.getEntity(); - - // Parse the response if present, return the empty value override if not - Optional parsedResponse = parseResponse(restConfig, httpRequestBase, httpEntity); - return parsedResponse.orElseGet(restConfig::getEmptyContentOverride); - } else { - throw new IOException(String.format("Stellar REST request to %s expected status code to be one of %s but " + - "failed with http status code %d: %s", - httpRequestBase.getURI().toString(), - restConfig.getResponseCodesAllowed().toString(), - statusCode, - EntityUtils.toString(response.getEntity()))); + + /** + * Only used for testing. + * + */ + protected static void setCloseableHttpClient(CloseableHttpClient httpClient) { + closeableHttpClient = httpClient; } - } - - /** - * Parses the Http response into a Map and checks for content length. - * @param restConfig - * @param httpUriRequest - * @param httpEntity - * @return - * @throws IOException - */ - protected static Optional parseResponse(RestConfig restConfig, HttpUriRequest httpUriRequest, HttpEntity httpEntity) throws IOException { - Optional parsedResponse = Optional.empty(); - if (httpEntity != null) { - int actualContentLength = 0; - String json = EntityUtils.toString(httpEntity); - if (json != null && !json.isEmpty()) { - actualContentLength = json.length(); - parsedResponse = Optional.of(JSONUtils.INSTANCE.load(json, JSONUtils.MAP_SUPPLIER)); - } - if (restConfig.verifyContentLength() && actualContentLength != httpEntity.getContentLength()) { - throw new IOException(String.format("Stellar REST request to %s returned incorrect or missing content length. " + - "Content length in the response was %d but the actual body content length was %d.", - httpUriRequest.getURI().toString(), - httpEntity.getContentLength(), - actualContentLength)); - } + + /** + * Only used for testing. + * + */ + protected static void setScheduledExecutorService(ScheduledExecutorService executorService) { + scheduledExecutorService = executorService; } - return parsedResponse; - } - - /** - * Only used for testing. - * @param httpClient - */ - protected static void setCloseableHttpClient(CloseableHttpClient httpClient) { - closeableHttpClient = httpClient; - } - - /** - * Only used for testing. - * @param executorService - */ - protected static void setScheduledExecutorService(ScheduledExecutorService executorService) { - scheduledExecutorService = executorService; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java index 87a3e3dc..e1a912c7 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java @@ -7,281 +7,285 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions; import com.google.common.collect.Iterables; -import org.apache.metron.stellar.dsl.BaseStellarFunction; -import org.apache.metron.stellar.dsl.Stellar; - import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; public class SetFunctions { - @Stellar(name="INIT" - , namespace="SET" - , description="Creates a new set" - , params = { "input (optional) - An initialization of the set"} - , returns = "A Set" - ) - public static class SetInit extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - LinkedHashSet ret = new LinkedHashSet<>(); - if (list.size() == 1) { - Object o = list.get(0); - if (o != null) { - if (o instanceof Iterable) { - Iterables.addAll(ret, (Iterable) o); - } else { - throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass()); - } - } + @Stellar(name = "INIT", + namespace = "SET", + description = "Creates a new set", + params = {"input (optional) - An initialization of the set"}, + returns = "A Set" + ) + public static class SetInit extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + LinkedHashSet ret = new LinkedHashSet<>(); + if (list.size() == 1) { + Object o = list.get(0); + if (o != null) { + if (o instanceof Iterable) { + Iterables.addAll(ret, (Iterable) o); + } else { + throw new IllegalArgumentException( + "Expected an Iterable, but " + o + " is of type " + o.getClass()); + } + } - } - return ret; + } + return ret; + } } - } - @Stellar(name = "ADD", - namespace = "SET", - description = "Adds to a set", - params = {"set - The set to add to", - "o - object to add to set" - }, - returns = "A Set" - ) - public static class SetAdd extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() < 1) { - return null; - } - LinkedHashSet ret = (LinkedHashSet)list.get(0); - if (ret == null) { - ret = new LinkedHashSet<>(); - } - for (int i = 1;i < list.size();++i) { - Object o = list.get(i); - if (o != null) { - ret.add(o); + @Stellar(name = "ADD", + namespace = "SET", + description = "Adds to a set", + params = {"set - The set to add to", + "o - object to add to set" + }, + returns = "A Set" + ) + public static class SetAdd extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() < 1) { + return null; + } + LinkedHashSet ret = (LinkedHashSet) list.get(0); + if (ret == null) { + ret = new LinkedHashSet<>(); + } + for (int i = 1; i < list.size(); ++i) { + Object o = list.get(i); + if (o != null) { + ret.add(o); + } + } + return ret; } - } - return ret; } - } - @Stellar(name="REMOVE" - , namespace="SET" - , description="Removes from a set" - , params = {"set - The set to add to" - ,"o - object to add to set" - } - , returns = "A Set" - ) - public static class SetRemove extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() < 1) { - return null; - } - LinkedHashSet ret = (LinkedHashSet)list.get(0); - if (ret == null) { - ret = new LinkedHashSet<>(); - } - for (int i = 1;i < list.size();++i) { - Object o = list.get(i); - if (o != null) { - ret.remove(o); + @Stellar(name = "REMOVE", + namespace = "SET", + description = "Removes from a set", + params = {"set - The set to add to", + "o - object to add to set" + }, + returns = "A Set" + ) + public static class SetRemove extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() < 1) { + return null; + } + LinkedHashSet ret = (LinkedHashSet) list.get(0); + if (ret == null) { + ret = new LinkedHashSet<>(); + } + for (int i = 1; i < list.size(); ++i) { + Object o = list.get(i); + if (o != null) { + ret.remove(o); + } + } + return ret; } - } - return ret; } - } - @Stellar(name = "MERGE", - namespace = "SET", - description = "Merges a list of sets", - params = {"sets - A collection of sets to merge"}, - returns = "A Set" - ) - public static class SetMerge extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() < 1) { - return null; - } - LinkedHashSet ret = new LinkedHashSet<>(); - Object o = list.get(0); - if (o != null) { - if (!(o instanceof Iterable)) { - throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass()); - } - Iterable sets = (Iterable) o; + @Stellar(name = "MERGE", + namespace = "SET", + description = "Merges a list of sets", + params = {"sets - A collection of sets to merge"}, + returns = "A Set" + ) + public static class SetMerge extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() < 1) { + return null; + } + LinkedHashSet ret = new LinkedHashSet<>(); + Object o = list.get(0); + if (o != null) { + if (!(o instanceof Iterable)) { + throw new IllegalArgumentException( + "Expected an Iterable, but " + o + " is of type " + o.getClass()); + } + Iterable sets = (Iterable) o; - for (Iterable s : sets) { - if (s != null) { - Iterables.addAll(ret, s); - } + for (Iterable s : sets) { + if (s != null) { + Iterables.addAll(ret, s); + } + } + } + return ret; } - } - return ret; } - } - @Stellar(name = "INIT", - namespace = "MULTISET", - description = "Creates an empty multiset, which is a map associating objects to their instance counts.", - params = { "input (optional) - An initialization of the multiset"}, - returns = "A multiset" - ) - public static class MultiSetInit extends BaseStellarFunction { - @Override - public Object apply(List list) { - LinkedHashMap ret = new LinkedHashMap<>(); - if (list.size() >= 1) { - Object o = list.get(0); - if (o != null) { - if (!(o instanceof Iterable)) { - throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass()); - } - for (Object obj : (Iterable) o) { - ret.merge(obj, 1, (k, one) -> k + one); - } + @Stellar(name = "INIT", + namespace = "MULTISET", + description = "Creates an empty multiset, which is a map associating objects to their instance counts.", + params = {"input (optional) - An initialization of the multiset"}, + returns = "A multiset" + ) + public static class MultiSetInit extends BaseStellarFunction { + @Override + public Object apply(List list) { + LinkedHashMap ret = new LinkedHashMap<>(); + if (list.size() >= 1) { + Object o = list.get(0); + if (o != null) { + if (!(o instanceof Iterable)) { + throw new IllegalArgumentException( + "Expected an Iterable, but " + o + " is of type " + o.getClass()); + } + for (Object obj : (Iterable) o) { + ret.merge(obj, 1, (k, one) -> k + one); + } + } + } + return ret; } - } - return ret; } - } - @Stellar(name="ADD" - , namespace="MULTISET" - , description="Adds to a multiset, which is a map associating objects to their instance counts." - , params = {"set - The multiset to add to" - ,"o - object to add to multiset" - } - , returns = "A multiset" - ) - public static class MultiSetAdd extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() < 1) { - return null; - } - LinkedHashMap ret = (LinkedHashMap)list.get(0); - if (ret == null) { - ret = new LinkedHashMap<>(); - } - for (int i = 1;i < list.size();++i) { - Object o = list.get(i); - if (o != null) { - ret.merge(o, 1, (k, one) -> k + one); + @Stellar(name = "ADD", + namespace = "MULTISET", + description = "Adds to a multiset, which is a map associating objects to their instance counts.", + params = {"set - The multiset to add to", + "o - object to add to multiset" + }, + returns = "A multiset" + ) + public static class MultiSetAdd extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() < 1) { + return null; + } + LinkedHashMap ret = (LinkedHashMap) list.get(0); + if (ret == null) { + ret = new LinkedHashMap<>(); + } + for (int i = 1; i < list.size(); ++i) { + Object o = list.get(i); + if (o != null) { + ret.merge(o, 1, (k, one) -> k + one); + } + } + return ret; } - } - return ret; } - } - @Stellar(name = "REMOVE", - namespace = "MULTISET", - description = "Removes from a multiset, which is a map associating objects to their instance counts.", - params = {"set - The multiset to add to", - "o - object to remove from multiset" - }, - returns = "A multiset" - ) - public static class MultiSetRemove extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if(list.size() < 1) { - return null; - } - LinkedHashMap ret = (LinkedHashMap)list.get(0); - if (ret == null) { - ret = new LinkedHashMap<>(); - } - for (int i = 1;i < list.size();++i) { - Object o = list.get(i); - if (o != null) { - Integer cnt = ret.get(o); - if (cnt == null) { - continue; - } - if (cnt == 1) { - ret.remove(o); - } else { - ret.put(o, cnt - 1); - } + @Stellar(name = "REMOVE", + namespace = "MULTISET", + description = "Removes from a multiset, which is a map associating objects to their instance counts.", + params = {"set - The multiset to add to", + "o - object to remove from multiset" + }, + returns = "A multiset" + ) + public static class MultiSetRemove extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() < 1) { + return null; + } + LinkedHashMap ret = (LinkedHashMap) list.get(0); + if (ret == null) { + ret = new LinkedHashMap<>(); + } + for (int i = 1; i < list.size(); ++i) { + Object o = list.get(i); + if (o != null) { + Integer cnt = ret.get(o); + if (cnt == null) { + continue; + } + if (cnt == 1) { + ret.remove(o); + } else { + ret.put(o, cnt - 1); + } + } + } + return ret; } - } - return ret; } - } - @Stellar(name = "MERGE", - namespace = "MULTISET", - description = "Merges a list of multisets, which is a map associating objects to their instance counts.", - params = {"sets - A collection of multisets to merge"}, - returns = "A multiset" - ) - public static class MultiSetMerge extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() < 1) { - return null; - } - LinkedHashMap ret = new LinkedHashMap<>(); - Iterable> maps = (Iterable>) list.get(0); - for (Map s : maps) { - if (s != null) { - for (Map.Entry kv : s.entrySet()) { - ret.merge(kv.getKey(), kv.getValue(), (k, cnt) -> k + cnt); - } + @Stellar(name = "MERGE", + namespace = "MULTISET", + description = "Merges a list of multisets, which is a map associating objects to their instance counts.", + params = {"sets - A collection of multisets to merge"}, + returns = "A multiset" + ) + public static class MultiSetMerge extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() < 1) { + return null; + } + LinkedHashMap ret = new LinkedHashMap<>(); + Iterable> maps = (Iterable>) list.get(0); + for (Map s : maps) { + if (s != null) { + for (Map.Entry kv : s.entrySet()) { + ret.merge(kv.getKey(), kv.getValue(), (k, cnt) -> k + cnt); + } + } + } + return ret; } - } - return ret; } - } - @Stellar(name="TO_SET" - , namespace="MULTISET" - , description="Create a set out of a multiset, which is a map associating objects to their instance counts." - , params = {"multiset - The multiset to convert." - } - , returns = "The set of objects in the multiset ignoring multiplicity" - ) - public static class MultiSetToSet extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List list) { - if (list.size() < 1) { - return null; - } - LinkedHashSet ret = new LinkedHashSet<>(); - if (list.size() == 1) { - Map multiset = (Map)list.get(0); - if (multiset != null) { - ret.addAll(multiset.keySet()); + @Stellar(name = "TO_SET", + namespace = "MULTISET", + description = "Create a set out of a multiset, which is a map associating objects to their instance counts.", + params = {"multiset - The multiset to convert."}, + returns = "The set of objects in the multiset ignoring multiplicity" + ) + public static class MultiSetToSet extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List list) { + if (list.size() < 1) { + return null; + } + LinkedHashSet ret = new LinkedHashSet<>(); + if (list.size() == 1) { + Map multiset = (Map) list.get(0); + if (multiset != null) { + ret.addAll(multiset.keySet()); + } + } + return ret; } - } - return ret; } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java index 5eddfa87..3ee91882 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions; import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE; @@ -22,7 +25,6 @@ import com.jakewharton.fliptables.FlipTable; import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; import java.lang.invoke.MethodHandles; @@ -47,256 +49,258 @@ import org.slf4j.LoggerFactory; public class ShellFunctions { - private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @SuppressWarnings("unchecked") - private static Map getVariables(Context context) { - return (Map) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get(); - } + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @Stellar( - namespace = "SHELL" - ,name = "MAP2TABLE" - ,description = "Take a map and return a table" - ,params = {"map - Map" - } - ,returns = "The map in table form" - ) - public static class Map2Table extends BaseStellarFunction { - - @Override @SuppressWarnings("unchecked") - public Object apply(List args) { - if (args.size() < 1) { - return null; - } - Map map = (Map) args.get(0); - if (map == null) { - map = new HashMap<>(); - } - String[] headers = {"KEY", "VALUE"}; - String[][] data = new String[map.size()][2]; - int i = 0; - for (Map.Entry kv : map.entrySet()) { - data[i++] = new String[]{kv.getKey().toString(), kv.getValue().toString()}; - } - return FlipTable.of(headers, data); + private static Map getVariables(Context context) { + return (Map) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get(); } - } - - @Stellar( - namespace = "SHELL" - ,name = "LIST_VARS" - ,description = "Return the variables in a tabular form" - ,params = { - "wrap : Length of string to wrap the columns" - } - ,returns = "A tabular representation of the variables." - ) - public static class ListVars implements StellarFunction { - @Override - public Object apply(List args, Context context) throws ParseException { - Map variables = getVariables(context); - String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"}; - String[][] data = new String[variables.size()][3]; - int wordWrap = -1; - if (args.size() > 0) { - wordWrap = ConversionUtils.convert(args.get(0), Integer.class); - } - int i = 0; - for (Map.Entry kv : variables.entrySet()) { - VariableResult result = kv.getValue(); - data[i++] = new String[]{toWrappedString(kv.getKey(), wordWrap), - toWrappedString(result.getResult(), wordWrap), - toWrappedString(result.getExpression().get(), wordWrap)}; - } - return FlipTable.of(headers, data); - } + @Stellar( + namespace = "SHELL", + name = "MAP2TABLE", + description = "Take a map and return a table", + params = { + "map - Map" + }, + returns = "The map in table form" + ) + public static class Map2Table extends BaseStellarFunction { - private static String toWrappedString(Object o, int wrap) { - String s = "" + o; - if(wrap <= 0) { - return s; - } - return WordUtils.wrap(s, wrap); + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + if (args.size() < 1) { + return null; + } + Map map = (Map) args.get(0); + if (map == null) { + map = new HashMap<>(); + } + String[] headers = {"KEY", "VALUE"}; + String[][] data = new String[map.size()][2]; + int i = 0; + for (Map.Entry kv : map.entrySet()) { + data[i++] = new String[] {kv.getKey().toString(), kv.getValue().toString()}; + } + return FlipTable.of(headers, data); + } } - @Override - public void initialize(Context context) { + @Stellar( + namespace = "SHELL", + name = "LIST_VARS", + description = "Return the variables in a tabular form", + params = { + "wrap : Length of string to wrap the columns" + }, + returns = "A tabular representation of the variables." + ) + public static class ListVars implements StellarFunction { - } + @Override + public Object apply(List args, Context context) throws ParseException { + Map variables = getVariables(context); + String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"}; + String[][] data = new String[variables.size()][3]; + int wordWrap = -1; + if (args.size() > 0) { + wordWrap = ConversionUtils.convert(args.get(0), Integer.class); + } + int i = 0; + for (Map.Entry kv : variables.entrySet()) { + VariableResult result = kv.getValue(); + data[i++] = new String[] {toWrappedString(kv.getKey(), wordWrap), + toWrappedString(result.getResult(), wordWrap), + toWrappedString(result.getExpression().get(), wordWrap)}; + } + return FlipTable.of(headers, data); + } - @Override - public boolean isInitialized() { - return true; - } - } + private static String toWrappedString(Object o, int wrap) { + String s = "" + o; + if (wrap <= 0) { + return s; + } + return WordUtils.wrap(s, wrap); + } - @Stellar( - namespace = "SHELL" - ,name = "VARS2MAP" - ,description = "Take a set of variables and return a map" - ,params = {"variables* - variable names to use to create map " - } - ,returns = "A map associating the variable name with the stellar expression." - ) - public static class Var2Map implements StellarFunction { + @Override + public void initialize(Context context) { - @Override - public Object apply(List args, Context context) throws ParseException { - Map variables = getVariables(context); - LinkedHashMap ret = new LinkedHashMap<>(); - for (Object arg : args) { - if (arg == null) { - continue; } - String variable = (String) arg; - VariableResult result = variables.get(variable); - if (result != null && result.getExpression().isPresent()) { - ret.put(variable, result.getExpression().orElseGet(() -> "")); + + @Override + public boolean isInitialized() { + return true; } - } - return ret; } - @Override - public void initialize(Context context) { + @Stellar( + namespace = "SHELL", + name = "VARS2MAP", + description = "Take a set of variables and return a map", + params = {"variables* - variable names to use to create map " + }, + returns = "A map associating the variable name with the stellar expression." + ) + public static class Var2Map implements StellarFunction { - } + @Override + public Object apply(List args, Context context) throws ParseException { + Map variables = getVariables(context); + LinkedHashMap ret = new LinkedHashMap<>(); + for (Object arg : args) { + if (arg == null) { + continue; + } + String variable = (String) arg; + VariableResult result = variables.get(variable); + if (result != null && result.getExpression().isPresent()) { + ret.put(variable, result.getExpression().orElseGet(() -> "")); + } + } + return ret; + } - @Override - public boolean isInitialized() { - return true; - } - } + @Override + public void initialize(Context context) { - @Stellar( - namespace = "SHELL" - ,name = "GET_EXPRESSION" - ,description = "Get a stellar expression from a variable" - ,params = {"variable - variable name" - } - ,returns = "The stellar expression associated with the variable." - ) - public static class GetExpression implements StellarFunction { + } - @Override - public Object apply(List args, Context context) throws ParseException { - Map variables = getVariables(context); - if (args.size() == 0) { - return null; - } - String variable = (String) args.get(0); - if (variable == null) { - return null; - } - VariableResult result = variables.get(variable); - if (result != null && result.getExpression().isPresent()) { - return result.getExpression().get(); - } - return null; + @Override + public boolean isInitialized() { + return true; + } } - @Override - public void initialize(Context context) { + @Stellar( + namespace = "SHELL", + name = "GET_EXPRESSION", + description = "Get a stellar expression from a variable", + params = { + "variable - variable name" + }, + returns = "The stellar expression associated with the variable." + ) + public static class GetExpression implements StellarFunction { - } + @Override + public Object apply(List args, Context context) throws ParseException { + Map variables = getVariables(context); + if (args.size() == 0) { + return null; + } + String variable = (String) args.get(0); + if (variable == null) { + return null; + } + VariableResult result = variables.get(variable); + if (result != null && result.getExpression().isPresent()) { + return result.getExpression().get(); + } + return null; + } - @Override - public boolean isInitialized() { - return true; - } - } + @Override + public void initialize(Context context) { - @Stellar( - namespace = "SHELL" - ,name = "EDIT" - ,description = "Open an editor (optionally initialized with text) and return " + - "whatever is saved from the editor. The editor to use is pulled " + - "from `EDITOR` or `VISUAL` environment variable." - ,params = { "string - (Optional) A string whose content is used to initialize the editor." - } - ,returns = "The content that the editor saved after editor exit." - ) - public static class Edit implements StellarFunction { + } - private String getEditor() { - // if we have editor in the system properties, it should - // override the env so we check that first - String editor = System.getProperty("EDITOR"); - if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { - editor = System.getenv().get("EDITOR"); - } - if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { - editor = System.getenv("VISUAL"); - } - if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { - editor = "/bin/vi"; - } - return editor; + @Override + public boolean isInitialized() { + return true; + } } - @Override - public Object apply(List args, Context context) throws ParseException { - File outFile = null; - String editor = getEditor(); - try { - outFile = File.createTempFile("stellar_shell", "out"); - if (args.size() > 0) { - String arg = (String) args.get(0); - try (PrintWriter pw = new PrintWriter(outFile,StandardCharsets.UTF_8.name())) { - IOUtils.write(arg, pw); - } - } - } catch (IOException e) { - String message = "Unable to create temp file: " + e.getMessage(); - LOG.error(message, e); - throw new IllegalStateException(message, e); - } - Optional console = context.getCapability(CONSOLE, false); - try { - PausableInput.INSTANCE.pause(); - //shut down the IO for the console - ProcessBuilder processBuilder = new ProcessBuilder(editor, outFile.getAbsolutePath()); - processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT); - processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); - processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); - try { - Process p = processBuilder.start(); - // wait for termination. - p.waitFor(); - try (BufferedReader br = Files.newBufferedReader(outFile.toPath(), StandardCharsets.UTF_8)) { - return IOUtils.toString(br).trim(); - } - } catch (Exception e) { - String message = "Unable to read output: " + e.getMessage(); - LOG.error(message, e); - return null; - } - } finally { - try { - PausableInput.INSTANCE.unpause(); - if (console.isPresent()) { - ((Console) console.get()).pushToInputStream("\b\n"); - } - } catch (IOException e) { - LOG.error("Unable to unpause: {}", e.getMessage(), e); + @Stellar( + namespace = "SHELL", + name = "EDIT", + description = "Open an editor (optionally initialized with text) and return " + + "whatever is saved from the editor. The editor to use is pulled " + + "from `EDITOR` or `VISUAL` environment variable.", + params = {"string - (Optional) A string whose content is used to initialize the editor." + }, + returns = "The content that the editor saved after editor exit." + ) + public static class Edit implements StellarFunction { + + private String getEditor() { + // if we have editor in the system properties, it should + // override the env so we check that first + String editor = System.getProperty("EDITOR"); + if (org.apache.commons.lang3.StringUtils.isEmpty(editor)) { + editor = System.getenv().get("EDITOR"); + } + if (org.apache.commons.lang3.StringUtils.isEmpty(editor)) { + editor = System.getenv("VISUAL"); + } + if (org.apache.commons.lang3.StringUtils.isEmpty(editor)) { + editor = "/bin/vi"; + } + return editor; } - if (outFile.exists()) { - outFile.delete(); + + @Override + public Object apply(List args, Context context) throws ParseException { + File outFile = null; + String editor = getEditor(); + try { + outFile = File.createTempFile("stellar_shell", "out"); + if (args.size() > 0) { + String arg = (String) args.get(0); + try (PrintWriter pw = new PrintWriter(outFile, StandardCharsets.UTF_8.name())) { + IOUtils.write(arg, pw); + } + } + } catch (IOException e) { + String message = "Unable to create temp file: " + e.getMessage(); + LOG.error(message, e); + throw new IllegalStateException(message, e); + } + Optional console = context.getCapability(CONSOLE, false); + try { + PausableInput.INSTANCE.pause(); + //shut down the IO for the console + ProcessBuilder processBuilder = new ProcessBuilder(editor, outFile.getAbsolutePath()); + processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT); + processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); + processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); + try { + Process p = processBuilder.start(); + // wait for termination. + p.waitFor(); + try (BufferedReader br = Files.newBufferedReader(outFile.toPath(), StandardCharsets.UTF_8)) { + return IOUtils.toString(br).trim(); + } + } catch (Exception e) { + String message = "Unable to read output: " + e.getMessage(); + LOG.error(message, e); + return null; + } + } finally { + try { + PausableInput.INSTANCE.unpause(); + if (console.isPresent()) { + ((Console) console.get()).pushToInputStream("\b\n"); + } + } catch (IOException e) { + LOG.error("Unable to unpause: {}", e.getMessage(), e); + } + if (outFile.exists()) { + outFile.delete(); + } + } } - } - } - @Override - public void initialize(Context context) { + @Override + public void initialize(Context context) { - } + } - @Override - public boolean isInitialized() { - return true; + @Override + public boolean isInitialized() { + return true; + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java index 89e5f61a..f826f3f3 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,593 +38,611 @@ public class StringFunctions { - @Stellar(name="ENDS_WITH" - ,description = "Determines whether a string ends with a specified suffix" - , params = { - "string - The string to test" - ,"suffix - The proposed suffix" + @Stellar(name = "ENDS_WITH", + description = "Determines whether a string ends with a specified suffix", + params = { + "string - The string to test", + "suffix - The proposed suffix" + }, + returns = "True if the string ends with the specified suffix and false if otherwise") + public static class EndsWith extends BaseStellarFunction { + @Override + public Object apply(List list) { + if (list.size() < 2) { + throw new IllegalStateException( + "ENDS_WITH expects two args: [string, suffix] where suffix is the string fragment that the string should end with"); } - , returns = "True if the string ends with the specified suffix and false if otherwise") - public static class EndsWith extends BaseStellarFunction { - @Override - public Object apply(List list) { - if(list.size() < 2) { - throw new IllegalStateException("ENDS_WITH expects two args: [string, suffix] where suffix is the string fragment that the string should end with"); - } - String prefix = (String) list.get(1); - String str = (String) list.get(0); - if(str == null || prefix == null) { - return false; - } - return str.endsWith(prefix); + String prefix = (String) list.get(1); + String str = (String) list.get(0); + if (str == null || prefix == null) { + return false; + } + return str.endsWith(prefix); + } } - } - @Stellar(name="STARTS_WITH" - ,description = "Determines whether a string starts with a prefix" - , params = { - "string - The string to test" - ,"prefix - The proposed prefix" + @Stellar(name = "STARTS_WITH", + description = "Determines whether a string starts with a prefix", + params = { + "string - The string to test", + "prefix - The proposed prefix" + }, + returns = "True if the string starts with the specified prefix and false if otherwise" + ) + public static class StartsWith extends BaseStellarFunction { + + @Override + public Object apply(List list) { + if (list.size() < 2) { + throw new IllegalStateException( + "STARTS_WITH expects two args: [string, prefix] where prefix is" + + "the string fragment that the string should start with"); } - , returns = "True if the string starts with the specified prefix and false if otherwise" - ) - public static class StartsWith extends BaseStellarFunction { - - @Override - public Object apply(List list) { - if(list.size() < 2) { - throw new IllegalStateException("STARTS_WITH expects two args: [string, prefix] where prefix is the string fragment that the string should start with"); - } - String prefix = (String) list.get(1); - String str = (String) list.get(0); - if(str == null || prefix == null) { - return false; - } - return str.startsWith(prefix); - } - } - - @Stellar( name="TO_LOWER" - , description = "Transforms the first argument to a lowercase string" - , params = { "input - String" } - , returns = "Lowercase string" - ) - public static class ToLower extends BaseStellarFunction { - @Override - public Object apply(List strings) { - return strings.get(0)==null?null:strings.get(0).toString().toLowerCase(); - } - } - - @Stellar( name="TO_UPPER" - , description = "Transforms the first argument to an uppercase string" - , params = { "input - String" } - , returns = "Uppercase string" - ) - public static class ToUpper extends BaseStellarFunction { - @Override - public Object apply(List strings) { - return strings.get(0)==null?null:strings.get(0).toString().toUpperCase(); - } - } - - @Stellar(name="TO_STRING" - , description = "Transforms the first argument to a string" - , params = { "input - Object" } - , returns = "String" - ) - public static class ToString extends BaseStellarFunction { - @Override - public Object apply(List strings) { - return strings.get(0)==null?null:strings.get(0).toString(); + String prefix = (String) list.get(1); + String str = (String) list.get(0); + if (str == null || prefix == null) { + return false; + } + return str.startsWith(prefix); + } } - } - - @Stellar(name="TRIM" - , description = "Trims whitespace from both sides of a string." - , params = { "input - String" } - , returns = "String" - ) - public static class Trim extends BaseStellarFunction { - @Override - public Object apply(List strings) { - return strings.get(0)==null?null:strings.get(0).toString().trim(); + + @Stellar(name = "TO_LOWER", + description = "Transforms the first argument to a lowercase string", + params = {"input - String"}, + returns = "Lowercase string" + ) + public static class ToLower extends BaseStellarFunction { + @Override + public Object apply(List strings) { + return strings.get(0) == null ? null : strings.get(0).toString().toLowerCase(); + } } - } - - @Stellar( name="JOIN" - , description="Joins the non-null items in the iterable as strings with the specified delimiter. Null items are dropped." - , params = { "iterable - Java iterable (e.g. List, LinkedHashSet, etc.) of items treated as strings", "delim - String delimiter"} - , returns = "String" - ) - public static class JoinFunction extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - Iterable arg1 = (Iterable) args.get(0); - String delim = (String) args.get(1); - return Joiner.on(delim).join(Iterables.filter(arg1, x -> x != null)); + + @Stellar(name = "TO_UPPER", + description = "Transforms the first argument to an uppercase string", + params = {"input - String"}, + returns = "Uppercase string" + ) + public static class ToUpper extends BaseStellarFunction { + @Override + public Object apply(List strings) { + return strings.get(0) == null ? null : strings.get(0).toString().toUpperCase(); + } } - } - - @Stellar(name="SPLIT" - , description="Splits the string by the delimiter." - , params = { "input - String to split", "delim - String delimiter"} - , returns = "List of strings" - ) - public static class SplitFunction extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - List ret = new ArrayList(); - Object o1 = args.get(0); - if (o1 != null) { - String arg1 = o1.toString(); - String delim = (String) args.get(1); - Iterables.addAll(ret, Splitter.on(delim).split(arg1)); - } - return ret; + + @Stellar(name = "TO_STRING", + description = "Transforms the first argument to a string", + params = {"input - Object"}, + returns = "String" + ) + public static class ToString extends BaseStellarFunction { + @Override + public Object apply(List strings) { + return strings.get(0) == null ? null : strings.get(0).toString(); + } } - } - - @Stellar(name = "GET_LAST", - description = "Returns the last element of the list", - params = { "input - List"}, - returns = "Last element of the list" - ) - public static class GetLast extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - List arg1 = (List) args.get(0); - return Iterables.getLast(arg1, null); + + @Stellar(name = "TRIM", + description = "Trims whitespace from both sides of a string.", + params = {"input - String"}, + returns = "String" + ) + public static class Trim extends BaseStellarFunction { + @Override + public Object apply(List strings) { + return strings.get(0) == null ? null : strings.get(0).toString().trim(); + } } - } - - @Stellar(name = "GET_FIRST", - description = "Returns the first element of the list", - params = { "input - List"}, - returns = "First element of the list" - ) - public static class GetFirst extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - List arg1 = (List) args.get(0); - return Iterables.getFirst(arg1, null); + + @Stellar(name = "JOIN", + description = "Joins the non-null items in the iterable as strings with the specified delimiter. Null items are dropped.", + params = {"iterable - Java iterable (e.g. List, LinkedHashSet, etc.) of items treated as strings", + "delim - String delimiter"}, + returns = "String" + ) + public static class JoinFunction extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + Iterable arg1 = (Iterable) args.get(0); + String delim = (String) args.get(1); + return Joiner.on(delim).join(Iterables.filter(arg1, x -> x != null)); + } } - } - - @Stellar(name = "GET", - description = "Returns the i'th element of the list ", - params = { "input - List", "i - The index (0-based)"}, - returns = "First element of the list" - ) - public static class Get extends BaseStellarFunction { - @Override - @SuppressWarnings("unchecked") - public Object apply(List args) { - List arg1 = (List) args.get(0); - int offset = (Integer) args.get(1); - if (offset < arg1.size()) { - return Iterables.get(arg1, offset); - } - return null; + + @Stellar(name = "SPLIT", + description = "Splits the string by the delimiter.", + params = {"input - String to split", "delim - String delimiter"}, + returns = "List of strings" + ) + public static class SplitFunction extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + List ret = new ArrayList(); + Object o1 = args.get(0); + if (o1 != null) { + String arg1 = o1.toString(); + String delim = (String) args.get(1); + Iterables.addAll(ret, Splitter.on(delim).split(arg1)); + } + return ret; + } } - } - - private enum FillDirection{ - LEFT, - RIGHT - } - - @Stellar(name="FILL_LEFT" - , description="Fills or pads a given string with a given character, to a given length on the left" - , params = { "input - string", "fill - the fill character", "len - the required length"} - , returns = "Filled String" - ) - public static class FillLeft extends BaseStellarFunction { - @Override - public Object apply(List args) { - if(args.size() < 3) { - throw new IllegalStateException("FILL_LEFT expects three args: [string,char,length] where char is the fill character string and length is the required length of the result"); - } - return fill(FillDirection.LEFT,args.get(0),args.get(1),args.get(2)); + + @Stellar(name = "GET_LAST", + description = "Returns the last element of the list", + params = {"input - List"}, + returns = "Last element of the list" + ) + public static class GetLast extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + List arg1 = (List) args.get(0); + return Iterables.getLast(arg1, null); + } } - } - - @Stellar(name="FILL_RIGHT" - , description="Fills or pads a given string with a given character, to a given length on the right" - , params = { "input - string", "fill - the fill character", "len - the required length"} - , returns = "Filled String" - ) - public static class FillRight extends BaseStellarFunction { - @Override - public Object apply(List args) { - if(args.size() < 3) { - throw new IllegalStateException("FILL_RIGHT expects three args: [string,char,length] where char is the fill character string and length is the required length of the result"); - } - return fill(FillDirection.RIGHT,args.get(0),args.get(1),args.get(2)); + + @Stellar(name = "GET_FIRST", + description = "Returns the first element of the list", + params = {"input - List"}, + returns = "First element of the list" + ) + public static class GetFirst extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + List arg1 = (List) args.get(0); + return Iterables.getFirst(arg1, null); + } } - } - private static Object fill(FillDirection direction, Object inputObject, Object fillObject, Object requiredLengthObject)throws ParseException{ - if(inputObject == null) { - return null; + @Stellar(name = "GET", + description = "Returns the i'th element of the list ", + params = {"input - List", "i - The index (0-based)"}, + returns = "First element of the list" + ) + public static class Get extends BaseStellarFunction { + @Override + @SuppressWarnings("unchecked") + public Object apply(List args) { + List arg1 = (List) args.get(0); + int offset = (Integer) args.get(1); + if (offset < arg1.size()) { + return Iterables.get(arg1, offset); + } + return null; + } } - String input = inputObject.toString(); - if(requiredLengthObject == null || fillObject == null) { - throw new IllegalStateException("Required Length and Fill String are both required"); + private enum FillDirection { + LEFT, + RIGHT } - String fill = fillObject.toString(); - if(org.apache.commons.lang.StringUtils.isEmpty(fill)){ - throw new IllegalStateException("The fill cannot be an empty string"); + @Stellar(name = "FILL_LEFT", + description = "Fills or pads a given string with a given character, to a given length on the left", + params = {"input - string", "fill - the fill character", "len - the required length"}, + returns = "Filled String" + ) + public static class FillLeft extends BaseStellarFunction { + @Override + public Object apply(List args) { + if (args.size() < 3) { + throw new IllegalStateException( + "FILL_LEFT expects three args: [string,char,length] where char is the fill character string " + + "and length is the required length of the result"); + } + return fill(FillDirection.LEFT, args.get(0), args.get(1), args.get(2)); + } } - fill = fill.substring(0,1); - Integer requiredLength = ConversionUtils.convert(requiredLengthObject,Integer.class); - if(requiredLength == null){ - throw new IllegalStateException("Required Length not a valid Integer: " + requiredLengthObject.toString()); + + @Stellar(name = "FILL_RIGHT", + description = "Fills or pads a given string with a given character, to a given length on the right", + params = {"input - string", "fill - the fill character", "len - the required length"}, + returns = "Filled String" + ) + public static class FillRight extends BaseStellarFunction { + @Override + public Object apply(List args) { + if (args.size() < 3) { + throw new IllegalStateException( + "FILL_RIGHT expects three args: [string,char,length] where char is the fill character string " + + "and length is the required length of the result"); + } + return fill(FillDirection.RIGHT, args.get(0), args.get(1), args.get(2)); + } } - if(direction == FillDirection.LEFT) { - return org.apache.commons.lang.StringUtils.leftPad(input,requiredLength,fill); + private static Object fill(FillDirection direction, Object inputObject, Object fillObject, + Object requiredLengthObject) throws ParseException { + if (inputObject == null) { + return null; + } + String input = inputObject.toString(); + + if (requiredLengthObject == null || fillObject == null) { + throw new IllegalStateException("Required Length and Fill String are both required"); + } + + String fill = fillObject.toString(); + if (org.apache.commons.lang.StringUtils.isEmpty(fill)) { + throw new IllegalStateException("The fill cannot be an empty string"); + } + fill = fill.substring(0, 1); + Integer requiredLength = ConversionUtils.convert(requiredLengthObject, Integer.class); + if (requiredLength == null) { + throw new IllegalStateException("Required Length not a valid Integer: " + requiredLengthObject); + } + + if (direction == FillDirection.LEFT) { + return org.apache.commons.lang.StringUtils.leftPad(input, requiredLength, fill); + } + return org.apache.commons.lang.StringUtils.rightPad(input, requiredLength, fill); } - return org.apache.commons.lang.StringUtils.rightPad(input,requiredLength,fill); - } - - @Stellar( namespace="STRING" - , name="ENTROPY" - , description = "Computes the base-2 shannon entropy of a string" - , params = { "input - String" } - , returns = "The base-2 shannon entropy of the string (https://en.wikipedia.org/wiki/Entropy_(information_theory)#Definition). The unit of this is bits." - ) - public static class Entropy extends BaseStellarFunction { - @Override - public Object apply(List strings) { - /* - Shannon entropy is defined as follows: - \Eta(X) = - \sum(p(x_i)*log_2(p(x_i)), i=0, n-1) where x_i are distinct characters in the string. - */ - Map frequency = new HashMap<>(); - if(strings.size() != 1) { - throw new IllegalArgumentException("STRING_ENTROPY expects exactly one argument which is a string."); - } - String input = ConversionUtils.convert(strings.get(0), String.class); - if(StringUtils.isEmpty(input)) { - return 0.0; - } - for(int i = 0;i < input.length();++i) { - char c = input.charAt(i); - frequency.put(c, frequency.getOrDefault(c, 0) + 1); - } - double ret = 0.0; - double log2 = Math.log(2); - for(Integer f : frequency.values()) { - double p = f.doubleValue()/input.length(); - ret -= p * Math.log(p) / log2; - } - return ret; + + @Stellar(namespace = "STRING", + name = "ENTROPY", + description = "Computes the base-2 shannon entropy of a string", + params = {"input - String"}, + returns = + "The base-2 shannon entropy of the string (https://en.wikipedia.org/wiki/Entropy_(information_theory)#Definition)." + + "The unit of this is bits." + ) + public static class Entropy extends BaseStellarFunction { + @Override + public Object apply(List strings) { + /* + Shannon entropy is defined as follows: + \Eta(X) = - \sum(p(x_i)*log_2(p(x_i)), i=0, n-1) where x_i are distinct characters in the string. + */ + Map frequency = new HashMap<>(); + if (strings.size() != 1) { + throw new IllegalArgumentException("STRING_ENTROPY expects exactly one argument which is a string."); + } + String input = ConversionUtils.convert(strings.get(0), String.class); + if (StringUtils.isEmpty(input)) { + return 0.0; + } + for (int i = 0; i < input.length(); ++i) { + char c = input.charAt(i); + frequency.put(c, frequency.getOrDefault(c, 0) + 1); + } + double ret = 0.0; + double log2 = Math.log(2); + for (Integer f : frequency.values()) { + double p = f.doubleValue() / input.length(); + ret -= p * Math.log(p) / log2; + } + return ret; + } } - } - @Stellar( name="FORMAT" - , description = "Returns a formatted string using the specified format string and arguments. Uses Java's string formatting conventions." - , params = { "format - string", "arguments... - object(s)" } - , returns = "A formatted string." - ) - public static class Format extends BaseStellarFunction { + @Stellar(name = "FORMAT", + description = "Returns a formatted string using the specified format string and arguments." + + "Uses Java's string formatting conventions.", + params = {"format - string", "arguments... - object(s)"}, + returns = "A formatted string." + ) + public static class Format extends BaseStellarFunction { - @Override - public Object apply(List args) { + @Override + public Object apply(List args) { - if(args.size() == 0) { - throw new IllegalArgumentException("[FORMAT] missing argument: format string"); - } + if (args.size() == 0) { + throw new IllegalArgumentException("[FORMAT] missing argument: format string"); + } - String format = ConversionUtils.convert(args.get(0), String.class); - Object[] formatArgs = args.subList(1, args.size()).toArray(); + String format = ConversionUtils.convert(args.get(0), String.class); + Object[] formatArgs = args.subList(1, args.size()).toArray(); - return String.format(format, formatArgs); + return String.format(format, formatArgs); + } } - } - @Stellar( name="SUBSTRING" - , description = "Returns a substring of a string" - , params = { + @Stellar(name = "SUBSTRING", + description = "Returns a substring of a string", + params = { "input - The string to take the substring of", "start - The starting position (0-based and inclusive)", "end? - The ending position (0-based and exclusive)" - } - , returns = "The substring of the input" - ) - public static class Substring extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - if(strings == null || strings.size() < 2 ) { - throw new IllegalArgumentException("SUBSTRING requires (at least) 2 arguments: the input and the start position (inclusive)"); - } - Object varObj = strings.get(0); - if(varObj != null && !(varObj instanceof String)) { - throw new IllegalArgumentException("SUBSTRING input must be a String"); - } - String var = varObj == null?null: (String) varObj; - Object startObj = strings.get(1); - if(startObj != null && !(startObj instanceof Number)) { - throw new IllegalArgumentException("SUBSTRING start must be an Number"); - } - Integer start = startObj == null?null:((Number)startObj).intValue(); - Integer end = null; - if(strings.size() > 2) { - Object endObj = strings.get(2); - if(endObj != null && !(endObj instanceof Number)) { - throw new IllegalArgumentException("SUBSTRING end must be an Number"); - } - end = endObj == null ? null : ((Number) endObj).intValue(); - } - if(var == null || start == null) { - return null; - } - else if(var.length() == 0) { - return var; - } - else { - if(end == null) { - return var.substring(start); - } - else { - return var.substring(start, end); + }, + returns = "The substring of the input" + ) + public static class Substring extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + if (strings == null || strings.size() < 2) { + throw new IllegalArgumentException( + "SUBSTRING requires (at least) 2 arguments: the input and the start position (inclusive)"); + } + Object varObj = strings.get(0); + if (varObj != null && !(varObj instanceof String)) { + throw new IllegalArgumentException("SUBSTRING input must be a String"); + } + String var = varObj == null ? null : (String) varObj; + Object startObj = strings.get(1); + if (startObj != null && !(startObj instanceof Number)) { + throw new IllegalArgumentException("SUBSTRING start must be an Number"); + } + Integer start = startObj == null ? null : ((Number) startObj).intValue(); + Integer end = null; + if (strings.size() > 2) { + Object endObj = strings.get(2); + if (endObj != null && !(endObj instanceof Number)) { + throw new IllegalArgumentException("SUBSTRING end must be an Number"); + } + end = endObj == null ? null : ((Number) endObj).intValue(); + } + if (var == null || start == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (end == null) { + return var.substring(start); + } else { + return var.substring(start, end); + } + } } - } - } - } - - @Stellar( name="CHOMP" - , description = "Removes one newline from end of a String if it's there, otherwise leave it alone. A newline is \"\\n\", \"\\r\", or \"\\r\\n\"" - , params = { "the String to chomp a newline from, may be null"} - , returns = "String without newline, null if null String input" - ) - public static class Chomp extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - if(strings == null || strings.size() == 0 ) { - throw new IllegalArgumentException("[CHOMP] missing argument: string to be chopped"); - } - String var = strings.get(0) == null?null: (String) strings.get(0); - if(var == null) { - return null; - } - else if(var.length() == 0) { - return var; - } - else { - return StringUtils.chomp(var); - } - } - } - @Stellar( name="CHOP" - , description = "Remove the last character from a String" - , params = { "the String to chop last character from, may be null"} - , returns = "String without last character, null if null String input" - ) - public static class Chop extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - if(strings == null || strings.size() == 0 ) { - throw new IllegalArgumentException("[CHOP] missing argument: string to be chopped"); - } - String var = strings.get(0) == null?null: (String) strings.get(0); - if(var == null) { - return null; - } - else if(var.length() == 0) { - return var; - } - else { - return StringUtils.chop(var); - } } - } - - @Stellar( name = "PREPEND_IF_MISSING" - , description = "Prepends the prefix to the start of the string if the string does not already start with any of the prefixes" - , params = { - "str - The string." - , "prefix - The string prefix to prepend to the start of the string" - , "additionalprefix - Optional - Additional string prefix that is valid" - } - , returns = "A new String if prefix was prepended, the same string otherwise." - ) - public static class PrependIfMissing extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - String prefixed; - switch (strings.size()) { - case 2: prefixed = StringUtils.prependIfMissing((String) strings.get(0), (String) strings.get(1)); - break; - case 3: prefixed = StringUtils.prependIfMissing((String) strings.get(0), (String) strings.get(1), (String) strings.get(2)); - break; - default: throw new IllegalArgumentException("[PREPEND_IF_MISSING] incorrect arguments: " + strings.toString() + "\nUsage: PREPEND_IF_MISSING [...]"); - } - return prefixed; - } - } - - @Stellar( name = "APPEND_IF_MISSING" - , description = "Appends the suffix to the end of the string if the string does not already end with any of the suffixes" - , params = { - "str - The string." - , "suffix - The string suffix to append to the end of the string" - , "additionalsuffix - Optional - Additional string suffix that is a valid terminator" - } - , returns = "A new String if suffix was appended, the same string otherwise." - ) - public static class AppendIfMissing extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - String suffixed; - switch (strings.size()) { - case 2: - suffixed = StringUtils.appendIfMissing((String) strings.get(0), (String) strings.get(1)); - break; - case 3: - suffixed = StringUtils.appendIfMissing((String) strings.get(0), (String) strings.get(1), (String) strings.get(2)); - break; - default: - throw new IllegalArgumentException("[APPEND_IF_MISSING] incorrect arguments. Usage: APPEND_IF_MISSING [...]"); - } - return suffixed; - } - } - - @Stellar( name = "COUNT_MATCHES" - , description = "Counts how many times the substring appears in the larger string" - , params = { - "str - the CharSequence to check, may be null" - , "sub - the substring to count, may be null" - } - , returns = "the number of non-overlapping occurrences, 0 if either CharSequence is null" - ) - public static class CountMatches extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - if(strings.size() != 2) { - throw new IllegalArgumentException("[COUNT_MATCHES] incorrect arguments. Usage: COUNT_MATCHES "); - } - - int matchcount; - matchcount = StringUtils.countMatches((String) strings.get(0), (String) strings.get(1)); - return matchcount; - } - } - - @Stellar(name = "TO_JSON_OBJECT" - , description = "Returns a JSON object for the specified JSON string" - , params = { - "str - the JSON String to convert, may be null" - } - , returns = "an Object containing the parsed JSON string" - ) - public static class ToJsonObject extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - if (strings == null || strings.size() == 0) { - throw new IllegalArgumentException("[TO_JSON_OBJECT] incorrect arguments. Usage: TO_JSON_OBJECT "); - } - String var = (strings.get(0) == null) ? null : (String) strings.get(0); - if (var == null) { - return null; - } else if (var.length() == 0) { - return var; - } else { - if (!(strings.get(0) instanceof String)) { - throw new ParseException("Valid JSON string not supplied"); + + @Stellar(name = "CHOMP", + description = "Removes one newline from end of a String if it's there, otherwise leave it alone." + + "A newline is \"\\n\", \"\\r\", or \"\\r\\n\"", + params = {"the String to chomp a newline from, may be null"}, + returns = "String without newline, null if null String input" + ) + public static class Chomp extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException("[CHOMP] missing argument: string to be chopped"); + } + String var = strings.get(0) == null ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + return StringUtils.chomp(var); + } } - // Return JSON Object - try { - return JSONUtils.INSTANCE.load((String) strings.get(0), Object.class); - } catch (JsonProcessingException ex) { - throw new ParseException("Valid JSON string not supplied", ex); - } catch (IOException e) { - e.printStackTrace(); + } + + @Stellar(name = "CHOP", + description = "Remove the last character from a String", + params = {"the String to chop last character from, may be null"}, + returns = "String without last character, null if null String input" + ) + public static class Chop extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException("[CHOP] missing argument: string to be chopped"); + } + String var = strings.get(0) == null ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + return StringUtils.chop(var); + } } - } - return new ParseException("Unable to parse JSON string"); } - } - - @Stellar(name = "TO_JSON_MAP" - , description = "Returns a MAP object for the specified JSON string" - , params = { - "str - the JSON String to convert, may be null" - } - , returns = "a MAP object containing the parsed JSON string" - ) - public static class ToJsonMap extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - if (strings == null || strings.size() == 0) { - throw new IllegalArgumentException("[TO_JSON_MAP] incorrect arguments. Usage: TO_JSON_MAP "); - } - String var = (strings.get(0) == null) ? null : (String) strings.get(0); - if (var == null) { - return null; - } else if (var.length() == 0) { - return var; - } else { - if (!(strings.get(0) instanceof String)) { - throw new ParseException("Valid JSON string not supplied"); + + @Stellar(name = "PREPEND_IF_MISSING", + description = "Prepends the prefix to the start of the string if the string does not already start with any of the prefixes", + params = { + "str - The string.", + "prefix - The string prefix to prepend to the start of the string", + "additionalprefix - Optional - Additional string prefix that is valid" + }, + returns = "A new String if prefix was prepended, the same string otherwise." + ) + public static class PrependIfMissing extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + String prefixed; + switch (strings.size()) { + case 2: + prefixed = StringUtils.prependIfMissing((String) strings.get(0), (String) strings.get(1)); + break; + case 3: + prefixed = StringUtils.prependIfMissing((String) strings.get(0), (String) strings.get(1), + (String) strings.get(2)); + break; + default: + throw new IllegalArgumentException( + "[PREPEND_IF_MISSING] incorrect arguments: " + strings + + "\nUsage: PREPEND_IF_MISSING [...]"); + } + return prefixed; } - // Return parsed JSON Object as a HashMap - String in = (String)strings.get(0); - try { - return (Map)JSONUtils.INSTANCE.load(in, JSONUtils.MAP_SUPPLIER); - } catch (JsonProcessingException ex) { - throw new ParseException(String.format("%s is not a valid JSON string", in), ex); - } catch (IOException ex) { - throw new ParseException(String.format("%s is not a valid JSON string", in), ex); + } + + @Stellar(name = "APPEND_IF_MISSING", + description = "Appends the suffix to the end of the string if the string does not already end with any of the suffixes", + params = { + "str - The string.", + "suffix - The string suffix to append to the end of the string", + "additionalsuffix - Optional - Additional string suffix that is a valid terminator" + }, + returns = "A new String if suffix was appended, the same string otherwise." + ) + public static class AppendIfMissing extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + String suffixed; + switch (strings.size()) { + case 2: + suffixed = StringUtils.appendIfMissing((String) strings.get(0), (String) strings.get(1)); + break; + case 3: + suffixed = StringUtils.appendIfMissing((String) strings.get(0), (String) strings.get(1), + (String) strings.get(2)); + break; + default: + throw new IllegalArgumentException( + "[APPEND_IF_MISSING] incorrect arguments. Usage: APPEND_IF_MISSING [...]"); + } + return suffixed; } - catch (ClassCastException ex) { - throw new ParseException(String.format("%s is not a valid JSON string, expected a map", in), ex); + } + + @Stellar(name = "COUNT_MATCHES", + description = "Counts how many times the substring appears in the larger string", + params = { + "str - the CharSequence to check, may be null", + "sub - the substring to count, may be null" + }, + returns = "the number of non-overlapping occurrences, 0 if either CharSequence is null" + ) + public static class CountMatches extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + if (strings.size() != 2) { + throw new IllegalArgumentException( + "[COUNT_MATCHES] incorrect arguments. Usage: COUNT_MATCHES "); + } + + int matchcount; + matchcount = StringUtils.countMatches((String) strings.get(0), (String) strings.get(1)); + return matchcount; } - } } - } - - @Stellar(name = "TO_JSON_LIST" - , description = "Returns a List object for the specified JSON string" - , params = { - "str - the JSON String to convert, may be null" - } - , returns = "a List object containing the parsed JSON string" - ) - public static class ToJsonList extends BaseStellarFunction { - - @Override - public Object apply(List strings) { - - if (strings == null || strings.size() == 0) { - throw new IllegalArgumentException("[TO_JSON_LIST] incorrect arguments. Usage: TO_JSON_LIST "); - } - String var = (strings.get(0) == null) ? null : (String) strings.get(0); - if (var == null) { - return null; - } else if (var.length() == 0) { - return var; - } else { - if (!(strings.get(0) instanceof String)) { - throw new ParseException("Valid JSON string not supplied"); + + @Stellar(name = "TO_JSON_OBJECT", + description = "Returns a JSON object for the specified JSON string", + params = { + "str - the JSON String to convert, may be null" + }, + returns = "an Object containing the parsed JSON string" + ) + public static class ToJsonObject extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException( + "[TO_JSON_OBJECT] incorrect arguments. Usage: TO_JSON_OBJECT "); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return JSON Object + try { + return JSONUtils.INSTANCE.load((String) strings.get(0), Object.class); + } catch (JsonProcessingException ex) { + throw new ParseException("Valid JSON string not supplied", ex); + } catch (IOException e) { + e.printStackTrace(); + } + } + return new ParseException("Unable to parse JSON string"); } - // Return parsed JSON Object as a List - String in = (String)strings.get(0); - try { - return (List) JSONUtils.INSTANCE.load(in, JSONUtils.LIST_SUPPLIER); - } catch (JsonProcessingException ex) { - throw new ParseException(String.format("%s is not a valid JSON string", in), ex); - } catch (IOException ex) { - throw new ParseException(String.format("%s is not a valid JSON string", in), ex); + } + + @Stellar(name = "TO_JSON_MAP", + description = "Returns a MAP object for the specified JSON string", + params = { + "str - the JSON String to convert, may be null" + }, + returns = "a MAP object containing the parsed JSON string" + ) + public static class ToJsonMap extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException( + "[TO_JSON_MAP] incorrect arguments. Usage: TO_JSON_MAP "); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return parsed JSON Object as a HashMap + String in = (String) strings.get(0); + try { + return JSONUtils.INSTANCE.load(in, JSONUtils.MAP_SUPPLIER); + } catch (JsonProcessingException ex) { + throw new ParseException(String.format("%s is not a valid JSON string", in), ex); + } catch (IOException ex) { + throw new ParseException(String.format("%s is not a valid JSON string", in), ex); + } catch (ClassCastException ex) { + throw new ParseException(String.format("%s is not a valid JSON string, expected a map", in), ex); + } + } } - catch (ClassCastException ex) { - throw new ParseException(String.format("%s is not a valid JSON string, expected a list", in), ex); + } + + @Stellar(name = "TO_JSON_LIST", + description = "Returns a List object for the specified JSON string", + params = { + "str - the JSON String to convert, may be null" + }, + returns = "a List object containing the parsed JSON string" + ) + public static class ToJsonList extends BaseStellarFunction { + + @Override + public Object apply(List strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException( + "[TO_JSON_LIST] incorrect arguments. Usage: TO_JSON_LIST "); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return parsed JSON Object as a List + String in = (String) strings.get(0); + try { + return JSONUtils.INSTANCE.load(in, JSONUtils.LIST_SUPPLIER); + } catch (JsonProcessingException ex) { + throw new ParseException(String.format("%s is not a valid JSON string", in), ex); + } catch (IOException ex) { + throw new ParseException(String.format("%s is not a valid JSON string", in), ex); + } catch (ClassCastException ex) { + throw new ParseException(String.format("%s is not a valid JSON string, expected a list", in), ex); + } + } } - } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/SystemFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/SystemFunctions.java index e1b41040..67536ee5 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/SystemFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/SystemFunctions.java @@ -7,81 +7,84 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.stellar.dsl.functions; -import org.apache.metron.stellar.dsl.BaseStellarFunction; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.common.system.Environment; +package org.apache.metron.stellar.dsl.functions; import java.util.List; import java.util.function.Function; +import org.apache.metron.stellar.common.system.Environment; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; public class SystemFunctions { - @Stellar(namespace = "SYSTEM", + @Stellar(namespace = "SYSTEM", name = "ENV_GET", description = "Returns the value associated with an environment variable", params = { - "env_var - Environment variable name to get the value for" + "env_var - Environment variable name to get the value for" }, returns = "String" - ) - public static class EnvGet extends BaseStellarFunction { - private Environment env; + ) + public static class EnvGet extends BaseStellarFunction { + private final Environment env; - public EnvGet() { - this(new Environment()); - } + public EnvGet() { + this(new Environment()); + } - public EnvGet(Environment env) { - this.env = env; - } + public EnvGet(Environment env) { + this.env = env; + } - @Override - public Object apply(List args) { - return extractTypeChecked(args, 0, String.class, x -> env.get((String) x.get(0))); + @Override + public Object apply(List args) { + return extractTypeChecked(args, 0, String.class, x -> env.get((String) x.get(0))); + } } - } - /** - * Extract type-checked value from an argument list using the specified type check and extraction function - * - * @param args Arguments to check - * @param i Index of argument to extract - * @param clazz Object type to verify - * @param extractFunc Function applied to extract the value from args - * @return value from args if passes type checks, null otherwise - */ - public static Object extractTypeChecked(List args, int i, Class clazz, Function, Object> extractFunc) { - if (args.size() < i + 1) { - return null; - } else if (clazz.isInstance(args.get(i))) { - return extractFunc.apply(args); - } else { - return null; + /** + * Extract type-checked value from an argument list using the specified type check and extraction function. + * + * @param args Arguments to check + * @param i Index of argument to extract + * @param clazz Object type to verify + * @param extractFunc Function applied to extract the value from args + * @return value from args if passes type checks, null otherwise + */ + public static Object extractTypeChecked(List args, int i, Class clazz, + Function, Object> extractFunc) { + if (args.size() < i + 1) { + return null; + } else if (clazz.isInstance(args.get(i))) { + return extractFunc.apply(args); + } else { + return null; + } } - } - @Stellar(namespace = "SYSTEM", + @Stellar(namespace = "SYSTEM", name = "PROPERTY_GET", description = "Returns the value associated with a Java system property", params = { - "key - Property to get the value for" + "key - Property to get the value for" }, returns = "String" - ) - public static class PropertyGet extends BaseStellarFunction { - @Override - public Object apply(List args) { - return extractTypeChecked(args, 0, String.class, x -> System.getProperty((String) args.get(0))); + ) + public static class PropertyGet extends BaseStellarFunction { + @Override + public Object apply(List args) { + return extractTypeChecked(args, 0, String.class, x -> System.getProperty((String) args.get(0))); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/TextFunctions.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/TextFunctions.java index 01e5da4f..dc8489fd 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/TextFunctions.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/TextFunctions.java @@ -5,8 +5,10 @@ * (the "License"); you may not use this file except in compliance with the License. You may obtain * a copy of the License at * + *

    * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under @@ -27,86 +29,82 @@ public class TextFunctions { - private static final List tagsList; + private static final List tagsList; - static { - List tags = new ArrayList<>(); - for (Locale locale : Locale.getAvailableLocales()) { - tags.add(locale.toLanguageTag()); + static { + List tags = new ArrayList<>(); + for (Locale locale : Locale.getAvailableLocales()) { + tags.add(locale.toLanguageTag()); + } + tagsList = ImmutableList.copyOf(tags); } - tagsList = ImmutableList.copyOf(tags); - } - @Stellar(name = "LANGS", - namespace = "FUZZY", - description = "Returns a list of IETF BCP 47 available to the system, such as en, fr, de. " - + "These values may be passed to FUZZY_SCORE", - params = {}, - returns = "A list of IEF BCP 47 language tag strings") - /** - * GetAvailableLanaguageTags exposes IEF BCP 47 lanaguage tags available to the system - */ - public static class GetAvailableLanaguageTags extends BaseStellarFunction { + /** + * GetAvailableLanaguageTags exposes IEF BCP 47 lanaguage tags available to the system. + */ + @Stellar(name = "LANGS", + namespace = "FUZZY", + description = "Returns a list of IETF BCP 47 available to the system, such as en, fr, de. " + + "These values may be passed to FUZZY_SCORE", + params = {}, + returns = "A list of IEF BCP 47 language tag strings") + public static class GetAvailableLanaguageTags extends BaseStellarFunction { - @Override - public Object apply(List list) { - return tagsList; + @Override + public Object apply(List list) { + return tagsList; + } } - } - @Stellar(name = "SCORE", - namespace = "FUZZY", - description = - "Returns the Fuzzy Score which indicates the similarity score between two Strings " - + - "One point is given for every matched character. Subsequent matches yield two bonus " - + - "points. A higher score indicates a higher similarity", - params = { - "string - The full term that should be matched against", - "string - The query that will be matched against a term", - "string - The IETF BCP 47 language code to use such as en, fr, de " - + - "( SEE FUZZY_LANGS and https://tools.ietf.org/html/bcp47)" - }, - returns = "integer representing the score") - /** - * FuzzyScoreFunction exposes the Apache Commons Text Similarity FuzzyScore through - * Stellar. - */ - public static class FuzzyScoreFunction extends BaseStellarFunction { + /** + * FuzzyScoreFunction exposes the Apache Commons Text Similarity FuzzyScore through Stellar. + */ + @Stellar(name = "SCORE", + namespace = "FUZZY", + description = + "Returns the Fuzzy Score which indicates the similarity score between two Strings " + + "One point is given for every matched character. Subsequent matches yield two bonus " + + "points. A higher score indicates a higher similarity", + params = { + "string - The full term that should be matched against", + "string - The query that will be matched against a term", + "string - The IETF BCP 47 language code to use such as en, fr, de " + + "( SEE FUZZY_LANGS and https://tools.ietf.org/html/bcp47)" + }, + returns = "integer representing the score") + public static class FuzzyScoreFunction extends BaseStellarFunction { - @Override - public Object apply(List list) { - if (list.size() < 3) { - throw new IllegalStateException("FUZZY_SCORE expects three args: [string, string, string]"); - } - Object oterm = list.get(0); - Object oquery = list.get(1); - Object olang = list.get(2); + @Override + public Object apply(List list) { + if (list.size() < 3) { + throw new IllegalStateException("FUZZY_SCORE expects three args: [string, string, string]"); + } + Object oterm = list.get(0); + Object oquery = list.get(1); + Object olang = list.get(2); - // return 0 here, validate will pass 3 nulls - // if we change validate to pass default of expected type, we can differentiate - if (!(oterm instanceof String) || !(oquery instanceof String) || !(olang instanceof String)) { - return 0; - } + // return 0 here, validate will pass 3 nulls + // if we change validate to pass default of expected type, we can differentiate + if (!(oterm instanceof String) || !(oquery instanceof String) || !(olang instanceof String)) { + return 0; + } - String term = (String) oterm; - String query = (String) oquery; - String lang = (String) olang; + String term = (String) oterm; + String query = (String) oquery; + String lang = (String) olang; - if (!tagsList.contains(lang)) { - throw new ParseException( - "FUZZY_SCORE requires a valid IETF BCP47 language code see FUZZY_LANGS and https://tools.ietf.org/html/bcp47"); - } - - if (StringUtils.isEmpty(term) || StringUtils.isEmpty(query)) { - return 0; - } + if (!tagsList.contains(lang)) { + throw new ParseException( + "FUZZY_SCORE requires a valid IETF BCP47 language code see FUZZY_LANGS and https://tools.ietf.org/html/bcp47"); + } - Locale locale = Locale.forLanguageTag(lang); - FuzzyScore score = new FuzzyScore(locale); - return score.fuzzyScore(term, query); + if (StringUtils.isEmpty(term) || StringUtils.isEmpty(query)) { + return 0; + } + + Locale locale = Locale.forLanguageTag(lang); + FuzzyScore score = new FuzzyScore(locale); + return score.fuzzyScore(term, query); + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java index 38a32d15..c1552169 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,204 +44,212 @@ * The base implementation of a function resolver that provides a means for lazy * initialization, thread-safety, and a mechanism for function resolution. * + *

    * Concrete function resolvers can override the `resolvables` method which * defines the classes that are interrogated further to discover Stellar functions. */ public abstract class BaseFunctionResolver implements FunctionResolver, Serializable { - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** - * Maps a function name to the metadata necessary to execute the Stellar function. - */ - private Supplier> functions; - - /** - * The Stellar execution context that can be used to inform the function resolution process. - */ - protected Context context; - - /** - * Indicates if closed has been called on this resolver. - */ - private boolean closed; - - public BaseFunctionResolver() { - // memoize provides lazy initialization and thread-safety (the ugly cast is necessary for serialization) - functions = Suppliers.memoize((Supplier> & Serializable) this::resolveFunctions); - closed = false; - } - - /** - * Returns a set of classes that should undergo further interrogation for resolution - * (aka discovery) of Stellar functions. - */ - public abstract Set> resolvables(); - - /** - * Provides metadata about each Stellar function that is resolvable. - */ - @Override - public Iterable getFunctionInfo() { - return functions.get().values(); - } - - /** - * The names of all Stellar functions that are resolvable. - */ - @Override - public Iterable getFunctions() { - return functions.get().keySet(); - } - - /** - * Initialize the function resolver. - * @param context Context used to initialize. - */ - @Override - public void initialize(Context context) { - this.context = context; - } - - /** - * Makes an attempt to close all Stellar functions. Calling close multiple times has no effect. - * @throws IOException Catches all exceptions and summarizes them. - */ - @Override - public void close() throws IOException { - if (!closed) { - LOG.info("Calling close() on Stellar functions."); - Map errors = new HashMap<>(); - for (StellarFunctionInfo info : getFunctionInfo()) { - try { - info.getFunction().close(); - } catch (Throwable t) { - errors.put(info.getName(), t); - } - } - if (!errors.isEmpty()) { - StringBuilder sb = new StringBuilder(); - sb.append("Unable to close Stellar functions:"); - for (Map.Entry e : errors.entrySet()) { - Throwable throwable = e.getValue(); - String eText = String - .format("Exception - Function: %s; Message: %s; Cause: %s", e.getKey(), - throwable.getMessage(), - throwable.getCause()); - sb.append(System.lineSeparator()); - sb.append(eText); - } - closed = true; - throw new IOException(sb.toString()); - } - closed = true; - } else { - LOG.info("close() already called on Stellar functions - skipping."); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Maps a function name to the metadata necessary to execute the Stellar function. + */ + private final Supplier> functions; + + /** + * The Stellar execution context that can be used to inform the function resolution process. + */ + protected Context context; + + /** + * Indicates if closed has been called on this resolver. + */ + private boolean closed; + + public BaseFunctionResolver() { + // memoize provides lazy initialization and thread-safety (the ugly cast is necessary for serialization) + functions = + Suppliers.memoize((Supplier> & Serializable) this::resolveFunctions); + closed = false; } - } - - /** - * Resolves a function by name. - * @param functionName The name of the function to resolve. - * @return The executable StellarFunction. - */ - @Override - public StellarFunction apply(String functionName) { - StellarFunctionInfo info = functions.get().get(functionName); - if(info == null) { - throw new IllegalStateException(format("Unknown function: `%s`", functionName)); + + /** + * Returns a set of classes that should undergo further interrogation for resolution + * (aka discovery) of Stellar functions. + */ + public abstract Set> resolvables(); + + /** + * Provides metadata about each Stellar function that is resolvable. + */ + @Override + public Iterable getFunctionInfo() { + return functions.get().values(); + } + + /** + * The names of all Stellar functions that are resolvable. + */ + @Override + public Iterable getFunctions() { + return functions.get().keySet(); + } + + /** + * Initialize the function resolver. + * + * @param context Context used to initialize. + */ + @Override + public void initialize(Context context) { + this.context = context; } - return info.getFunction(); - } - - /** - * Performs the core process of function resolution. - */ - protected Map resolveFunctions() { - - // maps a function name to its definition - Map functions = new HashMap<>(); - - for(Class clazz : resolvables()) { - StellarFunctionInfo fn = resolveFunction(clazz); - if(fn != null) { - // check for duplicate function names - StellarFunctionInfo fnSameName = functions.get(fn.getName()); - if (fnSameName != null && ObjectUtils.notEqual(fnSameName, fn)) { - LOG.warn("Namespace conflict: duplicate function names; `{}` implemented by [{}, {}]", - fn.getName(), fnSameName.getFunction(), fn.getFunction()); - } - functions.put(fn.getName(), fn); - } + /** + * Makes an attempt to close all Stellar functions. Calling close multiple times has no effect. + * + * @throws IOException Catches all exceptions and summarizes them. + */ + @Override + public void close() throws IOException { + if (!closed) { + LOG.info("Calling close() on Stellar functions."); + Map errors = new HashMap<>(); + for (StellarFunctionInfo info : getFunctionInfo()) { + try { + info.getFunction().close(); + } catch (Throwable t) { + errors.put(info.getName(), t); + } + } + if (!errors.isEmpty()) { + StringBuilder sb = new StringBuilder(); + sb.append("Unable to close Stellar functions:"); + for (Map.Entry e : errors.entrySet()) { + Throwable throwable = e.getValue(); + String exText = String + .format("Exception - Function: %s; Message: %s; Cause: %s", e.getKey(), + throwable.getMessage(), + throwable.getCause()); + sb.append(System.lineSeparator()); + sb.append(exText); + } + closed = true; + throw new IOException(sb.toString()); + } + closed = true; + } else { + LOG.info("close() already called on Stellar functions - skipping."); + } } - return functions; - } - - /** - * Resolves a Stellar function from a given class. - * @param clazz The class. - */ - public static StellarFunctionInfo resolveFunction(Class clazz) { - StellarFunctionInfo info = null; - - // the class must be annotated - if (clazz.isAnnotationPresent(Stellar.class)) { - - Stellar annotation = clazz.getAnnotation(Stellar.class); - String fullyQualifiedName = getNameFromAnnotation(annotation); - StellarFunction function = createFunction(clazz); - - if (fullyQualifiedName != null && function != null) { - info = new StellarFunctionInfo( - annotation.description(), - fullyQualifiedName, - annotation.params(), - annotation.returns(), - function); - } + /** + * Resolves a function by name. + * + * @param functionName The name of the function to resolve. + * @return The executable StellarFunction. + */ + @Override + public StellarFunction apply(String functionName) { + StellarFunctionInfo info = functions.get().get(functionName); + if (info == null) { + throw new IllegalStateException(format("Unknown function: `%s`", functionName)); + } + return info.getFunction(); } - return info; - } - - /** - * Returns the fully-qualified function name from a Stellar annotation. - * @param annotation The Stellar annotation. - */ - public static String getNameFromAnnotation(Stellar annotation) { - - // find the function name - String name = annotation.name(); - if(name == null || name.trim().length() == 0) { - return null; - } else { - name = name.trim(); + /** + * Performs the core process of function resolution. + */ + protected Map resolveFunctions() { + + // maps a function name to its definition + Map functions = new HashMap<>(); + + for (Class clazz : resolvables()) { + StellarFunctionInfo fn = resolveFunction(clazz); + if (fn != null) { + // check for duplicate function names + StellarFunctionInfo fnSameName = functions.get(fn.getName()); + if (fnSameName != null && ObjectUtils.notEqual(fnSameName, fn)) { + LOG.warn("Namespace conflict: duplicate function names; `{}` implemented by [{}, {}]", + fn.getName(), fnSameName.getFunction(), fn.getFunction()); + } + + functions.put(fn.getName(), fn); + } + } + + return functions; } - // find the function namespace - String namespace = annotation.namespace(); - if(namespace == null || namespace.length() == 0) { - namespace = null; - } else { - namespace = namespace.trim(); + /** + * Resolves a Stellar function from a given class. + * + * @param clazz The class. + */ + public static StellarFunctionInfo resolveFunction(Class clazz) { + StellarFunctionInfo info = null; + + // the class must be annotated + if (clazz.isAnnotationPresent(Stellar.class)) { + + Stellar annotation = clazz.getAnnotation(Stellar.class); + String fullyQualifiedName = getNameFromAnnotation(annotation); + StellarFunction function = createFunction(clazz); + + if (fullyQualifiedName != null && function != null) { + info = new StellarFunctionInfo( + annotation.description(), + fullyQualifiedName, + annotation.params(), + annotation.returns(), + function); + } + } + + return info; } - return Joiner.on("_").skipNulls().join(Arrays.asList(namespace, name)); - } + /** + * Returns the fully-qualified function name from a Stellar annotation. + * + * @param annotation The Stellar annotation. + */ + public static String getNameFromAnnotation(Stellar annotation) { + + // find the function name + String name = annotation.name(); + if (name == null || name.trim().length() == 0) { + return null; + } else { + name = name.trim(); + } + + // find the function namespace + String namespace = annotation.namespace(); + if (namespace == null || namespace.length() == 0) { + namespace = null; + } else { + namespace = namespace.trim(); + } - /** - * Instantiate the StellarFunction implementation class. - * @param clazz The class containing a Stellar function definition. - */ - public static StellarFunction createFunction(Class clazz) { - try { - return clazz.getConstructor().newInstance(); + return Joiner.on("_").skipNulls().join(Arrays.asList(namespace, name)); + } - } catch (Exception e) { - LOG.error("Unable to load {} because {}", clazz.getName(), e.getMessage(), e); - return null; + /** + * Instantiate the StellarFunction implementation class. + * + * @param clazz The class containing a Stellar function definition. + */ + public static StellarFunction createFunction(Class clazz) { + try { + return clazz.getConstructor().newInstance(); + + } catch (Exception e) { + LOG.error("Unable to load {} because {}", clazz.getName(), e.getMessage(), e); + return null; + } } - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java index 7a87f4d1..d2ae0859 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,6 +27,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -44,258 +47,268 @@ /** * Performs function resolution for Stellar by searching the classpath. * + *

    * By default, the entire classpath will be searched for Stellar functions. At times, * this can take quite a while. To shorten the search time, a property can be * defined to either include or exclude certain packages. The fewer packages there are * to search, the quicker the search will be. * + *

    * The properties are pulled from the Context's 'STELLAR_CONFIG'. In the REPL, this * is defined in a file called 'stellar.properties' on the classpath. * + *

    * The following property definition will include only Stellar functions that are * part of Apache Metron. * + *

    * stellar.function.resolver.includes = org.apache.metron.* * + *

    * The following property definition will exclude Stellar functions that are part of * Metron's management suite of function. * + *

    * stellar.function.resolver.excludes = org.apache.metron.management.* * + *

    * The following property definition would also exclude the Stellar functions that are * part of the management suite of functions. Of course, this may also exclude other * packages, but this serves as an example of the types of expression that can be used. * + *

    * stellar.function.resolver.excludes = org\\.management.* * */ public class ClasspathFunctionResolver extends BaseFunctionResolver { - public enum Config { - /** - * The set of paths. These paths are comma separated URLs with optional regex patterns at the end. - * e.g. hdfs://node1:8020/apps/metron/stellar/.*.jar,hdfs://node1:8020/apps/metron/my_org/.*.jar - * would signify all the jars under /apps/metron/stellar and /apps/metron/my_org in HDFS. - */ - STELLAR_VFS_PATHS("stellar.function.paths", ""), - /** - * The key for a global property that defines one or more regular expressions - * that specify what should be included when searching for Stellar functions. - */ - STELLAR_SEARCH_INCLUDES_KEY("stellar.function.resolver.includes", ""), - /** - * The key for a global property that defines one or more regular expressions - * that specify what should be excluded when searching for Stellar functions. - */ - STELLAR_SEARCH_EXCLUDES_KEY("stellar.function.resolver.excludes", ""), + public enum Config { + /** + * The set of paths. These paths are comma separated URLs with optional regex patterns at the end. + * e.g. hdfs://node1:8020/apps/metron/stellar/.*.jar,hdfs://node1:8020/apps/metron/my_org/.*.jar + * would signify all the jars under /apps/metron/stellar and /apps/metron/my_org in HDFS. + */ + STELLAR_VFS_PATHS("stellar.function.paths", ""), + /** + * The key for a global property that defines one or more regular expressions + * that specify what should be included when searching for Stellar functions. + */ + STELLAR_SEARCH_INCLUDES_KEY("stellar.function.resolver.includes", ""), + /** + * The key for a global property that defines one or more regular expressions + * that specify what should be excluded when searching for Stellar functions. + */ + STELLAR_SEARCH_EXCLUDES_KEY("stellar.function.resolver.excludes", ""), - ; - String param; - Object defaultValue; - Config(String param, String defaultValue) { - this.param = param; - this.defaultValue = defaultValue; - } + ; + String param; + Object defaultValue; - public String param() { - return param; - } + Config(String param, String defaultValue) { + this.param = param; + this.defaultValue = defaultValue; + } - public Object get(Map config) { - return config.getOrDefault(param, defaultValue); - } + public String param() { + return param; + } - public T get(Map config, Class clazz) { - return ConversionUtils.convert(get(config), clazz); + public Object get(Map config) { + return config.getOrDefault(param, defaultValue); + } + + public T get(Map config, Class clazz) { + return ConversionUtils.convert(get(config), clazz); + } } - } - /** - * The includes and excludes can include a list of multiple includes or excludes that - * are delimited by these values. - */ - private static final String STELLAR_SEARCH_DELIMS = "[,:]"; + /** + * The includes and excludes can include a list of multiple includes or excludes that + * are delimited by these values. + */ + private static final String STELLAR_SEARCH_DELIMS = "[,:]"; - /** - * Regular expressions defining packages that should be included in the Stellar function resolution - * process. - */ - private List includes; + /** + * Regular expressions defining packages that should be included in the Stellar function resolution + * process. + */ + private final List includes; - /** - * Regular expressions defining packages that should be excluded from the Stellar function resolution - * process. - */ - private List excludes; + /** + * Regular expressions defining packages that should be excluded from the Stellar function resolution + * process. + */ + private final List excludes; - /** - * Classloaders to try to load from - */ - private List classLoaders; + /** + * Classloaders to try to load from. + */ + private final List classLoaders; - public ClasspathFunctionResolver() { - this.includes = new ArrayList<>(); - this.excludes = new ArrayList<>(); - this.classLoaders = new ArrayList<>(); - } + public ClasspathFunctionResolver() { + this.includes = new ArrayList<>(); + this.excludes = new ArrayList<>(); + this.classLoaders = new ArrayList<>(); + } - /** - * Use one or more classloaders - * @param classloaders - */ - public void classLoaders(ClassLoader... classloaders) { - classLoaders.clear(); - Arrays.stream(classloaders).forEach(c -> classLoaders.add(c)); - } + /** + * Use one or more classloaders. + * + */ + public void classLoaders(ClassLoader... classloaders) { + classLoaders.clear(); + Arrays.stream(classloaders).forEach(c -> classLoaders.add(c)); + } - /** - * Includes one or more packages in the Stellar function resolution process. The packages - * to include can be specified with a regular expression. - * @param toInclude The regular expressions. - */ - public void include(String... toInclude) { - for(String incl : toInclude) { - includes.add(incl); + /** + * Includes one or more packages in the Stellar function resolution process. The packages + * to include can be specified with a regular expression. + * + * @param toInclude The regular expressions. + */ + public void include(String... toInclude) { + Collections.addAll(includes, toInclude); } - } - /** - * Excludes one or more packages from the Stellar function resolution process. The packages - * to exclude can be specified with a regular expression. - * @param toExclude The regular expressions defining packages that should be excluded. - */ - public void exclude(String... toExclude) { - for(String excl : toExclude) { - excludes.add(excl); + /** + * Excludes one or more packages from the Stellar function resolution process. The packages + * to exclude can be specified with a regular expression. + * + * @param toExclude The regular expressions defining packages that should be excluded. + */ + public void exclude(String... toExclude) { + Collections.addAll(excludes, toExclude); } - } - @Override - @SuppressWarnings("unchecked") - public void initialize(Context context) { - super.initialize(context); - if (context != null) { + @Override + @SuppressWarnings("unchecked") + public void initialize(Context context) { + super.initialize(context); + if (context != null) { - Optional optional = context.getCapability(STELLAR_CONFIG, false); - if (optional.isPresent()) { - Map stellarConfig = (Map) optional.get(); - if(LOG.isDebugEnabled()) { - LOG.debug("Setting up classloader using the following config: {}", stellarConfig); - } + Optional optional = context.getCapability(STELLAR_CONFIG, false); + if (optional.isPresent()) { + Map stellarConfig = (Map) optional.get(); + if (LOG.isDebugEnabled()) { + LOG.debug("Setting up classloader using the following config: {}", stellarConfig); + } - include(STELLAR_SEARCH_INCLUDES_KEY.get(stellarConfig, String.class).split(STELLAR_SEARCH_DELIMS)); - exclude(STELLAR_SEARCH_EXCLUDES_KEY.get(stellarConfig, String.class).split(STELLAR_SEARCH_DELIMS)); - Optional vfsLoader = Optional.empty(); - try { - vfsLoader = VFSClassloaderUtil.configureClassloader(STELLAR_VFS_PATHS.get(stellarConfig, String.class)); - if (vfsLoader.isPresent()) { - LOG.debug("CLASSLOADER LOADED WITH: {}", STELLAR_VFS_PATHS.get(stellarConfig, String.class)); - if(LOG.isDebugEnabled()) { - for (FileObject fo : ((VFSClassLoader) vfsLoader.get()).getFileObjects()) { - LOG.error("{} - {}", fo.getURL(), fo.exists()); - } + include(STELLAR_SEARCH_INCLUDES_KEY.get(stellarConfig, String.class).split(STELLAR_SEARCH_DELIMS)); + exclude(STELLAR_SEARCH_EXCLUDES_KEY.get(stellarConfig, String.class).split(STELLAR_SEARCH_DELIMS)); + Optional vfsLoader = Optional.empty(); + try { + vfsLoader = + VFSClassloaderUtil.configureClassloader(STELLAR_VFS_PATHS.get(stellarConfig, String.class)); + if (vfsLoader.isPresent()) { + LOG.debug("CLASSLOADER LOADED WITH: {}", STELLAR_VFS_PATHS.get(stellarConfig, String.class)); + if (LOG.isDebugEnabled()) { + for (FileObject fo : ((VFSClassLoader) vfsLoader.get()).getFileObjects()) { + LOG.error("{} - {}", fo.getURL(), fo.exists()); + } + } + classLoaders(vfsLoader.get()); + } + } catch (FileSystemException e) { + LOG.error("Unable to process filesystem: {}", e.getMessage(), e); + } + } else { + LOG.info("No stellar config set; I'm reverting to the context classpath with no restrictions."); + if (LOG.isDebugEnabled()) { + try { + throw new IllegalStateException("No config set, stacktrace follows."); + } catch (IllegalStateException ise) { + LOG.error(ise.getMessage(), ise); + } + } } - classLoaders(vfsLoader.get()); - } - } catch (FileSystemException e) { - LOG.error("Unable to process filesystem: {}", e.getMessage(), e); - } - } else { - LOG.info("No stellar config set; I'm reverting to the context classpath with no restrictions."); - if (LOG.isDebugEnabled()) { - try { - throw new IllegalStateException("No config set, stacktrace follows."); - } catch (IllegalStateException ise) { - LOG.error(ise.getMessage(), ise); - } + } else { + throw new IllegalStateException("CONTEXT IS NULL!"); } - } - } else { - throw new IllegalStateException("CONTEXT IS NULL!"); } - } - protected Iterable> getStellarClasses(ClassLoader cl) { - return ClassIndex.getAnnotated(Stellar.class, cl); - } + protected Iterable> getStellarClasses(ClassLoader cl) { + return ClassIndex.getAnnotated(Stellar.class, cl); + } - protected boolean includeClass(Class c, FilterBuilder filterBuilder) - { - boolean isAssignable = StellarFunction.class.isAssignableFrom(c); - boolean isFiltered = filterBuilder.apply(c.getCanonicalName()); - return isAssignable && isFiltered; - } + protected boolean includeClass(Class c, FilterBuilder filterBuilder) { + boolean isAssignable = StellarFunction.class.isAssignableFrom(c); + boolean isFiltered = filterBuilder.apply(c.getCanonicalName()); + return isAssignable && isFiltered; + } - /** - * Returns a set of classes that should undergo further interrogation for resolution - * (aka discovery) of Stellar functions. - */ - @Override - @SuppressWarnings("unchecked") - public Set> resolvables() { + /** + * Returns a set of classes that should undergo further interrogation for resolution + * (aka discovery) of Stellar functions. + */ + @Override + @SuppressWarnings("unchecked") + public Set> resolvables() { - ClassLoader[] cls = null; - if (this.classLoaders.size() == 0) { - LOG.warn("Using System classloader"); - cls = new ClassLoader[]{getClass().getClassLoader()}; - } else { - List classLoaderList = new ArrayList<>(); - for (int i = 0; i < this.classLoaders.size(); ++i) { - ClassLoader cl = this.classLoaders.get(i); - if (null != cl) { - LOG.debug("Using classloader: {}", cl.getClass().getCanonicalName()); - classLoaderList.add(cl); + ClassLoader[] cls = null; + if (this.classLoaders.size() == 0) { + LOG.warn("Using System classloader"); + cls = new ClassLoader[] {getClass().getClassLoader()}; } else { - LOG.error( - "This should not happen, so report a bug if you see this error - Classloader {} of {} was null. Classloader list is: {}", - i, this.classLoaders.size(), this.classLoaders); + List classLoaderList = new ArrayList<>(); + for (int i = 0; i < this.classLoaders.size(); ++i) { + ClassLoader cl = this.classLoaders.get(i); + if (null != cl) { + LOG.debug("Using classloader: {}", cl.getClass().getCanonicalName()); + classLoaderList.add(cl); + } else { + LOG.error( + "This should not happen, so report a bug if you see this error " + + "- Classloader {} of {} was null. Classloader list is: {}", + i, this.classLoaders.size(), this.classLoaders); + } + } + cls = classLoaderList.toArray(new ClassLoader[0]); } - } - cls = classLoaderList.toArray(new ClassLoader[0]); - } - FilterBuilder filterBuilder = new FilterBuilder(); - excludes.forEach(excl -> { - if (excl != null) { - filterBuilder.exclude(excl); - } - }); - includes.forEach(incl -> { - if (incl != null) { - filterBuilder.include(incl); - } - }); - Set classes = new HashSet<>(); - Set> ret = new HashSet<>(); - for (ClassLoader cl : cls) { - for (Class c : getStellarClasses(cl)) { - try { - LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), c.getCanonicalName()); - if (includeClass(c, filterBuilder)) { - String className = c.getName(); - if (!classes.contains(className)) { - LOG.debug("{}: Added class: {}", cl.getClass().getCanonicalName(), className); - ret.add((Class) c); - classes.add(className); + FilterBuilder filterBuilder = new FilterBuilder(); + excludes.forEach(excl -> { + if (excl != null) { + filterBuilder.exclude(excl); + } + }); + includes.forEach(incl -> { + if (incl != null) { + filterBuilder.include(incl); + } + }); + Set classes = new HashSet<>(); + Set> ret = new HashSet<>(); + for (ClassLoader cl : cls) { + for (Class c : getStellarClasses(cl)) { + try { + LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), c.getCanonicalName()); + if (includeClass(c, filterBuilder)) { + String className = c.getName(); + if (!classes.contains(className)) { + LOG.debug("{}: Added class: {}", cl.getClass().getCanonicalName(), className); + ret.add((Class) c); + classes.add(className); + } + } + } catch (Error le) { + //we have had some error loading a stellar function. This could mean that + //the classpath is unstable (e.g. old copies of jars are on the classpath). + try { + LOG.error("Skipping class " + c.getName() + ": " + le.getMessage() + + ", please check that there are not old versions of stellar functions on the classpath.", + le); + } catch (Error ie) { + //it's possible that getName() will throw an exception if the class is VERY malformed. + LOG.error("Skipping class: " + le.getMessage() + + ", please check that there are not old versions of stellar functions on the classpath.", + le); + } + } } - } - } catch (Error le) { - //we have had some error loading a stellar function. This could mean that - //the classpath is unstable (e.g. old copies of jars are on the classpath). - try { - LOG.error("Skipping class " + c.getName() + ": " + le.getMessage() - + ", please check that there are not old versions of stellar functions on the classpath.", le); - } catch (Error ie) { - //it's possible that getName() will throw an exception if the class is VERY malformed. - LOG.error("Skipping class: " + le.getMessage() - + ", please check that there are not old versions of stellar functions on the classpath.", le); - } } - } + return ret; } - return ret; - } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java index 40475865..40ec212b 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions.resolver; import java.io.Closeable; @@ -29,26 +32,28 @@ */ public interface FunctionResolver extends Function, Closeable { - /** - * Provides metadata about each Stellar function that is resolvable. - */ - Iterable getFunctionInfo(); - - /** - * The names of all Stellar functions that are resolvable. - */ - Iterable getFunctions(); - - /** - * Initialize the function resolver. - * @param context Context used to initialize. - */ - void initialize(Context context); - - /** - * Perform any cleanup necessary for the loaded Stellar functions. - */ - @Override - default void close() throws IOException {} + /** + * Provides metadata about each Stellar function that is resolvable. + */ + Iterable getFunctionInfo(); + + /** + * The names of all Stellar functions that are resolvable. + */ + Iterable getFunctions(); + + /** + * Initialize the function resolver. + * + * @param context Context used to initialize. + */ + void initialize(Context context); + + /** + * Perform any cleanup necessary for the loaded Stellar functions. + */ + @Override + default void close() throws IOException { + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/SimpleFunctionResolver.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/SimpleFunctionResolver.java index d2d0e62e..896592aa 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/SimpleFunctionResolver.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/SimpleFunctionResolver.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,6 +31,7 @@ * A simple Stellar function resolver that resolves functions from specific * classes rather than by searching the classpath. * + *

    * FunctionResolver functionResolver = new SimpleFunctionResolver() * .withClass(OneStellarFunction.class) * .withClass(AnotherStellarFunction.class) @@ -36,24 +39,25 @@ */ public class SimpleFunctionResolver extends BaseFunctionResolver { - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - /** - * The classes that will be further interrogated for Stellar functions. - */ - Set> classesToResolve = new HashSet<>(); + /** + * The classes that will be further interrogated for Stellar functions. + */ + Set> classesToResolve = new HashSet<>(); - @Override - public Set> resolvables() { - return classesToResolve; - } + @Override + public Set> resolvables() { + return classesToResolve; + } - /** - * Will attempt to resolve any Stellar functions defined within the specified class. - * @param clazz The class which may contain a Stellar function. - */ - public SimpleFunctionResolver withClass(Class clazz) { - this.classesToResolve.add(clazz); - return this; - } + /** + * Will attempt to resolve any Stellar functions defined within the specified class. + * + * @param clazz The class which may contain a Stellar function. + */ + public SimpleFunctionResolver withClass(Class clazz) { + this.classesToResolve.add(clazz); + return this; + } } diff --git a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/SingletonFunctionResolver.java b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/SingletonFunctionResolver.java index e3bd2c99..faa226f9 100644 --- a/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/SingletonFunctionResolver.java +++ b/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/SingletonFunctionResolver.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.stellar.dsl.functions.resolver; /** @@ -23,12 +26,13 @@ */ public class SingletonFunctionResolver extends ClasspathFunctionResolver { - private static SingletonFunctionResolver INSTANCE = new SingletonFunctionResolver(); + private static final SingletonFunctionResolver INSTANCE = new SingletonFunctionResolver(); - private SingletonFunctionResolver() {} + private SingletonFunctionResolver() { + } - public static FunctionResolver getInstance() { - return INSTANCE; - } + public static FunctionResolver getInstance() { + return INSTANCE; + } } diff --git a/flink-cyber/flink-stellar/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsIntegrationTest.java b/flink-cyber/flink-stellar/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsIntegrationTest.java index d7aa05e1..1325cd98 100644 --- a/flink-cyber/flink-stellar/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsIntegrationTest.java +++ b/flink-cyber/flink-stellar/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsIntegrationTest.java @@ -18,7 +18,29 @@ package org.apache.metron.stellar.dsl.functions; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.dsl.functions.RestConfig.EMPTY_CONTENT_OVERRIDE; +import static org.apache.metron.stellar.dsl.functions.RestConfig.PROXY_HOST; +import static org.apache.metron.stellar.dsl.functions.RestConfig.PROXY_PORT; +import static org.apache.metron.stellar.dsl.functions.RestConfig.RESPONSE_CODES_ALLOWED; +import static org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_GET_SETTINGS; +import static org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_POST_SETTINGS; +import static org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_SETTINGS; +import static org.apache.metron.stellar.dsl.functions.RestConfig.TIMEOUT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockserver.integration.ClientAndServer.startClientAndServer; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.io.FileUtils; import org.apache.metron.stellar.dsl.Context; @@ -31,20 +53,6 @@ import org.junit.rules.TemporaryFolder; import org.mockserver.integration.ClientAndServer; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; -import static org.apache.metron.stellar.dsl.functions.RestConfig.*; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockserver.integration.ClientAndServer.startClientAndServer; -import static org.mockserver.model.HttpRequest.request; -import static org.mockserver.model.HttpResponse.response; - @EnableRuleMigrationSupport public class RestFunctionsIntegrationTest { @@ -429,9 +437,9 @@ public void restPostShouldThrowExceptionOnMalformedJson() { ParseException e = assertThrows(ParseException.class, () -> run(String.format("REST_POST('%s', 'malformed json')", postUri), context)); assertEquals( String.format( - "Unable to parse REST_POST('http://localhost:1080/post', 'malformed json'): " + - "Unable to parse: REST_POST('%s', 'malformed json') due to: POST data 'malformed json' must be properly formatted JSON. " + - "Set the 'enforce.json' property to false to disable this check.", + "Unable to parse REST_POST('http://localhost:1080/post', 'malformed json'): " + + "Unable to parse: REST_POST('%s', 'malformed json') due to: POST data 'malformed json' must be properly formatted JSON. " + + "Set the 'enforce.json' property to false to disable this check.", postUri), e.getMessage()); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/Creator.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/Creator.java index b813dc57..63da82b4 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/Creator.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/Creator.java @@ -7,16 +7,19 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common; public interface Creator { - T create(); + T create(); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/aggregator/Aggregator.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/aggregator/Aggregator.java index d7ca31eb..f0cacd99 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/aggregator/Aggregator.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/aggregator/Aggregator.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,5 +24,5 @@ import java.util.Map; public interface Aggregator { - Double aggregate(List scores, Map config); + Double aggregate(List scores, Map config); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/aggregator/Aggregators.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/aggregator/Aggregators.java index bf2e4ebf..2e2b900f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/aggregator/Aggregators.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/aggregator/Aggregators.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,91 +20,92 @@ package org.apache.metron.common.aggregator; -import org.apache.metron.stellar.common.utils.ConversionUtils; - import java.util.List; import java.util.Map; import java.util.function.BinaryOperator; import java.util.function.Predicate; +import org.apache.metron.stellar.common.utils.ConversionUtils; public enum Aggregators implements Aggregator { - MAX( (numbers, config) -> accumulate(0d, (x,y) -> Math.max(x.doubleValue(),y.doubleValue()), numbers, config)) - ,MIN( (numbers, config) -> accumulate(Double.MAX_VALUE, (x,y) -> Math.min(x.doubleValue(),y.doubleValue()), numbers, config)) - ,SUM( (numbers, config) -> accumulate(0d, (x,y) -> x.doubleValue() + y.doubleValue(), numbers, config)) - ,MEAN( (numbers, config) -> scale(SUM.aggregate(numbers, config), numbers, n -> true)) - ,POSITIVE_MEAN( (numbers, config) -> positiveMean(numbers, config)) - ; - public static String NEGATIVE_VALUES_TRUMP_CONF = "negativeValuesTrump"; - Aggregator aggregator; - Aggregators(Aggregator agg) { - aggregator = agg; - } - public Aggregator getAggregator() { - return aggregator; - } + MAX((numbers, config) -> accumulate(0d, (x, y) -> Math.max(x.doubleValue(), y.doubleValue()), numbers, config)), + MIN((numbers, config) -> accumulate(Double.MAX_VALUE, (x, y) -> Math.min(x.doubleValue(), y.doubleValue()), numbers, + config)), + SUM((numbers, config) -> accumulate(0d, (x, y) -> x.doubleValue() + y.doubleValue(), numbers, config)), + MEAN((numbers, config) -> scale(SUM.aggregate(numbers, config), numbers, n -> true)), + POSITIVE_MEAN((numbers, config) -> positiveMean(numbers, config)); + public static String NEGATIVE_VALUES_TRUMP_CONF = "negativeValuesTrump"; + Aggregator aggregator; - private static double positiveMean(List list, Map config) { - Double ret = 0d; - int num = 0; - boolean negValuesTrump = doNegativeValuesTrump(config); - for(Number n : list) { - if(n.doubleValue() < 0) { - if(negValuesTrump) { - return Double.NEGATIVE_INFINITY; - } - } - else if(n.doubleValue() > 0) { - ret += n.doubleValue(); - num++; - } + Aggregators(Aggregator agg) { + aggregator = agg; } - return num > 0?ret/num:0d; - } + public Aggregator getAggregator() { + return aggregator; + } + + private static double positiveMean(List list, Map config) { + Double ret = 0d; + int num = 0; + boolean negValuesTrump = doNegativeValuesTrump(config); + for (Number n : list) { + if (n.doubleValue() < 0) { + if (negValuesTrump) { + return Double.NEGATIVE_INFINITY; + } + } else if (n.doubleValue() > 0) { + ret += n.doubleValue(); + num++; + } + } + return num > 0 ? ret / num : 0d; - private static boolean doNegativeValuesTrump(Map config) { - boolean negativeValuesTrump = true; - Object negValuesObj = config.get(NEGATIVE_VALUES_TRUMP_CONF); - if(negValuesObj != null) - { - Boolean b = ConversionUtils.convert(negValuesObj, Boolean.class); - if(b != null) { - negativeValuesTrump = b; - } } - return negativeValuesTrump; - } - private static double accumulate(double initial, BinaryOperator op, List list, Map config) { - if(list.isEmpty()) { - return 0d; + + private static boolean doNegativeValuesTrump(Map config) { + boolean negativeValuesTrump = true; + Object negValuesObj = config.get(NEGATIVE_VALUES_TRUMP_CONF); + if (negValuesObj != null) { + Boolean b = ConversionUtils.convert(negValuesObj, Boolean.class); + if (b != null) { + negativeValuesTrump = b; + } + } + return negativeValuesTrump; } - boolean negativeValuesTrump = doNegativeValuesTrump(config); - BinaryOperator binOp = op; - if(negativeValuesTrump) { - binOp =(x,y) -> { - if (y.doubleValue() < 0 || x.doubleValue() < 0) { - return Double.NEGATIVE_INFINITY; - } else { - return op.apply(x, y); + private static double accumulate(double initial, BinaryOperator op, List list, + Map config) { + if (list.isEmpty()) { + return 0d; + } + boolean negativeValuesTrump = doNegativeValuesTrump(config); + + BinaryOperator binOp = op; + if (negativeValuesTrump) { + binOp = (x, y) -> { + if (y.doubleValue() < 0 || x.doubleValue() < 0) { + return Double.NEGATIVE_INFINITY; + } else { + return op.apply(x, y); + } + }; } - }; + return list.stream() + .reduce(initial, binOp) + .doubleValue(); } - return list.stream() - .reduce(initial, binOp) - .doubleValue(); - } - private static double scale(double numberToScale, List list, Predicate filterFunc) { - double scale = list.stream().filter(filterFunc).count(); - if(scale < 1e-5) { - scale = 1; + private static double scale(double numberToScale, List list, Predicate filterFunc) { + double scale = list.stream().filter(filterFunc).count(); + if (scale < 1e-5) { + scale = 1; + } + return numberToScale / scale; } - return numberToScale / scale; - } - @Override - public Double aggregate(List scores, Map config) { - return aggregator.aggregate(scores, config); - } + @Override + public Double aggregate(List scores, Map config) { + return aggregator.aggregate(scores, config); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java index 1174630f..916ffd7c 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,118 +30,118 @@ */ public interface ConfigOption { - String getKey(); + String getKey(); - /** - * A default identity transformation. - * - * @return The transformed object - */ - default BiFunction transform() { - return (s, o) -> o; - } + /** + * A default identity transformation. + * + * @return The transformed object + */ + default BiFunction transform() { + return (s, o) -> o; + } - /** - * Returns true if the map contains the key for the defined config option. - */ - default boolean containsOption(Map map) { - return map.containsKey(getKey()); - } + /** + * Returns true if the map contains the key for the defined config option. + */ + default boolean containsOption(Map map) { + return map.containsKey(getKey()); + } - default void put(Map map, Object value) { - map.put(getKey(), value); - } + default void put(Map map, Object value) { + map.put(getKey(), value); + } - /** - * Retrieves a value from a {@link Map}. It will cast to a provided class. If nothing is found, - * a default will be returned. - * - * @param map The map to look for the key in - * @param clazz The class to cast the value to - * @param defaultValue The default value if no key is found - * @param The class type parameter for the class being casted to - * @return The value, or if not found, the default value - */ - default T getOrDefault(Map map, Class clazz, T defaultValue) { - T val; - return ((val = get(map, clazz)) == null ? defaultValue : val); - } + /** + * Retrieves a value from a {@link Map}. It will cast to a provided class. If nothing is found, + * a default will be returned. + * + * @param map The map to look for the key in + * @param clazz The class to cast the value to + * @param defaultValue The default value if no key is found + * @param The class type parameter for the class being casted to + * @return The value, or if not found, the default value + */ + default T getOrDefault(Map map, Class clazz, T defaultValue) { + T val; + return ((val = get(map, clazz)) == null ? defaultValue : val); + } - /** - * Retrieves a value from a {@link Map}. It will cast to a provided class. - * - * @param map The map to look for the key in - * @param clazz The class to cast the value to - * @param The class type parameter for the class being casted to - * @return The value if found, else null - */ - default T get(Map map, Class clazz) { - Object obj = map.get(getKey()); - if (clazz.isInstance(obj)) { - return clazz.cast(obj); - } else { - return ConversionUtils.convert(obj, clazz); + /** + * Retrieves a transformed result from a {@link Map}. It will cast to a provided class. If + * the transformation result is null, a default will be returned. + * + * @param map The map to look for the key in + * @param transform The transform to run on key-value + * @param clazz The class to cast the value to + * @param defaultValue The default value if no key is found + * @param The class type parameter for the class being casted to + * @return The transformed result of the lookup, or the default value if the result is null + */ + default T getOrDefault(Map map, BiFunction transform, + Class clazz, T defaultValue) { + T val; + return ((val = get(map, transform, clazz)) == null ? defaultValue : val); } - } - /** - * Retrieves a transformed result from a {@link Map}. It will cast to a provided class. If - * the transformation result is null, a default will be returned. - * - * @param map The map to look for the key in - * @param transform The transform to run on key-value - * @param clazz The class to cast the value to - * @param defaultValue The default value if no key is found - * @param The class type parameter for the class being casted to - * @return The transformed result of the lookup, or the default value if the result is null - */ - default T getOrDefault(Map map, BiFunction transform, - Class clazz, T defaultValue) { - T val; - return ((val = get(map, transform, clazz)) == null ? defaultValue : val); - } + /** + * Retrieves a value from a {@link Map}. It will cast to a provided class. + * + * @param map The map to look for the key in + * @param clazz The class to cast the value to + * @param The class type parameter for the class being casted to + * @return The value if found, else null + */ + default T get(Map map, Class clazz) { + Object obj = map.get(getKey()); + if (clazz.isInstance(obj)) { + return clazz.cast(obj); + } else { + return ConversionUtils.convert(obj, clazz); + } + } - /** - * Retrieves a transformed result from a {@link Map}. It will cast to a provided class. - * - * @param map The map to look for the key in - * @param transform The transform to run on key-value - * @param clazz The class to cast the value to - * @param The class type parameter for the class being casted to - * @return The transformed result of the lookup - */ - default T get(Map map, BiFunction transform, - Class clazz) { - return clazz.cast(transform.apply(getKey(), map.get(getKey()))); - } + /** + * Retrieves a transformed result from a {@link Map}. It will cast to a provided class. + * + * @param map The map to look for the key in + * @param transform The transform to run on key-value + * @param clazz The class to cast the value to + * @param The class type parameter for the class being casted to + * @return The transformed result of the lookup + */ + default T get(Map map, BiFunction transform, + Class clazz) { + return clazz.cast(transform.apply(getKey(), map.get(getKey()))); + } - /** - * Retrieves a transformed result from a {@link Map}. It will cast to a provided class. The - * transform is the one provided by the implementation. If the transformation result is null, - * a default will be returned. - * - * @param map The map to look for the key in - * @param clazz The class to cast the value to - * @param defaultValue The default value if no key is found - * @param The class type parameter for the class being casted to - * @return The transformed result of the lookup, or the default value if the result is null - */ - default T getTransformedOrDefault(Map map, Class clazz, T defaultValue) { - T val; - return ((val = getTransformed(map, clazz)) == null ? defaultValue : val); - } + /** + * Retrieves a transformed result from a {@link Map}. It will cast to a provided class. The + * transform is the one provided by the implementation. If the transformation result is null, + * a default will be returned. + * + * @param map The map to look for the key in + * @param clazz The class to cast the value to + * @param defaultValue The default value if no key is found + * @param The class type parameter for the class being casted to + * @return The transformed result of the lookup, or the default value if the result is null + */ + default T getTransformedOrDefault(Map map, Class clazz, T defaultValue) { + T val; + return ((val = getTransformed(map, clazz)) == null ? defaultValue : val); + } - /** - * Retrieves a transformed result from a {@link Map}. It will cast to a provided class. The - * transform is the one provided by the implementation. - * - * @param map The map to look for the key in - * @param clazz The class to cast the value to - * @param The class type parameter for the class being casted to - * @return The transformed result of the lookup, or the default value if the result is null - */ - default T getTransformed(Map map, Class clazz) { - return clazz.cast(transform().apply(getKey(), map.get(getKey()))); - } + /** + * Retrieves a transformed result from a {@link Map}. It will cast to a provided class. The + * transform is the one provided by the implementation. + * + * @param map The map to look for the key in + * @param clazz The class to cast the value to + * @param The class type parameter for the class being casted to + * @return The transformed result of the lookup, or the default value if the result is null + */ + default T getTransformed(Map map, Class clazz) { + return clazz.cast(transform().apply(getKey(), map.get(getKey()))); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/Configuration.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/Configuration.java index 03885426..9abc1f60 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/Configuration.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/Configuration.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration; import java.nio.file.Path; @@ -25,13 +28,14 @@ */ public class Configuration extends Configurations { - private Path configFileRoot; + private final Path configFileRoot; /** * Constructor for interacting with a file. + * * @param configFileRoot The config file path to use */ - public Configuration(Path configFileRoot){ + public Configuration(Path configFileRoot) { this.configFileRoot = configFileRoot; } @@ -39,6 +43,7 @@ public Configuration(Path configFileRoot){ /** * If there's a ZooKeeper client available, use that for updating configs, otherwise * update global configs from a file. + * * @throws Exception If there's an issue updating the config */ public void update() throws Exception { diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationOperations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationOperations.java index 4bb8efc8..a5e5a780 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationOperations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationOperations.java @@ -6,30 +6,36 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration; +import java.io.IOException; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.stellar.common.Constants; -import java.io.IOException; - public interface ConfigurationOperations { - String getTypeName(); - default String getDirectory() { - return getTypeName(); - } - default String getZookeeperRoot() { - return Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + getTypeName(); - } - Object deserialize(String s) throws IOException; - void writeSensorConfigToZookeeper(String sensorType, byte[] configData, CuratorFramework client) throws Exception; + String getTypeName(); + + default String getDirectory() { + return getTypeName(); + } + + default String getZookeeperRoot() { + return Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + getTypeName(); + } + + Object deserialize(String s) throws IOException; + + void writeSensorConfigToZookeeper(String sensorType, byte[] configData, CuratorFramework client) throws Exception; } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java index 1dfa544c..2754a752 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,51 +29,53 @@ */ public enum ConfigurationType implements Function, ConfigurationOperations { - GLOBAL(new GlobalConfigurationOperations()), - PARSER(new ParserConfigurationOperations()), - ENRICHMENT(new EnrichmentConfigurationOperations()), - INDEXING(new IndexingConfigurationOperations()); + GLOBAL(new GlobalConfigurationOperations()), + PARSER(new ParserConfigurationOperations()), + ENRICHMENT(new EnrichmentConfigurationOperations()), + INDEXING(new IndexingConfigurationOperations()); - ConfigurationOperations ops; + ConfigurationOperations ops; - ConfigurationType(ConfigurationOperations ops) { - this.ops = ops; - } + ConfigurationType(ConfigurationOperations ops) { + this.ops = ops; + } - public String getTypeName() { - return ops.getTypeName(); - } + public String getTypeName() { + return ops.getTypeName(); + } - public String getDirectory() { - return ops.getDirectory(); - } + public String getDirectory() { + return ops.getDirectory(); + } - /** - * Deserializes a string according for the config type. - * - * @param s The string to be deserialized - * @return The deserialized string - * @throws RuntimeException If the string cannot be deserialized - */ - public Object deserialize(String s) { - try { - return ops.deserialize(s); - } catch (IOException e) { - throw new RuntimeException("Unable to load " + s, e); + /** + * Deserializes a string according for the config type. + * + * @param s The string to be deserialized + * @return The deserialized string + * @throws RuntimeException If the string cannot be deserialized + */ + public Object deserialize(String s) { + try { + return ops.deserialize(s); + } catch (IOException e) { + throw new RuntimeException("Unable to load " + s, e); + } } - } - @Override - public Object apply(String s) { - return deserialize(s); - } + @Override + public Object apply(String s) { + return deserialize(s); + } - @Override - public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, - CuratorFramework client) throws Exception { + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { - } + } - public String getZookeeperRoot() {return null;} + public String getZookeeperRoot() { + return null; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java index ba336f18..dc50364d 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration; import java.io.ByteArrayInputStream; @@ -26,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.stellar.common.utils.ConversionUtils; @@ -36,93 +40,103 @@ * Allows for retrieval and update of configurations, particularly global configurations. */ public class Configurations implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private List validations = new ArrayList<>(); - protected Map configurations = new ConcurrentHashMap<>(); - - public Map getConfigurations() { - return configurations; - } - - @SuppressWarnings("unchecked") - public Map getGlobalConfig() { - return getGlobalConfig(true); - } - - public Map getGlobalConfig(boolean emptyMapOnNonExistent) { - return (Map) getConfigurations().getOrDefault(ConfigurationType.GLOBAL.getTypeName(), emptyMapOnNonExistent?new HashMap():null); - } - - public List getFieldValidations() { - return validations; - } - - public void updateGlobalConfig(byte[] data) throws IOException { - if (data == null) throw new IllegalStateException("global config data cannot be null"); - updateGlobalConfig(new ByteArrayInputStream(data)); - } - - public void updateGlobalConfig(InputStream io) throws IOException { - Map globalConfig = JSONUtils.INSTANCE.load(io, JSONUtils.MAP_SUPPLIER); - updateGlobalConfig(globalConfig); - } - - /** - * Updates the global config from a provided map. - * - * @param globalConfig The map to update the config to - */ - public void updateGlobalConfig(Map globalConfig) { - if(globalConfig != null) { - getConfigurations().put(ConfigurationType.GLOBAL.getTypeName(), globalConfig); - validations = FieldValidator.readValidations(getGlobalConfig()); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private List validations = new ArrayList<>(); + protected Map configurations = new ConcurrentHashMap<>(); + + public Map getConfigurations() { + return configurations; + } + + @SuppressWarnings("unchecked") + public Map getGlobalConfig() { + return getGlobalConfig(true); + } + + public Map getGlobalConfig(boolean emptyMapOnNonExistent) { + return (Map) getConfigurations().getOrDefault(ConfigurationType.GLOBAL.getTypeName(), + emptyMapOnNonExistent ? new HashMap() : null); + } + + public List getFieldValidations() { + return validations; + } + + public void updateGlobalConfig(byte[] data) throws IOException { + if (data == null) { + throw new IllegalStateException("global config data cannot be null"); + } + updateGlobalConfig(new ByteArrayInputStream(data)); + } + + public void updateGlobalConfig(InputStream io) throws IOException { + Map globalConfig = JSONUtils.INSTANCE.load(io, JSONUtils.MAP_SUPPLIER); + updateGlobalConfig(globalConfig); + } + + /** + * Updates the global config from a provided map. + * + * @param globalConfig The map to update the config to + */ + public void updateGlobalConfig(Map globalConfig) { + if (globalConfig != null) { + getConfigurations().put(ConfigurationType.GLOBAL.getTypeName(), globalConfig); + validations = FieldValidator.readValidations(getGlobalConfig()); + } + } + + public void deleteGlobalConfig() { + getConfigurations().remove(ConfigurationType.GLOBAL.getTypeName()); + } + + /** + * Retrieves a key from a map, casts it to a provided class. If there is no entry for the key, + * a default is returned. + * + * @param key The key to retrieve from the map + * @param map The map to retrieve the key from + * @param defaultValue The default value to return if no value is found + * @param clazz The class to cast the result to + * @param The type value of the class being casted to + * @return The casted value if found, the default value otherwise + */ + public static T getAs(String key, Map map, T defaultValue, Class clazz) { + return map == null ? defaultValue + : ConversionUtils.convert(map.getOrDefault(key, defaultValue), clazz); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Configurations that = (Configurations) o; + + if (!Objects.equals(validations, that.validations)) { + return false; + } + return getConfigurations() != null ? getConfigurations().equals(that.getConfigurations()) + : that.getConfigurations() == null; + + } + + @Override + public int hashCode() { + int result = validations != null ? validations.hashCode() : 0; + result = 31 * result + (getConfigurations() != null ? getConfigurations().hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "Configurations{" + + "validations=" + validations + + ", configurations=" + getConfigurations() + + '}'; } - } - - public void deleteGlobalConfig() { - getConfigurations().remove(ConfigurationType.GLOBAL.getTypeName()); - } - - /** - * Retrieves a key from a map, casts it to a provided class. If there is no entry for the key, - * a default is returned. - * - * @param key The key to retrieve from the map - * @param map The map to retrieve the key from - * @param defaultValue The default value to return if no value is found - * @param clazz The class to cast the result to - * @param The type value of the class being casted to - * @return The casted value if found, the default value otherwise - */ - public static T getAs(String key, Map map, T defaultValue, Class clazz) { - return map == null ? defaultValue - : ConversionUtils.convert(map.getOrDefault(key, defaultValue), clazz); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Configurations that = (Configurations) o; - - if (validations != null ? !validations.equals(that.validations) : that.validations != null) return false; - return getConfigurations() != null ? getConfigurations().equals(that.getConfigurations()) : that.getConfigurations() == null; - - } - - @Override - public int hashCode() { - int result = validations != null ? validations.hashCode() : 0; - result = 31 * result + (getConfigurations() != null ? getConfigurations().hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "Configurations{" + - "validations=" + validations + - ", configurations=" + getConfigurations()+ - '}'; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java index 3ce32f58..d2e43924 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

    * http://www.apache.org/licenses/LICENSE-2.0 + * *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,14 +17,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration; -import org.apache.commons.io.FilenameUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.StellarFunctions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; +import static org.apache.metron.common.configuration.ConfigurationType.GLOBAL; +import static org.apache.metron.common.configuration.ConfigurationType.INDEXING; +import static org.apache.metron.common.configuration.ConfigurationType.PARSER; import java.io.File; import java.io.IOException; @@ -31,8 +32,12 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; - -import static org.apache.metron.common.configuration.ConfigurationType.*; +import org.apache.commons.io.FilenameUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A Utility class for managing various configs, including global configs, sensor specific configs, @@ -49,20 +54,20 @@ public class ConfigurationsUtils { * @param globalConfig Optional global configuration to be used */ public static void setupStellarStatically(CuratorFramework client, Optional globalConfig) { - /* - In order to validate stellar functions, the function resolver must be initialized. Otherwise, - those utilities that require validation cannot validate the stellar expressions necessarily. - */ + /* + In order to validate stellar functions, the function resolver must be initialized. Otherwise, + those utilities that require validation cannot validate the stellar expressions necessarily. + */ Context.Builder builder = new Context.Builder() - .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client); + .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client); if (globalConfig.isPresent()) { builder = builder - .with(Context.Capabilities.GLOBAL_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())) - .with(Context.Capabilities.STELLAR_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())); + .with(Context.Capabilities.GLOBAL_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())) + .with(Context.Capabilities.STELLAR_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())); } else { builder = builder - .with(Context.Capabilities.STELLAR_CONFIG, () -> new HashMap<>()); + .with(Context.Capabilities.STELLAR_CONFIG, () -> new HashMap<>()); } Context stellarContext = builder.build(); StellarFunctions.FUNCTION_RESOLVER().initialize(stellarContext); @@ -125,7 +130,8 @@ public static Map readSensorIndexingConfigsFromFile(String rootP * @return map of file names to the contents of that file as a byte array * @throws IOException If there's an issue reading the configs */ - public static Map readSensorConfigsFromFile(String rootPath, ConfigurationType configType) throws IOException { + public static Map readSensorConfigsFromFile(String rootPath, ConfigurationType configType) + throws IOException { return readSensorConfigsFromFile(rootPath, configType, Optional.empty()); } @@ -141,7 +147,8 @@ public static Map readSensorConfigsFromFile(String rootPath, Con * @throws IOException If there's an issue reading the configs */ public static Map readSensorConfigsFromFile(String rootPath, - ConfigurationType configType, Optional configName) throws IOException { + ConfigurationType configType, + Optional configName) throws IOException { Map sensorConfigs = new HashMap<>(); File configPath = new File(rootPath, configType.getDirectory()); if (configPath.exists() && configPath.isDirectory()) { @@ -149,13 +156,13 @@ public static Map readSensorConfigsFromFile(String rootPath, if (!configName.isPresent()) { for (File file : children) { sensorConfigs.put(FilenameUtils.removeExtension(file.getName()), - Files.readAllBytes(file.toPath())); + Files.readAllBytes(file.toPath())); } } else { for (File file : children) { if (FilenameUtils.removeExtension(file.getName()).equals(configName.get())) { sensorConfigs.put(FilenameUtils.removeExtension(file.getName()), - Files.readAllBytes(file.toPath())); + Files.readAllBytes(file.toPath())); } } if (sensorConfigs.isEmpty()) { @@ -175,7 +182,8 @@ public static Map readSensorConfigsFromFile(String rootPath, * @param defaultFieldName The default to use if config is null or key not found * @return The config value or the default if config is null or key not found */ - public static String getFieldName(Map globalConfig, String globalConfigKey, String defaultFieldName) { + public static String getFieldName(Map globalConfig, String globalConfigKey, + String defaultFieldName) { if (globalConfig == null) { return defaultFieldName; } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurationOperations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurationOperations.java index 96211bec..b29593de 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurationOperations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurationOperations.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,19 +27,19 @@ public class EnrichmentConfigurationOperations implements ConfigurationOperations { - @Override - public String getTypeName() { - return "enrichments"; - } + @Override + public String getTypeName() { + return "enrichments"; + } - @Override - public Object deserialize(String s) throws IOException { - return JSONUtils.INSTANCE.load(s, SensorEnrichmentConfig.class); - } + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, SensorEnrichmentConfig.class); + } - @Override - public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, - CuratorFramework client) throws Exception { - } + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java index 5325b5f0..88aae6b3 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java @@ -7,99 +7,101 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.configuration; -import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; -import org.apache.metron.common.utils.JSONUtils; +package org.apache.metron.common.configuration; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.utils.JSONUtils; /** * Allows for retrieval and update of enrichment configurations. Some fields are pulled from * global config and provided here for convenience. */ public class EnrichmentConfigurations extends Configurations { - // Writer batch params - public static final Integer DEFAULT_KAFKA_BATCH_SIZE = 15; - public static final String BATCH_SIZE_CONF = "enrichment.writer.batchSize"; - public static final String BATCH_TIMEOUT_CONF = "enrichment.writer.batchTimeout"; - // Enrichment list table params - assumes HBase implementation - public static final String TABLE_PROVIDER = "enrichment.list.hbase.provider.impl"; - public static final String TABLE_NAME = "enrichment.list.hbase.table"; - public static final String COLUMN_FAMILY = "enrichment.list.hbase.cf"; + // Writer batch params + public static final Integer DEFAULT_KAFKA_BATCH_SIZE = 15; + public static final String BATCH_SIZE_CONF = "enrichment.writer.batchSize"; + public static final String BATCH_TIMEOUT_CONF = "enrichment.writer.batchTimeout"; + // Enrichment list table params - assumes HBase implementation + public static final String TABLE_PROVIDER = "enrichment.list.hbase.provider.impl"; + public static final String TABLE_NAME = "enrichment.list.hbase.table"; + public static final String COLUMN_FAMILY = "enrichment.list.hbase.cf"; - public SensorEnrichmentConfig getSensorEnrichmentConfig(String sensorType) { - return (SensorEnrichmentConfig) getConfigurations().get(getKey(sensorType)); - } + public SensorEnrichmentConfig getSensorEnrichmentConfig(String sensorType) { + return (SensorEnrichmentConfig) getConfigurations().get(getKey(sensorType)); + } - public void updateSensorEnrichmentConfig(String sensorType, byte[] data) throws IOException { - updateSensorEnrichmentConfig(sensorType, new ByteArrayInputStream(data)); - } + public void updateSensorEnrichmentConfig(String sensorType, byte[] data) throws IOException { + updateSensorEnrichmentConfig(sensorType, new ByteArrayInputStream(data)); + } - public void updateSensorEnrichmentConfig(String sensorType, InputStream io) throws IOException { - SensorEnrichmentConfig sensorEnrichmentConfig = JSONUtils.INSTANCE.load(io, SensorEnrichmentConfig.class); - updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfig); - } + public void updateSensorEnrichmentConfig(String sensorType, InputStream io) throws IOException { + SensorEnrichmentConfig sensorEnrichmentConfig = JSONUtils.INSTANCE.load(io, SensorEnrichmentConfig.class); + updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfig); + } - public void updateSensorEnrichmentConfig(String sensorType, SensorEnrichmentConfig sensorEnrichmentConfig) { - getConfigurations().put(getKey(sensorType), sensorEnrichmentConfig); - } + public void updateSensorEnrichmentConfig(String sensorType, SensorEnrichmentConfig sensorEnrichmentConfig) { + getConfigurations().put(getKey(sensorType), sensorEnrichmentConfig); + } - public void delete(String sensorType) { - getConfigurations().remove(getKey(sensorType)); - } + public void delete(String sensorType) { + getConfigurations().remove(getKey(sensorType)); + } - /** - * Pulled from global config. - * Note: enrichment writes out to 1 kafka topic, so it is not pulling this config by sensor. - * - * @return batch size for writing to kafka - * @see org.apache.metron.common.configuration.EnrichmentConfigurations#BATCH_SIZE_CONF - */ - public int getBatchSize() { - return getAs(BATCH_SIZE_CONF, getGlobalConfig(true), DEFAULT_KAFKA_BATCH_SIZE, Integer.class); - } + /** + * Pulled from global config. + * Note: enrichment writes out to 1 kafka topic, so it is not pulling this config by sensor. + * + * @return batch size for writing to kafka + * @see org.apache.metron.common.configuration.EnrichmentConfigurations#BATCH_SIZE_CONF + */ + public int getBatchSize() { + return getAs(BATCH_SIZE_CONF, getGlobalConfig(true), DEFAULT_KAFKA_BATCH_SIZE, Integer.class); + } - /** - * Pulled from global config - * Note: enrichment writes out to 1 kafka topic, so it is not pulling this config by sensor. - * - * @return batch timeout for writing to kafka - * @see org.apache.metron.common.configuration.EnrichmentConfigurations#BATCH_TIMEOUT_CONF - */ - public int getBatchTimeout() { - return getAs(BATCH_TIMEOUT_CONF, getGlobalConfig(true), 0, Integer.class); - } + /** + * Pulled from global config + * Note: enrichment writes out to 1 kafka topic, so it is not pulling this config by sensor. + * + * @return batch timeout for writing to kafka + * @see org.apache.metron.common.configuration.EnrichmentConfigurations#BATCH_TIMEOUT_CONF + */ + public int getBatchTimeout() { + return getAs(BATCH_TIMEOUT_CONF, getGlobalConfig(true), 0, Integer.class); + } - /** - * Gets the sensor names that have associated enrichments. - * - * @return List of sensor names - */ - public List getTypes() { - List ret = new ArrayList<>(); - for(String keyedSensor : getConfigurations().keySet()) { - if(!keyedSensor.isEmpty() && keyedSensor.startsWith(ConfigurationType.ENRICHMENT.getTypeName())) { - ret.add(keyedSensor.substring(ConfigurationType.ENRICHMENT.getTypeName().length() + 1)); - } + /** + * Gets the sensor names that have associated enrichments. + * + * @return List of sensor names + */ + public List getTypes() { + List ret = new ArrayList<>(); + for (String keyedSensor : getConfigurations().keySet()) { + if (!keyedSensor.isEmpty() && keyedSensor.startsWith(ConfigurationType.ENRICHMENT.getTypeName())) { + ret.add(keyedSensor.substring(ConfigurationType.ENRICHMENT.getTypeName().length() + 1)); + } + } + return ret; } - return ret; - } - public static String getKey(String sensorType) { - return ConfigurationType.ENRICHMENT.getTypeName() + "." + sensorType; - } + public static String getKey(String sensorType) { + return ConfigurationType.ENRICHMENT.getTypeName() + "." + sensorType; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java index 2c592d03..895ae111 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,170 +37,178 @@ * Performs an {@link FieldTransformation} on Json. */ public class FieldTransformer implements Serializable { - private List input = new ArrayList<>(); - private List output; - private FieldTransformation transformation; - private String transformationName; - private LinkedHashMap config = new LinkedHashMap<>(); - private boolean initialized = false; - public FieldTransformer() { - } - - public List getInput() { - return input; - } - - /** - * Setter for input fields. Will handle either a plain string or a list of strings. - * - * @param inputFields The input fields to the transformation. Can be either a string or list of - * strings. - */ - public void setInput(Object inputFields) { - if(inputFields instanceof String) { - this.input= ImmutableList.of(inputFields.toString()); - } - else if(inputFields instanceof List) { - this.input= (List)inputFields; - } - } - - public List getOutput() { - return output; - } - - /** - * Setter for output fields. Will handle either a plain string or a list of strings. - * - * @param outputField The output fields of the transformation. Can be either a string or list of - * strings. - */ - public void setOutput(Object outputField) { - if(outputField instanceof String) { - this.output = ImmutableList.of(outputField.toString()); - } - else if(outputField instanceof List) { - this.output = (List)outputField; - } - } - - public Map getConfig() { - return config; - } - - public void setConfig(LinkedHashMap config) { - this.config = config; - } - - public String getTransformation() { - return transformationName; - } - - @JsonIgnore - public FieldTransformation getFieldTransformation() { - return transformation; - } - - public void setTransformation(String transformation) { - this.transformationName = transformation; - this.transformation = FieldTransformations.get(transformation); - } - - /** - * Initializes the FieldTransformer and does some basic checking of input and output fields. - */ - public void initAndValidate() { - if(!initialized) { - if (getTransformation() == null) { - throw new IllegalStateException("Mapping cannot be null."); - } - - if (output == null || output.isEmpty()) { - if (input == null || input.isEmpty()) { - //both are empty, so let's set them both to null - output = null; - input = null; + private List input = new ArrayList<>(); + private List output; + private FieldTransformation transformation; + private String transformationName; + private LinkedHashMap config = new LinkedHashMap<>(); + private boolean initialized = false; + + public FieldTransformer() { + } + + public List getInput() { + return input; + } + + /** + * Setter for input fields. Will handle either a plain string or a list of strings. + * + * @param inputFields The input fields to the transformation. Can be either a string or list of + * strings. + */ + public void setInput(Object inputFields) { + if (inputFields instanceof String) { + this.input = ImmutableList.of(inputFields.toString()); + } else if (inputFields instanceof List) { + this.input = (List) inputFields; + } + } + + public List getOutput() { + return output; + } + + /** + * Setter for output fields. Will handle either a plain string or a list of strings. + * + * @param outputField The output fields of the transformation. Can be either a string or list of + * strings. + */ + public void setOutput(Object outputField) { + if (outputField instanceof String) { + this.output = ImmutableList.of(outputField.toString()); + } else if (outputField instanceof List) { + this.output = (List) outputField; + } + } + + public Map getConfig() { + return config; + } + + public void setConfig(LinkedHashMap config) { + this.config = config; + } + + public String getTransformation() { + return transformationName; + } + + @JsonIgnore + public FieldTransformation getFieldTransformation() { + return transformation; + } + + public void setTransformation(String transformation) { + this.transformationName = transformation; + this.transformation = FieldTransformations.get(transformation); + } + + /** + * Initializes the FieldTransformer and does some basic checking of input and output fields. + */ + public void initAndValidate() { + if (!initialized) { + if (getTransformation() == null) { + throw new IllegalStateException("Mapping cannot be null."); + } + + if (output == null || output.isEmpty()) { + if (input == null || input.isEmpty()) { + //both are empty, so let's set them both to null + output = null; + input = null; + } else { + output = input; + } + } + initialized = true; + } + } + + /** + * Performs the actual transformation on Json input. Returns a map of fields to values. + * + * @param input The input Json to be transformed + * @param context The Stellar context of the transformation + * @param sensorConfig Map of the sensor config + * @return A map of field -> value + */ + public Map transform(JSONObject input, Context context, Map... sensorConfig) { + if (getInput() == null || getInput().isEmpty()) { + return transformation.map(input, getOutput(), config, context, sensorConfig); } else { - output = input; + Map in = new HashMap<>(); + for (String inputField : getInput()) { + in.put(inputField, input.get(inputField)); + } + return transformation.map(in, getOutput(), config, context, sensorConfig); + } + } + + /** + * Performs the actual transformation on Json input in place, removing or adding keys as + * necessary. + * + * @param message The input Json to be transformed + * @param context The Stellar context of the transformation + * @param sensorConfig Map of the sensor config + */ + public void transformAndUpdate(JSONObject message, Context context, Map... sensorConfig) { + Map currentValue = transform(message, context, sensorConfig); + if (currentValue != null) { + for (Map.Entry kv : currentValue.entrySet()) { + if (kv.getValue() == null) { + message.remove(kv.getKey()); + } else { + message.put(kv.getKey(), kv.getValue()); + } + } + } + } + + @Override + public String toString() { + return "MappingHandler{" + + "input=" + input + + ", output='" + output + '\'' + + ", transformation=" + transformation + + ", config=" + config + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + FieldTransformer that = (FieldTransformer) o; + + if (getInput() != null ? !getInput().equals(that.getInput()) : that.getInput() != null) { + return false; } - } - initialized = true; - } - } - - /** - * Performs the actual transformation on Json input. Returns a map of fields to values. - * - * @param input The input Json to be transformed - * @param context The Stellar context of the transformation - * @param sensorConfig Map of the sensor config - * @return A map of field -> value - */ - public Map transform(JSONObject input, Context context, Map... sensorConfig) { - if(getInput() == null || getInput().isEmpty()) { - return transformation.map(input, getOutput(), config, context, sensorConfig); - } - else { - Map in = new HashMap<>(); - for(String inputField : getInput()) { - in.put(inputField, input.get(inputField)); - } - return transformation.map(in, getOutput(), config, context, sensorConfig); - } - } - - /** - * Performs the actual transformation on Json input in place, removing or adding keys as - * necessary. - * - * @param message The input Json to be transformed - * @param context The Stellar context of the transformation - * @param sensorConfig Map of the sensor config - */ - public void transformAndUpdate(JSONObject message, Context context, Map... sensorConfig) { - Map currentValue = transform(message, context, sensorConfig); - if(currentValue != null) { - for(Map.Entry kv : currentValue.entrySet()) { - if(kv.getValue() == null) { - message.remove(kv.getKey()); + if (getOutput() != null ? !getOutput().equals(that.getOutput()) : that.getOutput() != null) { + return false; } - else { - message.put(kv.getKey(), kv.getValue()); + if (getTransformation() != null ? !getTransformation().equals(that.getTransformation()) + : that.getTransformation() != null) { + return false; } - } - } - } - - @Override - public String toString() { - return "MappingHandler{" + - "input=" + input + - ", output='" + output + '\'' + - ", transformation=" + transformation + - ", config=" + config + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - FieldTransformer that = (FieldTransformer) o; - - if (getInput() != null ? !getInput().equals(that.getInput()) : that.getInput() != null) return false; - if (getOutput() != null ? !getOutput().equals(that.getOutput()) : that.getOutput() != null) return false; - if (getTransformation() != null ? !getTransformation().equals(that.getTransformation()) : that.getTransformation() != null) return false; - return getConfig() != null ? getConfig().equals(that.getConfig()) : that.getConfig() == null; - - } - - @Override - public int hashCode() { - int result = getInput() != null ? getInput().hashCode() : 0; - result = 31 * result + (getOutput() != null ? getOutput().hashCode() : 0); - result = 31 * result + (getTransformation() != null ? getTransformation().hashCode() : 0); - result = 31 * result + (getConfig() != null ? getConfig().hashCode() : 0); - return result; - } + return getConfig() != null ? getConfig().equals(that.getConfig()) : that.getConfig() == null; + + } + + @Override + public int hashCode() { + int result = getInput() != null ? getInput().hashCode() : 0; + result = 31 * result + (getOutput() != null ? getOutput().hashCode() : 0); + result = 31 * result + (getTransformation() != null ? getTransformation().hashCode() : 0); + result = 31 * result + (getConfig() != null ? getConfig().hashCode() : 0); + return result; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java index 28b5bcff..f7e4702e 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/FieldValidator.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,154 +37,157 @@ */ public class FieldValidator implements Serializable { - public enum Config { - FIELD_VALIDATIONS("fieldValidations") - ,VALIDATION("validation") - ,INPUT("input") - ,CONFIG("config") - ; - String key; - Config(String key) { - this.key = key; + public enum Config { + FIELD_VALIDATIONS("fieldValidations"), VALIDATION("validation"), INPUT("input"), CONFIG("config"); + String key; + + Config(String key) { + this.key = key; + } + + /** + * Retrieves the value of the key from the provided config, and casts it to the provided class. + * + * @param config The config to retrieve the value associated with the key from + * @param clazz The class to cast to + * @param The type parameter of the class to cast to + * @return The value, casted appropriately + */ + public T get(Map config, Class clazz) { + Object o = config.get(key); + if (o == null) { + return null; + } + return clazz.cast(o); + } } + private final FieldValidation validation; + private final List input; + private Map config; + /** - * Retrieves the value of the key from the provided config, and casts it to the provided class. + * Constructor for a FieldValidator. + * + * @param o Should be a map, otherwise exception is thrown. From the map, retrieve the various + * necessary components, e.g. input, and ensure they are read appropriately. * - * @param config The config to retrieve the value associated with the key from - * @param clazz The class to cast to - * @param The type parameter of the class to cast to - * @return The value, casted appropriately */ - public T get(Map config, Class clazz) { - Object o = config.get(key); - if(o == null) { - return null; - } - return clazz.cast(o); + public FieldValidator(Object o) { + if (o instanceof Map) { + Map validatorConfig = (Map) o; + Object inputObj = Config.INPUT.get(validatorConfig, Object.class); + if (inputObj instanceof String) { + input = ImmutableList.of(inputObj.toString()); + } else if (inputObj instanceof List) { + input = new ArrayList<>(); + for (Object inputO : (List) inputObj) { + input.add(inputO.toString()); + } + } else { + input = new ArrayList<>(); + } + config = Config.CONFIG.get(validatorConfig, Map.class); + if (config == null) { + config = new HashMap<>(); + } + String validator = Config.VALIDATION.get(validatorConfig, String.class); + if (validator == null) { + throw new IllegalStateException("Validation not set."); + } + validation = FieldValidations.get(validator); + } else { + throw new IllegalStateException("Unable to configure field validations"); + } + } + + public FieldValidation getValidation() { + return validation; } - } - private FieldValidation validation; - private List input; - private Map config; - - /** - * Constructor for a FieldValidator. - * - * @param o Should be a map, otherwise exception is thrown. From the map, retrieve the various - * necessary components, e.g. input, and ensure they are read appropriately. - * - */ - public FieldValidator(Object o) { - if(o instanceof Map) { - Map validatorConfig = (Map) o; - Object inputObj = Config.INPUT.get(validatorConfig, Object.class); - if(inputObj instanceof String) { - input = ImmutableList.of(inputObj.toString()); - } - else if(inputObj instanceof List) { - input = new ArrayList<>(); - for(Object inputO : (List)inputObj) { - input.add(inputO.toString()); + + public List getInput() { + return input; + } + + public Map getConfig() { + return config; + } + + /** + * Runs a validation against Json input data. + * + * @param inputData The Json data being validated + * @param globalConfig The global config feeding the validation + * @param context The Stellar context of the validation + * @return true if valid, false otherwise + */ + public boolean isValid(JSONObject inputData, Map globalConfig, Context context) { + Map in = inputData; + if (input != null && !input.isEmpty()) { + in = new HashMap<>(); + for (String i : input) { + Object o = inputData.get(i); + in.put(i, o); + } } - } else { - input = new ArrayList<>(); - } - config = Config.CONFIG.get(validatorConfig, Map.class); - if(config == null) { - config = new HashMap<>(); - } - String validator = Config.VALIDATION.get(validatorConfig, String.class); - if(validator == null) { - throw new IllegalStateException("Validation not set."); - } - validation= FieldValidations.get(validator); + return validation.isValid(in, config, globalConfig, context); } - else { - throw new IllegalStateException("Unable to configure field validations"); + + /** + * Reads the validations from the global config and returns a list of them. + * + * @param globalConfig The config to read field validations from + * @return A list of validations + */ + public static List readValidations(Map globalConfig) { + List validators = new ArrayList<>(); + List validations = (List) Config.FIELD_VALIDATIONS.get(globalConfig, List.class); + if (validations != null) { + for (Object o : validations) { + FieldValidator f = new FieldValidator(o); + f.getValidation().initialize(f.getConfig(), globalConfig); + validators.add(new FieldValidator(o)); + } + } + return validators; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + FieldValidator that = (FieldValidator) o; + + if (getValidation() != null ? !getValidation().equals(that.getValidation()) : that.getValidation() != null) { + return false; + } + if (getInput() != null ? !getInput().equals(that.getInput()) : that.getInput() != null) { + return false; + } + return getConfig() != null ? getConfig().equals(that.getConfig()) : that.getConfig() == null; + } - } - - public FieldValidation getValidation() { - return validation; - } - - public List getInput() { - return input; - } - - public Map getConfig() { - return config; - } - - /** - * Runs a validation against Json input data. - * - * @param inputData The Json data being validated - * @param globalConfig The global config feeding the validation - * @param context The Stellar context of the validation - * @return true if valid, false otherwise - */ - public boolean isValid(JSONObject inputData, Map globalConfig, Context context) { - Map in = inputData; - if(input != null && !input.isEmpty()) { - in = new HashMap<>(); - for(String i : input) { - Object o = inputData.get(i); - in.put(i,o); - } + + @Override + public int hashCode() { + int result = getValidation() != null ? getValidation().hashCode() : 0; + result = 31 * result + (getInput() != null ? getInput().hashCode() : 0); + result = 31 * result + (getConfig() != null ? getConfig().hashCode() : 0); + return result; } - return validation.isValid(in, config, globalConfig, context); - } - - /** - * Reads the validations from the global config and returns a list of them. - * - * @param globalConfig The config to read field validations from - * @return A list of validations - */ - public static List readValidations(Map globalConfig) { - List validators = new ArrayList<>(); - List validations = (List) Config.FIELD_VALIDATIONS.get(globalConfig, List.class); - if(validations != null) { - for (Object o : validations) { - FieldValidator f = new FieldValidator(o); - f.getValidation().initialize(f.getConfig(), globalConfig); - validators.add(new FieldValidator(o)); - } + + @Override + public String toString() { + return "FieldValidator{" + + "validation=" + validation + + ", input=" + input + + ", config=" + config + + '}'; } - return validators; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - FieldValidator that = (FieldValidator) o; - - if (getValidation() != null ? !getValidation().equals(that.getValidation()) : that.getValidation() != null) - return false; - if (getInput() != null ? !getInput().equals(that.getInput()) : that.getInput() != null) return false; - return getConfig() != null ? getConfig().equals(that.getConfig()) : that.getConfig() == null; - - } - - @Override - public int hashCode() { - int result = getValidation() != null ? getValidation().hashCode() : 0; - result = 31 * result + (getInput() != null ? getInput().hashCode() : 0); - result = 31 * result + (getConfig() != null ? getConfig().hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "FieldValidator{" + - "validation=" + validation + - ", input=" + input + - ", config=" + config + - '}'; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java index 581ec3ef..3f41c161 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,31 +21,30 @@ package org.apache.metron.common.configuration; import java.io.IOException; -import java.util.Map; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.utils.JSONUtils; public class GlobalConfigurationOperations implements ConfigurationOperations { - @Override - public String getTypeName() { - return "global"; - } - - @Override - public String getDirectory() { - return "."; - } - - @Override - public Object deserialize(String s) throws IOException { - return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER); - } - - @Override - public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, - CuratorFramework client) { - throw new UnsupportedOperationException("Global configs are not per-sensor"); - } + @Override + public String getTypeName() { + return "global"; + } + + @Override + public String getDirectory() { + return "."; + } + + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER); + } + + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) { + throw new UnsupportedOperationException("Global configs are not per-sensor"); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java index 3f6d5320..009a447d 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,26 +20,25 @@ package org.apache.metron.common.configuration; +import java.io.IOException; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.utils.JSONUtils; -import java.io.IOException; - public class IndexingConfigurationOperations implements ConfigurationOperations { - @Override - public String getTypeName() { - return "indexing"; - } + @Override + public String getTypeName() { + return "indexing"; + } - @Override - public Object deserialize(String s) throws IOException { - return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER); - } + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER); + } - @Override - public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, - CuratorFramework client) throws Exception { - } + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java index 5001767c..65f03007 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration; import java.io.ByteArrayInputStream; @@ -30,323 +33,322 @@ * Allows for retrieval and update of indexing configurations. */ public class IndexingConfigurations extends Configurations { - public static final String BATCH_SIZE_CONF = "batchSize"; - public static final String BATCH_TIMEOUT_CONF = "batchTimeout"; - public static final String ENABLED_CONF = "enabled"; - public static final String INDEX_CONF = "index"; - public static final String OUTPUT_PATH_FUNCTION_CONF = "outputPathFunction"; - public static final String FIELD_NAME_CONVERTER_CONF = "fieldNameConverter"; - public static final String SET_DOCUMENT_ID_CONF = "setDocumentId"; - public static final String GLOBAL_ELASTICSEARCH_SET_DOCUMENT_ID_CONF = "indexing.writer.elasticsearch.setDocumentId"; - - /** - * Gets the indexing config for a specific sensor. - * - * @param sensorType The sensor to retrieve config for - * @param emptyMapOnNonExistent If true and the config doesn't exist return empty map, else null - * @return Map of the config key -> value. Value on missing depends on emptyMapOnNonExistent - */ - public Map getSensorIndexingConfig(String sensorType, boolean emptyMapOnNonExistent) { - Map ret = (Map) getConfigurations().get(getKey(sensorType)); - if(ret == null) { - return emptyMapOnNonExistent?new HashMap<>():null; + public static final String BATCH_SIZE_CONF = "batchSize"; + public static final String BATCH_TIMEOUT_CONF = "batchTimeout"; + public static final String ENABLED_CONF = "enabled"; + public static final String INDEX_CONF = "index"; + public static final String OUTPUT_PATH_FUNCTION_CONF = "outputPathFunction"; + public static final String FIELD_NAME_CONVERTER_CONF = "fieldNameConverter"; + public static final String SET_DOCUMENT_ID_CONF = "setDocumentId"; + public static final String GLOBAL_ELASTICSEARCH_SET_DOCUMENT_ID_CONF = + "indexing.writer.elasticsearch.setDocumentId"; + + /** + * Gets the indexing config for a specific sensor. + * + * @param sensorType The sensor to retrieve config for + * @param emptyMapOnNonExistent If true and the config doesn't exist return empty map, else null + * @return Map of the config key -> value. Value on missing depends on emptyMapOnNonExistent + */ + public Map getSensorIndexingConfig(String sensorType, boolean emptyMapOnNonExistent) { + Map ret = (Map) getConfigurations().get(getKey(sensorType)); + if (ret == null) { + return emptyMapOnNonExistent ? new HashMap<>() : null; + } else { + return ret; + } + } + + public Map getSensorIndexingConfig(String sensorType) { + return getSensorIndexingConfig(sensorType, true); + } + + /** + * Gets the sensor indexing config for a given writer. + * + * @param sensorType The sensor to retrieve configs for + * @param writerName The particular writer to get configurations for + * @return A Map of the configuration + */ + public Map getSensorIndexingConfig(String sensorType, String writerName) { + String key = getKey(sensorType); + Map ret = (Map) getConfigurations().get(key); + if (ret == null) { + return new HashMap(); + } else { + Map writerConfig = (Map) ret.get(writerName); + return writerConfig != null ? writerConfig : new HashMap<>(); + } + } + + /** + * Gets the list of sensor types that indexing configurations exist for. + * + * @return List of sensor types + */ + public List getTypes() { + List ret = new ArrayList<>(); + for (String keyedSensor : getConfigurations().keySet()) { + if (!keyedSensor.isEmpty() && keyedSensor.startsWith(ConfigurationType.INDEXING.getTypeName())) { + ret.add(keyedSensor.substring(ConfigurationType.INDEXING.getTypeName().length() + 1)); + } + } + return ret; + } + + public void delete(String sensorType) { + getConfigurations().remove(getKey(sensorType)); + } + + public void updateSensorIndexingConfig(String sensorType, byte[] data) throws IOException { + updateSensorIndexingConfig(sensorType, new ByteArrayInputStream(data)); + } + + public void updateSensorIndexingConfig(String sensorType, InputStream io) throws IOException { + Map sensorIndexingConfig = JSONUtils.INSTANCE.load(io, JSONUtils.MAP_SUPPLIER); + updateSensorIndexingConfig(sensorType, sensorIndexingConfig); + } + + public void updateSensorIndexingConfig(String sensorType, Map sensorIndexingConfig) { + getConfigurations().put(getKey(sensorType), sensorIndexingConfig); + } + + public static String getKey(String sensorType) { + return ConfigurationType.INDEXING.getTypeName() + "." + sensorType; + } + + /** + * Determines if a configuration is default or not. In particular, this means the config is null + * for the sensor/writer combo. + * + * @param sensorName The sensor to check for default + * @param writerName The specific writer to check for default + * @return True if default, false otherwise. + */ + public boolean isDefault(String sensorName, String writerName) { + Map ret = (Map) getConfigurations().get(getKey(sensorName)); + if (ret == null) { + return true; + } else { + Map writerConfig = (Map) ret.get(writerName); + return writerConfig == null; + } + } + + public int getBatchSize(String sensorName, String writerName) { + return getBatchSize(getSensorIndexingConfig(sensorName, writerName)); + } + + /** + * Retrieves the batch size value from the config. + * + * @param conf The configuration to retrieve from + * @return The batch size if defined, 1 by default + */ + public static int getBatchSize(Map conf) { + return getAs(BATCH_SIZE_CONF, + conf, + 1, + Integer.class + ); + } + + public int getBatchTimeout(String sensorName, String writerName) { + return getBatchTimeout(getSensorIndexingConfig(sensorName, writerName)); + } + + /** + * Retrieves the batch timeout value from the config. + * + * @param conf The configuration to retrieve from + * @return The batch timeout if defined, 0 by default + */ + public static int getBatchTimeout(Map conf) { + return getAs(BATCH_TIMEOUT_CONF, + conf, + 0, + Integer.class + ); + } + + /** + * Returns all configured values of batchTimeout, for all configured sensors, + * but only for the specific writer identified by {@code writerName}. So, if it is + * an hdfs writer, it will return the batchTimeouts for hdfs writers for all the sensors. + * The goal is to return to a {@link org.apache.metron.common.bolt.ConfiguredBolt} + * the set of all and only batchTimeouts relevant to that ConfiguredBolt. + * + * @param writerName The name of the writer to look up. + * @return list of integer batchTimeouts, one per configured sensor + */ + public List getAllConfiguredTimeouts(String writerName) { + // The configuration infrastructure was not designed to enumerate sensors, so we synthesize. + // Since getKey is in this same class, we know we can pass it a null string to get the key prefix + // for all sensor types within this capability. We then enumerate all keys in configurations.keySet + // and select those that match the key prefix, as being sensor keys. The suffix substring of + // each such key is used as a sensor name to query the batchTimeout settings, if any. + String keyPrefixString = getKey(""); + int prefixStringLength = keyPrefixString.length(); + List configuredBatchTimeouts = new ArrayList<>(); + for (String sensorKeyString : getConfigurations().keySet()) { + if (sensorKeyString.startsWith(keyPrefixString)) { + String configuredSensorName = sensorKeyString.substring(prefixStringLength); + configuredBatchTimeouts.add(getBatchTimeout(configuredSensorName, writerName)); + } + } + return configuredBatchTimeouts; + } + + public String getIndex(String sensorName, String writerName) { + return getIndex(getSensorIndexingConfig(sensorName, writerName), sensorName); + } + + /** + * Retrieves the index value from the config. + * + * @param conf The configuration to retrieve from + * @param sensorName The name of the sensor to retrieve the index for + * @return The index if defined, the sensor name by default + */ + public static String getIndex(Map conf, String sensorName) { + return getAs(INDEX_CONF, + conf, + sensorName, + String.class + ); + } + + public boolean isEnabled(String sensorName, String writerName) { + return isEnabled(getSensorIndexingConfig(sensorName, writerName)); + } + + /** + * Retrieves the enabled value from the config. + * + * @param conf The configuration to retrieve from + * @return True if this configuration is enabled, false otherwise + */ + public static boolean isEnabled(Map conf) { + return getAs(ENABLED_CONF, + conf, + true, + Boolean.class + ); + } + + public String getOutputPathFunction(String sensorName, String writerName) { + return getOutputPathFunction(getSensorIndexingConfig(sensorName, writerName), sensorName); } - else { - return ret; + + /** + * Retrieves the output path function value from the config. + * + * @param conf The configuration to retrieve from + * @param sensorName Unused + * @return The output path function if defined, empty string otherwise + */ + public static String getOutputPathFunction(Map conf, String sensorName) { + return getAs(OUTPUT_PATH_FUNCTION_CONF, + conf, + "", + String.class + ); } - } - - public Map getSensorIndexingConfig(String sensorType) { - return getSensorIndexingConfig(sensorType, true); - } - - /** - * Gets the list of sensor types that indexing configurations exist for. - * - * @return List of sensor types - */ - public List getTypes() { - List ret = new ArrayList<>(); - for(String keyedSensor : getConfigurations().keySet()) { - if(!keyedSensor.isEmpty() && keyedSensor.startsWith(ConfigurationType.INDEXING.getTypeName())) { - ret.add(keyedSensor.substring(ConfigurationType.INDEXING.getTypeName().length() + 1)); - } + + public String getFieldNameConverter(String sensorName, String writerName) { + return getFieldNameConverter(getSensorIndexingConfig(sensorName, writerName), sensorName); } - return ret; - } - - public void delete(String sensorType) { - getConfigurations().remove(getKey(sensorType)); - } - - /** - * Gets the sensor indexing config for a given writer. - * - * @param sensorType The sensor to retrieve configs for - * @param writerName The particular writer to get configurations for - * @return A Map of the configuration - */ - public Map getSensorIndexingConfig(String sensorType, String writerName) { - String key = getKey(sensorType); - Map ret = (Map) getConfigurations().get(key); - if(ret == null) { - return new HashMap(); + + /** + * Retrieves the field name converter value from the config. + * + * @param conf The configuration to retrieve from + * @param sensorName Unused + * @return The field name converter if defined, empty string otherwise + */ + public static String getFieldNameConverter(Map conf, String sensorName) { + return getAs(FIELD_NAME_CONVERTER_CONF, conf, "", String.class); } - else { - Map writerConfig = (Map)ret.get(writerName); - return writerConfig != null?writerConfig:new HashMap<>(); + + public boolean isSetDocumentId(String sensorName, String writerName) { + return isSetDocumentId(getGlobalConfig(true), getSensorIndexingConfig(sensorName, writerName)); } - } - - public void updateSensorIndexingConfig(String sensorType, byte[] data) throws IOException { - updateSensorIndexingConfig(sensorType, new ByteArrayInputStream(data)); - } - - public void updateSensorIndexingConfig(String sensorType, InputStream io) throws IOException { - Map sensorIndexingConfig = JSONUtils.INSTANCE.load(io, JSONUtils.MAP_SUPPLIER); - updateSensorIndexingConfig(sensorType, sensorIndexingConfig); - } - - public void updateSensorIndexingConfig(String sensorType, Map sensorIndexingConfig) { - getConfigurations().put(getKey(sensorType), sensorIndexingConfig); - } - - public static String getKey(String sensorType) { - return ConfigurationType.INDEXING.getTypeName() + "." + sensorType; - } - - /** - * Determines if a configuration is default or not. In particular, this means the config is null - * for the sensor/writer combo. - * - * @param sensorName The sensor to check for default - * @param writerName The specific writer to check for default - * @return True if default, false otherwise. - */ - public boolean isDefault(String sensorName, String writerName) { - Map ret = (Map) getConfigurations().get(getKey(sensorName)); - if(ret == null) { - return true; + + /** + * Determines if the Metron generated id should be used when indexing. + * + * @param globalConf The global config + * @param sensorConf The indexing config for a given sensor + * @return True if the Metron generated id should be used as the id, False otherwise + */ + public static boolean isSetDocumentId(Map globalConf, Map sensorConf) { + return getAs(SET_DOCUMENT_ID_CONF, sensorConf, + getAs(GLOBAL_ELASTICSEARCH_SET_DOCUMENT_ID_CONF, globalConf, false, Boolean.class), Boolean.class); } - else { - Map writerConfig = (Map)ret.get(writerName); - return writerConfig != null?false:true; + + /** + * Sets the enabled flag in the config. + * + * @param conf The configuration map to set enabled in. If null replaced with empty map. + * @param enabled True if enabled, false otherwise + * @return The configuration with the enabled value set + */ + public static Map setEnabled(Map conf, boolean enabled) { + Map ret = conf == null ? new HashMap<>() : conf; + ret.put(ENABLED_CONF, enabled); + return ret; } - } - - public int getBatchSize(String sensorName, String writerName ) { - return getBatchSize(getSensorIndexingConfig(sensorName, writerName)); - } - - public int getBatchTimeout(String sensorName, String writerName ) { - return getBatchTimeout(getSensorIndexingConfig(sensorName, writerName)); - } - - /** - * Returns all configured values of batchTimeout, for all configured sensors, - * but only for the specific writer identified by {@code writerName}. So, if it is - * an hdfs writer, it will return the batchTimeouts for hdfs writers for all the sensors. - * The goal is to return to a {@link org.apache.metron.common.bolt.ConfiguredBolt} - * the set of all and only batchTimeouts relevant to that ConfiguredBolt. - * - * @param writerName The name of the writer to look up. - * @return list of integer batchTimeouts, one per configured sensor - */ - public List getAllConfiguredTimeouts(String writerName) { - // The configuration infrastructure was not designed to enumerate sensors, so we synthesize. - // Since getKey is in this same class, we know we can pass it a null string to get the key prefix - // for all sensor types within this capability. We then enumerate all keys in configurations.keySet - // and select those that match the key prefix, as being sensor keys. The suffix substring of - // each such key is used as a sensor name to query the batchTimeout settings, if any. - String keyPrefixString = getKey(""); - int prefixStringLength = keyPrefixString.length(); - List configuredBatchTimeouts = new ArrayList<>(); - for (String sensorKeyString : getConfigurations().keySet()) { - if (sensorKeyString.startsWith(keyPrefixString)) { - String configuredSensorName = sensorKeyString.substring(prefixStringLength); - configuredBatchTimeouts.add(getBatchTimeout(configuredSensorName, writerName)); - } + + /** + * Sets the batch size in the config. + * + * @param conf The configuration map to set enabled in. If null, replaced with empty map. + * @param batchSize The desired batch size + * @return The configuration with the batch size value set + */ + public static Map setBatchSize(Map conf, int batchSize) { + Map ret = conf == null ? new HashMap<>() : conf; + ret.put(BATCH_SIZE_CONF, batchSize); + return ret; + } + + /** + * Sets the batch timeout in the config. + * + * @param conf The configuration map to set enabled in. If null, replaced with empty map. + * @param batchTimeout The desired batch timeout + * @return The configuration with the batch timeout value set + */ + public static Map setBatchTimeout(Map conf, int batchTimeout) { + Map ret = conf == null ? new HashMap<>() : conf; + ret.put(BATCH_TIMEOUT_CONF, batchTimeout); + return ret; + } + + /** + * Sets the index in the config. + * + * @param conf The configuration map to set enabled in. If null, replaced with empty map. + * @param index The desired index + * @return The configuration with the index value set + */ + public static Map setIndex(Map conf, String index) { + Map ret = conf == null ? new HashMap<>() : conf; + ret.put(INDEX_CONF, index); + return ret; + } + + /** + * Sets the field name converter in the config. + * + * @param conf The configuration map to set enabled in. If null, replaced with empty map. + * @param index The desired index + * @return The configuration with the field name converter value set + */ + public static Map setFieldNameConverter(Map conf, String index) { + Map ret = conf == null ? new HashMap<>() : conf; + ret.put(FIELD_NAME_CONVERTER_CONF, index); + return ret; } - return configuredBatchTimeouts; - } - - public String getIndex(String sensorName, String writerName) { - return getIndex(getSensorIndexingConfig(sensorName, writerName), sensorName); - } - - public boolean isEnabled(String sensorName, String writerName) { - return isEnabled(getSensorIndexingConfig(sensorName, writerName)); - } - - public String getOutputPathFunction(String sensorName, String writerName) { - return getOutputPathFunction(getSensorIndexingConfig(sensorName, writerName), sensorName); - } - - public String getFieldNameConverter(String sensorName, String writerName) { - return getFieldNameConverter(getSensorIndexingConfig(sensorName, writerName), sensorName); - } - - public boolean isSetDocumentId(String sensorName, String writerName) { - return isSetDocumentId(getGlobalConfig(true), getSensorIndexingConfig(sensorName, writerName)); - } - - /** - * Retrieves the enabled value from the config. - * - * @param conf The configuration to retrieve from - * @return True if this configuration is enabled, false otherwise - */ - public static boolean isEnabled(Map conf) { - return getAs( ENABLED_CONF - ,conf - , true - , Boolean.class - ); - } - - /** - * Retrieves the batch size value from the config. - * - * @param conf The configuration to retrieve from - * @return The batch size if defined, 1 by default - */ - public static int getBatchSize(Map conf) { - return getAs( BATCH_SIZE_CONF - ,conf - , 1 - , Integer.class - ); - } - - /** - * Retrieves the batch timeout value from the config. - * - * @param conf The configuration to retrieve from - * @return The batch timeout if defined, 0 by default - */ - public static int getBatchTimeout(Map conf) { - return getAs( BATCH_TIMEOUT_CONF - ,conf - , 0 - , Integer.class - ); - } - - /** - * Retrieves the index value from the config. - * - * @param conf The configuration to retrieve from - * @param sensorName The name of the sensor to retrieve the index for - * @return The index if defined, the sensor name by default - */ - public static String getIndex(Map conf, String sensorName) { - return getAs( INDEX_CONF - ,conf - , sensorName - , String.class - ); - } - - /** - * Retrieves the output path function value from the config. - * - * @param conf The configuration to retrieve from - * @param sensorName Unused - * @return The output path function if defined, empty string otherwise - */ - public static String getOutputPathFunction(Map conf, String sensorName) { - return getAs(OUTPUT_PATH_FUNCTION_CONF - ,conf - , "" - , String.class - ); - } - - /** - * Retrieves the field name converter value from the config. - * - * @param conf The configuration to retrieve from - * @param sensorName Unused - * @return The field name converter if defined, empty string otherwise - */ - public static String getFieldNameConverter(Map conf, String sensorName) { - return getAs(FIELD_NAME_CONVERTER_CONF, conf, "", String.class); - } - - /** - * Determines if the Metron generated id should be used when indexing - * - * @param globalConf The global config - * @param sensorConf The indexing config for a given sensor - * @return True if the Metron generated id should be used as the id, False otherwise - */ - public static boolean isSetDocumentId(Map globalConf, Map sensorConf) { - return getAs(SET_DOCUMENT_ID_CONF, sensorConf, getAs(GLOBAL_ELASTICSEARCH_SET_DOCUMENT_ID_CONF, globalConf, false, Boolean.class), Boolean.class); - } - - /** - * Sets the enabled flag in the config. - * - * @param conf The configuration map to set enabled in. If null replaced with empty map. - * @param enabled True if enabled, false otherwise - * @return The configuration with the enabled value set - */ - public static Map setEnabled(Map conf, boolean enabled) { - Map ret = conf == null?new HashMap<>():conf; - ret.put(ENABLED_CONF, enabled); - return ret; - } - - /** - * Sets the batch size in the config. - * - * @param conf The configuration map to set enabled in. If null, replaced with empty map. - * @param batchSize The desired batch size - * @return The configuration with the batch size value set - */ - public static Map setBatchSize(Map conf, int batchSize) { - Map ret = conf == null?new HashMap<>():conf; - ret.put(BATCH_SIZE_CONF, batchSize); - return ret; - } - - /** - * Sets the batch timeout in the config. - * - * @param conf The configuration map to set enabled in. If null, replaced with empty map. - * @param batchTimeout The desired batch timeout - * @return The configuration with the batch timeout value set - */ - public static Map setBatchTimeout(Map conf, int batchTimeout) { - Map ret = conf == null?new HashMap<>():conf; - ret.put(BATCH_TIMEOUT_CONF, batchTimeout); - return ret; - } - - /** - * Sets the index in the config. - * - * @param conf The configuration map to set enabled in. If null, replaced with empty map. - * @param index The desired index - * @return The configuration with the index value set - */ - public static Map setIndex(Map conf, String index) { - Map ret = conf == null?new HashMap<>():conf; - ret.put(INDEX_CONF, index); - return ret; - } - - /** - * Sets the field name converter in the config. - * - * @param conf The configuration map to set enabled in. If null, replaced with empty map. - * @param index The desired index - * @return The configuration with the field name converter value set - */ - public static Map setFieldNameConverter(Map conf, String index) { - Map ret = conf == null ? new HashMap<>(): conf; - ret.put(FIELD_NAME_CONVERTER_CONF, index); - return ret; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurationOperations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurationOperations.java index f75a1305..153337d4 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurationOperations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurationOperations.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,26 +20,25 @@ package org.apache.metron.common.configuration; +import java.io.IOException; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.utils.JSONUtils; -import java.io.IOException; - public class ParserConfigurationOperations implements ConfigurationOperations { - @Override - public String getTypeName() { - return "parsers"; - } + @Override + public String getTypeName() { + return "parsers"; + } - @Override - public Object deserialize(String s) throws IOException { - return JSONUtils.INSTANCE.load(s, SensorParserConfig.class); - } + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, SensorParserConfig.class); + } - @Override - public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, - CuratorFramework client) throws Exception { - } + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java index 7a638df8..3a2f3d75 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java @@ -7,19 +7,20 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration; import com.fasterxml.jackson.core.type.TypeReference; -import org.apache.metron.common.utils.JSONUtils; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -28,64 +29,69 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.metron.common.utils.JSONUtils; /** * Allows for retrieval and update of parsing configurations. */ public class ParserConfigurations extends Configurations { - public static final Integer DEFAULT_KAFKA_BATCH_SIZE = 15; - public static final String PARSER_GROUPS_CONF = "parser.groups"; + public static final Integer DEFAULT_KAFKA_BATCH_SIZE = 15; + public static final String PARSER_GROUPS_CONF = "parser.groups"; - public SensorParserConfig getSensorParserConfig(String sensorType) { - return (SensorParserConfig) getConfigurations().get(getKey(sensorType)); - } + public SensorParserConfig getSensorParserConfig(String sensorType) { + return (SensorParserConfig) getConfigurations().get(getKey(sensorType)); + } - public void updateSensorParserConfig(String sensorType, byte[] data) throws IOException { - updateSensorParserConfig(sensorType, new ByteArrayInputStream(data)); - } + public void updateSensorParserConfig(String sensorType, byte[] data) throws IOException { + updateSensorParserConfig(sensorType, new ByteArrayInputStream(data)); + } - public void updateSensorParserConfig(String sensorType, InputStream io) throws IOException { - SensorParserConfig sensorParserConfig = JSONUtils.INSTANCE.load(io, SensorParserConfig.class); - updateSensorParserConfig(sensorType, sensorParserConfig); - } + public void updateSensorParserConfig(String sensorType, InputStream io) throws IOException { + SensorParserConfig sensorParserConfig = JSONUtils.INSTANCE.load(io, SensorParserConfig.class); + updateSensorParserConfig(sensorType, sensorParserConfig); + } - public void updateSensorParserConfig(String sensorType, SensorParserConfig sensorParserConfig) { - sensorParserConfig.init(); - getConfigurations().put(getKey(sensorType), sensorParserConfig); - } + public void updateSensorParserConfig(String sensorType, SensorParserConfig sensorParserConfig) { + sensorParserConfig.init(); + getConfigurations().put(getKey(sensorType), sensorParserConfig); + } - /** - * Retrieves all the sensor groups from the global config - * @return Map of sensor groups with the group name as the key - */ - public Map getSensorParserGroups() { - Object groups = getGlobalConfig(true).getOrDefault(PARSER_GROUPS_CONF, new ArrayList<>()); - Collection sensorParserGroups = JSONUtils.INSTANCE.getMapper() - .convertValue(groups, new TypeReference>() {{}}); - return sensorParserGroups.stream() - .collect(Collectors.toMap(SensorParserGroup::getName, sensorParserGroup -> sensorParserGroup)); - } + /** + * Retrieves all the sensor groups from the global config. + * + * @return Map of sensor groups with the group name as the key + */ + public Map getSensorParserGroups() { + Object groups = getGlobalConfig(true).getOrDefault(PARSER_GROUPS_CONF, new ArrayList<>()); + Collection sensorParserGroups = + JSONUtils.INSTANCE.getMapper() + .convertValue(groups, + new TypeReference>() {}); + return sensorParserGroups.stream() + .collect(Collectors.toMap(SensorParserGroup::getName, + sensorParserGroup -> sensorParserGroup)); + } - /** - * Gets the list of sensor types that parsing configurations exist for. - * - * @return List of sensor types - */ - public List getTypes() { - List ret = new ArrayList<>(); - for(String keyedSensor : getConfigurations().keySet()) { - if(!keyedSensor.isEmpty() && keyedSensor.startsWith(ConfigurationType.PARSER.getTypeName())) { - ret.add(keyedSensor.substring(ConfigurationType.PARSER.getTypeName().length() + 1)); - } + /** + * Gets the list of sensor types that parsing configurations exist for. + * + * @return List of sensor types + */ + public List getTypes() { + List ret = new ArrayList<>(); + for (String keyedSensor : getConfigurations().keySet()) { + if (!keyedSensor.isEmpty() && keyedSensor.startsWith(ConfigurationType.PARSER.getTypeName())) { + ret.add(keyedSensor.substring(ConfigurationType.PARSER.getTypeName().length() + 1)); + } + } + return ret; } - return ret; - } - public void delete(String sensorType) { - getConfigurations().remove(getKey(sensorType)); - } + public void delete(String sensorType) { + getConfigurations().remove(getKey(sensorType)); + } - public static String getKey(String sensorType) { - return ConfigurationType.PARSER.getTypeName() + "." + sensorType; - } + public static String getKey(String sensorType) { + return ConfigurationType.PARSER.getTypeName() + "." + sensorType; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java index 2ed4b87b..9ee0e1c7 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java @@ -7,32 +7,34 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration; import com.fasterxml.jackson.core.JsonProcessingException; -import java.nio.charset.StandardCharsets; -import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.lang.builder.HashCodeBuilder; -import org.apache.commons.lang3.builder.ToStringBuilder; -import org.apache.metron.common.message.metadata.RawMessageStrategy; -import org.apache.metron.common.message.metadata.RawMessageStrategies; -import org.apache.metron.common.utils.JSONUtils; - import java.io.IOException; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.metron.common.message.metadata.RawMessageStrategies; +import org.apache.metron.common.message.metadata.RawMessageStrategy; +import org.apache.metron.common.utils.JSONUtils; /** * The configuration object that defines a parser for a given sensor. Each @@ -40,516 +42,517 @@ */ public class SensorParserConfig implements Serializable { - /** - * The class name of the parser. - */ - private String parserClassName; - - /** - * Allows logic to be defined to filter or ignore messages. Messages that have been - * filtered will not be parsed. - * - *

    This should be a fully qualified name of a class that implements the - * org.apache.metron.parsers.interfaces.MessageFilter interface. - */ - private String filterClassName; - - /** - * The input topic containing the sensor telemetry to parse. - */ - private String sensorTopic; - - /** - * The output topic where the parsed telemetry will be written. - */ - private String outputTopic; - - /** - * The error topic where errors are written to. - */ - private String errorTopic; - - /** - * The fully qualified name of a class used to write messages - * to the output topic. - * - *

    A sensible default is provided. - */ - private String writerClassName; - - /** - * The fully qualified name of a class used to write messages - * to the error topic. - * - *

    A sensible default is provided. - */ - private String errorWriterClassName; - - /** - * Determines if parser metadata is made available to the parser's field - * transformations. If true, the parser field transformations can access - * parser metadata values. - * - *

    The default is dependent upon the raw message strategy used: - *

      - *
    • The default strategy sets this to false and metadata is not read by default.
      • - *
    • The envelope strategy sets this to true and metadata is read by default.
      • - *
    - */ - private Boolean readMetadata = null; - - /** - * Determines if parser metadata is automatically merged into the message. If - * true, parser metadata values will appear as fields within the message. - * - *

    The default is dependent upon the raw message strategy used: - *

      - *
    • The default strategy sets this to false and metadata is not merged by default.
      • - *
    • The envelope strategy sets this to true and metadata is merged by default.
      • - *
    - */ - private Boolean mergeMetadata = null; - - /** - * The number of workers for the topology. - * - *

    This property can be overridden on the CLI. - */ - private Integer numWorkers = null; - - /** - * The number of ackers for the topology. - * - *

    This property can be overridden on the CLI. - */ - private Integer numAckers= null; - - /** - * The parallelism of the Kafka spout. - * If multiple sensors are specified, each sensor will use it's own configured value. - * - *

    This property can be overridden on the CLI. - */ - private Integer spoutParallelism = 1; - - /** - * The number of tasks for the Kafka spout. - * If multiple sensors are specified, each sensor will use it's own configured value. - * - *

    This property can be overridden on the CLI. - */ - private Integer spoutNumTasks = 1; - - /** - * The parallelism of the parser bolt. - * If multiple sensors are defined, the last one's config will win. - * - *

    This property can be overridden on the CLI. - */ - private Integer parserParallelism = 1; - - /** - * The number of tasks for the parser bolt. - * If multiple sensors are defined, the last one's config will win. - * - *

    This property can be overridden on the CLI. - */ - private Integer parserNumTasks = 1; - - /** - * The parallelism of the error writer bolt. - * - *

    This property can be overridden on the CLI. - */ - private Integer errorWriterParallelism = 1; - - /** - * The number of tasks for the error writer bolt. - * - *

    This property can be overridden on the CLI. - */ - private Integer errorWriterNumTasks = 1; - - /** - * Configuration properties passed to the Kafka spout. - * - *

    This property can be overridden on the CLI. - */ - private Map spoutConfig = new HashMap<>(); - - /** - * The Kafka security protocol. - * If multiple sensors are defined, any non PLAINTEXT configuration will be used. - * - *

    This property can be overridden on the CLI. This property can also be overridden by the spout config. - */ - private String securityProtocol = null; - - /** - * Configuration properties passed to the storm topology. - * - *

    This property can be overridden on the CLI. - */ - private Map stormConfig = new HashMap<>(); - - /** - * Configuration for the parser. - */ - private Map parserConfig = new HashMap<>(); - - /** - * The field transformations applied to the parsed messages. These allow fields - * of the parsed message to be transformed. - */ - private List fieldTransformations = new ArrayList<>(); - - /** - * Configures the cache that backs stellar field transformations. - * If there are multiple sensors, the configs are merged, and the last non-empty config wins. - * - *

      - *
    • stellar.cache.maxSize - The maximum number of elements in the cache. - *
    • stellar.cache.maxTimeRetain - The maximum amount of time an element is kept in the cache (in minutes). - *
    - */ - private Map cacheConfig = new HashMap<>(); - - /** - * Return the raw message supplier. This is the strategy to use to extract the raw message and metadata from - * the tuple. - */ - private RawMessageStrategy rawMessageStrategy = RawMessageStrategies.DEFAULT; - - /** - * The config for the raw message supplier. - */ - private Map rawMessageStrategyConfig = new HashMap<>(); - - public RawMessageStrategy getRawMessageStrategy() { - return rawMessageStrategy; - } - - public void setRawMessageStrategy(String rawMessageSupplierName) { - this.rawMessageStrategy = RawMessageStrategies.valueOf(rawMessageSupplierName); - } - - public Map getRawMessageStrategyConfig() { - return rawMessageStrategyConfig; - } - - public void setRawMessageStrategyConfig(Map rawMessageStrategyConfig) { - this.rawMessageStrategyConfig = rawMessageStrategyConfig; - } - - public Map getCacheConfig() { - return cacheConfig; - } - - public void setCacheConfig(Map cacheConfig) { - this.cacheConfig = cacheConfig; - } - - public Integer getNumWorkers() { - return numWorkers; - } - - public void setNumWorkers(Integer numWorkers) { - this.numWorkers = numWorkers; - } - - public Integer getNumAckers() { - return numAckers; - } - - public void setNumAckers(Integer numAckers) { - this.numAckers = numAckers; - } - - public Integer getSpoutParallelism() { - return spoutParallelism; - } - - public void setSpoutParallelism(Integer spoutParallelism) { - this.spoutParallelism = spoutParallelism; - } - - public Integer getSpoutNumTasks() { - return spoutNumTasks; - } - - public void setSpoutNumTasks(Integer spoutNumTasks) { - this.spoutNumTasks = spoutNumTasks; - } - - public Integer getParserParallelism() { - return parserParallelism; - } - - public void setParserParallelism(Integer parserParallelism) { - this.parserParallelism = parserParallelism; - } - - public Integer getParserNumTasks() { - return parserNumTasks; - } - - public void setParserNumTasks(Integer parserNumTasks) { - this.parserNumTasks = parserNumTasks; - } - - public Integer getErrorWriterParallelism() { - return errorWriterParallelism; - } - - public void setErrorWriterParallelism(Integer errorWriterParallelism) { - this.errorWriterParallelism = errorWriterParallelism; - } - - public Integer getErrorWriterNumTasks() { - return errorWriterNumTasks; - } - - public void setErrorWriterNumTasks(Integer errorWriterNumTasks) { - this.errorWriterNumTasks = errorWriterNumTasks; - } + /** + * The class name of the parser. + */ + private String parserClassName; + + /** + * Allows logic to be defined to filter or ignore messages. Messages that have been + * filtered will not be parsed. + * + *

    This should be a fully qualified name of a class that implements the + * org.apache.metron.parsers.interfaces.MessageFilter interface. + */ + private String filterClassName; + + /** + * The input topic containing the sensor telemetry to parse. + */ + private String sensorTopic; + + /** + * The output topic where the parsed telemetry will be written. + */ + private String outputTopic; + + /** + * The error topic where errors are written to. + */ + private String errorTopic; + + /** + * The fully qualified name of a class used to write messages + * to the output topic. + * + *

    A sensible default is provided. + */ + private String writerClassName; + + /** + * The fully qualified name of a class used to write messages + * to the error topic. + * + *

    A sensible default is provided. + */ + private String errorWriterClassName; + + /** + * Determines if parser metadata is made available to the parser's field + * transformations. If true, the parser field transformations can access + * parser metadata values. + * + *

    The default is dependent upon the raw message strategy used: + *

      + *
    • The default strategy sets this to false and metadata is not read by default.
      • + *
    • The envelope strategy sets this to true and metadata is read by default.
      • + *
    + */ + private Boolean readMetadata = null; + + /** + * Determines if parser metadata is automatically merged into the message. If + * true, parser metadata values will appear as fields within the message. + * + *

    The default is dependent upon the raw message strategy used: + *

      + *
    • The default strategy sets this to false and metadata is not merged by default.
      • + *
    • The envelope strategy sets this to true and metadata is merged by default.
      • + *
    + */ + private Boolean mergeMetadata = null; + + /** + * The number of workers for the topology. + * + *

    This property can be overridden on the CLI. + */ + private Integer numWorkers = null; + + /** + * The number of ackers for the topology. + * + *

    This property can be overridden on the CLI. + */ + private Integer numAckers = null; + + /** + * The parallelism of the Kafka spout. + * If multiple sensors are specified, each sensor will use it's own configured value. + * + *

    This property can be overridden on the CLI. + */ + private Integer spoutParallelism = 1; + + /** + * The number of tasks for the Kafka spout. + * If multiple sensors are specified, each sensor will use it's own configured value. + * + *

    This property can be overridden on the CLI. + */ + private Integer spoutNumTasks = 1; + + /** + * The parallelism of the parser bolt. + * If multiple sensors are defined, the last one's config will win. + * + *

    This property can be overridden on the CLI. + */ + private Integer parserParallelism = 1; + + /** + * The number of tasks for the parser bolt. + * If multiple sensors are defined, the last one's config will win. + * + *

    This property can be overridden on the CLI. + */ + private Integer parserNumTasks = 1; + + /** + * The parallelism of the error writer bolt. + * + *

    This property can be overridden on the CLI. + */ + private Integer errorWriterParallelism = 1; + + /** + * The number of tasks for the error writer bolt. + * + *

    This property can be overridden on the CLI. + */ + private Integer errorWriterNumTasks = 1; + + /** + * Configuration properties passed to the Kafka spout. + * + *

    This property can be overridden on the CLI. + */ + private Map spoutConfig = new HashMap<>(); + + /** + * The Kafka security protocol. + * If multiple sensors are defined, any non PLAINTEXT configuration will be used. + * + *

    This property can be overridden on the CLI. This property can also be overridden by the spout config. + */ + private String securityProtocol = null; + + /** + * Configuration properties passed to the storm topology. + * + *

    This property can be overridden on the CLI. + */ + private Map stormConfig = new HashMap<>(); + + /** + * Configuration for the parser. + */ + private Map parserConfig = new HashMap<>(); + + /** + * The field transformations applied to the parsed messages. These allow fields + * of the parsed message to be transformed. + */ + private List fieldTransformations = new ArrayList<>(); + + /** + * Configures the cache that backs stellar field transformations. + * If there are multiple sensors, the configs are merged, and the last non-empty config wins. + * + *

      + *
    • stellar.cache.maxSize - The maximum number of elements in the cache. + *
    • stellar.cache.maxTimeRetain - The maximum amount of time an element is kept in the cache (in minutes). + *
    + */ + private Map cacheConfig = new HashMap<>(); + + /** + * Return the raw message supplier. This is the strategy to use to extract the raw message and metadata from + * the tuple. + */ + private RawMessageStrategy rawMessageStrategy = RawMessageStrategies.DEFAULT; + + /** + * The config for the raw message supplier. + */ + private Map rawMessageStrategyConfig = new HashMap<>(); + + public RawMessageStrategy getRawMessageStrategy() { + return rawMessageStrategy; + } + + public void setRawMessageStrategy(String rawMessageSupplierName) { + this.rawMessageStrategy = RawMessageStrategies.valueOf(rawMessageSupplierName); + } - public Map getSpoutConfig() { - return spoutConfig; - } + public Map getRawMessageStrategyConfig() { + return rawMessageStrategyConfig; + } - public void setSpoutConfig(Map spoutConfig) { - this.spoutConfig = spoutConfig; - } + public void setRawMessageStrategyConfig(Map rawMessageStrategyConfig) { + this.rawMessageStrategyConfig = rawMessageStrategyConfig; + } - public String getSecurityProtocol() { - return securityProtocol; - } + public Map getCacheConfig() { + return cacheConfig; + } - public void setSecurityProtocol(String securityProtocol) { - this.securityProtocol = securityProtocol; - } + public void setCacheConfig(Map cacheConfig) { + this.cacheConfig = cacheConfig; + } - public Map getStormConfig() { - return stormConfig; - } - - public void setStormConfig(Map stormConfig) { - this.stormConfig = stormConfig; - } - - public Boolean getMergeMetadata() { - return Optional.ofNullable(mergeMetadata).orElse(getRawMessageStrategy().mergeMetadataDefault()); - } - - public void setMergeMetadata(Boolean mergeMetadata) { - this.mergeMetadata = mergeMetadata; - } - - public Boolean getReadMetadata() { - return Optional.ofNullable(readMetadata).orElse(getRawMessageStrategy().readMetadataDefault()); - } - - public void setReadMetadata(Boolean readMetadata) { - this.readMetadata = readMetadata; - } - - public String getErrorWriterClassName() { - return errorWriterClassName; - } - - public void setErrorWriterClassName(String errorWriterClassName) { - this.errorWriterClassName = errorWriterClassName; - } + public Integer getNumWorkers() { + return numWorkers; + } - public String getWriterClassName() { - return writerClassName; - } - - public void setWriterClassName(String classNames) { - this.writerClassName = classNames; - } - - public List getFieldTransformations() { - return fieldTransformations; - } - - public void setFieldTransformations(List fieldTransformations) { - this.fieldTransformations = fieldTransformations; - } - - public String getFilterClassName() { - return filterClassName; - } - - public void setFilterClassName(String filterClassName) { - this.filterClassName = filterClassName; - } - - public String getParserClassName() { - return parserClassName; - } - - public void setParserClassName(String parserClassName) { - this.parserClassName = parserClassName; - } - - public String getSensorTopic() { - return sensorTopic; - } - - public void setSensorTopic(String sensorTopic) { - this.sensorTopic = sensorTopic; - } - - public String getOutputTopic() { - return outputTopic; - } - - public void setOutputTopic(String outputTopic) { - this.outputTopic = outputTopic; - } - - public String getErrorTopic() { - return errorTopic; - } - - public void setErrorTopic(String errorTopic) { - this.errorTopic = errorTopic; - } - - public Map getParserConfig() { - return parserConfig; - } - - public void setParserConfig(Map parserConfig) { - this.parserConfig = parserConfig; - } - - /** - * Creates a SensorParserConfig from the raw bytes of a Json string. - * - * @param config The raw bytes value of the config as a Json string - * @return SensorParserConfig containing the configuration - * @throws IOException If the config cannot be loaded - */ - public static SensorParserConfig fromBytes(byte[] config) throws IOException { - SensorParserConfig ret = JSONUtils.INSTANCE.load(new String(config, StandardCharsets.UTF_8), SensorParserConfig.class); - ret.init(); - return ret; - } - - /** - * Init method that retrieves, initializes, and validates field transformations for this config. - */ - public void init() { - for(FieldTransformer h : getFieldTransformations()) { - h.initAndValidate(); - } - } - - public String toJSON() throws JsonProcessingException { - return JSONUtils.INSTANCE.toJSON(this, true); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - SensorParserConfig that = (SensorParserConfig) o; - return new EqualsBuilder() - .append(parserClassName, that.parserClassName) - .append(filterClassName, that.filterClassName) - .append(sensorTopic, that.sensorTopic) - .append(outputTopic, that.outputTopic) - .append(errorTopic, that.errorTopic) - .append(writerClassName, that.writerClassName) - .append(errorWriterClassName, that.errorWriterClassName) - .append(getReadMetadata(), that.getReadMetadata()) - .append(getMergeMetadata(), that.getMergeMetadata()) - .append(numWorkers, that.numWorkers) - .append(numAckers, that.numAckers) - .append(spoutParallelism, that.spoutParallelism) - .append(spoutNumTasks, that.spoutNumTasks) - .append(parserParallelism, that.parserParallelism) - .append(parserNumTasks, that.parserNumTasks) - .append(errorWriterParallelism, that.errorWriterParallelism) - .append(errorWriterNumTasks, that.errorWriterNumTasks) - .append(spoutConfig, that.spoutConfig) - .append(securityProtocol, that.securityProtocol) - .append(stormConfig, that.stormConfig) - .append(cacheConfig, that.cacheConfig) - .append(parserConfig, that.parserConfig) - .append(fieldTransformations, that.fieldTransformations) - .append(rawMessageStrategy, that.rawMessageStrategy) - .append(rawMessageStrategyConfig, that.rawMessageStrategyConfig) - .isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(parserClassName) - .append(filterClassName) - .append(sensorTopic) - .append(outputTopic) - .append(errorTopic) - .append(writerClassName) - .append(errorWriterClassName) - .append(getReadMetadata()) - .append(getMergeMetadata()) - .append(numWorkers) - .append(numAckers) - .append(spoutParallelism) - .append(spoutNumTasks) - .append(parserParallelism) - .append(parserNumTasks) - .append(errorWriterParallelism) - .append(errorWriterNumTasks) - .append(spoutConfig) - .append(securityProtocol) - .append(stormConfig) - .append(cacheConfig) - .append(parserConfig) - .append(fieldTransformations) - .append(rawMessageStrategy) - .append(rawMessageStrategyConfig) - .toHashCode(); - } - - @Override - public String toString() { - return new ToStringBuilder(this) - .append("parserClassName", parserClassName) - .append("filterClassName", filterClassName) - .append("sensorTopic", sensorTopic) - .append("outputTopic", outputTopic) - .append("errorTopic", errorTopic) - .append("writerClassName", writerClassName) - .append("errorWriterClassName", errorWriterClassName) - .append("readMetadata", getReadMetadata()) - .append("mergeMetadata", getMergeMetadata()) - .append("numWorkers", numWorkers) - .append("numAckers", numAckers) - .append("spoutParallelism", spoutParallelism) - .append("spoutNumTasks", spoutNumTasks) - .append("parserParallelism", parserParallelism) - .append("parserNumTasks", parserNumTasks) - .append("errorWriterParallelism", errorWriterParallelism) - .append("errorWriterNumTasks", errorWriterNumTasks) - .append("spoutConfig", spoutConfig) - .append("securityProtocol", securityProtocol) - .append("stormConfig", stormConfig) - .append("cacheConfig", cacheConfig) - .append("parserConfig", parserConfig) - .append("fieldTransformations", fieldTransformations) - .append("rawMessageStrategy", rawMessageStrategy) - .append("rawMessageStrategyConfig", rawMessageStrategyConfig) - .toString(); - } + public void setNumWorkers(Integer numWorkers) { + this.numWorkers = numWorkers; + } + + public Integer getNumAckers() { + return numAckers; + } + + public void setNumAckers(Integer numAckers) { + this.numAckers = numAckers; + } + + public Integer getSpoutParallelism() { + return spoutParallelism; + } + + public void setSpoutParallelism(Integer spoutParallelism) { + this.spoutParallelism = spoutParallelism; + } + + public Integer getSpoutNumTasks() { + return spoutNumTasks; + } + + public void setSpoutNumTasks(Integer spoutNumTasks) { + this.spoutNumTasks = spoutNumTasks; + } + + public Integer getParserParallelism() { + return parserParallelism; + } + + public void setParserParallelism(Integer parserParallelism) { + this.parserParallelism = parserParallelism; + } + + public Integer getParserNumTasks() { + return parserNumTasks; + } + + public void setParserNumTasks(Integer parserNumTasks) { + this.parserNumTasks = parserNumTasks; + } + + public Integer getErrorWriterParallelism() { + return errorWriterParallelism; + } + + public void setErrorWriterParallelism(Integer errorWriterParallelism) { + this.errorWriterParallelism = errorWriterParallelism; + } + + public Integer getErrorWriterNumTasks() { + return errorWriterNumTasks; + } + + public void setErrorWriterNumTasks(Integer errorWriterNumTasks) { + this.errorWriterNumTasks = errorWriterNumTasks; + } + + public Map getSpoutConfig() { + return spoutConfig; + } + + public void setSpoutConfig(Map spoutConfig) { + this.spoutConfig = spoutConfig; + } + + public String getSecurityProtocol() { + return securityProtocol; + } + + public void setSecurityProtocol(String securityProtocol) { + this.securityProtocol = securityProtocol; + } + + public Map getStormConfig() { + return stormConfig; + } + + public void setStormConfig(Map stormConfig) { + this.stormConfig = stormConfig; + } + + public Boolean getMergeMetadata() { + return Optional.ofNullable(mergeMetadata).orElse(getRawMessageStrategy().mergeMetadataDefault()); + } + + public void setMergeMetadata(Boolean mergeMetadata) { + this.mergeMetadata = mergeMetadata; + } + + public Boolean getReadMetadata() { + return Optional.ofNullable(readMetadata).orElse(getRawMessageStrategy().readMetadataDefault()); + } + + public void setReadMetadata(Boolean readMetadata) { + this.readMetadata = readMetadata; + } + + public String getErrorWriterClassName() { + return errorWriterClassName; + } + + public void setErrorWriterClassName(String errorWriterClassName) { + this.errorWriterClassName = errorWriterClassName; + } + + public String getWriterClassName() { + return writerClassName; + } + + public void setWriterClassName(String classNames) { + this.writerClassName = classNames; + } + + public List getFieldTransformations() { + return fieldTransformations; + } + + public void setFieldTransformations(List fieldTransformations) { + this.fieldTransformations = fieldTransformations; + } + + public String getFilterClassName() { + return filterClassName; + } + + public void setFilterClassName(String filterClassName) { + this.filterClassName = filterClassName; + } + + public String getParserClassName() { + return parserClassName; + } + + public void setParserClassName(String parserClassName) { + this.parserClassName = parserClassName; + } + + public String getSensorTopic() { + return sensorTopic; + } + + public void setSensorTopic(String sensorTopic) { + this.sensorTopic = sensorTopic; + } + + public String getOutputTopic() { + return outputTopic; + } + + public void setOutputTopic(String outputTopic) { + this.outputTopic = outputTopic; + } + + public String getErrorTopic() { + return errorTopic; + } + + public void setErrorTopic(String errorTopic) { + this.errorTopic = errorTopic; + } + + public Map getParserConfig() { + return parserConfig; + } + + public void setParserConfig(Map parserConfig) { + this.parserConfig = parserConfig; + } + + /** + * Creates a SensorParserConfig from the raw bytes of a Json string. + * + * @param config The raw bytes value of the config as a Json string + * @return SensorParserConfig containing the configuration + * @throws IOException If the config cannot be loaded + */ + public static SensorParserConfig fromBytes(byte[] config) throws IOException { + SensorParserConfig ret = + JSONUtils.INSTANCE.load(new String(config, StandardCharsets.UTF_8), SensorParserConfig.class); + ret.init(); + return ret; + } + + /** + * Init method that retrieves, initializes, and validates field transformations for this config. + */ + public void init() { + for (FieldTransformer h : getFieldTransformations()) { + h.initAndValidate(); + } + } + + public String toJSON() throws JsonProcessingException { + return JSONUtils.INSTANCE.toJSON(this, true); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + SensorParserConfig that = (SensorParserConfig) o; + return new EqualsBuilder() + .append(parserClassName, that.parserClassName) + .append(filterClassName, that.filterClassName) + .append(sensorTopic, that.sensorTopic) + .append(outputTopic, that.outputTopic) + .append(errorTopic, that.errorTopic) + .append(writerClassName, that.writerClassName) + .append(errorWriterClassName, that.errorWriterClassName) + .append(getReadMetadata(), that.getReadMetadata()) + .append(getMergeMetadata(), that.getMergeMetadata()) + .append(numWorkers, that.numWorkers) + .append(numAckers, that.numAckers) + .append(spoutParallelism, that.spoutParallelism) + .append(spoutNumTasks, that.spoutNumTasks) + .append(parserParallelism, that.parserParallelism) + .append(parserNumTasks, that.parserNumTasks) + .append(errorWriterParallelism, that.errorWriterParallelism) + .append(errorWriterNumTasks, that.errorWriterNumTasks) + .append(spoutConfig, that.spoutConfig) + .append(securityProtocol, that.securityProtocol) + .append(stormConfig, that.stormConfig) + .append(cacheConfig, that.cacheConfig) + .append(parserConfig, that.parserConfig) + .append(fieldTransformations, that.fieldTransformations) + .append(rawMessageStrategy, that.rawMessageStrategy) + .append(rawMessageStrategyConfig, that.rawMessageStrategyConfig) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(parserClassName) + .append(filterClassName) + .append(sensorTopic) + .append(outputTopic) + .append(errorTopic) + .append(writerClassName) + .append(errorWriterClassName) + .append(getReadMetadata()) + .append(getMergeMetadata()) + .append(numWorkers) + .append(numAckers) + .append(spoutParallelism) + .append(spoutNumTasks) + .append(parserParallelism) + .append(parserNumTasks) + .append(errorWriterParallelism) + .append(errorWriterNumTasks) + .append(spoutConfig) + .append(securityProtocol) + .append(stormConfig) + .append(cacheConfig) + .append(parserConfig) + .append(fieldTransformations) + .append(rawMessageStrategy) + .append(rawMessageStrategyConfig) + .toHashCode(); + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .append("parserClassName", parserClassName) + .append("filterClassName", filterClassName) + .append("sensorTopic", sensorTopic) + .append("outputTopic", outputTopic) + .append("errorTopic", errorTopic) + .append("writerClassName", writerClassName) + .append("errorWriterClassName", errorWriterClassName) + .append("readMetadata", getReadMetadata()) + .append("mergeMetadata", getMergeMetadata()) + .append("numWorkers", numWorkers) + .append("numAckers", numAckers) + .append("spoutParallelism", spoutParallelism) + .append("spoutNumTasks", spoutNumTasks) + .append("parserParallelism", parserParallelism) + .append("parserNumTasks", parserNumTasks) + .append("errorWriterParallelism", errorWriterParallelism) + .append("errorWriterNumTasks", errorWriterNumTasks) + .append("spoutConfig", spoutConfig) + .append("securityProtocol", securityProtocol) + .append("stormConfig", stormConfig) + .append("cacheConfig", cacheConfig) + .append("parserConfig", parserConfig) + .append("fieldTransformations", fieldTransformations) + .append("rawMessageStrategy", rawMessageStrategy) + .append("rawMessageStrategyConfig", rawMessageStrategyConfig) + .toString(); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserGroup.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserGroup.java index e5538c1b..06f94d47 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserGroup.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserGroup.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,56 +30,60 @@ */ public class SensorParserGroup { - private String name; - private String description; - private Set sensors = new HashSet<>(); + private String name; + private String description; + private Set sensors = new HashSet<>(); - public String getName() { - return name; - } + public String getName() { + return name; + } - public void setName(String name) { - this.name = name; - } + public void setName(String name) { + this.name = name; + } - public String getDescription() { - return description; - } + public String getDescription() { + return description; + } - public void setDescription(String description) { - this.description = description; - } + public void setDescription(String description) { + this.description = description; + } - public Set getSensors() { - return sensors; - } + public Set getSensors() { + return sensors; + } - public void setSensors(Set sensors) { - this.sensors = sensors; - } + public void setSensors(Set sensors) { + this.sensors = sensors; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SensorParserGroup that = (SensorParserGroup) o; - return Objects.equals(name, that.name) && - Objects.equals(description, that.description) && - Objects.equals(sensors, that.sensors); - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SensorParserGroup that = (SensorParserGroup) o; + return Objects.equals(name, that.name) + && Objects.equals(description, that.description) + && Objects.equals(sensors, that.sensors); + } - @Override - public int hashCode() { + @Override + public int hashCode() { - return Objects.hash(name, description, sensors); - } + return Objects.hash(name, description, sensors); + } - @Override - public String toString() { - return "SensorParserGroup{" + - "name='" + name + '\'' + - ", description='" + description + '\'' + - ", sensors=" + sensors + - '}'; - } + @Override + public String toString() { + return "SensorParserGroup{" + + "name='" + name + '\'' + + ", description='" + description + '\'' + + ", sensors=" + sensors + + '}'; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java index 344c6f6f..997aa28e 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/EnrichmentConfig.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,93 +21,99 @@ package org.apache.metron.common.configuration.enrichment; import com.fasterxml.jackson.annotation.JsonIgnore; -import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; - import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; /** * Holds the enrichment configuration. */ public class EnrichmentConfig { - private Map fieldMap = new HashMap<>(); - private Map enrichmentConfigs = new HashMap<>(); - private Map> fieldToTypeMap = new HashMap<>(); - private Map config = new HashMap<>(); - - public Map getConfig() { - return config; - } - - public void setConfig(Map config) { - this.config = config; - } - - public Map getFieldMap() { - return fieldMap; - } - - @JsonIgnore - public Map getEnrichmentConfigs() { - return enrichmentConfigs; - } - - /** - * Builds enrichment configs map of enrichment type to {@link ConfigHandler}, based on a - * provided map. - * - * @param fieldMap Map of enrichment bolts names to configuration handlers which know how to - * split the message up. - */ - public void setFieldMap(Map fieldMap) { - this.fieldMap = fieldMap; - for(Map.Entry kv : fieldMap.entrySet()) { - if(kv.getValue() instanceof List) { - enrichmentConfigs.put(kv.getKey(), new ConfigHandler((List)kv.getValue())); - } - else { - enrichmentConfigs.put(kv.getKey(), new ConfigHandler(kv.getKey(), (Map)kv.getValue())); - } + private Map fieldMap = new HashMap<>(); + private final Map enrichmentConfigs = new HashMap<>(); + private Map> fieldToTypeMap = new HashMap<>(); + private Map config = new HashMap<>(); + + public Map getConfig() { + return config; + } + + public void setConfig(Map config) { + this.config = config; + } + + public Map getFieldMap() { + return fieldMap; + } + + @JsonIgnore + public Map getEnrichmentConfigs() { + return enrichmentConfigs; + } + + /** + * Builds enrichment configs map of enrichment type to {@link ConfigHandler}, based on a + * provided map. + * + * @param fieldMap Map of enrichment bolts names to configuration handlers which know how to + * split the message up. + */ + public void setFieldMap(Map fieldMap) { + this.fieldMap = fieldMap; + for (Map.Entry kv : fieldMap.entrySet()) { + if (kv.getValue() instanceof List) { + enrichmentConfigs.put(kv.getKey(), new ConfigHandler((List) kv.getValue())); + } else { + enrichmentConfigs.put(kv.getKey(), new ConfigHandler(kv.getKey(), (Map) kv.getValue())); + } + } + } + + public Map> getFieldToTypeMap() { + return fieldToTypeMap; + } + + public void setFieldToTypeMap(Map> fieldToTypeMap) { + this.fieldToTypeMap = fieldToTypeMap; + } + + @Override + public String toString() { + return "EnrichmentConfig{" + + "fieldMap=" + fieldMap + + ", fieldToTypeMap=" + fieldToTypeMap + + ", config=" + config + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + EnrichmentConfig that = (EnrichmentConfig) o; + + if (getFieldMap() != null ? !getFieldMap().equals(that.getFieldMap()) : that.getFieldMap() != null) { + return false; + } + if (getFieldToTypeMap() != null ? !getFieldToTypeMap().equals(that.getFieldToTypeMap()) + : that.getFieldToTypeMap() != null) { + return false; + } + return getConfig() != null ? getConfig().equals(that.getConfig()) : that.getConfig() == null; + + } + + @Override + public int hashCode() { + int result = getFieldMap() != null ? getFieldMap().hashCode() : 0; + result = 31 * result + (getFieldToTypeMap() != null ? getFieldToTypeMap().hashCode() : 0); + result = 31 * result + (getConfig() != null ? getConfig().hashCode() : 0); + return result; } - } - - public Map> getFieldToTypeMap() { - return fieldToTypeMap; - } - - public void setFieldToTypeMap(Map> fieldToTypeMap) { - this.fieldToTypeMap = fieldToTypeMap; - } - - @Override - public String toString() { - return "EnrichmentConfig{" + - "fieldMap=" + fieldMap + - ", fieldToTypeMap=" + fieldToTypeMap + - ", config=" + config + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - EnrichmentConfig that = (EnrichmentConfig) o; - - if (getFieldMap() != null ? !getFieldMap().equals(that.getFieldMap()) : that.getFieldMap() != null) return false; - if (getFieldToTypeMap() != null ? !getFieldToTypeMap().equals(that.getFieldToTypeMap()) : that.getFieldToTypeMap() != null) - return false; - return getConfig() != null ? getConfig().equals(that.getConfig()) : that.getConfig() == null; - - } - - @Override - public int hashCode() { - int result = getFieldMap() != null ? getFieldMap().hashCode() : 0; - result = 31 * result + (getFieldToTypeMap() != null ? getFieldToTypeMap().hashCode() : 0); - result = 31 * result + (getConfig() != null ? getConfig().hashCode() : 0); - return result; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java index 1338ba22..8ac4a49d 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java @@ -7,99 +7,108 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.enrichment; import com.fasterxml.jackson.core.JsonProcessingException; -import java.nio.charset.StandardCharsets; -import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig; -import org.apache.metron.common.utils.JSONUtils; - import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; +import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig; +import org.apache.metron.common.utils.JSONUtils; /** * Allows for retrieval and update of enrichment configurations. */ public class SensorEnrichmentConfig { - private EnrichmentConfig enrichment = new EnrichmentConfig(); - private ThreatIntelConfig threatIntel = new ThreatIntelConfig(); - private Map configuration = new HashMap<>(); - - public Map getConfiguration() { - return configuration; - } - - public void setConfiguration(Map configuration) { - this.configuration = configuration; - } - - public EnrichmentConfig getEnrichment() { - return enrichment; - } - - public void setEnrichment(EnrichmentConfig enrichment) { - this.enrichment = enrichment; - } - - public ThreatIntelConfig getThreatIntel() { - return threatIntel; - } - - public void setThreatIntel(ThreatIntelConfig threatIntel) { - this.threatIntel = threatIntel; - } - - - - @Override - public String toString() { - return "SensorEnrichmentConfig{" + - ", enrichment=" + enrichment + - ", threatIntel=" + threatIntel + - ", configuration=" + configuration + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - SensorEnrichmentConfig that = (SensorEnrichmentConfig) o; - - if (getEnrichment() != null ? !getEnrichment().equals(that.getEnrichment()) : that.getEnrichment() != null) - return false; - if (getThreatIntel() != null ? !getThreatIntel().equals(that.getThreatIntel()) : that.getThreatIntel() != null) - return false; - return getConfiguration() != null ? getConfiguration().equals(that.getConfiguration()) : that.getConfiguration() == null; - - } - - @Override - public int hashCode() { - int result = getEnrichment() != null ? getEnrichment().hashCode() : 0; - result = 31 * result + (getEnrichment() != null ? getEnrichment().hashCode() : 0); - result = 31 * result + (getThreatIntel() != null ? getThreatIntel().hashCode() : 0); - result = 31 * result + (getConfiguration() != null ? getConfiguration().hashCode() : 0); - return result; - } - - public static SensorEnrichmentConfig fromBytes(byte[] config) throws IOException { - return JSONUtils.INSTANCE.load(new String(config, StandardCharsets.UTF_8), SensorEnrichmentConfig.class); - } - - public String toJSON() throws JsonProcessingException { - return JSONUtils.INSTANCE.toJSON(this, true); - } + private EnrichmentConfig enrichment = new EnrichmentConfig(); + private ThreatIntelConfig threatIntel = new ThreatIntelConfig(); + private Map configuration = new HashMap<>(); + + public Map getConfiguration() { + return configuration; + } + + public void setConfiguration(Map configuration) { + this.configuration = configuration; + } + + public EnrichmentConfig getEnrichment() { + return enrichment; + } + + public void setEnrichment(EnrichmentConfig enrichment) { + this.enrichment = enrichment; + } + + public ThreatIntelConfig getThreatIntel() { + return threatIntel; + } + + public void setThreatIntel(ThreatIntelConfig threatIntel) { + this.threatIntel = threatIntel; + } + + + @Override + public String toString() { + return "SensorEnrichmentConfig{" + + ", enrichment=" + enrichment + + ", threatIntel=" + threatIntel + + ", configuration=" + configuration + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + SensorEnrichmentConfig that = (SensorEnrichmentConfig) o; + + if (getEnrichment() != null ? !getEnrichment().equals(that.getEnrichment()) : that.getEnrichment() != null) { + return false; + } + if (getThreatIntel() != null ? !getThreatIntel().equals(that.getThreatIntel()) + : that.getThreatIntel() != null) { + return false; + } + return getConfiguration() != null ? getConfiguration().equals(that.getConfiguration()) + : that.getConfiguration() == null; + + } + + @Override + public int hashCode() { + int result = getEnrichment() != null ? getEnrichment().hashCode() : 0; + result = 31 * result + (getEnrichment() != null ? getEnrichment().hashCode() : 0); + result = 31 * result + (getThreatIntel() != null ? getThreatIntel().hashCode() : 0); + result = 31 * result + (getConfiguration() != null ? getConfiguration().hashCode() : 0); + return result; + } + + public static SensorEnrichmentConfig fromBytes(byte[] config) throws IOException { + return JSONUtils.INSTANCE.load(new String(config, StandardCharsets.UTF_8), SensorEnrichmentConfig.class); + } + + public String toJSON() throws JsonProcessingException { + return JSONUtils.INSTANCE.toJSON(this, true); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java index 6b70ae32..81b24b1d 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,175 +21,177 @@ package org.apache.metron.common.configuration.enrichment; import com.google.common.base.Joiner; -import org.apache.metron.common.utils.LazyLogger; -import org.apache.metron.common.utils.LazyLoggerFactory; -import org.apache.metron.stellar.common.Constants; - import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import org.apache.metron.common.utils.LazyLogger; +import org.apache.metron.common.utils.LazyLoggerFactory; +import org.apache.metron.stellar.common.Constants; public class SensorEnrichmentUpdateConfig { - protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static class FieldList { + Type type; + Map> fieldToEnrichmentTypes; - public static class FieldList { - Type type; - Map> fieldToEnrichmentTypes; + public Type getType() { + return type; + } + + public void setType(Type type) { + this.type = type; + } + + public Map> getFieldToEnrichmentTypes() { + return fieldToEnrichmentTypes; + } - public Type getType() { - return type; + public void setFieldToEnrichmentTypes(Map> fieldToEnrichmentTypes) { + this.fieldToEnrichmentTypes = fieldToEnrichmentTypes; + } } - public void setType(Type type) { - this.type = type; + public String zkQuorum; + public Map sensorToFieldList; + + public String getZkQuorum() { + return zkQuorum; } - public Map> getFieldToEnrichmentTypes() { - return fieldToEnrichmentTypes; + public void setZkQuorum(String zkQuorum) { + this.zkQuorum = zkQuorum; } - public void setFieldToEnrichmentTypes(Map> fieldToEnrichmentTypes) { - this.fieldToEnrichmentTypes = fieldToEnrichmentTypes; + public Map getSensorToFieldList() { + return sensorToFieldList; } - } - public String zkQuorum; - public Map sensorToFieldList; - - public String getZkQuorum() { - return zkQuorum; - } - - public void setZkQuorum(String zkQuorum) { - this.zkQuorum = zkQuorum; - } - - public Map getSensorToFieldList() { - return sensorToFieldList; - } - - public void setSensorToFieldList(Map sensorToFieldList) { - this.sensorToFieldList = sensorToFieldList; - } - - public static interface SourceConfigHandler { - SensorEnrichmentConfig readConfig(String sensor) throws Exception; - void persistConfig(String sensor, SensorEnrichmentConfig config) throws Exception; - } - - - - /** - * Updates the sensor configs with the provided @{link SourceConfigHandler} and the provided - * {@code sensorToFieldList}. - * - * @param scHandler Handles retrieval of configs - * @param sensorToFieldList Map from sensor to @{link FieldList} - * @throws Exception If there's an issue updating config - */ - public static void updateSensorConfigs( SourceConfigHandler scHandler - , Map sensorToFieldList - ) throws Exception - { - Map sourceConfigsChanged = new HashMap<>(); - for (Map.Entry kv : sensorToFieldList.entrySet()) { - SensorEnrichmentConfig config = findConfigBySensorType(scHandler, sourceConfigsChanged, kv.getKey()); - Map fieldMap = null; - Map> fieldToTypeMap = null; - List fieldList = null; - if(kv.getValue().type == Type.THREAT_INTEL) { - fieldMap = config.getThreatIntel().getFieldMap(); - if(fieldMap!= null) { - fieldList = (List)fieldMap.get(Constants.SIMPLE_HBASE_THREAT_INTEL); - } else { - fieldMap = new HashMap<>(); - } - if(fieldList == null) { - fieldList = new ArrayList<>(); - fieldMap.put(Constants.SIMPLE_HBASE_THREAT_INTEL, fieldList); - } - fieldToTypeMap = config.getThreatIntel().getFieldToTypeMap(); - if(fieldToTypeMap == null) { - fieldToTypeMap = new HashMap<>(); - config.getThreatIntel().setFieldToTypeMap(fieldToTypeMap); - } - } - else if(kv.getValue().type == Type.ENRICHMENT) { - fieldMap = config.getEnrichment().getFieldMap(); - if(fieldMap!= null) { - fieldList = (List)fieldMap.get(Constants.SIMPLE_HBASE_ENRICHMENT); - } else { - fieldMap = new HashMap<>(); - } - if(fieldList == null) { - fieldList = new ArrayList<>(); - fieldMap.put(Constants.SIMPLE_HBASE_ENRICHMENT, fieldList); - } - fieldToTypeMap = config.getEnrichment().getFieldToTypeMap(); - if(fieldToTypeMap == null) { - fieldToTypeMap = new HashMap<>(); - config.getEnrichment().setFieldToTypeMap(fieldToTypeMap); - } - } - if(fieldToTypeMap == null || fieldMap == null) { - LOG.debug("fieldToTypeMap is null or fieldMap is null, so skipping"); - continue; - } - //Add the additional fields to the field list associated with the hbase adapter - { - HashSet fieldSet = new HashSet<>(fieldList); - List additionalFields = new ArrayList<>(); - for (String field : kv.getValue().getFieldToEnrichmentTypes().keySet()) { - if (!fieldSet.contains(field)) { - additionalFields.add(field); - } - } - //adding only the ones that we don't already have to the field list - if (additionalFields.size() > 0) { - LOG.debug("Adding additional fields: {}", () -> Joiner.on(',').join(additionalFields)); - fieldList.addAll(additionalFields); - sourceConfigsChanged.put(kv.getKey(), config); + + public void setSensorToFieldList(Map sensorToFieldList) { + this.sensorToFieldList = sensorToFieldList; + } + + public interface SourceConfigHandler { + SensorEnrichmentConfig readConfig(String sensor) throws Exception; + + void persistConfig(String sensor, SensorEnrichmentConfig config) throws Exception; + } + + + /** + * Updates the sensor configs with the provided @{link SourceConfigHandler} and the provided + * {@code sensorToFieldList}. + * + * @param scHandler Handles retrieval of configs + * @param sensorToFieldList Map from sensor to @{link FieldList} + * @throws Exception If there's an issue updating config + */ + public static void updateSensorConfigs(SourceConfigHandler scHandler, + Map sensorToFieldList) throws Exception { + Map sourceConfigsChanged = new HashMap<>(); + for (Map.Entry kv : sensorToFieldList.entrySet()) { + SensorEnrichmentConfig config = findConfigBySensorType(scHandler, sourceConfigsChanged, kv.getKey()); + Map fieldMap = null; + Map> fieldToTypeMap = null; + List fieldList = null; + if (kv.getValue().type == Type.THREAT_INTEL) { + fieldMap = config.getThreatIntel().getFieldMap(); + if (fieldMap != null) { + fieldList = (List) fieldMap.get(Constants.SIMPLE_HBASE_THREAT_INTEL); + } else { + fieldMap = new HashMap<>(); + } + if (fieldList == null) { + fieldList = new ArrayList<>(); + fieldMap.put(Constants.SIMPLE_HBASE_THREAT_INTEL, fieldList); + } + fieldToTypeMap = config.getThreatIntel().getFieldToTypeMap(); + if (fieldToTypeMap == null) { + fieldToTypeMap = new HashMap<>(); + config.getThreatIntel().setFieldToTypeMap(fieldToTypeMap); + } + } else if (kv.getValue().type == Type.ENRICHMENT) { + fieldMap = config.getEnrichment().getFieldMap(); + if (fieldMap != null) { + fieldList = (List) fieldMap.get(Constants.SIMPLE_HBASE_ENRICHMENT); + } else { + fieldMap = new HashMap<>(); + } + if (fieldList == null) { + fieldList = new ArrayList<>(); + fieldMap.put(Constants.SIMPLE_HBASE_ENRICHMENT, fieldList); + } + fieldToTypeMap = config.getEnrichment().getFieldToTypeMap(); + if (fieldToTypeMap == null) { + fieldToTypeMap = new HashMap<>(); + config.getEnrichment().setFieldToTypeMap(fieldToTypeMap); + } + } + if (fieldToTypeMap == null || fieldMap == null) { + LOG.debug("fieldToTypeMap is null or fieldMap is null, so skipping"); + continue; + } + //Add the additional fields to the field list associated with the hbase adapter + { + HashSet fieldSet = new HashSet<>(fieldList); + List additionalFields = new ArrayList<>(); + for (String field : kv.getValue().getFieldToEnrichmentTypes().keySet()) { + if (!fieldSet.contains(field)) { + additionalFields.add(field); + } + } + //adding only the ones that we don't already have to the field list + if (additionalFields.size() > 0) { + LOG.debug("Adding additional fields: {}", () -> Joiner.on(',').join(additionalFields)); + fieldList.addAll(additionalFields); + sourceConfigsChanged.put(kv.getKey(), config); + } + } + //Add the additional enrichment types to the mapping between the fields + { + for (Map.Entry> fieldToType : kv.getValue().getFieldToEnrichmentTypes() + .entrySet()) { + String field = fieldToType.getKey(); + final HashSet types = new HashSet<>(fieldToType.getValue()); + int sizeBefore = 0; + if (fieldToTypeMap.containsKey(field)) { + List typeList = fieldToTypeMap.get(field); + sizeBefore = new HashSet<>(typeList).size(); + types.addAll(typeList); + } + int sizeAfter = types.size(); + boolean changed = sizeBefore != sizeAfter; + if (changed) { + fieldToTypeMap.put(field, new ArrayList() { + { + addAll(types); + } + }); + sourceConfigsChanged.put(kv.getKey(), config); + } + } + } } - } - //Add the additional enrichment types to the mapping between the fields - { - for(Map.Entry> fieldToType : kv.getValue().getFieldToEnrichmentTypes().entrySet()) { - String field = fieldToType.getKey(); - final HashSet types = new HashSet<>(fieldToType.getValue()); - int sizeBefore = 0; - if(fieldToTypeMap.containsKey(field)) { - List typeList = (List)fieldToTypeMap.get(field); - sizeBefore = new HashSet<>(typeList).size(); - types.addAll(typeList); - } - int sizeAfter = types.size(); - boolean changed = sizeBefore != sizeAfter; - if(changed) { - fieldToTypeMap.put(field, new ArrayList() {{ - addAll(types); - }}); - sourceConfigsChanged.put(kv.getKey(), config); - } + for (Map.Entry kv : sourceConfigsChanged.entrySet()) { + scHandler.persistConfig(kv.getKey(), kv.getValue()); } - } - } - for(Map.Entry kv : sourceConfigsChanged.entrySet()) { - scHandler.persistConfig(kv.getKey(), kv.getValue()); } - } - - private static SensorEnrichmentConfig findConfigBySensorType(SourceConfigHandler scHandler, Map sourceConfigsChanged, String key) throws Exception { - SensorEnrichmentConfig config = sourceConfigsChanged.get(key); - if(config == null) { - config = scHandler.readConfig(key); - if(LOG.isDebugEnabled()) { - LOG.debug(config.toJSON()); - } + + private static SensorEnrichmentConfig findConfigBySensorType(SourceConfigHandler scHandler, + Map sourceConfigsChanged, + String key) throws Exception { + SensorEnrichmentConfig config = sourceConfigsChanged.get(key); + if (config == null) { + config = scHandler.readConfig(key); + if (LOG.isDebugEnabled()) { + LOG.debug(config.toJSON()); + } + } + return config; } - return config; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/Type.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/Type.java index dc56495e..8f804326 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/Type.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/Type.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +21,5 @@ package org.apache.metron.common.configuration.enrichment; public enum Type { - THREAT_INTEL - ,ENRICHMENT + THREAT_INTEL, ENRICHMENT } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/Config.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/Config.java index 909ecc11..70918bde 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/Config.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/Config.java @@ -6,68 +6,70 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.configuration.enrichment.handler; +package org.apache.metron.common.configuration.enrichment.handler; -import com.google.common.collect.ImmutableList; -import org.json.simple.JSONObject; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; +import org.json.simple.JSONObject; public interface Config { - /** - * Split a message by the fields. Certain configs will do this differently than others, but - * these are the messages sent to the enrichment adapter downstream. - * - * @param message The Json message to be split - * @param fields The fields to split by - * @param fieldToEnrichmentKey A function to get the enrichment key - * @param config The config to use - * @return A list of Json objects that have been split from the message. - */ - List splitByFields( JSONObject message - , Object fields - , Function fieldToEnrichmentKey - , Iterable> config - ); + /** + * Split a message by the fields. Certain configs will do this differently than others, but + * these are the messages sent to the enrichment adapter downstream. + * + * @param message The Json message to be split + * @param fields The fields to split by + * @param fieldToEnrichmentKey A function to get the enrichment key + * @param config The config to use + * @return A list of Json objects that have been split from the message. + */ + List splitByFields(JSONObject message, + Object fields, + Function fieldToEnrichmentKey, + Iterable> config + ); - default List splitByFields( JSONObject message - , Object fields - , Function fieldToEnrichmentKey - , ConfigHandler handler - ) { - return splitByFields(message, fields, fieldToEnrichmentKey, handler.getType().toConfig(handler.getConfig())); - } + default List splitByFields(JSONObject message, + Object fields, + Function fieldToEnrichmentKey, + ConfigHandler handler + ) { + return splitByFields(message, fields, fieldToEnrichmentKey, handler.getType().toConfig(handler.getConfig())); + } - /** - * Return the subgroups for a given enrichment. This will allow the join bolt to know when the join is complete. - * NOTE: this implies that a given enrichment may have a 1 to many relationship with subgroups. - * @param config An iterable of config entries - * @return The list of subgroups - */ - List getSubgroups(Iterable> config); + /** + * Return the subgroups for a given enrichment. This will allow the join bolt to know when the join is complete. + * NOTE: this implies that a given enrichment may have a 1 to many relationship with subgroups. + * + * @param config An iterable of config entries + * @return The list of subgroups + */ + List getSubgroups(Iterable> config); - default List getSubgroups(ConfigHandler handler) { - return getSubgroups(handler.getType().toConfig(handler.getConfig())); - } + default List getSubgroups(ConfigHandler handler) { + return getSubgroups(handler.getType().toConfig(handler.getConfig())); + } - /** - * Convert a config object (currently either a map or list is supported) to a list of configs. - * @param c Either a map or list representing the enrichment adapter configuration. - * @return an iterable of config entries - */ - Iterable> toConfig(Object c); + /** + * Convert a config object (currently either a map or list is supported) to a list of configs. + * + * @param c Either a map or list representing the enrichment adapter configuration. + * @return an iterable of config entries + */ + Iterable> toConfig(Object c); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/ConfigHandler.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/ConfigHandler.java index 050fef3e..8f7d6ecf 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/ConfigHandler.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/ConfigHandler.java @@ -7,80 +7,91 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.enrichment.handler; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * This is the core logic of how to configure enrichments. The default type of enrichment configuration is a simple list * however more complex enrichment adapters require more complex configuration (e.g. stellar). */ public class ConfigHandler { - private Object config; - private Configs type = Configs.LIST; + private Object config; + private Configs type = Configs.LIST; - /** - * Constructs new instance. - * - * @param enrichment The enrichment being handled - * @param obj Map that contains the config and may contain the type of enrichment, - * (e.g. Stellar). Otherwise, use it will use the uppercased enrichment as the type. - */ - public ConfigHandler(String enrichment, Map obj) { - config = obj.get("config"); - if(obj.containsKey("type")) { - type = Configs.valueOf((String) obj.get("type")); + /** + * Constructs new instance. + * + * @param enrichment The enrichment being handled + * @param obj Map that contains the config and may contain the type of enrichment, + * (e.g. Stellar). Otherwise, use it will use the uppercased enrichment as the type. + */ + public ConfigHandler(String enrichment, Map obj) { + config = obj.get("config"); + if (obj.containsKey("type")) { + type = Configs.valueOf((String) obj.get("type")); + } else { + //TODO: make this more adaptable + type = Configs.valueOf(enrichment.toUpperCase()); + } } - else { - //TODO: make this more adaptable - type = Configs.valueOf(enrichment.toUpperCase()); + + public ConfigHandler(List obj) { + config = new HashMap<>(); + type = Configs.LIST; } - } - public ConfigHandler(List obj) { - config = new HashMap<>(); - type = Configs.LIST; - } - public Object getConfig() { - return config; - } + public Object getConfig() { + return config; + } - public void setConfig(Object config) { - this.config = config; - } + public void setConfig(Object config) { + this.config = config; + } - public Configs getType() { - return type; - } + public Configs getType() { + return type; + } - public void setType(Configs retriever) { - this.type = retriever; - } + public void setType(Configs retriever) { + this.type = retriever; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - ConfigHandler that = (ConfigHandler) o; + ConfigHandler that = (ConfigHandler) o; - if (getConfig() != null ? !getConfig().equals(that.getConfig()) : that.getConfig() != null) return false; - return getType() != null ? getType().equals(that.getType()) : that.getType() == null; + if (getConfig() != null ? !getConfig().equals(that.getConfig()) : that.getConfig() != null) { + return false; + } + return getType() != null ? getType().equals(that.getType()) : that.getType() == null; - } + } - @Override - public int hashCode() { - int result = getConfig() != null ? getConfig().hashCode() : 0; - result = 31 * result + (getType() != null ? getType().hashCode() : 0); - return result; - } + @Override + public int hashCode() { + int result = getConfig() != null ? getConfig().hashCode() : 0; + result = 31 * result + (getType() != null ? getType().hashCode() : 0); + return result; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/Configs.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/Configs.java index dced5648..567b70b7 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/Configs.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/Configs.java @@ -7,50 +7,48 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.configuration.enrichment.handler; -import org.json.simple.JSONObject; +package org.apache.metron.common.configuration.enrichment.handler; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; +import org.json.simple.JSONObject; public enum Configs implements Config { - STELLAR(new StellarConfig()) - ,LIST(new ListConfig()) - ; - Config configCreator; - Configs(Config retriever) { - this.configCreator = retriever; - } - - @Override - public List splitByFields(JSONObject message - , Object fields - , Function fieldToEnrichmentKey - , Iterable> config - ) - { - return configCreator.splitByFields(message, fields, fieldToEnrichmentKey, config); - } - - @Override - public List getSubgroups(Iterable> config) { - return configCreator.getSubgroups(config); - } - - @Override - public Iterable> toConfig(Object c) { - return configCreator.toConfig(c); - } + STELLAR(new StellarConfig()), LIST(new ListConfig()); + Config configCreator; + + Configs(Config retriever) { + this.configCreator = retriever; + } + + @Override + public List splitByFields(JSONObject message, + Object fields, + Function fieldToEnrichmentKey, + Iterable> config) { + return configCreator.splitByFields(message, fields, fieldToEnrichmentKey, config); + } + + @Override + public List getSubgroups(Iterable> config) { + return configCreator.getSubgroups(config); + } + + @Override + public Iterable> toConfig(Object c) { + return configCreator.toConfig(c); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/ListConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/ListConfig.java index 852418ab..b056db50 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/ListConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/ListConfig.java @@ -7,57 +7,55 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.enrichment.handler; import com.google.common.collect.ImmutableList; -import org.json.simple.JSONObject; - import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; +import org.json.simple.JSONObject; public class ListConfig implements Config { - @Override - public List splitByFields( JSONObject message - , Object fieldsObj - , Function fieldToEnrichmentKey - , Iterable> config - ) - { - List fields = (List)fieldsObj; - JSONObject enrichmentObject = new JSONObject(); - if (fields != null && fields.size() > 0) { - for (String field : fields) { - enrichmentObject.put(fieldToEnrichmentKey.apply(field), message.get(field)); - } + @Override + public List splitByFields(JSONObject message, + Object fieldsObj, + Function fieldToEnrichmentKey, + Iterable> config) { + List fields = (List) fieldsObj; + JSONObject enrichmentObject = new JSONObject(); + if (fields != null && fields.size() > 0) { + for (String field : fields) { + enrichmentObject.put(fieldToEnrichmentKey.apply(field), message.get(field)); + } + } + return ImmutableList.of(enrichmentObject); } - return ImmutableList.of(enrichmentObject); - } - - @Override - public List getSubgroups(Iterable> config) { - return ImmutableList.of(""); - } - @Override - public Iterable> toConfig(Object c) { - if(c instanceof Map) { - return ((Map)c).entrySet(); + @Override + public List getSubgroups(Iterable> config) { + return ImmutableList.of(""); } - else { - return new ArrayList<>(); + + @Override + public Iterable> toConfig(Object c) { + if (c instanceof Map) { + return ((Map) c).entrySet(); + } else { + return new ArrayList<>(); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java index 9f96a228..3f8d5ca5 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java @@ -7,16 +7,27 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.enrichment.handler; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; import org.apache.metron.stellar.common.StellarAssignment; import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.VariableResolver; @@ -24,134 +35,117 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; -import java.util.function.Function; - public class StellarConfig implements Config { - protected static final Logger _LOG = LoggerFactory.getLogger(StellarConfig.class); - @Override - public List getSubgroups(Iterable> config) { - boolean includeEmpty = false; - List ret = new ArrayList<>(); - for(Map.Entry kv : config) { - if(kv.getValue() instanceof String) { - includeEmpty = true; - } - else if(kv.getValue() instanceof Map || kv.getValue() instanceof List) { - ret.add(kv.getKey()); - } - } - if(includeEmpty) { - ret.add(""); - } - return ret; - } + protected static final Logger _LOG = LoggerFactory.getLogger(StellarConfig.class); - @Override - public Iterable> toConfig(Object c) { - if(c instanceof Map) { - return ((Map)c).entrySet(); - } - else if(c instanceof Collection) { - List> ret = new ArrayList<>(); - for(Object o : (Collection)c) { - if(o instanceof String) { - StellarAssignment assignment = StellarAssignment.from((String)o); - ret.add(assignment); - } - else if(o instanceof Map.Entry) { - ret.add((Map.Entry)o); + @Override + public List getSubgroups(Iterable> config) { + boolean includeEmpty = false; + List ret = new ArrayList<>(); + for (Map.Entry kv : config) { + if (kv.getValue() instanceof String) { + includeEmpty = true; + } else if (kv.getValue() instanceof Map || kv.getValue() instanceof List) { + ret.add(kv.getKey()); + } } - else { - throw new IllegalStateException("Expected " + c + " to be a list of strings, but got non-string."); + if (includeEmpty) { + ret.add(""); } - } - return ret; + return ret; } - throw new IllegalStateException("Unable to convert config " + c - + " to stellar config. Expected List or Map"); - } - @Override - public List splitByFields( JSONObject message - , Object fields - , Function fieldToEnrichmentKey - , Iterable> config - ) - { - StellarProcessor processor = new StellarProcessor(); - List messages = new ArrayList<>(); - List defaultStellarStatementGroup = new ArrayList<>(); - for(Map.Entry kv : config) { - if(kv.getValue() instanceof String) { - defaultStellarStatementGroup.add((String)kv.getValue()); - } - else if(kv.getValue() instanceof Map) { - JSONObject ret = new JSONObject(); - ret.put(kv.getKey(), getMessage(getFields(processor, (Map)kv.getValue()), message)); - messages.add(ret); - } - else if(kv.getValue() instanceof List) { - JSONObject ret = new JSONObject(); - ret.put(kv.getKey(), getMessage(getFields(processor, (List)kv.getValue()), message)); - messages.add(ret); - } + @Override + public Iterable> toConfig(Object c) { + if (c instanceof Map) { + return ((Map) c).entrySet(); + } else if (c instanceof Collection) { + List> ret = new ArrayList<>(); + for (Object o : (Collection) c) { + if (o instanceof String) { + StellarAssignment assignment = StellarAssignment.from((String) o); + ret.add(assignment); + } else if (o instanceof Map.Entry) { + ret.add((Map.Entry) o); + } else { + throw new IllegalStateException("Expected " + c + " to be a list of strings, but got non-string."); + } + } + return ret; + } + throw new IllegalStateException("Unable to convert config " + c + + " to stellar config. Expected List or Map"); } - if(defaultStellarStatementGroup.size() > 0) - { - JSONObject ret = new JSONObject(); - ret.put("", getMessage(getFields(processor, defaultStellarStatementGroup), message)); - messages.add(ret); + + @Override + public List splitByFields(JSONObject message, + Object fields, + Function fieldToEnrichmentKey, + Iterable> config) { + StellarProcessor processor = new StellarProcessor(); + List messages = new ArrayList<>(); + List defaultStellarStatementGroup = new ArrayList<>(); + for (Map.Entry kv : config) { + if (kv.getValue() instanceof String) { + defaultStellarStatementGroup.add((String) kv.getValue()); + } else if (kv.getValue() instanceof Map) { + JSONObject ret = new JSONObject(); + ret.put(kv.getKey(), getMessage(getFields(processor, (Map) kv.getValue()), message)); + messages.add(ret); + } else if (kv.getValue() instanceof List) { + JSONObject ret = new JSONObject(); + ret.put(kv.getKey(), getMessage(getFields(processor, (List) kv.getValue()), message)); + messages.add(ret); + } + } + if (defaultStellarStatementGroup.size() > 0) { + JSONObject ret = new JSONObject(); + ret.put("", getMessage(getFields(processor, defaultStellarStatementGroup), message)); + messages.add(ret); + } + _LOG.debug("Stellar enrichment split: {}", messages); + return messages; } - _LOG.debug("Stellar enrichment split: {}", messages ); - return messages; - } - private Set getFields(StellarProcessor processor - , List stellarStatementGroup - ) - { - Set stellarFields = new HashSet<>(); - for(String stellarStatementExpr: stellarStatementGroup) { - StellarAssignment assignment = StellarAssignment.from(stellarStatementExpr); - if(assignment.getStatement() != null) { - Set variables = processor.variablesUsed(assignment.getStatement()); - if (variables != null) { - stellarFields.addAll(variables); + private Set getFields(StellarProcessor processor, + List stellarStatementGroup) { + Set stellarFields = new HashSet<>(); + for (String stellarStatementExpr : stellarStatementGroup) { + StellarAssignment assignment = StellarAssignment.from(stellarStatementExpr); + if (assignment.getStatement() != null) { + Set variables = processor.variablesUsed(assignment.getStatement()); + if (variables != null) { + stellarFields.addAll(variables); + } + } } - } + return stellarFields; } - return stellarFields; - } - private Set getFields( StellarProcessor processor - , Map stellarStatementGroup - ) { - Set stellarFields = new HashSet<>(); - for (String stellarStatement : stellarStatementGroup.values()) { - Set variables = processor.variablesUsed(stellarStatement); - if (variables != null) { - stellarFields.addAll(variables); - } + private Set getFields(StellarProcessor processor, + Map stellarStatementGroup) { + Set stellarFields = new HashSet<>(); + for (String stellarStatement : stellarStatementGroup.values()) { + Set variables = processor.variablesUsed(stellarStatement); + if (variables != null) { + stellarFields.addAll(variables); + } + } + return stellarFields; } - return stellarFields; - } - private Map getMessage( Set stellarFields - , JSONObject message - ) - { + private Map getMessage(Set stellarFields, + JSONObject message) { - Map messageSegment = new HashMap<>(); - if(stellarFields.contains(VariableResolver.ALL_FIELDS)) { - //we need to include all of the fields here. - messageSegment.putAll(message); - } - else { - for (String variable : stellarFields) { - messageSegment.put(variable, message.get(variable)); - } + Map messageSegment = new HashMap<>(); + if (stellarFields.contains(VariableResolver.ALL_FIELDS)) { + //we need to include all of the fields here. + messageSegment.putAll(message); + } else { + for (String variable : stellarFields) { + messageSegment.put(variable, message.get(variable)); + } + } + return messageSegment; } - return messageSegment; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java index be961253..ad17cf88 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.enrichment.threatintel; import com.fasterxml.jackson.annotation.JsonProperty; @@ -34,122 +37,127 @@ */ public class RiskLevelRule { - /** - * The name of the rule. This field is optional. - */ - private String name; - - /** - * A description of the rule. This field is optional. - */ - private String comment; - - /** - * A predicate, in the form of a Stellar expression, that determines whether - * the rule is applied to an alert or not. This field is required. - */ - private String rule; - - /** - * A Stellar expression that when evaluated results in a numeric score. The expression - * can refer to fields within the message undergoing triage. - */ - private String scoreExpression; - - /** - * Allows a rule author to provide contextual information when a rule is applied - * to a message. This can assist a SOC analyst when actioning a threat. - * - *

    This is expected to be a valid Stellar expression and can refer to any of the - * fields within the message itself. - */ - private String reason; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getComment() { - return comment; - } - - public void setComment(String comment) { - this.comment = comment; - } - - public String getRule() { - return rule; - } - - public void setRule(String rule) { - this.rule = rule; - } - - @JsonProperty("score") - public String getScoreExpression() { - return scoreExpression; - } - - /** - * Sets the score expression based on an input object, taking care to properly handle numbers or - * strings. - * - * @param scoreExpression The raw object containing the score expression. - */ - @JsonProperty("score") - public void setScoreExpression(Object scoreExpression) { - if(scoreExpression instanceof Number) { - // a numeric value was provided - scoreExpression = Number.class.cast(scoreExpression).toString(); - - } else if (scoreExpression instanceof String) { - // a stellar expression was provided - scoreExpression = String.class.cast(scoreExpression); - - } else { - throw new IllegalArgumentException(String.format("Expected 'score' to be number or string, but got '%s'", scoreExpression)); + /** + * The name of the rule. This field is optional. + */ + private String name; + + /** + * A description of the rule. This field is optional. + */ + private String comment; + + /** + * A predicate, in the form of a Stellar expression, that determines whether + * the rule is applied to an alert or not. This field is required. + */ + private String rule; + + /** + * A Stellar expression that when evaluated results in a numeric score. The expression + * can refer to fields within the message undergoing triage. + */ + private String scoreExpression; + + /** + * Allows a rule author to provide contextual information when a rule is applied + * to a message. This can assist a SOC analyst when actioning a threat. + * + *

    This is expected to be a valid Stellar expression and can refer to any of the + * fields within the message itself. + */ + private String reason; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getRule() { + return rule; + } + + public void setRule(String rule) { + this.rule = rule; + } + + @JsonProperty("score") + public String getScoreExpression() { + return scoreExpression; + } + + /** + * Sets the score expression based on an input object, taking care to properly handle numbers or + * strings. + * + * @param scoreExpression The raw object containing the score expression. + */ + @JsonProperty("score") + public void setScoreExpression(Object scoreExpression) { + if (scoreExpression instanceof Number) { + // a numeric value was provided + scoreExpression = ((Number) scoreExpression).toString(); + + } else if (scoreExpression instanceof String) { + // a stellar expression was provided + scoreExpression = (String) scoreExpression; + + } else { + throw new IllegalArgumentException( + String.format("Expected 'score' to be number or string, but got '%s'", scoreExpression)); + } + + this.scoreExpression = scoreExpression.toString(); + } + + public String getReason() { + return reason; + } + + public void setReason(String reason) { + this.reason = reason; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RiskLevelRule)) { + return false; + } + RiskLevelRule that = (RiskLevelRule) o; + return Objects.equals(name, that.name) + && Objects.equals(comment, that.comment) + && Objects.equals(rule, that.rule) + && Objects.equals(scoreExpression, that.scoreExpression) + && Objects.equals(reason, that.reason); } - this.scoreExpression = scoreExpression.toString(); - } - - public String getReason() { - return reason; - } - - public void setReason(String reason) { - this.reason = reason; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof RiskLevelRule)) return false; - RiskLevelRule that = (RiskLevelRule) o; - return Objects.equals(name, that.name) && - Objects.equals(comment, that.comment) && - Objects.equals(rule, that.rule) && - Objects.equals(scoreExpression, that.scoreExpression) && - Objects.equals(reason, that.reason); - } - - @Override - public int hashCode() { - return Objects.hash(name, comment, rule, scoreExpression, reason); - } - - @Override - public String toString() { - return "RiskLevelRule{" + - "name='" + name + '\'' + - ", comment='" + comment + '\'' + - ", rule='" + rule + '\'' + - ", scoreExpression='" + scoreExpression + '\'' + - ", reason='" + reason + '\'' + - '}'; - } + @Override + public int hashCode() { + return Objects.hash(name, comment, rule, scoreExpression, reason); + } + + @Override + public String toString() { + return "RiskLevelRule{" + + "name='" + name + '\'' + + ", comment='" + comment + '\'' + + ", rule='" + rule + '\'' + + ", scoreExpression='" + scoreExpression + '\'' + + ", reason='" + reason + '\'' + + '}'; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RuleScore.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RuleScore.java index fb09c95c..27a401a2 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RuleScore.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RuleScore.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.enrichment.threatintel; import java.util.Objects; @@ -31,75 +34,75 @@ */ public class RuleScore { - /** - * The rule that when applied to a message resulted in this score. - */ - RiskLevelRule rule; - - /** - * Allows a rule author to provide contextual information when a rule is applied - * to a message. This can assist a SOC analyst when actioning a threat. - * - *

    This is the result of executing the 'reason' Stellar expression from the - * associated RiskLevelRule. - */ - private String reason; + /** + * The rule that when applied to a message resulted in this score. + */ + RiskLevelRule rule; - /** - * The numeric score which is the result of executing the {@link RiskLevelRule} score Stellar expression. - */ - private Number score; + /** + * Allows a rule author to provide contextual information when a rule is applied + * to a message. This can assist a SOC analyst when actioning a threat. + * + *

    This is the result of executing the 'reason' Stellar expression from the + * associated RiskLevelRule. + */ + private final String reason; - /** - * Constructs a RuleScore. - * - * @param rule The threat triage rule that when applied resulted in this score. - * @param reason The result of executing the rule's 'reason' expression. Provides context to why a rule was applied. - * @param score The result of executing the rule's 'score' expression. - */ - public RuleScore(RiskLevelRule rule, String reason, Number score) { - this.rule = rule; - this.reason = reason; - this.score = score; - } + /** + * The numeric score which is the result of executing the {@link RiskLevelRule} score Stellar expression. + */ + private final Number score; - public String getReason() { - return reason; - } + /** + * Constructs a RuleScore. + * + * @param rule The threat triage rule that when applied resulted in this score. + * @param reason The result of executing the rule's 'reason' expression. Provides context to why a rule was applied. + * @param score The result of executing the rule's 'score' expression. + */ + public RuleScore(RiskLevelRule rule, String reason, Number score) { + this.rule = rule; + this.reason = reason; + this.score = score; + } - public RiskLevelRule getRule() { - return rule; - } + public String getReason() { + return reason; + } - public Number getScore() { - return score; - } + public RiskLevelRule getRule() { + return rule; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; + public Number getScore() { + return score; } - if (!(o instanceof RuleScore)) { - return false; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RuleScore)) { + return false; + } + RuleScore ruleScore = (RuleScore) o; + return Objects.equals(rule, ruleScore.rule) + && Objects.equals(reason, ruleScore.reason) + && Objects.equals(score, ruleScore.score); } - RuleScore ruleScore = (RuleScore) o; - return Objects.equals(rule, ruleScore.rule) && - Objects.equals(reason, ruleScore.reason) && - Objects.equals(score, ruleScore.score); - } - @Override - public int hashCode() { - return Objects.hash(rule, reason, score); - } + @Override + public int hashCode() { + return Objects.hash(rule, reason, score); + } - @Override - public String toString() { - return "RuleScore{" + - "rule=" + rule + - ", reason='" + reason + '\'' + - ", score=" + score + - '}'; - } + @Override + public String toString() { + return "RuleScore{" + + "rule=" + rule + + ", reason='" + reason + '\'' + + ", score=" + score + + '}'; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java index 3b34a0af..387e6c0b 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatIntelConfig.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,39 +23,46 @@ import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; public class ThreatIntelConfig extends EnrichmentConfig { - private ThreatTriageConfig triageConfig = new ThreatTriageConfig(); + private ThreatTriageConfig triageConfig = new ThreatTriageConfig(); - public ThreatTriageConfig getTriageConfig() { - return triageConfig; - } + public ThreatTriageConfig getTriageConfig() { + return triageConfig; + } - public void setTriageConfig(ThreatTriageConfig triageConfig) { - this.triageConfig = triageConfig; - } + public void setTriageConfig(ThreatTriageConfig triageConfig) { + this.triageConfig = triageConfig; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } - ThreatIntelConfig that = (ThreatIntelConfig) o; + ThreatIntelConfig that = (ThreatIntelConfig) o; - return getTriageConfig() != null ? getTriageConfig().equals(that.getTriageConfig()) : that.getTriageConfig() == null; + return getTriageConfig() != null ? getTriageConfig().equals(that.getTriageConfig()) + : that.getTriageConfig() == null; - } + } - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (getTriageConfig() != null ? getTriageConfig().hashCode() : 0); - return result; - } + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (getTriageConfig() != null ? getTriageConfig().hashCode() : 0); + return result; + } - @Override - public String toString() { - return "ThreatIntelConfig{" + - "triageConfig=" + triageConfig + - '}'; - } + @Override + public String toString() { + return "ThreatIntelConfig{" + + "triageConfig=" + triageConfig + + '}'; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatScore.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatScore.java index 9e4a9882..d649154b 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatScore.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatScore.java @@ -6,19 +6,23 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.enrichment.threatintel; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * The overall threat score which is arrived at by aggregating the individual @@ -35,60 +39,66 @@ */ public class ThreatScore { - /** - * The numeric threat score resulting from aggregating - * all of the individual scores from each applied rule. - */ - private Double score; + /** + * The numeric threat score resulting from aggregating + * all of the individual scores from each applied rule. + */ + private Double score; - /** - * The individual rule scores produced by applying each rule to the message. - */ - private List ruleScores; + /** + * The individual rule scores produced by applying each rule to the message. + */ + private final List ruleScores; - public ThreatScore() { - this.ruleScores = new ArrayList<>(); - } + public ThreatScore() { + this.ruleScores = new ArrayList<>(); + } - public Double getScore() { - return score; - } + public Double getScore() { + return score; + } - public void setScore(Double score) { - this.score = score; - } + public void setScore(Double score) { + this.score = score; + } - public List getRuleScores() { - return ruleScores; - } + public List getRuleScores() { + return ruleScores; + } - public void addRuleScore(RuleScore score) { - this.ruleScores.add(score); - } + public void addRuleScore(RuleScore score) { + this.ruleScores.add(score); + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - ThreatScore that = (ThreatScore) o; + ThreatScore that = (ThreatScore) o; - if (score != null ? !score.equals(that.score) : that.score != null) return false; - return ruleScores != null ? ruleScores.equals(that.ruleScores) : that.ruleScores == null; - } + if (!Objects.equals(score, that.score)) { + return false; + } + return Objects.equals(ruleScores, that.ruleScores); + } - @Override - public int hashCode() { - int result = score != null ? score.hashCode() : 0; - result = 31 * result + (ruleScores != null ? ruleScores.hashCode() : 0); - return result; - } + @Override + public int hashCode() { + int result = score != null ? score.hashCode() : 0; + result = 31 * result + (ruleScores != null ? ruleScores.hashCode() : 0); + return result; + } - @Override - public String toString() { - return "ThreatScore{" + - "score=" + score + - ", ruleScores=" + ruleScores + - '}'; - } + @Override + public String toString() { + return "ThreatScore{" + + "score=" + score + + ", ruleScores=" + ruleScores + + '}'; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java index 0582a705..a9e13eec 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,105 +21,111 @@ package org.apache.metron.common.configuration.enrichment.threatintel; import com.google.common.base.Joiner; -import org.apache.metron.common.aggregator.Aggregators; - import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import org.apache.metron.common.aggregator.Aggregators; public class ThreatTriageConfig { - private List riskLevelRules = new ArrayList<>(); - private Aggregators aggregator = Aggregators.MAX; - private Map aggregationConfig = new HashMap<>(); - - public List getRiskLevelRules() { - return riskLevelRules; - } - - /** - * Given a list of @{link RiskLevelRule}, builds up the necessary context to evaluate them. - * This includes validation of the Stellar expression contained. - * - * @param riskLevelRules The list of {@link RiskLevelRule}s to be evaluated - */ - public void setRiskLevelRules(List riskLevelRules) { - List rules = new ArrayList<>(); - Set ruleIndex = new HashSet<>(); - - for(RiskLevelRule rule : riskLevelRules) { - if(rule.getRule() == null || rule.getScoreExpression() == null) { - throw new IllegalStateException("Risk level rules must contain both a rule and a score."); - } - if(ruleIndex.contains(rule.getRule())) { - continue; - } else { - ruleIndex.add(rule.getRule()); - } - rules.add(rule); + private List riskLevelRules = new ArrayList<>(); + private Aggregators aggregator = Aggregators.MAX; + private Map aggregationConfig = new HashMap<>(); + + public List getRiskLevelRules() { + return riskLevelRules; + } + + /** + * Given a list of @{link RiskLevelRule}, builds up the necessary context to evaluate them. + * This includes validation of the Stellar expression contained. + * + * @param riskLevelRules The list of {@link RiskLevelRule}s to be evaluated + */ + public void setRiskLevelRules(List riskLevelRules) { + List rules = new ArrayList<>(); + Set ruleIndex = new HashSet<>(); + + for (RiskLevelRule rule : riskLevelRules) { + if (rule.getRule() == null || rule.getScoreExpression() == null) { + throw new IllegalStateException("Risk level rules must contain both a rule and a score."); + } + if (ruleIndex.contains(rule.getRule())) { + continue; + } else { + ruleIndex.add(rule.getRule()); + } + rules.add(rule); + } + this.riskLevelRules = rules; } - this.riskLevelRules = rules; - } - - public Aggregators getAggregator() { - return aggregator; - } - - /** - * Sets an aggregator by name from {@link Aggregators}. - * - * @param aggregator The aggregator name to grab - */ - public void setAggregator(String aggregator) { - try { - this.aggregator = Aggregators.valueOf(aggregator); + + public Aggregators getAggregator() { + return aggregator; } - catch(IllegalArgumentException iae) { - throw new IllegalArgumentException("Unable to load aggregator of " + aggregator - + ". Valid aggregators are " + Joiner.on(',').join(Aggregators.values()) - ); + + /** + * Sets an aggregator by name from {@link Aggregators}. + * + * @param aggregator The aggregator name to grab + */ + public void setAggregator(String aggregator) { + try { + this.aggregator = Aggregators.valueOf(aggregator); + } catch (IllegalArgumentException iae) { + throw new IllegalArgumentException("Unable to load aggregator of " + aggregator + + ". Valid aggregators are " + Joiner.on(',').join(Aggregators.values()) + ); + } + } + + public Map getAggregationConfig() { + return aggregationConfig; + } + + public void setAggregationConfig(Map aggregationConfig) { + this.aggregationConfig = aggregationConfig; + } + + @Override + public String toString() { + return "ThreatTriageConfig{" + + "riskLevelRules=" + riskLevelRules + + ", aggregator=" + aggregator + + ", aggregationConfig=" + aggregationConfig + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ThreatTriageConfig that = (ThreatTriageConfig) o; + + if (!Objects.equals(riskLevelRules, that.riskLevelRules)) { + return false; + } + if (aggregator != that.aggregator) { + return false; + } + return Objects.equals(aggregationConfig, that.aggregationConfig); + + } + + @Override + public int hashCode() { + int result = riskLevelRules != null ? riskLevelRules.hashCode() : 0; + result = 31 * result + (aggregator != null ? aggregator.hashCode() : 0); + result = 31 * result + (aggregationConfig != null ? aggregationConfig.hashCode() : 0); + return result; } - } - - public Map getAggregationConfig() { - return aggregationConfig; - } - - public void setAggregationConfig(Map aggregationConfig) { - this.aggregationConfig = aggregationConfig; - } - - @Override - public String toString() { - return "ThreatTriageConfig{" + - "riskLevelRules=" + riskLevelRules + - ", aggregator=" + aggregator + - ", aggregationConfig=" + aggregationConfig + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ThreatTriageConfig that = (ThreatTriageConfig) o; - - if (riskLevelRules != null ? !riskLevelRules.equals(that.riskLevelRules) : that.riskLevelRules != null) - return false; - if (aggregator != that.aggregator) return false; - return aggregationConfig != null ? aggregationConfig.equals(that.aggregationConfig) : that.aggregationConfig == null; - - } - - @Override - public int hashCode() { - int result = riskLevelRules != null ? riskLevelRules.hashCode() : 0; - result = 31 * result + (aggregator != null ? aggregator.hashCode() : 0); - result = 31 * result + (aggregationConfig != null ? aggregationConfig.hashCode() : 0); - return result; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java index 1313eb2b..c34539a2 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java @@ -6,309 +6,312 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.profiler; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; -import java.nio.charset.StandardCharsets; -import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.lang.builder.HashCodeBuilder; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.apache.metron.common.utils.JSONUtils; - import java.io.IOException; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.metron.common.utils.JSONUtils; /** * The definition of a single Profile. */ public class ProfileConfig implements Serializable { - /** - * A unique name identifying the profile. The field is treated as a string. - */ - private String profile; - - /** - * A separate profile is maintained for each of these. This is effectively the - * entity that the profile is describing. The field is expected to contain a - * Stellar expression whose result is the entity name. For example, if `ip_src_addr` - * then a separate profile would be maintained for each unique IP source address in - * the data; 10.0.0.1, 10.0.0.2, etc. - */ - private String foreach; - - /** - * An expression that determines if a message should be applied to the profile. A - * Stellar expression is expected that when executed returns a boolean. A message - * is only applied to a profile if this condition is true. This allows a profile - * to filter the messages that it receives. - */ - private String onlyif = "true"; - - /** - * A set of expressions that is executed at the start of a window period. A map is - * expected where the key is the variable name and the value is a Stellar expression. - * The map can contain 0 or more variables/expressions. At the start of each window - * period the expression is executed once and stored in a variable with the given - * name. - */ - private Map init = new HashMap<>(); - - /** - * A set of expressions that is executed when a message is applied to the profile. - * A map is expected where the key is the variable name and the value is a Stellar - * expression. The map can include 0 or more variables/expressions. - */ - private Map update = new HashMap<>(); - - /** - * A list of Stellar expressions that is executed in order and used to group the - * resulting profile data. - */ - private List groupBy = new ArrayList<>(); - - /** - * Stellar expression(s) that are executed when the window period expires. The - * expression(s) are expected to in some way summarize the messages that were applied - * to the profile over the window period. - */ - private ProfileResult result; - - /** - * How long the data created by this Profile will be retained. After this period of time the - * profile data will be purged and no longer accessible. - */ - private Long expires; - - public ProfileConfig() { - } - - /** - * A profile definition requires at the very least the profile name, the foreach, and result - * expressions. - * @param profile The name of the profile. - * @param foreach The foreach expression of the profile. - * @param result The result expression of the profile. - */ - public ProfileConfig( + /** + * A unique name identifying the profile. The field is treated as a string. + */ + private String profile; + + /** + * A separate profile is maintained for each of these. This is effectively the + * entity that the profile is describing. The field is expected to contain a + * Stellar expression whose result is the entity name. For example, if `ip_src_addr` + * then a separate profile would be maintained for each unique IP source address in + * the data; 10.0.0.1, 10.0.0.2, etc. + */ + private String foreach; + + /** + * An expression that determines if a message should be applied to the profile. A + * Stellar expression is expected that when executed returns a boolean. A message + * is only applied to a profile if this condition is true. This allows a profile + * to filter the messages that it receives. + */ + private String onlyif = "true"; + + /** + * A set of expressions that is executed at the start of a window period. A map is + * expected where the key is the variable name and the value is a Stellar expression. + * The map can contain 0 or more variables/expressions. At the start of each window + * period the expression is executed once and stored in a variable with the given + * name. + */ + private Map init = new HashMap<>(); + + /** + * A set of expressions that is executed when a message is applied to the profile. + * A map is expected where the key is the variable name and the value is a Stellar + * expression. The map can include 0 or more variables/expressions. + */ + private Map update = new HashMap<>(); + + /** + * A list of Stellar expressions that is executed in order and used to group the + * resulting profile data. + */ + private List groupBy = new ArrayList<>(); + + /** + * Stellar expression(s) that are executed when the window period expires. The + * expression(s) are expected to in some way summarize the messages that were applied + * to the profile over the window period. + */ + private ProfileResult result; + + /** + * How long the data created by this Profile will be retained. After this period of time the + * profile data will be purged and no longer accessible. + */ + private Long expires; + + public ProfileConfig() { + } + + /** + * A profile definition requires at the very least the profile name, the foreach, and result + * expressions. + * + * @param profile The name of the profile. + * @param foreach The foreach expression of the profile. + * @param result The result expression of the profile. + */ + public ProfileConfig( @JsonProperty(value = "profile", required = true) String profile, @JsonProperty(value = "foreach", required = true) String foreach, - @JsonProperty(value = "result", required = true) ProfileResult result) { - - this.profile = profile; - this.foreach = foreach; - this.result = result; - } - - public String getProfile() { - return profile; - } - - public void setProfile(String profile) { - this.profile = profile; - } - - public ProfileConfig withProfile(String profile) { - this.profile = profile; - return this; - } - - public String getForeach() { - return foreach; - } - - public void setForeach(String foreach) { - this.foreach = foreach; - } - - public ProfileConfig withForeach(String foreach) { - this.foreach = foreach; - return this; - } - - public String getOnlyif() { - return onlyif; - } - - public void setOnlyif(String onlyif) { - this.onlyif = onlyif; - } - - public ProfileConfig withOnlyif(String onlyif) { - this.onlyif = onlyif; - return this; - } - - public Map getInit() { - return init; - } - - public void setInit(Map init) { - this.init = init; - } - - public ProfileConfig withInit(Map init) { - this.init.putAll(init); - return this; - } - - public ProfileConfig withInit(String var, String expression) { - this.init.put(var, expression); - return this; - } - - public Map getUpdate() { - return update; - } - - public void setUpdate(Map update) { - this.update = update; - } - - public ProfileConfig withUpdate(Map update) { - this.update.putAll(update); - return this; - } - - public ProfileConfig withUpdate(String var, String expression) { - this.update.put(var, expression); - return this; - } - - public List getGroupBy() { - return groupBy; - } - - public void setGroupBy(List groupBy) { - this.groupBy = groupBy; - } - - public ProfileConfig withGroupBy(List groupBy) { - this.groupBy = groupBy; - return this; - } - - public ProfileResult getResult() { - return result; - } - - public void setResult(ProfileResult result) { - this.result = result; - } - - public ProfileConfig withResult(String profileExpression) { - this.result = new ProfileResult(profileExpression); - return this; - } - - public Long getExpires() { - return expires; - } - - public void setExpires(Long expiresDays) { - this.expires = expiresDays; - } - - public ProfileConfig withExpires(Long expiresDays) { - this.expires = TimeUnit.DAYS.toMillis(expiresDays); - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - ProfileConfig that = (ProfileConfig) o; - return new EqualsBuilder() - .append(profile, that.profile) - .append(foreach, that.foreach) - .append(onlyif, that.onlyif) - .append(init, that.init) - .append(update, that.update) - .append(groupBy, that.groupBy) - .append(result, that.result) - .append(expires, that.expires) - .isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(profile) - .append(foreach) - .append(onlyif) - .append(init) - .append(update) - .append(groupBy) - .append(result) - .append(expires) - .toHashCode(); - } - - @Override - public String toString() { - return new ToStringBuilder(this) - .append("profile", profile) - .append("foreach", foreach) - .append("onlyif", onlyif) - .append("init", init) - .append("update", update) - .append("groupBy", groupBy) - .append("result", result) - .append("expires", expires) - .toString(); - } - - /** - * Deserialize a {@link ProfileConfig}. - * - * @param bytes Raw bytes containing a UTF-8 JSON String. - * @return The Profile definition. - * @throws IOException If unable to deserialize the bytes into a {@link ProfileConfig} - */ - public static ProfileConfig fromBytes(byte[] bytes) throws IOException { - return JSONUtils.INSTANCE.load(new String(bytes, StandardCharsets.UTF_8), ProfileConfig.class); - } - - /** - * Deserialize a {@link ProfileConfig}. - * - * @param json A String containing JSON. - * @return The Profile definition. - * @throws IOException If unable to deserialize the string into a {@link ProfileConfig} - */ - public static ProfileConfig fromJSON(String json) throws IOException { - return JSONUtils.INSTANCE.load(json, ProfileConfig.class); - } - - /** - * Serialize the profile definition to a JSON string. - * - * @return The Profiler configuration serialized as a JSON string. - * @throws JsonProcessingException If there's an error converting this to Json - */ - public String toJSON() throws JsonProcessingException { - return JSONUtils.INSTANCE.toJSON(this, true); - } + @JsonProperty(value = "result", required = true) ProfileResult result) { + + this.profile = profile; + this.foreach = foreach; + this.result = result; + } + + public String getProfile() { + return profile; + } + + public void setProfile(String profile) { + this.profile = profile; + } + + public ProfileConfig withProfile(String profile) { + this.profile = profile; + return this; + } + + public String getForeach() { + return foreach; + } + + public void setForeach(String foreach) { + this.foreach = foreach; + } + + public ProfileConfig withForeach(String foreach) { + this.foreach = foreach; + return this; + } + + public String getOnlyif() { + return onlyif; + } + + public void setOnlyif(String onlyif) { + this.onlyif = onlyif; + } + + public ProfileConfig withOnlyif(String onlyif) { + this.onlyif = onlyif; + return this; + } + + public Map getInit() { + return init; + } + + public void setInit(Map init) { + this.init = init; + } + + public ProfileConfig withInit(Map init) { + this.init.putAll(init); + return this; + } + + public ProfileConfig withInit(String var, String expression) { + this.init.put(var, expression); + return this; + } + + public Map getUpdate() { + return update; + } + + public void setUpdate(Map update) { + this.update = update; + } + + public ProfileConfig withUpdate(Map update) { + this.update.putAll(update); + return this; + } + + public ProfileConfig withUpdate(String var, String expression) { + this.update.put(var, expression); + return this; + } + + public List getGroupBy() { + return groupBy; + } + + public void setGroupBy(List groupBy) { + this.groupBy = groupBy; + } + + public ProfileConfig withGroupBy(List groupBy) { + this.groupBy = groupBy; + return this; + } + + public ProfileResult getResult() { + return result; + } + + public void setResult(ProfileResult result) { + this.result = result; + } + + public ProfileConfig withResult(String profileExpression) { + this.result = new ProfileResult(profileExpression); + return this; + } + + public Long getExpires() { + return expires; + } + + public void setExpires(Long expiresDays) { + this.expires = expiresDays; + } + + public ProfileConfig withExpires(Long expiresDays) { + this.expires = TimeUnit.DAYS.toMillis(expiresDays); + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + ProfileConfig that = (ProfileConfig) o; + return new EqualsBuilder() + .append(profile, that.profile) + .append(foreach, that.foreach) + .append(onlyif, that.onlyif) + .append(init, that.init) + .append(update, that.update) + .append(groupBy, that.groupBy) + .append(result, that.result) + .append(expires, that.expires) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(profile) + .append(foreach) + .append(onlyif) + .append(init) + .append(update) + .append(groupBy) + .append(result) + .append(expires) + .toHashCode(); + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .append("profile", profile) + .append("foreach", foreach) + .append("onlyif", onlyif) + .append("init", init) + .append("update", update) + .append("groupBy", groupBy) + .append("result", result) + .append("expires", expires) + .toString(); + } + + /** + * Deserialize a {@link ProfileConfig}. + * + * @param bytes Raw bytes containing a UTF-8 JSON String. + * @return The Profile definition. + * @throws IOException If unable to deserialize the bytes into a {@link ProfileConfig} + */ + public static ProfileConfig fromBytes(byte[] bytes) throws IOException { + return JSONUtils.INSTANCE.load(new String(bytes, StandardCharsets.UTF_8), ProfileConfig.class); + } + + /** + * Deserialize a {@link ProfileConfig}. + * + * @param json A String containing JSON. + * @return The Profile definition. + * @throws IOException If unable to deserialize the string into a {@link ProfileConfig} + */ + public static ProfileConfig fromJSON(String json) throws IOException { + return JSONUtils.INSTANCE.load(json, ProfileConfig.class); + } + + /** + * Serialize the profile definition to a JSON string. + * + * @return The Profiler configuration serialized as a JSON string. + * @throws JsonProcessingException If there's an error converting this to Json + */ + public String toJSON() throws JsonProcessingException { + return JSONUtils.INSTANCE.toJSON(this, true); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java index 70c9dd1e..5b833a00 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java @@ -7,107 +7,115 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.profiler; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.io.Serializable; +import java.util.Objects; /** * Defines the 'result' field of a Profile definition. */ public class ProfileResult implements Serializable { - /** - * A Stellar expression that is executed to produce - * a measurement that is persisted in the profile store. - */ - @JsonProperty("profile") - private ProfileResultExpressions profileExpressions; - - /** - * A set of named Stellar expressions that are executed - * to produce a measurement that can be used for threat - * triage. - */ - @JsonProperty("triage") - private ProfileTriageExpressions triageExpressions; - - public ProfileResult() { - // no-arg constructor required for kryo serialization in storm - } - - @JsonCreator - public ProfileResult( + /** + * A Stellar expression that is executed to produce + * a measurement that is persisted in the profile store. + */ + @JsonProperty("profile") + private ProfileResultExpressions profileExpressions; + + /** + * A set of named Stellar expressions that are executed + * to produce a measurement that can be used for threat + * triage. + */ + @JsonProperty("triage") + private ProfileTriageExpressions triageExpressions; + + public ProfileResult() { + // no-arg constructor required for kryo serialization in storm + } + + @JsonCreator + public ProfileResult( @JsonProperty(value = "profile", required = true) ProfileResultExpressions profileExpressions, @JsonProperty(value = "triage") ProfileTriageExpressions triageExpressions) { - this.profileExpressions = profileExpressions; - this.triageExpressions = triageExpressions != null ? triageExpressions : new ProfileTriageExpressions(); - } - - /** - * Allows a single result expression to be interpreted as a 'profile' expression. - * - *

    The profile definition - * 

    {@code {..., "result": "2 + 2" }}
    - * is equivalent to - * 
    {@code {..., "result": { "profile": "2 + 2" }}}
    - * - * @param expression The result expression. - */ - public ProfileResult(String expression) { - this.profileExpressions = new ProfileResultExpressions(expression); - this.triageExpressions = new ProfileTriageExpressions(); - } - - public ProfileResultExpressions getProfileExpressions() { - return profileExpressions; - } - - public void setProfileExpressions(ProfileResultExpressions profileExpressions) { - this.profileExpressions = profileExpressions; - } - - public ProfileTriageExpressions getTriageExpressions() { - return triageExpressions; - } - - public void setTriageExpressions(ProfileTriageExpressions triageExpressions) { - this.triageExpressions = triageExpressions; - } - - @Override - public String toString() { - return "ProfileResult{" + - "profileExpressions=" + profileExpressions + - ", triageExpressions=" + triageExpressions + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ProfileResult that = (ProfileResult) o; - if (profileExpressions != null ? !profileExpressions.equals(that.profileExpressions) : that.profileExpressions != null) - return false; - return triageExpressions != null ? triageExpressions.equals(that.triageExpressions) : that.triageExpressions == null; - } - - @Override - public int hashCode() { - int result = profileExpressions != null ? profileExpressions.hashCode() : 0; - result = 31 * result + (triageExpressions != null ? triageExpressions.hashCode() : 0); - return result; - } + this.profileExpressions = profileExpressions; + this.triageExpressions = triageExpressions != null ? triageExpressions : new ProfileTriageExpressions(); + } + + /** + * Allows a single result expression to be interpreted as a 'profile' expression. + * + *

    The profile definition + * 

    {@code {..., "result": "2 + 2" }}
    + * is equivalent to + * 
    {@code {..., "result": { "profile": "2 + 2" }}}
    + * + * @param expression The result expression. + */ + public ProfileResult(String expression) { + this.profileExpressions = new ProfileResultExpressions(expression); + this.triageExpressions = new ProfileTriageExpressions(); + } + + public ProfileResultExpressions getProfileExpressions() { + return profileExpressions; + } + + public void setProfileExpressions(ProfileResultExpressions profileExpressions) { + this.profileExpressions = profileExpressions; + } + + public ProfileTriageExpressions getTriageExpressions() { + return triageExpressions; + } + + public void setTriageExpressions(ProfileTriageExpressions triageExpressions) { + this.triageExpressions = triageExpressions; + } + + @Override + public String toString() { + return "ProfileResult{" + + "profileExpressions=" + profileExpressions + + ", triageExpressions=" + triageExpressions + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ProfileResult that = (ProfileResult) o; + if (!Objects.equals(profileExpressions, that.profileExpressions)) { + return false; + } + return Objects.equals(triageExpressions, that.triageExpressions); + } + + @Override + public int hashCode() { + int result = profileExpressions != null ? profileExpressions.hashCode() : 0; + result = 31 * result + (triageExpressions != null ? triageExpressions.hashCode() : 0); + return result; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java index 2af83b5b..8b639f09 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java @@ -6,21 +6,24 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.profiler; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; - import java.io.Serializable; +import java.util.Objects; /** * A Stellar expression that is executed to produce a single @@ -28,43 +31,47 @@ */ public class ProfileResultExpressions implements Serializable { - private String expression; + private String expression; - public ProfileResultExpressions() { - // no-arg constructor required for kryo serialization in storm - } + public ProfileResultExpressions() { + // no-arg constructor required for kryo serialization in storm + } - @JsonCreator - public ProfileResultExpressions(String expression) { - this.expression = expression; - } + @JsonCreator + public ProfileResultExpressions(String expression) { + this.expression = expression; + } - @JsonValue - public String getExpression() { - return expression; - } + @JsonValue + public String getExpression() { + return expression; + } - public void setExpression(String expression) { - this.expression = expression; - } + public void setExpression(String expression) { + this.expression = expression; + } - @Override - public String toString() { - return "ProfileResultExpressions{" + - "expression='" + expression + '\'' + - '}'; - } + @Override + public String toString() { + return "ProfileResultExpressions{" + + "expression='" + expression + '\'' + + '}'; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ProfileResultExpressions that = (ProfileResultExpressions) o; - return expression != null ? expression.equals(that.expression) : that.expression == null; - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ProfileResultExpressions that = (ProfileResultExpressions) o; + return Objects.equals(expression, that.expression); + } - @Override - public int hashCode() { - return expression != null ? expression.hashCode() : 0; - } + @Override + public int hashCode() { + return expression != null ? expression.hashCode() : 0; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java index c958efd5..427dcc35 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileTriageExpressions.java @@ -7,21 +7,23 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.profiler; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; - import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -35,65 +37,71 @@ */ public class ProfileTriageExpressions implements Serializable { - /** - * A set of named Stellar expressions. The name of the expression - * serves as the key and the value is the expression itself. - * - *

    Evaluating the expression(s) must result in a basic data type - * or map of basic data types that can be serialized. - */ - @JsonIgnore - private Map expressions; + /** + * A set of named Stellar expressions. The name of the expression + * serves as the key and the value is the expression itself. + * + *

    Evaluating the expression(s) must result in a basic data type + * or map of basic data types that can be serialized. + */ + @JsonIgnore + private Map expressions; - @JsonCreator - public ProfileTriageExpressions(Map expressions) { - this.expressions = expressions; - } + @JsonCreator + public ProfileTriageExpressions(Map expressions) { + this.expressions = expressions; + } - @JsonCreator - public ProfileTriageExpressions() { - this.expressions = new HashMap<>(); - } + @JsonCreator + public ProfileTriageExpressions() { + this.expressions = new HashMap<>(); + } - /** - * Returns the expression associated with a given name. - * @param name The name of the expression. - * @return A Stellar expression. - */ - public String getExpression(String name) { - return expressions.get(name); - } + /** + * Returns the expression associated with a given name. + * + * @param name The name of the expression. + * @return A Stellar expression. + */ + public String getExpression(String name) { + return expressions.get(name); + } - @JsonAnyGetter - public Map getExpressions() { - return expressions; - } + @JsonAnyGetter + public Map getExpressions() { + return expressions; + } - @JsonAnySetter - public void setExpressions(Map expressions) { - this.expressions = expressions; - } + @JsonAnySetter + public void setExpressions(Map expressions) { + this.expressions = expressions; + } - @Override - public String toString() { - return "ProfileTriageExpressions{" + - "expressions=" + expressions + - '}'; - } + @Override + public String toString() { + return "ProfileTriageExpressions{" + + "expressions=" + expressions + + '}'; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - ProfileTriageExpressions that = (ProfileTriageExpressions) o; + ProfileTriageExpressions that = (ProfileTriageExpressions) o; - return getExpressions() != null ? getExpressions().equals(that.getExpressions()) : that.getExpressions() == null; + return getExpressions() != null ? getExpressions().equals(that.getExpressions()) + : that.getExpressions() == null; - } + } - @Override - public int hashCode() { - return getExpressions() != null ? getExpressions().hashCode() : 0; - } + @Override + public int hashCode() { + return getExpressions() != null ? getExpressions().hashCode() : 0; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java index 33cd1f70..8562bbd6 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java @@ -6,21 +6,29 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.profiler; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.IOException; +import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -28,135 +36,129 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - /** * The configuration object for the Profiler, which may contain many Profile definitions. */ -@JsonSerialize(include=Inclusion.NON_NULL) +@JsonSerialize(include = Inclusion.NON_NULL) public class ProfilerConfig implements Serializable { - /** - * One or more profile definitions. - */ - private List profiles = new ArrayList<>(); - - /** - * The name of a field containing the timestamp that is used to - * generate profiles. - * - *

    By default, the processing time of the Profiler is used rather - * than event time; a value contained within the message itself. - * - *

    The field must contain a timestamp in epoch milliseconds. - * - *

    If a message does NOT contain this field, it will be dropped - * and not included in any profiles. - */ - private String timestampField = null; - - public List getProfiles() { - return profiles; - } - - public void setProfiles(List profiles) { - this.profiles = profiles; - } - - public ProfilerConfig withProfile(ProfileConfig profileConfig) { - this.profiles.add(profileConfig); - return this; - } - - @JsonGetter("timestampField") - public String getTimestampFieldForJson() { - return timestampField; - } - - public Optional getTimestampField() { - return Optional.ofNullable(timestampField); - } - - @JsonSetter("timestampField") - public void setTimestampField(String timestampField) { - this.timestampField = timestampField; - } - - public void setTimestampField(Optional timestampField) { - this.timestampField = timestampField.orElse(null); - } - - public ProfilerConfig withTimestampField(Optional timestampField) { - this.timestampField = timestampField.orElse(null); - return this; - } - - @Override - public String toString() { - return new ToStringBuilder(this) - .append("profiles", profiles) - .append("timestampField", timestampField) - .toString(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; + /** + * One or more profile definitions. + */ + private List profiles = new ArrayList<>(); + + /** + * The name of a field containing the timestamp that is used to + * generate profiles. + * + *

    By default, the processing time of the Profiler is used rather + * than event time; a value contained within the message itself. + * + *

    The field must contain a timestamp in epoch milliseconds. + * + *

    If a message does NOT contain this field, it will be dropped + * and not included in any profiles. + */ + private String timestampField = null; + + public List getProfiles() { + return profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } + + public ProfilerConfig withProfile(ProfileConfig profileConfig) { + this.profiles.add(profileConfig); + return this; + } + + @JsonGetter("timestampField") + public String getTimestampFieldForJson() { + return timestampField; } - if (o == null || getClass() != o.getClass()) { - return false; + public Optional getTimestampField() { + return Optional.ofNullable(timestampField); } - ProfilerConfig that = (ProfilerConfig) o; - return new EqualsBuilder() - .append(profiles, that.profiles) - .append(timestampField, that.timestampField) - .isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(profiles) - .append(timestampField) - .toHashCode(); - } - - /** - * Deserialize a {@link ProfilerConfig}. - * - * @param bytes Raw bytes containing a UTF-8 JSON String. - * @return The Profiler configuration. - * @throws IOException If there's an error deserializing the raw bytes - */ - public static ProfilerConfig fromBytes(byte[] bytes) throws IOException { - return JSONUtils.INSTANCE.load(new String(bytes, StandardCharsets.UTF_8), ProfilerConfig.class); - } - - /** - * Deserialize a {@link ProfilerConfig}. - * - * @param json A String containing JSON. - * @return The Profiler configuration. - * @throws IOException If there's an error deserializing the string - */ - public static ProfilerConfig fromJSON(String json) throws IOException { - return JSONUtils.INSTANCE.load(json, ProfilerConfig.class); - } - - /** - * Serialize a {@link ProfilerConfig} to a JSON string. - * - * @return The Profiler configuration serialized as a JSON string. - * @throws JsonProcessingException If an error occurs serializing this as a Json string - */ - public String toJSON() throws JsonProcessingException { - return JSONUtils.INSTANCE.toJSON(this, true); - } + @JsonSetter("timestampField") + public void setTimestampField(String timestampField) { + this.timestampField = timestampField; + } + + public void setTimestampField(Optional timestampField) { + this.timestampField = timestampField.orElse(null); + } + + public ProfilerConfig withTimestampField(Optional timestampField) { + this.timestampField = timestampField.orElse(null); + return this; + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .append("profiles", profiles) + .append("timestampField", timestampField) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + ProfilerConfig that = (ProfilerConfig) o; + return new EqualsBuilder() + .append(profiles, that.profiles) + .append(timestampField, that.timestampField) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(profiles) + .append(timestampField) + .toHashCode(); + } + + /** + * Deserialize a {@link ProfilerConfig}. + * + * @param bytes Raw bytes containing a UTF-8 JSON String. + * @return The Profiler configuration. + * @throws IOException If there's an error deserializing the raw bytes + */ + public static ProfilerConfig fromBytes(byte[] bytes) throws IOException { + return JSONUtils.INSTANCE.load(new String(bytes, StandardCharsets.UTF_8), ProfilerConfig.class); + } + + /** + * Deserialize a {@link ProfilerConfig}. + * + * @param json A String containing JSON. + * @return The Profiler configuration. + * @throws IOException If there's an error deserializing the string + */ + public static ProfilerConfig fromJSON(String json) throws IOException { + return JSONUtils.INSTANCE.load(json, ProfilerConfig.class); + } + + /** + * Serialize a {@link ProfilerConfig} to a JSON string. + * + * @return The Profiler configuration serialized as a JSON string. + * @throws JsonProcessingException If an error occurs serializing this as a Json string + */ + public String toJSON() throws JsonProcessingException { + return JSONUtils.INSTANCE.toJSON(this, true); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ConfigurationStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ConfigurationStrategy.java index a6112130..24548cb3 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ConfigurationStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ConfigurationStrategy.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.configuration.writer; import java.util.function.Supplier; @@ -25,20 +28,22 @@ public interface ConfigurationStrategy { - /** - * Create a specific writer configuration. - * @param writer provided for the underlying creator to access metadata as needed - * @param configs a {@link WriterConfiguration} will typically access the pass configs - * @return A {@link WriterConfiguration} created from the configs - */ - WriterConfiguration createWriterConfig(BulkMessageWriter writer, Configurations configs); + /** + * Create a specific writer configuration. + * + * @param writer provided for the underlying creator to access metadata as needed + * @param configs a {@link WriterConfiguration} will typically access the pass configs + * @return A {@link WriterConfiguration} created from the configs + */ + WriterConfiguration createWriterConfig(BulkMessageWriter writer, Configurations configs); - /** - * Create specific {@link ConfigurationsUpdater} for the type of config extending Configurations. - * @param reloadable setup as a callback by the updater - * @param configSupplier supplies config to the updater - * @return updater - */ - ConfigurationsUpdater createUpdater(Reloadable reloadable, Supplier configSupplier); + /** + * Create specific {@link ConfigurationsUpdater} for the type of config extending Configurations. + * + * @param reloadable setup as a callback by the updater + * @param configSupplier supplies config to the updater + * @return updater + */ + ConfigurationsUpdater createUpdater(Reloadable reloadable, Supplier configSupplier); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ConfigurationsStrategies.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ConfigurationsStrategies.java index 3b0dae3f..38c224ae 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ConfigurationsStrategies.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ConfigurationsStrategies.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +20,7 @@ package org.apache.metron.common.configuration.writer; +import java.util.function.Supplier; import org.apache.metron.common.configuration.Configurations; import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.IndexingConfigurations; @@ -29,8 +32,6 @@ import org.apache.metron.common.zookeeper.configurations.ParserUpdater; import org.apache.metron.common.zookeeper.configurations.Reloadable; -import java.util.function.Supplier; - /** * Strategy pattern implementation that couples factories for WriterConfiguration and * ConfigurationsUpdater together for a particular type. @@ -40,85 +41,86 @@ */ public enum ConfigurationsStrategies implements ConfigurationStrategy { - PARSERS(new ConfigurationStrategy() { - - @Override - public WriterConfiguration createWriterConfig(BulkMessageWriter writer, - Configurations configs) { - if (configs instanceof ParserConfigurations) { - return new ParserWriterConfiguration((ParserConfigurations) configs); - } else { - throw new IllegalArgumentException( - "Expected config of type ParserConfigurations but found " + configs.getClass()); - } - } - - @Override - public ConfigurationsUpdater createUpdater(Reloadable reloadable, - Supplier configSupplier) { - return new ParserUpdater(reloadable, configSupplier); - } - }), - - ENRICHMENT(new ConfigurationStrategy() { - - @Override - public WriterConfiguration createWriterConfig(BulkMessageWriter writer, - Configurations configs) { - if (configs instanceof EnrichmentConfigurations) { - return new EnrichmentWriterConfiguration((EnrichmentConfigurations) configs); - } else { - throw new IllegalArgumentException( - "Expected config of type EnrichmentConfigurations but found " + configs.getClass()); - } - } - - @Override - public ConfigurationsUpdater createUpdater(Reloadable reloadable, - Supplier configSupplier) { - return new EnrichmentUpdater(reloadable, configSupplier); + PARSERS(new ConfigurationStrategy() { + + @Override + public WriterConfiguration createWriterConfig(BulkMessageWriter writer, + Configurations configs) { + if (configs instanceof ParserConfigurations) { + return new ParserWriterConfiguration((ParserConfigurations) configs); + } else { + throw new IllegalArgumentException( + "Expected config of type ParserConfigurations but found " + configs.getClass()); + } + } + + @Override + public ConfigurationsUpdater createUpdater(Reloadable reloadable, + Supplier configSupplier) { + return new ParserUpdater(reloadable, configSupplier); + } + }), + + ENRICHMENT(new ConfigurationStrategy() { + + @Override + public WriterConfiguration createWriterConfig(BulkMessageWriter writer, + Configurations configs) { + if (configs instanceof EnrichmentConfigurations) { + return new EnrichmentWriterConfiguration((EnrichmentConfigurations) configs); + } else { + throw new IllegalArgumentException( + "Expected config of type EnrichmentConfigurations but found " + configs.getClass()); + } + } + + @Override + public ConfigurationsUpdater createUpdater(Reloadable reloadable, + Supplier configSupplier) { + return new EnrichmentUpdater(reloadable, configSupplier); + } + }), + + INDEXING(new ConfigurationStrategy() { + + @Override + public WriterConfiguration createWriterConfig(BulkMessageWriter writer, + Configurations configs) { + if (configs instanceof IndexingConfigurations) { + return new IndexingWriterConfiguration(writer.getName(), (IndexingConfigurations) configs); + } else { + throw new IllegalArgumentException( + "Expected config of type IndexingConfigurations but found " + configs.getClass()); + } + } + + @Override + public ConfigurationsUpdater createUpdater(Reloadable reloadable, + Supplier configSupplier) { + return new IndexingUpdater(reloadable, configSupplier); + } + }); + + private final ConfigurationStrategy strategy; + + ConfigurationsStrategies(ConfigurationStrategy strategy) { + this.strategy = strategy; } - }), - - INDEXING(new ConfigurationStrategy() { @Override - public WriterConfiguration createWriterConfig(BulkMessageWriter writer, - Configurations configs) { - if (configs instanceof IndexingConfigurations) { - return new IndexingWriterConfiguration(writer.getName(), (IndexingConfigurations) configs); - } else { - throw new IllegalArgumentException( - "Expected config of type IndexingConfigurations but found " + configs.getClass()); - } + public WriterConfiguration createWriterConfig(BulkMessageWriter writer, Configurations configs) { + return strategy.createWriterConfig(writer, configs); } + /** + * Config updater. + * + * @param reloadable callback + * @param configSupplier Supplier provides config of type {@code } + * @return Config updater + */ @Override - public ConfigurationsUpdater createUpdater(Reloadable reloadable, - Supplier configSupplier) { - return new IndexingUpdater(reloadable, configSupplier); + public ConfigurationsUpdater createUpdater(Reloadable reloadable, Supplier configSupplier) { + return strategy.createUpdater(reloadable, configSupplier); } - }); - - private ConfigurationStrategy strategy; - - ConfigurationsStrategies(ConfigurationStrategy strategy) { - this.strategy = strategy; - } - - @Override - public WriterConfiguration createWriterConfig(BulkMessageWriter writer, Configurations configs) { - return strategy.createWriterConfig(writer, configs); - } - - /** - * Config updater. - * @param reloadable callback - * @param configSupplier Supplier provides config of type {@code } - * @return Config updater - */ - @Override - public ConfigurationsUpdater createUpdater(Reloadable reloadable, Supplier configSupplier) { - return strategy.createUpdater(reloadable, configSupplier); - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/EnrichmentWriterConfiguration.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/EnrichmentWriterConfiguration.java index 7000154a..9c25df42 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/EnrichmentWriterConfiguration.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/EnrichmentWriterConfiguration.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,8 +20,7 @@ package org.apache.metron.common.configuration.writer; -import static java.util.Arrays.asList; - +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -31,80 +32,85 @@ */ public class EnrichmentWriterConfiguration implements WriterConfiguration { - private Optional config; + private final Optional config; - public EnrichmentWriterConfiguration(EnrichmentConfigurations config) { - this.config = Optional.ofNullable(config); - } + public EnrichmentWriterConfiguration(EnrichmentConfigurations config) { + this.config = Optional.ofNullable(config); + } - /** - * Batch size for writing. - * @param sensorName n/a - * @return batch size in # messages - */ - @Override - public int getBatchSize(String sensorName) { - return config.orElse(new EnrichmentConfigurations()).getBatchSize(); - } + /** + * Batch size for writing. + * + * @param sensorName n/a + * @return batch size in # messages + */ + @Override + public int getBatchSize(String sensorName) { + return config.orElse(new EnrichmentConfigurations()).getBatchSize(); + } - /** - * Timeout for this writer. - * @param sensorName n/a - * @return timeout in ms - */ - @Override - public int getBatchTimeout(String sensorName) { - return config.orElse(new EnrichmentConfigurations()).getBatchTimeout(); - } + /** + * Timeout for this writer. + * + * @param sensorName n/a + * @return timeout in ms + */ + @Override + public int getBatchTimeout(String sensorName) { + return config.orElse(new EnrichmentConfigurations()).getBatchTimeout(); + } - /** - * Timeout for this writer. - * @return single item list with this writer's timeout - */ - @Override - public List getAllConfiguredTimeouts() { - return asList(getBatchTimeout(null)); - } + /** + * Timeout for this writer. + * + * @return single item list with this writer's timeout + */ + @Override + public List getAllConfiguredTimeouts() { + return Collections.singletonList(getBatchTimeout(null)); + } - /** - * n/a for enrichment. - * @param sensorName n/a - * @return null - */ - @Override - public String getIndex(String sensorName) { - return null; - } + /** + * n/a for enrichment. + * + * @param sensorName n/a + * @return null + */ + @Override + public String getIndex(String sensorName) { + return null; + } - /** - * Always enabled in enrichment. - * @param sensorName n/a - * @return true - */ - @Override - public boolean isEnabled(String sensorName) { - return true; - } + /** + * Always enabled in enrichment. + * + * @param sensorName n/a + * @return true + */ + @Override + public boolean isEnabled(String sensorName) { + return true; + } - @Override - public Map getSensorConfig(String sensorName) { - return config.orElse(new EnrichmentConfigurations()).getSensorEnrichmentConfig(sensorName) - .getConfiguration(); - } + @Override + public Map getSensorConfig(String sensorName) { + return config.orElse(new EnrichmentConfigurations()).getSensorEnrichmentConfig(sensorName) + .getConfiguration(); + } - @Override - public Map getGlobalConfig() { - return config.orElse(new EnrichmentConfigurations()).getGlobalConfig(); - } + @Override + public Map getGlobalConfig() { + return config.orElse(new EnrichmentConfigurations()).getGlobalConfig(); + } - @Override - public boolean isDefault(String sensorName) { - return false; - } + @Override + public boolean isDefault(String sensorName) { + return false; + } - @Override - public String getFieldNameConverter(String sensorName) { - // not applicable - return null; - } + @Override + public String getFieldNameConverter(String sensorName) { + // not applicable + return null; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/IndexingWriterConfiguration.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/IndexingWriterConfiguration.java index fbd11788..cdc0c399 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/IndexingWriterConfiguration.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/IndexingWriterConfiguration.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,68 +20,67 @@ package org.apache.metron.common.configuration.writer; -import org.apache.metron.common.configuration.IndexingConfigurations; - import java.util.List; import java.util.Map; import java.util.Optional; +import org.apache.metron.common.configuration.IndexingConfigurations; -public class IndexingWriterConfiguration implements WriterConfiguration{ - private Optional config; - private String writerName; +public class IndexingWriterConfiguration implements WriterConfiguration { + private final Optional config; + private final String writerName; - public IndexingWriterConfiguration(String writerName, IndexingConfigurations config) { - this.config = Optional.ofNullable(config); - this.writerName = writerName; - } + public IndexingWriterConfiguration(String writerName, IndexingConfigurations config) { + this.config = Optional.ofNullable(config); + this.writerName = writerName; + } - @Override - public int getBatchSize(String sensorName) { - return config.orElse(new IndexingConfigurations()).getBatchSize(sensorName, writerName); - } + @Override + public int getBatchSize(String sensorName) { + return config.orElse(new IndexingConfigurations()).getBatchSize(sensorName, writerName); + } - @Override - public int getBatchTimeout(String sensorName) { - return config.orElse(new IndexingConfigurations()).getBatchTimeout(sensorName, writerName); - } + @Override + public int getBatchTimeout(String sensorName) { + return config.orElse(new IndexingConfigurations()).getBatchTimeout(sensorName, writerName); + } - @Override - public List getAllConfiguredTimeouts() { - return config.orElse(new IndexingConfigurations()).getAllConfiguredTimeouts(writerName); - } + @Override + public List getAllConfiguredTimeouts() { + return config.orElse(new IndexingConfigurations()).getAllConfiguredTimeouts(writerName); + } - @Override - public String getIndex(String sensorName) { - return config.orElse(new IndexingConfigurations()).getIndex(sensorName, writerName); - } + @Override + public String getIndex(String sensorName) { + return config.orElse(new IndexingConfigurations()).getIndex(sensorName, writerName); + } - @Override - public boolean isEnabled(String sensorName) { - return config.orElse(new IndexingConfigurations()).isEnabled(sensorName, writerName); - } + @Override + public boolean isEnabled(String sensorName) { + return config.orElse(new IndexingConfigurations()).isEnabled(sensorName, writerName); + } - @Override - public Map getSensorConfig(String sensorName) { - return config.orElse(new IndexingConfigurations()).getSensorIndexingConfig(sensorName, writerName); - } + @Override + public Map getSensorConfig(String sensorName) { + return config.orElse(new IndexingConfigurations()).getSensorIndexingConfig(sensorName, writerName); + } - @Override - public Map getGlobalConfig() { - return config.orElse(new IndexingConfigurations()).getGlobalConfig(); - } + @Override + public Map getGlobalConfig() { + return config.orElse(new IndexingConfigurations()).getGlobalConfig(); + } - @Override - public boolean isDefault(String sensorName) { - return config.orElse(new IndexingConfigurations()).isDefault(sensorName, writerName); - } + @Override + public boolean isDefault(String sensorName) { + return config.orElse(new IndexingConfigurations()).isDefault(sensorName, writerName); + } - @Override - public String getFieldNameConverter(String sensorName) { - return config.orElse(new IndexingConfigurations()).getFieldNameConverter(sensorName, writerName); - } + @Override + public String getFieldNameConverter(String sensorName) { + return config.orElse(new IndexingConfigurations()).getFieldNameConverter(sensorName, writerName); + } - @Override - public boolean isSetDocumentId(String sensorName) { - return config.orElse(new IndexingConfigurations()).isSetDocumentId(sensorName, writerName); - } + @Override + public boolean isSetDocumentId(String sensorName) { + return config.orElse(new IndexingConfigurations()).isSetDocumentId(sensorName, writerName); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ParserWriterConfiguration.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ParserWriterConfiguration.java index a4f3f667..744615f2 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ParserWriterConfiguration.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/ParserWriterConfiguration.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,97 +20,103 @@ package org.apache.metron.common.configuration.writer; -import org.apache.metron.common.configuration.IndexingConfigurations; -import org.apache.metron.common.configuration.ParserConfigurations; -import org.apache.metron.stellar.common.utils.ConversionUtils; - import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.metron.common.configuration.IndexingConfigurations; +import org.apache.metron.common.configuration.ParserConfigurations; +import org.apache.metron.stellar.common.utils.ConversionUtils; /* * Note that parsers can also be used for streaming enrichments, which means broader scope than * Kafka alone. */ public class ParserWriterConfiguration implements WriterConfiguration { - private ParserConfigurations config; - public ParserWriterConfiguration(ParserConfigurations config) { - this.config = config; - } - @Override - public int getBatchSize(String sensorName) { - if(config != null - && config.getSensorParserConfig(sensorName) != null - && config.getSensorParserConfig(sensorName).getParserConfig() != null - ) { - Object batchObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.BATCH_SIZE_CONF); - return batchObj == null ? ParserConfigurations.DEFAULT_KAFKA_BATCH_SIZE : ConversionUtils.convert(batchObj, Integer.class); + private final ParserConfigurations config; + + public ParserWriterConfiguration(ParserConfigurations config) { + this.config = config; } - return 1; - } - @Override - public int getBatchTimeout(String sensorName) { - if(config != null - && config.getSensorParserConfig(sensorName) != null - && config.getSensorParserConfig(sensorName).getParserConfig() != null - ) { - Object batchObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.BATCH_TIMEOUT_CONF); - return batchObj == null ? 0 : ConversionUtils.convert(batchObj, Integer.class); + @Override + public int getBatchSize(String sensorName) { + if (config != null + && config.getSensorParserConfig(sensorName) != null + && config.getSensorParserConfig(sensorName).getParserConfig() != null + ) { + Object batchObj = config.getSensorParserConfig(sensorName).getParserConfig() + .get(IndexingConfigurations.BATCH_SIZE_CONF); + return batchObj == null ? ParserConfigurations.DEFAULT_KAFKA_BATCH_SIZE + : ConversionUtils.convert(batchObj, Integer.class); + } + return 1; } - return 0; - } - @Override - public List getAllConfiguredTimeouts() { - // TODO - stub implementation pending METRON-750 - return new ArrayList(); - } + @Override + public int getBatchTimeout(String sensorName) { + if (config != null + && config.getSensorParserConfig(sensorName) != null + && config.getSensorParserConfig(sensorName).getParserConfig() != null + ) { + Object batchObj = config.getSensorParserConfig(sensorName).getParserConfig() + .get(IndexingConfigurations.BATCH_TIMEOUT_CONF); + return batchObj == null ? 0 : ConversionUtils.convert(batchObj, Integer.class); + } + return 0; + } - @Override - public String getIndex(String sensorName) { - if(config != null && config.getSensorParserConfig(sensorName) != null - && config.getSensorParserConfig(sensorName).getParserConfig() != null - ) { - Object indexObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.INDEX_CONF); - if(indexObj != null) { - return indexObj.toString(); - } - return null; + @Override + public List getAllConfiguredTimeouts() { + // TODO - stub implementation pending METRON-750 + return new ArrayList(); } - return sensorName; - } - @Override - public boolean isEnabled(String sensorName) { - if(config != null - && config.getSensorParserConfig(sensorName) != null - && config.getSensorParserConfig(sensorName).getParserConfig() != null - ) { - Object enabledObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.ENABLED_CONF); - return enabledObj == null ? true : ConversionUtils.convert(enabledObj, Boolean.class); + @Override + public String getIndex(String sensorName) { + if (config != null && config.getSensorParserConfig(sensorName) != null + && config.getSensorParserConfig(sensorName).getParserConfig() != null + ) { + Object indexObj = + config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.INDEX_CONF); + if (indexObj != null) { + return indexObj.toString(); + } + return null; + } + return sensorName; } - return true; - } - @Override - public Map getSensorConfig(String sensorName) { - return config.getSensorParserConfig(sensorName).getParserConfig(); - } + @Override + public boolean isEnabled(String sensorName) { + if (config != null + && config.getSensorParserConfig(sensorName) != null + && config.getSensorParserConfig(sensorName).getParserConfig() != null + ) { + Object enabledObj = + config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.ENABLED_CONF); + return enabledObj == null || ConversionUtils.convert(enabledObj, Boolean.class); + } + return true; + } - @Override - public Map getGlobalConfig() { - return config.getGlobalConfig(); - } + @Override + public Map getSensorConfig(String sensorName) { + return config.getSensorParserConfig(sensorName).getParserConfig(); + } - @Override - public boolean isDefault(String sensorName) { - return false; - } + @Override + public Map getGlobalConfig() { + return config.getGlobalConfig(); + } - @Override - public String getFieldNameConverter(String sensorName) { - // not applicable - return null; - } + @Override + public boolean isDefault(String sensorName) { + return false; + } + + @Override + public String getFieldNameConverter(String sensorName) { + // not applicable + return null; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/SingleBatchConfigurationFacade.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/SingleBatchConfigurationFacade.java index 2d5a62c5..49828f08 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/SingleBatchConfigurationFacade.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/SingleBatchConfigurationFacade.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,55 +25,56 @@ import java.util.Map; public class SingleBatchConfigurationFacade implements WriterConfiguration { - private WriterConfiguration config; - public SingleBatchConfigurationFacade(WriterConfiguration config) { - this.config = config; - } + private final WriterConfiguration config; - @Override - public int getBatchSize(String sensorName) { - return 1; - } + public SingleBatchConfigurationFacade(WriterConfiguration config) { + this.config = config; + } - @Override - public int getBatchTimeout(String sensorName) { - return 0; - } + @Override + public int getBatchSize(String sensorName) { + return 1; + } - @Override - public List getAllConfiguredTimeouts() { - // null implementation since batching is disabled - return new ArrayList(); - } + @Override + public int getBatchTimeout(String sensorName) { + return 0; + } - @Override - public String getIndex(String sensorName) { - return config.getIndex(sensorName); - } + @Override + public List getAllConfiguredTimeouts() { + // null implementation since batching is disabled + return new ArrayList(); + } - @Override - public boolean isEnabled(String sensorName) { - return true; - } + @Override + public String getIndex(String sensorName) { + return config.getIndex(sensorName); + } - @Override - public Map getSensorConfig(String sensorName) { - return config.getSensorConfig(sensorName); - } + @Override + public boolean isEnabled(String sensorName) { + return true; + } - @Override - public Map getGlobalConfig() { - return config.getGlobalConfig(); - } + @Override + public Map getSensorConfig(String sensorName) { + return config.getSensorConfig(sensorName); + } - @Override - public boolean isDefault(String sensorName) { - return false; - } + @Override + public Map getGlobalConfig() { + return config.getGlobalConfig(); + } - @Override - public String getFieldNameConverter(String sensorName) { - // not applicable - return null; - } + @Override + public boolean isDefault(String sensorName) { + return false; + } + + @Override + public String getFieldNameConverter(String sensorName) { + // not applicable + return null; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/WriterConfiguration.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/WriterConfiguration.java index 5c932da1..0e366045 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/WriterConfiguration.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/configuration/writer/WriterConfiguration.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,11 +20,10 @@ package org.apache.metron.common.configuration.writer; -import org.apache.metron.common.field.FieldNameConverter; - import java.io.Serializable; import java.util.List; import java.util.Map; +import org.apache.metron.common.field.FieldNameConverter; /** * Configures a writer to write messages to an endpoint. @@ -33,83 +34,86 @@ */ public interface WriterConfiguration extends Serializable { - /** - * Defines the maximum batch size for a given sensor. - * - * @param sensorName The name of the sensor. - * @return The batch size for the sensor. - */ - int getBatchSize(String sensorName); + /** + * Defines the maximum batch size for a given sensor. + * + * @param sensorName The name of the sensor. + * @return The batch size for the sensor. + */ + int getBatchSize(String sensorName); - /** - * Defines the batch timeout for a given sensor. Even if the maximum - * batch size has not been reached, the messages will be written when - * the timeout is reached. - * - * @param sensorName The name of the sensor. - * @return The batch timeout for the sensor. - */ - int getBatchTimeout(String sensorName); + /** + * Defines the batch timeout for a given sensor. Even if the maximum + * batch size has not been reached, the messages will be written when + * the timeout is reached. + * + * @param sensorName The name of the sensor. + * @return The batch timeout for the sensor. + */ + int getBatchTimeout(String sensorName); - /** - * Returns the batch timeouts for all of the currently configured sensors. - * @return All of the batch timeouts. - */ - List getAllConfiguredTimeouts(); + /** + * Returns the batch timeouts for all of the currently configured sensors. + * + * @return All of the batch timeouts. + */ + List getAllConfiguredTimeouts(); - /** - * The name of the index to write to for a given sensor. - * - * @param sensorName The name of the sensor. - * @return The name of the index to write to - */ - String getIndex(String sensorName); + /** + * The name of the index to write to for a given sensor. + * + * @param sensorName The name of the sensor. + * @return The name of the index to write to + */ + String getIndex(String sensorName); - /** - * Returns true, if this writer is enabled for the given sensor. - * - * @param sensorName The name of the sensor. - * @return True, if this writer is enabled. Otherwise, false. - */ - boolean isEnabled(String sensorName); + /** + * Returns true, if this writer is enabled for the given sensor. + * + * @param sensorName The name of the sensor. + * @return True, if this writer is enabled. Otherwise, false. + */ + boolean isEnabled(String sensorName); - /** - * Returns the sensor config for a specific sensor. - * - * @param sensorName The name of a sensor. - * @return a map containing the config - */ - Map getSensorConfig(String sensorName); + /** + * Returns the sensor config for a specific sensor. + * + * @param sensorName The name of a sensor. + * @return a map containing the config + */ + Map getSensorConfig(String sensorName); - /** - * Returns the global configuration. - * @return The global configuration. - */ - Map getGlobalConfig(); + /** + * Returns the global configuration. + * + * @return The global configuration. + */ + Map getGlobalConfig(); - /** - * Returns true, if the current writer configuration is set to all default values. - * - * @param sensorName The name of the sensor. - * @return True, if the writer is using all default values. Otherwise, false. - */ - boolean isDefault(String sensorName); + /** + * Returns true, if the current writer configuration is set to all default values. + * + * @param sensorName The name of the sensor. + * @return True, if the writer is using all default values. Otherwise, false. + */ + boolean isDefault(String sensorName); - /** - * Return the {@link FieldNameConverter} to use - * when writing messages. - * - * @param sensorName The name of the sensor; - * @return The {@link FieldNameConverter} - */ - String getFieldNameConverter(String sensorName); + /** + * Return the {@link FieldNameConverter} to use + * when writing messages. + * + * @param sensorName The name of the sensor; + * @return The {@link FieldNameConverter} + */ + String getFieldNameConverter(String sensorName); - /** - * Returns true, if the current writer configuration is set to use the GUID generated by Metron as the id - * @param sensorName The name of the sensor. - * @return True, if writer is configured to use GUID generated by Metron, false otherwise (and by default) - */ - default boolean isSetDocumentId(String sensorName) { - return false; - } + /** + * Returns true, if the current writer configuration is set to use the GUID generated by Metron as the id. + * + * @param sensorName The name of the sensor. + * @return True, if writer is configured to use GUID generated by Metron, false otherwise (and by default) + */ + default boolean isSetDocumentId(String sensorName) { + return false; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java index 9d946bc7..0b4896ec 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,11 +20,12 @@ package org.apache.metron.common.csv; +import static org.apache.commons.lang3.StringUtils.isEmpty; + import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.opencsv.CSVParser; import com.opencsv.CSVParserBuilder; - import java.io.IOException; import java.io.Serializable; import java.util.AbstractMap; @@ -30,8 +33,6 @@ import java.util.List; import java.util.Map; -import static org.apache.commons.lang3.StringUtils.isEmpty; - /** * Based on the parser config field `COLUMNS_KEY` the log line is * transformed into a intermediate result using the CSV parser. @@ -40,131 +41,127 @@ * will be trimmed before adding to `values`. */ public class CSVConverter implements Serializable { - public static final String COLUMNS_KEY="columns"; - public static final String SEPARATOR_KEY="separator"; - protected Map columnMap = new HashMap<>(); - protected CSVParser parser; - - public Map getColumnMap() { - return columnMap; - } + public static final String COLUMNS_KEY = "columns"; + public static final String SEPARATOR_KEY = "separator"; + protected Map columnMap = new HashMap<>(); + protected CSVParser parser; - public CSVParser getParser() { - return parser; - } - - /** - * Converts a CSV line to a map of column name -> value. - * - * @param line The line to be turned into a map - * @return A map from column name -> value in the line - * @throws IOException If there's an issue parsing the line - */ - public Map toMap(String line) throws IOException { - if(ignore(line)) { - return null; - } - String[] tokens = parser.parseLine(line); - Map values = new HashMap<>(); - for(Map.Entry kv : columnMap.entrySet()) { - values.put(kv.getKey().trim(), tokens[kv.getValue()].trim()); - } - return values; - } - - /** - * Initializes the CSVConverter based on the provided config. The config should contain - * an entry for {@code columns}, and can optionally contain a {@code separator}. - * - * @param config The configuration used for setup - */ - public void initialize(Map config) { - if(config.containsKey(COLUMNS_KEY)) { - columnMap = getColumnMap(config); - } - else { - throw new IllegalStateException("CSVExtractor requires " + COLUMNS_KEY + " configuration"); + public Map getColumnMap() { + return columnMap; } - char separator = ','; - if(config.containsKey(SEPARATOR_KEY)) { - separator = config.get(SEPARATOR_KEY).toString().charAt(0); + /** + * Retrieves the column map based on the columns key in the config map. + * + *

    If the data is a string, the default is to split on ',' as a list of column names. + * + *

    If the data is a list, the default is to used as the list of column names. + * + *

    If the data is a map, it must be in the form columnName -> position. + * + *

    For string and list, the data can be in the form columnName:position, and extracted + * appropriately. + * + * @param config Uses the columns key if defined to retrieve columns. + * @return A map from column -> position + */ + public static Map getColumnMap(Map config) { + Map columnMap = new HashMap<>(); + if (config.containsKey(COLUMNS_KEY)) { + Object columnsObj = config.get(COLUMNS_KEY); + if (columnsObj instanceof String) { + String columns = (String) columnsObj; + int i = 0; + for (String column : Splitter.on(',').split(columns)) { + Map.Entry e = getColumnMapEntry(column, i++); + columnMap.put(e.getKey(), e.getValue()); + } + } else if (columnsObj instanceof List) { + List columns = (List) columnsObj; + int i = 0; + for (Object column : columns) { + Map.Entry e = getColumnMapEntry(column.toString(), i++); + columnMap.put(e.getKey(), e.getValue()); + } + } else if (columnsObj instanceof Map) { + Map map = (Map) columnsObj; + for (Map.Entry e : map.entrySet()) { + columnMap.put(e.getKey().toString(), Integer.parseInt(e.getValue().toString())); + } + } + } + return columnMap; } - parser = new CSVParserBuilder().withSeparator(separator) - .build(); - } - protected boolean ignore(String line) { - if(null == line) { - return true; + public CSVParser getParser() { + return parser; } - String trimmedLine = line.trim(); - return trimmedLine.startsWith("#") || isEmpty(trimmedLine); - } - /** - * Given a column determines the column -> position. If the column contains data in the format - * {@code columnName:position}, use that. Otherwise, use the provided value of i. - * - * @param column The data in the column - * @param i The default position to use - * @return A map entry of column value -> position - */ - public static Map.Entry getColumnMapEntry(String column, int i) { - if(column.contains(":")) { - Iterable tokens = Splitter.on(':').split(column); - String col = Iterables.getFirst(tokens, null); - Integer pos = Integer.parseInt(Iterables.getLast(tokens)); - return new AbstractMap.SimpleEntry<>(col, pos); - } - else { - return new AbstractMap.SimpleEntry<>(column, i); + /** + * Converts a CSV line to a map of column name -> value. + * + * @param line The line to be turned into a map + * @return A map from column name -> value in the line + * @throws IOException If there's an issue parsing the line + */ + public Map toMap(String line) throws IOException { + if (ignore(line)) { + return null; + } + String[] tokens = parser.parseLine(line); + Map values = new HashMap<>(); + for (Map.Entry kv : columnMap.entrySet()) { + values.put(kv.getKey().trim(), tokens[kv.getValue()].trim()); + } + return values; } - } + /** + * Initializes the CSVConverter based on the provided config. The config should contain + * an entry for {@code columns}, and can optionally contain a {@code separator}. + * + * @param config The configuration used for setup + */ + public void initialize(Map config) { + if (config.containsKey(COLUMNS_KEY)) { + columnMap = getColumnMap(config); + } else { + throw new IllegalStateException("CSVExtractor requires " + COLUMNS_KEY + " configuration"); + } + char separator = ','; + if (config.containsKey(SEPARATOR_KEY)) { + separator = config.get(SEPARATOR_KEY).toString().charAt(0); - /** - * Retrieves the column map based on the columns key in the config map. - * - *

    If the data is a string, the default is to split on ',' as a list of column names. - * - *

    If the data is a list, the default is to used as the list of column names. - * - *

    If the data is a map, it must be in the form columnName -> position. - * - *

    For string and list, the data can be in the form columnName:position, and extracted - * appropriately. - * - * @param config Uses the columns key if defined to retrieve columns. - * @return A map from column -> position - */ - public static Map getColumnMap(Map config) { - Map columnMap = new HashMap<>(); - if(config.containsKey(COLUMNS_KEY)) { - Object columnsObj = config.get(COLUMNS_KEY); - if(columnsObj instanceof String) { - String columns = (String)columnsObj; - int i = 0; - for (String column : Splitter.on(',').split(columns)) { - Map.Entry e = getColumnMapEntry(column, i++); - columnMap.put(e.getKey(), e.getValue()); } - } - else if(columnsObj instanceof List) { - List columns = (List)columnsObj; - int i = 0; - for(Object column : columns) { - Map.Entry e = getColumnMapEntry(column.toString(), i++); - columnMap.put(e.getKey(), e.getValue()); + parser = new CSVParserBuilder().withSeparator(separator) + .build(); + } + + protected boolean ignore(String line) { + if (null == line) { + return true; } - } - else if(columnsObj instanceof Map) { - Map map = (Map)columnsObj; - for(Map.Entry e : map.entrySet()) { - columnMap.put(e.getKey().toString(), Integer.parseInt(e.getValue().toString())); + String trimmedLine = line.trim(); + return trimmedLine.startsWith("#") || isEmpty(trimmedLine); + } + + /** + * Given a column determines the column -> position. If the column contains data in the format + * {@code columnName:position}, use that. Otherwise, use the provided value of i. + * + * @param column The data in the column + * @param i The default position to use + * @return A map entry of column value -> position + */ + public static Map.Entry getColumnMapEntry(String column, int i) { + if (column.contains(":")) { + Iterable tokens = Splitter.on(':').split(column); + String col = Iterables.getFirst(tokens, null); + Integer pos = Integer.parseInt(Iterables.getLast(tokens)); + return new AbstractMap.SimpleEntry<>(col, pos); + } else { + return new AbstractMap.SimpleEntry<>(column, i); } - } + } - return columnMap; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java index c83fdfbc..4ac31f23 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.error; import static java.nio.charset.StandardCharsets.UTF_8; @@ -33,7 +36,6 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; - import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.metron.common.utils.HashUtils; import org.apache.metron.stellar.common.Constants; @@ -42,207 +44,217 @@ public class MetronError { - private String message; - private Throwable throwable; - private Set sensorTypes = Collections.singleton(ERROR_TYPE); - private ErrorType errorType = ErrorType.DEFAULT_ERROR; - private Set errorFields; - private List rawMessages; - private Map metadata = new HashMap<>(); - - public MetronError withMessage(String message) { - this.message = message; - return this; - } - - public MetronError withThrowable(Throwable throwable) { - this.throwable = throwable; - return this; - } - - public MetronError withSensorType(Set sensorTypes) { - this.sensorTypes = sensorTypes; - return this; - } - - public MetronError withErrorType(ErrorType errorType) { - this.errorType = errorType; - return this; - } - - public MetronError withErrorFields(Set errorFields) { - this.errorFields = errorFields; - return this; - } - - public MetronError withMetadata(Map metadata) { - this.metadata.putAll(metadata); - return this; - } - - /** - * Adds a rawMessage to the error. Calls can be chained, as the method returns this. - * - * @param rawMessage The raw message to add - * @return this, to allow for call chaining - */ - public MetronError addRawMessage(Object rawMessage) { - if (rawMessage != null) { - if (this.rawMessages == null) { - this.rawMessages = new ArrayList<>(); - } - this.rawMessages.add(rawMessage); - } - return this; - } - - public MetronError withRawMessages(List rawMessages) { - this.rawMessages = rawMessages; - return this; - } - - public Optional getThrowable() { - return throwable != null ? Optional.of(throwable) : Optional.empty(); - } - - public List getRawMessages() { - return rawMessages; - } - - /** - * Serializes the MetronError into a JSON object. - * - * @return The resulting json object - */ - @SuppressWarnings({"unchecked"}) - public JSONObject getJSONObject() { - JSONObject errorMessage = new JSONObject(); - errorMessage.put(Constants.GUID, UUID.randomUUID().toString()); - errorMessage.put(Constants.SENSOR_TYPE, ERROR_TYPE); - errorMessage.put(Constants.ErrorFields.ERROR_TYPE.getName(), errorType.getType()); - addFailedSensorType(errorMessage); - addMessageString(errorMessage); - addStacktrace(errorMessage); - addTimestamp(errorMessage); - addHostname(errorMessage); - addRawMessages(errorMessage); - addErrorHash(errorMessage); - addMetadata(errorMessage); - - return errorMessage; - } - - private void addFailedSensorType(JSONObject errorMessage) { - if (sensorTypes.size() == 1) { - errorMessage.put(ErrorFields.FAILED_SENSOR_TYPE.getName(), sensorTypes.iterator().next()); - } else { - errorMessage.put(ErrorFields.FAILED_SENSOR_TYPE.getName(), new JSONArray().addAll(sensorTypes)); - } - } - - @SuppressWarnings({"unchecked"}) - private void addMessageString(JSONObject errorMessage) { - if (message != null) { - errorMessage.put(ErrorFields.MESSAGE.getName(), message); - } else if (throwable != null) { - errorMessage.put(ErrorFields.MESSAGE.getName(), throwable.getMessage()); - } - } - - @SuppressWarnings({"unchecked"}) - private void addStacktrace(JSONObject errorMessage) { - if (throwable != null) { - String stackTrace = ExceptionUtils.getStackTrace(throwable); - String exception = throwable.toString(); - errorMessage.put(ErrorFields.EXCEPTION.getName(), exception); - errorMessage.put(ErrorFields.STACK.getName(), stackTrace); - } - } - - @SuppressWarnings({"unchecked"}) - private void addTimestamp(JSONObject errorMessage) { - errorMessage.put(ErrorFields.TIMESTAMP.getName(), System.currentTimeMillis()); - } - - @SuppressWarnings({"unchecked"}) - private void addHostname(JSONObject errorMessage) { - try { - errorMessage.put(ErrorFields.HOSTNAME.getName(), InetAddress.getLocalHost().getHostName()); - } catch (UnknownHostException ex) { - // Leave the hostname field off if it cannot be found - } - } - - @SuppressWarnings({"unchecked"}) - private void addRawMessages(JSONObject errorMessage) { - if(rawMessages != null) { - for(int i = 0; i < rawMessages.size(); i++) { - Object rawMessage = rawMessages.get(i); - // If multiple messages are included add an index to the field name, otherwise leave it off - String rawMessageField = rawMessages.size() == 1 ? ErrorFields.RAW_MESSAGE.getName() : ErrorFields.RAW_MESSAGE.getName() + "_" + i; - // It's unclear if we need a rawMessageBytes field so commenting out for now - //String rawMessageBytesField = rawMessages.size() == 1 ? ErrorFields.RAW_MESSAGE_BYTES.getName() : ErrorFields.RAW_MESSAGE_BYTES.getName() + "_" + i; - if(rawMessage instanceof byte[]) { - errorMessage.put(rawMessageField, new String((byte[])rawMessage, UTF_8)); - //errorMessage.put(rawMessageBytesField, com.google.common.primitives.Bytes.asList((byte[])rawMessage)); - } else if (rawMessage instanceof JSONObject) { - JSONObject rawMessageJSON = (JSONObject) rawMessage; - String rawMessageJSONString = rawMessageJSON.toJSONString(); - errorMessage.put(rawMessageField, rawMessageJSONString); - //errorMessage.put(rawMessageBytesField, com.google.common.primitives.Bytes.asList(rawMessageJSONString.getBytes(UTF_8))); - } else { - errorMessage.put(rawMessageField, rawMessage.toString()); - //errorMessage.put(rawMessageBytesField, com.google.common.primitives.Bytes.asList(rawMessage.toString().getBytes(UTF_8))); + private String message; + private Throwable throwable; + private Set sensorTypes = Collections.singleton(ERROR_TYPE); + private ErrorType errorType = ErrorType.DEFAULT_ERROR; + private Set errorFields; + private List rawMessages; + private final Map metadata = new HashMap<>(); + + public MetronError withMessage(String message) { + this.message = message; + return this; + } + + public MetronError withThrowable(Throwable throwable) { + this.throwable = throwable; + return this; + } + + public MetronError withSensorType(Set sensorTypes) { + this.sensorTypes = sensorTypes; + return this; + } + + public MetronError withErrorType(ErrorType errorType) { + this.errorType = errorType; + return this; + } + + public MetronError withErrorFields(Set errorFields) { + this.errorFields = errorFields; + return this; + } + + public MetronError withMetadata(Map metadata) { + this.metadata.putAll(metadata); + return this; + } + + /** + * Adds a rawMessage to the error. Calls can be chained, as the method returns this. + * + * @param rawMessage The raw message to add + * @return this, to allow for call chaining + */ + public MetronError addRawMessage(Object rawMessage) { + if (rawMessage != null) { + if (this.rawMessages == null) { + this.rawMessages = new ArrayList<>(); + } + this.rawMessages.add(rawMessage); } - } - } - } - - @SuppressWarnings({"unchecked"}) - private void addErrorHash(JSONObject errorMessage) { - if (rawMessages != null && rawMessages.size() == 1) { - Object rawMessage = rawMessages.get(0); - if (rawMessage instanceof JSONObject) { - JSONObject rawJSON = (JSONObject) rawMessage; - if (errorFields != null) { - errorMessage.put(ErrorFields.ERROR_FIELDS.getName(), String.join(",", errorFields)); - errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash(rawJSON, errorFields)); + return this; + } + + public MetronError withRawMessages(List rawMessages) { + this.rawMessages = rawMessages; + return this; + } + + public Optional getThrowable() { + return throwable != null ? Optional.of(throwable) : Optional.empty(); + } + + public List getRawMessages() { + return rawMessages; + } + + /** + * Serializes the MetronError into a JSON object. + * + * @return The resulting json object + */ + @SuppressWarnings({"unchecked"}) + public JSONObject getJSONObject() { + JSONObject errorMessage = new JSONObject(); + errorMessage.put(Constants.GUID, UUID.randomUUID().toString()); + errorMessage.put(Constants.SENSOR_TYPE, ERROR_TYPE); + errorMessage.put(Constants.ErrorFields.ERROR_TYPE.getName(), errorType.getType()); + addFailedSensorType(errorMessage); + addMessageString(errorMessage); + addStacktrace(errorMessage); + addTimestamp(errorMessage); + addHostname(errorMessage); + addRawMessages(errorMessage); + addErrorHash(errorMessage); + addMetadata(errorMessage); + + return errorMessage; + } + + private void addFailedSensorType(JSONObject errorMessage) { + if (sensorTypes.size() == 1) { + errorMessage.put(ErrorFields.FAILED_SENSOR_TYPE.getName(), sensorTypes.iterator().next()); } else { - errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash(rawJSON)); + errorMessage.put(ErrorFields.FAILED_SENSOR_TYPE.getName(), new JSONArray().addAll(sensorTypes)); } - } else if (rawMessage instanceof byte[]) { - errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash((byte[])rawMessage)); - } else { - errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash(rawMessage.toString().getBytes(UTF_8))); - } - } - } - - private void addMetadata(JSONObject errorMessage) { - if(metadata != null && metadata.keySet().size() > 0) { - // add each metadata element directly to the message. each metadata key already has - // a standard prefix, no need to add another prefix to avoid collisions. this mimics - // the behavior of merging metadata. - errorMessage.putAll(metadata); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof MetronError)) return false; - MetronError that = (MetronError) o; - return Objects.equals(message, that.message) && - Objects.equals(throwable, that.throwable) && - Objects.equals(sensorTypes, that.sensorTypes) && - errorType == that.errorType && - Objects.equals(errorFields, that.errorFields) && - Objects.equals(rawMessages, that.rawMessages) && - Objects.equals(metadata, that.metadata); - } - - @Override - public int hashCode() { - return Objects.hash(message, throwable, sensorTypes, errorType, errorFields, rawMessages, metadata); - } + } + + @SuppressWarnings({"unchecked"}) + private void addMessageString(JSONObject errorMessage) { + if (message != null) { + errorMessage.put(ErrorFields.MESSAGE.getName(), message); + } else if (throwable != null) { + errorMessage.put(ErrorFields.MESSAGE.getName(), throwable.getMessage()); + } + } + + @SuppressWarnings({"unchecked"}) + private void addStacktrace(JSONObject errorMessage) { + if (throwable != null) { + String stackTrace = ExceptionUtils.getStackTrace(throwable); + String exception = throwable.toString(); + errorMessage.put(ErrorFields.EXCEPTION.getName(), exception); + errorMessage.put(ErrorFields.STACK.getName(), stackTrace); + } + } + + @SuppressWarnings({"unchecked"}) + private void addTimestamp(JSONObject errorMessage) { + errorMessage.put(ErrorFields.TIMESTAMP.getName(), System.currentTimeMillis()); + } + + @SuppressWarnings({"unchecked"}) + private void addHostname(JSONObject errorMessage) { + try { + errorMessage.put(ErrorFields.HOSTNAME.getName(), InetAddress.getLocalHost().getHostName()); + } catch (UnknownHostException ex) { + // Leave the hostname field off if it cannot be found + } + } + + @SuppressWarnings({"unchecked"}) + private void addRawMessages(JSONObject errorMessage) { + if (rawMessages != null) { + for (int i = 0; i < rawMessages.size(); i++) { + Object rawMessage = rawMessages.get(i); + // If multiple messages are included add an index to the field name, otherwise leave it off + String rawMessageField = rawMessages.size() == 1 ? ErrorFields.RAW_MESSAGE.getName() + : ErrorFields.RAW_MESSAGE.getName() + "_" + i; + // It's unclear if we need a rawMessageBytes field so commenting out for now + //String rawMessageBytesField = rawMessages.size() == 1 + // ? ErrorFields.RAW_MESSAGE_BYTES.getName() + // : ErrorFields.RAW_MESSAGE_BYTES.getName() + "_" + i; + if (rawMessage instanceof byte[]) { + errorMessage.put(rawMessageField, new String((byte[]) rawMessage, UTF_8)); + //errorMessage.put(rawMessageBytesField, com.google.common.primitives.Bytes.asList((byte[])rawMessage)); + } else if (rawMessage instanceof JSONObject) { + JSONObject rawMessageJSON = (JSONObject) rawMessage; + String rawMessageJSONString = rawMessageJSON.toJSONString(); + errorMessage.put(rawMessageField, rawMessageJSONString); + //errorMessage.put(rawMessageBytesField, + // com.google.common.primitives.Bytes.asList(rawMessageJSONString.getBytes(UTF_8))); + } else { + errorMessage.put(rawMessageField, rawMessage.toString()); + //errorMessage.put(rawMessageBytesField, + // com.google.common.primitives.Bytes.asList(rawMessage.toString().getBytes(UTF_8))); + } + } + } + } + + @SuppressWarnings({"unchecked"}) + private void addErrorHash(JSONObject errorMessage) { + if (rawMessages != null && rawMessages.size() == 1) { + Object rawMessage = rawMessages.get(0); + if (rawMessage instanceof JSONObject) { + JSONObject rawJSON = (JSONObject) rawMessage; + if (errorFields != null) { + errorMessage.put(ErrorFields.ERROR_FIELDS.getName(), String.join(",", errorFields)); + errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash(rawJSON, errorFields)); + } else { + errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash(rawJSON)); + } + } else if (rawMessage instanceof byte[]) { + errorMessage.put(ErrorFields.ERROR_HASH.getName(), HashUtils.getMessageHash((byte[]) rawMessage)); + } else { + errorMessage.put(ErrorFields.ERROR_HASH.getName(), + HashUtils.getMessageHash(rawMessage.toString().getBytes(UTF_8))); + } + } + } + + private void addMetadata(JSONObject errorMessage) { + if (metadata != null && metadata.keySet().size() > 0) { + // add each metadata element directly to the message. each metadata key already has + // a standard prefix, no need to add another prefix to avoid collisions. this mimics + // the behavior of merging metadata. + errorMessage.putAll(metadata); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MetronError)) { + return false; + } + MetronError that = (MetronError) o; + return Objects.equals(message, that.message) + && Objects.equals(throwable, that.throwable) + && Objects.equals(sensorTypes, that.sensorTypes) + && errorType == that.errorType + && Objects.equals(errorFields, that.errorFields) + && Objects.equals(rawMessages, that.rawMessages) + && Objects.equals(metadata, that.metadata); + } + + @Override + public int hashCode() { + return Objects.hash(message, throwable, sensorTypes, errorType, errorFields, rawMessages, metadata); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/DeDotFieldNameConverter.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/DeDotFieldNameConverter.java index a571ce5d..a3ec1762 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/DeDotFieldNameConverter.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/DeDotFieldNameConverter.java @@ -7,21 +7,23 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.field; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +package org.apache.metron.common.field; import java.io.Serializable; import java.lang.invoke.MethodHandles; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A {@link FieldNameConverter} that replaces all field names containing dots @@ -29,18 +31,18 @@ */ public class DeDotFieldNameConverter implements FieldNameConverter, Serializable { - private static final long serialVersionUID = -3126840090749760299L; - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final long serialVersionUID = -3126840090749760299L; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @Override - public String convert(String originalField) { + @Override + public String convert(String originalField) { - String newField = originalField.replace(".",":"); + String newField = originalField.replace(".", ":"); - if(LOG.isDebugEnabled() && originalField.contains(".")) { - LOG.debug("Renamed dotted field; original={}, new={}", originalField, newField); - } + if (LOG.isDebugEnabled() && originalField.contains(".")) { + LOG.debug("Renamed dotted field; original={}, new={}", originalField, newField); + } - return newField; - } + return newField; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/FieldNameConverter.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/FieldNameConverter.java index 0c15ec44..e9ec24b6 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/FieldNameConverter.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/FieldNameConverter.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.field; /** @@ -22,11 +25,11 @@ */ public interface FieldNameConverter { - /** - * Convert the field name. - * - * @param originalField The original field name. - * @return The new field name. - */ - String convert(String originalField); + /** + * Convert the field name. + * + * @param originalField The original field name. + * @return The new field name. + */ + String convert(String originalField); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/FieldNameConverters.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/FieldNameConverters.java index c5e71d25..c05d5a5f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/FieldNameConverters.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/FieldNameConverters.java @@ -18,6 +18,7 @@ package org.apache.metron.common.field; +import java.lang.invoke.MethodHandles; import org.apache.commons.lang.ClassUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -25,8 +26,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; - /** * Enumerates a set of {@link FieldNameConverter} implementations. * @@ -37,80 +36,81 @@ */ public enum FieldNameConverters implements FieldNameConverter { - /** - * A {@link FieldNameConverter} that does not rename any fields. All field - * names remain unchanged. - */ - NOOP(new NoopFieldNameConverter()), - - /** - * A {@link FieldNameConverter} that replaces all field names containing dots - * with colons. - */ - DEDOT(new DeDotFieldNameConverter()); - - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private FieldNameConverter converter; - - FieldNameConverters(FieldNameConverter converter) { - this.converter = converter; - } - - /** - * Returns a shared instance of the {@link FieldNameConverter}. - * - * @return A shared {@link FieldNameConverter} instance. - */ - public FieldNameConverter get() { - return converter; - } - - /** - * Allows the {@link FieldNameConverters} enums to be used directly as a {@link FieldNameConverter}. - * - * {@code - * FieldNameConverter converter = FieldNameConverters.DEDOT; - * } - * - * @param originalField The original field name. - * @return the converted field name - */ - @Override - public String convert(String originalField) { - return converter.convert(originalField); - } - - /** - * Create a new {@link FieldNameConverter} for a given sensor type and config. - * - * @param sensorType The type of sensor. - * @param config The writer configuration. - * @return The new {@link FieldNameConverter} - */ - public static FieldNameConverter create(String sensorType, WriterConfiguration config) { - FieldNameConverter result = null; - - // which field name converter has been configured? - String converterName = config.getFieldNameConverter(sensorType); - if(StringUtils.isNotBlank(converterName)) { - try { - result = FieldNameConverters.valueOf(converterName); - - } catch (IllegalArgumentException e) { - LOG.error("Invalid field name converter, using default; configured={}, knownValues={}, error={}", - converterName, FieldNameConverters.values(), ExceptionUtils.getRootCauseMessage(e)); - } + /** + * A {@link FieldNameConverter} that does not rename any fields. All field + * names remain unchanged. + */ + NOOP(new NoopFieldNameConverter()), + + /** + * A {@link FieldNameConverter} that replaces all field names containing dots + * with colons. + */ + DEDOT(new DeDotFieldNameConverter()); + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final FieldNameConverter converter; + + FieldNameConverters(FieldNameConverter converter) { + this.converter = converter; } - if(result == null) { - // if no converter defined or an invalid converter is defined, default to 'DEDOT' - result = FieldNameConverters.DEDOT; + /** + * Returns a shared instance of the {@link FieldNameConverter}. + * + * @return A shared {@link FieldNameConverter} instance. + */ + public FieldNameConverter get() { + return converter; } - LOG.debug("Created field name converter; sensorType={}, configured={}, class={}", - sensorType, converterName, ClassUtils.getShortClassName(result, "null")); + /** + * Allows the {@link FieldNameConverters} enums to be used directly as a {@link FieldNameConverter}. + * + *

    + * {@code + * FieldNameConverter converter = FieldNameConverters.DEDOT; + * } + * + * @param originalField The original field name. + * @return the converted field name + */ + @Override + public String convert(String originalField) { + return converter.convert(originalField); + } - return result; - } + /** + * Create a new {@link FieldNameConverter} for a given sensor type and config. + * + * @param sensorType The type of sensor. + * @param config The writer configuration. + * @return The new {@link FieldNameConverter} + */ + public static FieldNameConverter create(String sensorType, WriterConfiguration config) { + FieldNameConverter result = null; + + // which field name converter has been configured? + String converterName = config.getFieldNameConverter(sensorType); + if (StringUtils.isNotBlank(converterName)) { + try { + result = FieldNameConverters.valueOf(converterName); + + } catch (IllegalArgumentException e) { + LOG.error("Invalid field name converter, using default; configured={}, knownValues={}, error={}", + converterName, FieldNameConverters.values(), ExceptionUtils.getRootCauseMessage(e)); + } + } + + if (result == null) { + // if no converter defined or an invalid converter is defined, default to 'DEDOT' + result = FieldNameConverters.DEDOT; + } + + LOG.debug("Created field name converter; sensorType={}, configured={}, class={}", + sensorType, converterName, ClassUtils.getShortClassName(result, "null")); + + return result; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/NoopFieldNameConverter.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/NoopFieldNameConverter.java index a7f3f7c5..d7a0f13f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/NoopFieldNameConverter.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/NoopFieldNameConverter.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.field; /** @@ -23,10 +26,10 @@ */ public class NoopFieldNameConverter implements FieldNameConverter { - @Override - public String convert(String originalField) { + @Override + public String convert(String originalField) { - // no change to the field name - return originalField; - } + // no change to the field name + return originalField; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java index 138e2281..cc530fb0 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,18 +20,17 @@ package org.apache.metron.common.field.transformation; -import org.apache.metron.stellar.dsl.Context; - import java.io.Serializable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.dsl.Context; public interface FieldTransformation extends Serializable { - Map map( Map input - , List outputField - , LinkedHashMap fieldMappingConfig - , Context context - , Map... sensorConfig - ); + Map map(Map input, + List outputField, + LinkedHashMap fieldMappingConfig, + Context context, + Map... sensorConfig + ); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java index d30b71a5..776f558b 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,34 +23,30 @@ import org.apache.metron.common.utils.ReflectionUtils; public enum FieldTransformations { - IP_PROTOCOL(new IPProtocolTransformation()) - ,REMOVE(new RemoveTransformation()) - ,STELLAR(new StellarTransformation()) - ,SELECT(new SelectTransformation()) - ,RENAME(new RenameTransformation()) - ,REGEX_SELECT(new RegexSelectTransformation()) - ; - FieldTransformation mapping; - FieldTransformations(FieldTransformation mapping) { - this.mapping = mapping; - } + IP_PROTOCOL(new IPProtocolTransformation()), REMOVE(new RemoveTransformation()), + STELLAR(new StellarTransformation()), SELECT(new SelectTransformation()), RENAME(new RenameTransformation()), + REGEX_SELECT(new RegexSelectTransformation()); + FieldTransformation mapping; - /** - * Gets a {@link FieldTransformation} based on the provided mapping. - * - * @param mapping The name of the mapping to pull - * @return The field transformation associated with the mapping - */ - public static FieldTransformation get(String mapping) { - try { - return FieldTransformations.valueOf(mapping).mapping; + FieldTransformations(FieldTransformation mapping) { + this.mapping = mapping; } - catch(Exception ex) { - return ReflectionUtils.createInstance(mapping); + + /** + * Gets a {@link FieldTransformation} based on the provided mapping. + * + * @param mapping The name of the mapping to pull + * @return The field transformation associated with the mapping + */ + public static FieldTransformation get(String mapping) { + try { + return FieldTransformations.valueOf(mapping).mapping; + } catch (Exception ex) { + return ReflectionUtils.createInstance(mapping); + } } - } - public Class getMappingClass() { - return mapping.getClass(); - } + public Class getMappingClass() { + return mapping.getClass(); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java index 0518d6b0..9a26c7a0 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,202 +21,201 @@ package org.apache.metron.common.field.transformation; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; -import org.apache.metron.stellar.common.utils.ConversionUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -@Stellar(name="PROTOCOL_TO_NAME" - , description="Converts the IANA protocol number to the protocol name" - , params = { - "IANA Number" - } - , returns = "The protocol name associated with the IANA number." - ) +@Stellar(name = "PROTOCOL_TO_NAME", + description = "Converts the IANA protocol number to the protocol name", + params = { + "IANA Number" + }, + returns = "The protocol name associated with the IANA number." +) public class IPProtocolTransformation extends SimpleFieldTransformation implements StellarFunction { - private final static Map PROTOCOLS = new HashMap<>(); + private static final Map PROTOCOLS = new HashMap<>(); - static { - PROTOCOLS.put(1, "ICMP"); - PROTOCOLS.put(2, "IGMP"); - PROTOCOLS.put(3, "GGP"); - PROTOCOLS.put(4, "IP-in-IP"); - PROTOCOLS.put(5, "ST"); - PROTOCOLS.put(6, "TCP"); - PROTOCOLS.put(7, "CBT"); - PROTOCOLS.put(8, "EGP"); - PROTOCOLS.put(9, "IGP"); - PROTOCOLS.put(10, "BBN-RCC-MON"); - PROTOCOLS.put(11, "NVP-II"); - PROTOCOLS.put(12, "PUP"); - PROTOCOLS.put(13, "ARGUS"); - PROTOCOLS.put(14, "EMCON"); - PROTOCOLS.put(15, "XNET"); - PROTOCOLS.put(16, "CHAOS"); - PROTOCOLS.put(17, "UDP"); - PROTOCOLS.put(18, "MUX"); - PROTOCOLS.put(19, "DCN-MEAS"); - PROTOCOLS.put(20, "HMP"); - PROTOCOLS.put(21, "PRM"); - PROTOCOLS.put(22, "XNS-IDP"); - PROTOCOLS.put(23, "TRUNK-1"); - PROTOCOLS.put(24, "TRUNK-2"); - PROTOCOLS.put(25, "LEAF-1"); - PROTOCOLS.put(26, "LEAF-2"); - PROTOCOLS.put(27, "RDP"); - PROTOCOLS.put(28, "IRTP"); - PROTOCOLS.put(29, "ISO-TP4"); - PROTOCOLS.put(30, "NETBLT"); - PROTOCOLS.put(31, "MFE-NSP"); - PROTOCOLS.put(32, "MERIT-INP"); - PROTOCOLS.put(33, "DCCP"); - PROTOCOLS.put(34, "3PC"); - PROTOCOLS.put(35, "IDPR"); - PROTOCOLS.put(36, "XTP"); - PROTOCOLS.put(37, "DDP"); - PROTOCOLS.put(38, "IDPR-CMTP"); - PROTOCOLS.put(39, "TP++"); - PROTOCOLS.put(40, "IL"); - PROTOCOLS.put(41, "IPv6"); - PROTOCOLS.put(42, "SDRP"); - PROTOCOLS.put(43, "IPv6-Route"); - PROTOCOLS.put(44, "IPv6-Frag"); - PROTOCOLS.put(45, "IDRP"); - PROTOCOLS.put(46, "RSVP"); - PROTOCOLS.put(47, "GRE"); - PROTOCOLS.put(48, "MHRP"); - PROTOCOLS.put(49, "BNA"); - PROTOCOLS.put(50, "ESP"); - PROTOCOLS.put(51, "AH"); - PROTOCOLS.put(52, "I-NLSP"); - PROTOCOLS.put(53, "SWIPE"); - PROTOCOLS.put(54, "NARP"); - PROTOCOLS.put(55, "MOBILE"); - PROTOCOLS.put(56, "TLSP"); - PROTOCOLS.put(57, "SKIP"); - PROTOCOLS.put(58, "IPv6-ICMP"); - PROTOCOLS.put(59, "IPv6-NoNxt"); - PROTOCOLS.put(60, "IPv6-Opts"); - PROTOCOLS.put(62, "CFTP"); - PROTOCOLS.put(64, "SAT-EXPAK"); - PROTOCOLS.put(65, "KRYPTOLAN"); - PROTOCOLS.put(66, "RVD"); - PROTOCOLS.put(67, "IPPC"); - PROTOCOLS.put(68, "SAT-MON"); - PROTOCOLS.put(70, "VISA"); - PROTOCOLS.put(71, "IPCU"); - PROTOCOLS.put(72, "CPNX"); - PROTOCOLS.put(73, "CPHB"); - PROTOCOLS.put(74, "WSN"); - PROTOCOLS.put(75, "PVP"); - PROTOCOLS.put(76, "BR-SAT-MON"); - PROTOCOLS.put(77, "SUN-ND"); - PROTOCOLS.put(78, "WB-MON"); - PROTOCOLS.put(79, "WB-EXPAK"); - PROTOCOLS.put(80, "ISO-IP"); - PROTOCOLS.put(81, "VMTP"); - PROTOCOLS.put(82, "SECURE-VMTP"); - PROTOCOLS.put(83, "VINES"); - PROTOCOLS.put(84, "TTP"); - PROTOCOLS.put(85, "NSFNET-IGP"); - PROTOCOLS.put(86, "DGP"); - PROTOCOLS.put(87, "TCF"); - PROTOCOLS.put(88, ""); - PROTOCOLS.put(89, "OSPF"); - PROTOCOLS.put(90, "Sprite-RP"); - PROTOCOLS.put(91, "LARP"); - PROTOCOLS.put(92, "MTP"); - PROTOCOLS.put(93, "AX.25"); - PROTOCOLS.put(94, "IPIP"); - PROTOCOLS.put(95, "MICP"); - PROTOCOLS.put(96, "SCC-SP"); - PROTOCOLS.put(97, "ETHERIP"); - PROTOCOLS.put(98, "ENCAP"); - PROTOCOLS.put(100, "GMTP"); - PROTOCOLS.put(101, "IFMP"); - PROTOCOLS.put(102, "PNNI"); - PROTOCOLS.put(103, "PIM"); - PROTOCOLS.put(104, "ARIS"); - PROTOCOLS.put(105, "SCPS"); - PROTOCOLS.put(106, "QNX"); - PROTOCOLS.put(107, "A/N"); - PROTOCOLS.put(108, "IPComp"); - PROTOCOLS.put(109, "SNP"); - PROTOCOLS.put(110, "Compaq-Peer"); - PROTOCOLS.put(111, "IPX-in-IP"); - PROTOCOLS.put(112, "VRRP"); - PROTOCOLS.put(113, "PGM"); - PROTOCOLS.put(115, "L2TP"); - PROTOCOLS.put(116, "DDX"); - PROTOCOLS.put(117, "IATP"); - PROTOCOLS.put(118, "STP"); - PROTOCOLS.put(119, "SRP"); - PROTOCOLS.put(120, "UTI"); - PROTOCOLS.put(121, "SMP"); - PROTOCOLS.put(122, "SM"); - PROTOCOLS.put(123, "PTP"); - PROTOCOLS.put(124, "IS-IS"); - PROTOCOLS.put(125, "FIRE"); - PROTOCOLS.put(126, "CRTP"); - PROTOCOLS.put(127, "CRUDP"); - PROTOCOLS.put(128, "SSCOPMCE"); - PROTOCOLS.put(129, "IPLT"); - PROTOCOLS.put(130, "SPS"); - PROTOCOLS.put(131, "PIPE"); - PROTOCOLS.put(132, "SCTP"); - PROTOCOLS.put(133, "FC"); - PROTOCOLS.put(134, "RSVP-E2E-IGNORE"); - PROTOCOLS.put(135, "Mobility Header"); - PROTOCOLS.put(136, "UDPLite"); - PROTOCOLS.put(137, "MPLS-in-IP"); - PROTOCOLS.put(138, "manet"); - PROTOCOLS.put(139, "HIP"); - PROTOCOLS.put(140, "Shim6"); - PROTOCOLS.put(141, "WESP"); - PROTOCOLS.put(142, "ROHC"); - } + static { + PROTOCOLS.put(1, "ICMP"); + PROTOCOLS.put(2, "IGMP"); + PROTOCOLS.put(3, "GGP"); + PROTOCOLS.put(4, "IP-in-IP"); + PROTOCOLS.put(5, "ST"); + PROTOCOLS.put(6, "TCP"); + PROTOCOLS.put(7, "CBT"); + PROTOCOLS.put(8, "EGP"); + PROTOCOLS.put(9, "IGP"); + PROTOCOLS.put(10, "BBN-RCC-MON"); + PROTOCOLS.put(11, "NVP-II"); + PROTOCOLS.put(12, "PUP"); + PROTOCOLS.put(13, "ARGUS"); + PROTOCOLS.put(14, "EMCON"); + PROTOCOLS.put(15, "XNET"); + PROTOCOLS.put(16, "CHAOS"); + PROTOCOLS.put(17, "UDP"); + PROTOCOLS.put(18, "MUX"); + PROTOCOLS.put(19, "DCN-MEAS"); + PROTOCOLS.put(20, "HMP"); + PROTOCOLS.put(21, "PRM"); + PROTOCOLS.put(22, "XNS-IDP"); + PROTOCOLS.put(23, "TRUNK-1"); + PROTOCOLS.put(24, "TRUNK-2"); + PROTOCOLS.put(25, "LEAF-1"); + PROTOCOLS.put(26, "LEAF-2"); + PROTOCOLS.put(27, "RDP"); + PROTOCOLS.put(28, "IRTP"); + PROTOCOLS.put(29, "ISO-TP4"); + PROTOCOLS.put(30, "NETBLT"); + PROTOCOLS.put(31, "MFE-NSP"); + PROTOCOLS.put(32, "MERIT-INP"); + PROTOCOLS.put(33, "DCCP"); + PROTOCOLS.put(34, "3PC"); + PROTOCOLS.put(35, "IDPR"); + PROTOCOLS.put(36, "XTP"); + PROTOCOLS.put(37, "DDP"); + PROTOCOLS.put(38, "IDPR-CMTP"); + PROTOCOLS.put(39, "TP++"); + PROTOCOLS.put(40, "IL"); + PROTOCOLS.put(41, "IPv6"); + PROTOCOLS.put(42, "SDRP"); + PROTOCOLS.put(43, "IPv6-Route"); + PROTOCOLS.put(44, "IPv6-Frag"); + PROTOCOLS.put(45, "IDRP"); + PROTOCOLS.put(46, "RSVP"); + PROTOCOLS.put(47, "GRE"); + PROTOCOLS.put(48, "MHRP"); + PROTOCOLS.put(49, "BNA"); + PROTOCOLS.put(50, "ESP"); + PROTOCOLS.put(51, "AH"); + PROTOCOLS.put(52, "I-NLSP"); + PROTOCOLS.put(53, "SWIPE"); + PROTOCOLS.put(54, "NARP"); + PROTOCOLS.put(55, "MOBILE"); + PROTOCOLS.put(56, "TLSP"); + PROTOCOLS.put(57, "SKIP"); + PROTOCOLS.put(58, "IPv6-ICMP"); + PROTOCOLS.put(59, "IPv6-NoNxt"); + PROTOCOLS.put(60, "IPv6-Opts"); + PROTOCOLS.put(62, "CFTP"); + PROTOCOLS.put(64, "SAT-EXPAK"); + PROTOCOLS.put(65, "KRYPTOLAN"); + PROTOCOLS.put(66, "RVD"); + PROTOCOLS.put(67, "IPPC"); + PROTOCOLS.put(68, "SAT-MON"); + PROTOCOLS.put(70, "VISA"); + PROTOCOLS.put(71, "IPCU"); + PROTOCOLS.put(72, "CPNX"); + PROTOCOLS.put(73, "CPHB"); + PROTOCOLS.put(74, "WSN"); + PROTOCOLS.put(75, "PVP"); + PROTOCOLS.put(76, "BR-SAT-MON"); + PROTOCOLS.put(77, "SUN-ND"); + PROTOCOLS.put(78, "WB-MON"); + PROTOCOLS.put(79, "WB-EXPAK"); + PROTOCOLS.put(80, "ISO-IP"); + PROTOCOLS.put(81, "VMTP"); + PROTOCOLS.put(82, "SECURE-VMTP"); + PROTOCOLS.put(83, "VINES"); + PROTOCOLS.put(84, "TTP"); + PROTOCOLS.put(85, "NSFNET-IGP"); + PROTOCOLS.put(86, "DGP"); + PROTOCOLS.put(87, "TCF"); + PROTOCOLS.put(88, ""); + PROTOCOLS.put(89, "OSPF"); + PROTOCOLS.put(90, "Sprite-RP"); + PROTOCOLS.put(91, "LARP"); + PROTOCOLS.put(92, "MTP"); + PROTOCOLS.put(93, "AX.25"); + PROTOCOLS.put(94, "IPIP"); + PROTOCOLS.put(95, "MICP"); + PROTOCOLS.put(96, "SCC-SP"); + PROTOCOLS.put(97, "ETHERIP"); + PROTOCOLS.put(98, "ENCAP"); + PROTOCOLS.put(100, "GMTP"); + PROTOCOLS.put(101, "IFMP"); + PROTOCOLS.put(102, "PNNI"); + PROTOCOLS.put(103, "PIM"); + PROTOCOLS.put(104, "ARIS"); + PROTOCOLS.put(105, "SCPS"); + PROTOCOLS.put(106, "QNX"); + PROTOCOLS.put(107, "A/N"); + PROTOCOLS.put(108, "IPComp"); + PROTOCOLS.put(109, "SNP"); + PROTOCOLS.put(110, "Compaq-Peer"); + PROTOCOLS.put(111, "IPX-in-IP"); + PROTOCOLS.put(112, "VRRP"); + PROTOCOLS.put(113, "PGM"); + PROTOCOLS.put(115, "L2TP"); + PROTOCOLS.put(116, "DDX"); + PROTOCOLS.put(117, "IATP"); + PROTOCOLS.put(118, "STP"); + PROTOCOLS.put(119, "SRP"); + PROTOCOLS.put(120, "UTI"); + PROTOCOLS.put(121, "SMP"); + PROTOCOLS.put(122, "SM"); + PROTOCOLS.put(123, "PTP"); + PROTOCOLS.put(124, "IS-IS"); + PROTOCOLS.put(125, "FIRE"); + PROTOCOLS.put(126, "CRTP"); + PROTOCOLS.put(127, "CRUDP"); + PROTOCOLS.put(128, "SSCOPMCE"); + PROTOCOLS.put(129, "IPLT"); + PROTOCOLS.put(130, "SPS"); + PROTOCOLS.put(131, "PIPE"); + PROTOCOLS.put(132, "SCTP"); + PROTOCOLS.put(133, "FC"); + PROTOCOLS.put(134, "RSVP-E2E-IGNORE"); + PROTOCOLS.put(135, "Mobility Header"); + PROTOCOLS.put(136, "UDPLite"); + PROTOCOLS.put(137, "MPLS-in-IP"); + PROTOCOLS.put(138, "manet"); + PROTOCOLS.put(139, "HIP"); + PROTOCOLS.put(140, "Shim6"); + PROTOCOLS.put(141, "WESP"); + PROTOCOLS.put(142, "ROHC"); + } - @Override - public Map map(Object value, String outputField) { - Map ret = new HashMap<>(); - if(value != null && value instanceof Number) { - int protocolNum = ((Number)value).intValue(); - ret.put(outputField, PROTOCOLS.get(protocolNum)); + @Override + public Map map(Object value, String outputField) { + Map ret = new HashMap<>(); + if (value != null && value instanceof Number) { + int protocolNum = ((Number) value).intValue(); + ret.put(outputField, PROTOCOLS.get(protocolNum)); + } + return ret; } - return ret; - } - @Override - public Object apply(List objects, Context context) throws ParseException { - Object keyObj = objects.get(0); - if(keyObj == null) { - return keyObj; - } - Integer key = ConversionUtils.convert(keyObj, Integer.class); - if(key == null ) { - return keyObj; + @Override + public Object apply(List objects, Context context) throws ParseException { + Object keyObj = objects.get(0); + if (keyObj == null) { + return keyObj; + } + Integer key = ConversionUtils.convert(keyObj, Integer.class); + if (key == null) { + return keyObj; + } + Object ret = PROTOCOLS.get(key); + if (ret == null) { + return keyObj; + } + return ret; } - Object ret = PROTOCOLS.get(key); - if(ret == null ) { - return keyObj; - } - return ret; - } - @Override - public void initialize(Context context) { + @Override + public void initialize(Context context) { - } + } - @Override - public boolean isInitialized() { - return true; - } + @Override + public boolean isInitialized() { + return true; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java index ed8cd330..aac438d3 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java @@ -7,91 +7,91 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.field.transformation; import com.google.common.collect.Iterables; -import org.apache.metron.stellar.common.utils.PatternCache; -import org.apache.metron.stellar.dsl.Context; - import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import org.apache.metron.stellar.common.utils.PatternCache; +import org.apache.metron.stellar.dsl.Context; public class RegexSelectTransformation implements FieldTransformation { - @Override - public Map map( Map input - , List outputField - , LinkedHashMap fieldMappingConfig - , Context context - , Map... sensorConfig - ) { - String outField = null; - if(!(outputField == null || outputField.isEmpty())) { - outField = outputField.get(0); - } - String inVal = null; - if(!(input == null || input.isEmpty() || input.size() > 1)) { - Object inValObj = Iterables.getFirst(input.entrySet(), null).getValue(); - if(inValObj != null) { - inVal = inValObj.toString(); - } - } - Map ret = new HashMap<>(1); - if(outField == null || inVal == null) { - //in the situation where we don't have valid input or output, then we want to do nothing - return ret; - } - for(Map.Entry valToRegex : fieldMappingConfig.entrySet()) { - if(isMatch(valToRegex.getValue(), inVal)) { - ret.put(outField, valToRegex.getKey()); - break; - } - } - return ret; - } - - /** - * Return true if there is a regex match or false otherwise. - * @param regexes This could be either a list of regexes or a single regex - * @param field The field to match - * @return True if match, false otherwise - */ - private static boolean isMatch(Object regexes, String field) { - if(regexes instanceof String) { - return isMatch((String)regexes, field); - } - else if(regexes instanceof List) { - for(Object regex : (List)regexes) { - if(isMatch(regex.toString(), field)) { - return true; + @Override + public Map map(Map input, + List outputField, + LinkedHashMap fieldMappingConfig, + Context context, + Map... sensorConfig) { + String outField = null; + if (!(outputField == null || outputField.isEmpty())) { + outField = outputField.get(0); + } + String inVal = null; + if (!(input == null || input.isEmpty() || input.size() > 1)) { + Object inValObj = Iterables.getFirst(input.entrySet(), null).getValue(); + if (inValObj != null) { + inVal = inValObj.toString(); + } + } + Map ret = new HashMap<>(1); + if (outField == null || inVal == null) { + //in the situation where we don't have valid input or output, then we want to do nothing + return ret; } - } + for (Map.Entry valToRegex : fieldMappingConfig.entrySet()) { + if (isMatch(valToRegex.getValue(), inVal)) { + ret.put(outField, valToRegex.getKey()); + break; + } + } + return ret; } - return false; - } - private static boolean isMatch(String regex, String field) { - try { - Pattern p = PatternCache.INSTANCE.getPattern(regex); - if (p == null) { + /** + * Return true if there is a regex match or false otherwise. + * + * @param regexes This could be either a list of regexes or a single regex + * @param field The field to match + * @return True if match, false otherwise + */ + private static boolean isMatch(Object regexes, String field) { + if (regexes instanceof String) { + return isMatch((String) regexes, field); + } else if (regexes instanceof List) { + for (Object regex : (List) regexes) { + if (isMatch(regex.toString(), field)) { + return true; + } + } + } return false; - } - return p.asPredicate().test(field); } - catch(PatternSyntaxException pse) { - return false; + + private static boolean isMatch(String regex, String field) { + try { + Pattern p = PatternCache.INSTANCE.getPattern(regex); + if (p == null) { + return false; + } + return p.asPredicate().test(field); + } catch (PatternSyntaxException pse) { + return false; + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java index 1c875b42..56d2984f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,68 +20,72 @@ package org.apache.metron.common.field.transformation; -import org.apache.metron.stellar.dsl.*; -import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.stellar.common.StellarPredicateProcessor; - import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.common.StellarPredicateProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; public class RemoveTransformation implements FieldTransformation { - public static final String CONDITION_CONF = "condition"; - public static final StellarPredicateProcessor PASSTHROUGH_PROCESSOR = new StellarPredicateProcessor() { - @Override - public Boolean parse(String rule, VariableResolver resolver, FunctionResolver functionResolver, Context context) { - return true; - } + public static final String CONDITION_CONF = "condition"; + public static final StellarPredicateProcessor PASSTHROUGH_PROCESSOR = new StellarPredicateProcessor() { + @Override + public Boolean parse(String rule, VariableResolver resolver, FunctionResolver functionResolver, + Context context) { + return true; + } - @Override - public boolean validate(String rule) throws ParseException { - return true; - } + @Override + public boolean validate(String rule) throws ParseException { + return true; + } - @Override - public boolean validate(String rule, boolean throwException, Context context) throws ParseException { - return true; - } - }; - private String getCondition(Map fieldMappingConfig) { - Object conditionObj = fieldMappingConfig.get(CONDITION_CONF); + @Override + public boolean validate(String rule, boolean throwException, Context context) throws ParseException { + return true; + } + }; - if(conditionObj == null || !(conditionObj instanceof String)) { - return null; - } - return conditionObj.toString(); - } + private String getCondition(Map fieldMappingConfig) { + Object conditionObj = fieldMappingConfig.get(CONDITION_CONF); - private StellarPredicateProcessor getPredicateProcessor(String condition) - { - if(condition == null) { - return PASSTHROUGH_PROCESSOR; + if (conditionObj == null || !(conditionObj instanceof String)) { + return null; + } + return conditionObj.toString(); } - else { - return new StellarPredicateProcessor(); + + private StellarPredicateProcessor getPredicateProcessor(String condition) { + if (condition == null) { + return PASSTHROUGH_PROCESSOR; + } else { + return new StellarPredicateProcessor(); + } } - } - @Override - public Map map( Map input - , final List outputFields - , LinkedHashMap fieldMappingConfig - , Context context - , Map... sensorConfig - ) { - String condition = getCondition(fieldMappingConfig); - StellarPredicateProcessor processor = getPredicateProcessor(condition); - if(processor.parse(condition, new MapVariableResolver(input), StellarFunctions.FUNCTION_RESOLVER(), context)) { - return new HashMap() {{ - for(String outputField : outputFields) { - put(outputField, null); + @Override + public Map map(Map input, + final List outputFields, + LinkedHashMap fieldMappingConfig, + Context context, + Map... sensorConfig) { + String condition = getCondition(fieldMappingConfig); + StellarPredicateProcessor processor = getPredicateProcessor(condition); + if (processor.parse(condition, new MapVariableResolver(input), StellarFunctions.FUNCTION_RESOLVER(), context)) { + return new HashMap() { + { + for (String outputField : outputFields) { + put(outputField, null); + } + } + }; } - }}; + return null; } - return null; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java index f8b9374d..eebd2de3 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,38 +20,34 @@ package org.apache.metron.common.field.transformation; -import org.apache.metron.stellar.dsl.Context; - import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.dsl.Context; -public class RenameTransformation implements FieldTransformation{ - @Override - public Map map( Map input - , List outputField - , LinkedHashMap fieldMappingConfig - , Context context - , Map... sensorConfig - ) - { - if(fieldMappingConfig == null || fieldMappingConfig.isEmpty()) { - return input; - } - Map ret = new HashMap<>(); - for(Map.Entry kv : input.entrySet()) { - Object renamed = fieldMappingConfig.get(kv.getKey()); - if(renamed != null) { - //if we're renaming, then we want to copy the field to the new name - ret.put(renamed.toString(), kv.getValue()); - //and remove the old field - ret.put(kv.getKey(), null); - } - else { - ret.put(kv.getKey(), kv.getValue()); - } +public class RenameTransformation implements FieldTransformation { + @Override + public Map map(Map input, + List outputField, + LinkedHashMap fieldMappingConfig, + Context context, + Map... sensorConfig) { + if (fieldMappingConfig == null || fieldMappingConfig.isEmpty()) { + return input; + } + Map ret = new HashMap<>(); + for (Map.Entry kv : input.entrySet()) { + Object renamed = fieldMappingConfig.get(kv.getKey()); + if (renamed != null) { + //if we're renaming, then we want to copy the field to the new name + ret.put(renamed.toString(), kv.getValue()); + //and remove the old field + ret.put(kv.getKey(), null); + } else { + ret.put(kv.getKey(), kv.getValue()); + } + } + return ret; } - return ret; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/SelectTransformation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/SelectTransformation.java index 6f0defd4..2097228d 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/SelectTransformation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/SelectTransformation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,34 +26,34 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; - import org.apache.metron.stellar.dsl.Context; public class SelectTransformation implements FieldTransformation { - private static final List systemFields = Arrays.asList("timestamp", "original_string", "source.type"); - - @Override - public Map map(Map input, List outputField, - LinkedHashMap fieldMappingConfig, Context context, Map... sensorConfig) { - // note that we have to set the value to null for any field not in the output - // list, since FieldTransformer will otherwise put them back again from the - // original output. - - // note, this cannot be implemented with streams because HashMap merge guards - // against null values - - HashMap output = new HashMap(); - for (Entry e : input.entrySet()) { - if (outputField.contains(e.getKey())) { - output.put(e.getKey(), e.getValue()); - } else { - if (!systemFields.contains(e.getKey())) { - output.put(e.getKey(), null); - } - } - } - return output; - } + private static final List systemFields = Arrays.asList("timestamp", "original_string", "source.type"); + + @Override + public Map map(Map input, List outputField, + LinkedHashMap fieldMappingConfig, Context context, + Map... sensorConfig) { + // note that we have to set the value to null for any field not in the output + // list, since FieldTransformer will otherwise put them back again from the + // original output. + + // note, this cannot be implemented with streams because HashMap merge guards + // against null values + + HashMap output = new HashMap(); + for (Entry e : input.entrySet()) { + if (outputField.contains(e.getKey())) { + output.put(e.getKey(), e.getValue()); + } else { + if (!systemFields.contains(e.getKey())) { + output.put(e.getKey(), null); + } + } + } + return output; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java index 9ccbdc7c..536a4afd 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,27 +21,24 @@ package org.apache.metron.common.field.transformation; import com.google.common.collect.Iterables; -import org.apache.metron.stellar.dsl.Context; - import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.metron.stellar.dsl.Context; public abstract class SimpleFieldTransformation implements FieldTransformation { - @Override - public Map map (Map input - , List outputField - , LinkedHashMap fieldMappingConfig - , Context context - , Map... sensorConfig - ) - { - Object value = (input == null) - ? null - : Iterables.getFirst(input.values(), null) - ; - return map(value, outputField.get(0)); - } + @Override + public Map map(Map input, + List outputField, + LinkedHashMap fieldMappingConfig, + Context context, + Map... sensorConfig + ) { + Object value = (input == null) + ? null + : Iterables.getFirst(input.values(), null); + return map(value, outputField.get(0)); + } - public abstract Map map(Object input, String outputField); + public abstract Map map(Object input, String outputField); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java index bb7501df..d787a90f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,64 +20,61 @@ package org.apache.metron.common.field.transformation; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.metron.stellar.common.CachingStellarProcessor; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; -import org.apache.metron.stellar.dsl.VariableResolver; -import org.apache.metron.stellar.common.StellarProcessor; - -import java.util.*; public class StellarTransformation implements FieldTransformation { - @Override - public Map map( Map input - , List outputField - , LinkedHashMap fieldMappingConfig - , Context context - , Map... sensorConfig - ) - { - Map ret = new HashMap<>(); - Map intermediateVariables = new HashMap<>(); - Set outputs = new HashSet<>(outputField); - MapVariableResolver resolver = new MapVariableResolver(ret, intermediateVariables, input); - resolver.add(sensorConfig); - StellarProcessor processor = new CachingStellarProcessor(); - for(Map.Entry kv : fieldMappingConfig.entrySet()) { - String oField = kv.getKey(); - Object transformObj = kv.getValue(); - if(transformObj != null) { - try { - Object o = processor.parse(transformObj.toString(), resolver, StellarFunctions.FUNCTION_RESOLVER(), context); - if (o != null) { - if(outputs.contains(oField)) { - ret.put(oField, o); - } - else { - intermediateVariables.put(oField, o); + @Override + public Map map(Map input, + List outputField, + LinkedHashMap fieldMappingConfig, + Context context, + Map... sensorConfig) { + Map ret = new HashMap<>(); + Map intermediateVariables = new HashMap<>(); + Set outputs = new HashSet<>(outputField); + MapVariableResolver resolver = new MapVariableResolver(ret, intermediateVariables, input); + resolver.add(sensorConfig); + StellarProcessor processor = new CachingStellarProcessor(); + for (Map.Entry kv : fieldMappingConfig.entrySet()) { + String objField = kv.getKey(); + Object transformObj = kv.getValue(); + if (transformObj != null) { + try { + Object o = processor.parse(transformObj.toString(), resolver, StellarFunctions.FUNCTION_RESOLVER(), + context); + if (o != null) { + if (outputs.contains(objField)) { + ret.put(objField, o); + } else { + intermediateVariables.put(objField, o); + } + } else { + if (outputs.contains(objField)) { + ret.put(objField, o); + } + if (o != null) { + intermediateVariables.put(objField, o); + } else { + // remove here, in case there are other statements + intermediateVariables.remove(objField); + } + } + } catch (Exception ex) { + throw new IllegalStateException("Unable to process transformation: " + transformObj + + " for " + objField + " because " + ex.getMessage(), ex); + } } - } - else { - if(outputs.contains(oField)) { - ret.put(oField, o); - } - if( o != null ) { - intermediateVariables.put(oField, o); - } else { - // remove here, in case there are other statements - intermediateVariables.remove(oField); - } - } - } - catch(Exception ex) { - throw new IllegalStateException( "Unable to process transformation: " + transformObj.toString() - + " for " + oField + " because " + ex.getMessage() - , ex - ); } - } + return ret; } - return ret; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidation.java index 767cfc11..620f3b6f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,16 +20,15 @@ package org.apache.metron.common.field.validation; -import org.apache.metron.stellar.dsl.Context; - import java.io.Serializable; import java.util.Map; +import org.apache.metron.stellar.dsl.Context; public interface FieldValidation extends Serializable { - boolean isValid( Map input - , Map validationConfig - , Map globalConfig - , Context context - ); - void initialize(Map validationConfig, Map globalConfig); + boolean isValid(Map input, + Map validationConfig, + Map globalConfig, + Context context); + + void initialize(Map validationConfig, Map globalConfig); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidations.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidations.java index cc378a8b..ee761366 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidations.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/FieldValidations.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,33 +31,26 @@ import org.apache.metron.common.utils.ReflectionUtils; public enum FieldValidations { - STELLAR(new QueryValidation()) - ,IP(new IPValidation()) - ,DOMAIN(new DomainValidation()) - ,EMAIL(new EmailValidation()) - ,URL(new URLValidation()) - ,DATE(new DateValidation()) - ,INTEGER(new IntegerValidation()) - ,REGEX_MATCH(new RegexValidation()) - ,NOT_EMPTY(new NotEmptyValidation()) - ; - private FieldValidation validation; - FieldValidations(FieldValidation validation) { - this.validation = validation; - } + STELLAR(new QueryValidation()), IP(new IPValidation()), DOMAIN(new DomainValidation()), + EMAIL(new EmailValidation()), URL(new URLValidation()), DATE(new DateValidation()), + INTEGER(new IntegerValidation()), REGEX_MATCH(new RegexValidation()), NOT_EMPTY(new NotEmptyValidation()); + private final FieldValidation validation; - /** - * Gets a {@link FieldValidation} based on the provided validation. - * - * @param validation The validation to pull - * @return The field validation associated with the mapping - */ - public static FieldValidation get(String validation) { - try { - return FieldValidations.valueOf(validation).validation; + FieldValidations(FieldValidation validation) { + this.validation = validation; } - catch(Exception ex) { - return ReflectionUtils.createInstance(validation); + + /** + * Gets a {@link FieldValidation} based on the provided validation. + * + * @param validation The validation to pull + * @return The field validation associated with the mapping + */ + public static FieldValidation get(String validation) { + try { + return FieldValidations.valueOf(validation).validation; + } catch (Exception ex) { + return ReflectionUtils.createInstance(validation); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/QueryValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/QueryValidation.java index 074e20ad..e1d92163 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/QueryValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/QueryValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,58 +20,56 @@ package org.apache.metron.common.field.validation; +import java.util.Map; +import org.apache.metron.stellar.common.StellarPredicateProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; -import org.apache.metron.stellar.common.StellarPredicateProcessor; - -import java.util.Map; public class QueryValidation implements FieldValidation { - private enum Config { - CONDITION("condition") - ; - String key; - Config(String key) { - this.key = key; - } - public T get(Map config, Class clazz) { - Object o = config.get(key); - if(o == null) { - return null; - } - return clazz.cast(o); - } - } + private enum Config { + CONDITION("condition"); + String key; - @Override - public boolean isValid( Map input - , Map validationConfig - , Map globalConfig - , Context context - ) { - String condition = Config.CONDITION.get(validationConfig, String.class); - if(condition == null) { - return true; - } - else { - StellarPredicateProcessor processor = new StellarPredicateProcessor(); - return processor.parse(condition, new MapVariableResolver(input, validationConfig, globalConfig), StellarFunctions.FUNCTION_RESOLVER(), context); - } - } + Config(String key) { + this.key = key; + } - @Override - public void initialize(Map validationConfig, Map globalConfig) { - String condition = Config.CONDITION.get(validationConfig, String.class); - if(condition == null) { - throw new IllegalStateException("You must specify a condition."); + public T get(Map config, Class clazz) { + Object o = config.get(key); + if (o == null) { + return null; + } + return clazz.cast(o); + } } - try { - new StellarPredicateProcessor().validate(condition); + + @Override + public boolean isValid(Map input, + Map validationConfig, + Map globalConfig, + Context context) { + String condition = Config.CONDITION.get(validationConfig, String.class); + if (condition == null) { + return true; + } else { + StellarPredicateProcessor processor = new StellarPredicateProcessor(); + return processor.parse(condition, new MapVariableResolver(input, validationConfig, globalConfig), + StellarFunctions.FUNCTION_RESOLVER(), context); + } } - catch(Exception e) { - throw new IllegalStateException("Invalid condition: " + condition, e); + + @Override + public void initialize(Map validationConfig, Map globalConfig) { + String condition = Config.CONDITION.get(validationConfig, String.class); + if (condition == null) { + throw new IllegalStateException("You must specify a condition."); + } + try { + new StellarPredicateProcessor().validate(condition); + } catch (Exception e) { + throw new IllegalStateException("Invalid condition: " + condition, e); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/SimpleValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/SimpleValidation.java index 2909df3b..193e926c 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/SimpleValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/SimpleValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,59 +20,56 @@ package org.apache.metron.common.field.validation; -import org.apache.metron.stellar.dsl.Context; - import java.util.List; import java.util.Map; import java.util.function.Predicate; +import org.apache.metron.stellar.dsl.Context; public abstract class SimpleValidation implements FieldValidation, Predicate> { - @Override - public boolean isValid( Map input - , Map validationConfig - , Map globalConfig - , Context context - ) - { - Predicate predicate = getPredicate(); - if(isNonExistentOk()) { - for (Object o : input.values()) { - if (o != null && !predicate.test(o.toString())) { - return false; + @Override + public boolean isValid(Map input, + Map validationConfig, + Map globalConfig, + Context context) { + Predicate predicate = getPredicate(); + if (isNonExistentOk()) { + for (Object o : input.values()) { + if (o != null && !predicate.test(o.toString())) { + return false; + } + } + } else { + for (Object o : input.values()) { + if (o == null || !predicate.test(o.toString())) { + return false; + } + } } - } + return true; } - else { - for (Object o : input.values()) { - if (o == null || !predicate.test(o.toString())) { - return false; + + @Override + public boolean test(List input) { + if (input.isEmpty()) { + return false; + } + Predicate predicate = getPredicate(); + for (Object o : input) { + if (o == null || !predicate.test(o)) { + return false; + } } - } + return true; } - return true; - } - @Override - public boolean test(List input) { - if(input.isEmpty()) { - return false; - } - Predicate predicate = getPredicate(); - for(Object o : input) { - if(o == null || !predicate.test(o)){ - return false; - } - } - return true; - } + @Override + public void initialize(Map validationConfig, Map globalConfig) { - @Override - public void initialize(Map validationConfig, Map globalConfig) { + } - } + public abstract Predicate getPredicate(); - public abstract Predicate getPredicate(); - protected boolean isNonExistentOk() { - return true; - } + protected boolean isNonExistentOk() { + return true; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/DomainValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/DomainValidation.java index 88d95109..8077215b 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/DomainValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/DomainValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,31 +20,31 @@ package org.apache.metron.common.field.validation.network; +import java.util.function.Predicate; import org.apache.commons.validator.routines.DomainValidator; +import org.apache.metron.common.field.validation.SimpleValidation; import org.apache.metron.stellar.dsl.Predicate2StellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.common.field.validation.SimpleValidation; - -import java.util.function.Predicate; public class DomainValidation extends SimpleValidation { - @Stellar(name="IS_DOMAIN" - ,description = "Tests if a string refers to a valid domain name. Domain names are evaluated according" + - " to the standards RFC1034 section 3, and RFC1123 section 2.1." - ,params = { - "address - The string to test" - } - , returns = "True if the string refers to a valid domain name and false if otherwise") - public static class IS_DOMAIN extends Predicate2StellarFunction { + @SuppressWarnings("checkstyle:TypeName") + @Stellar(name = "IS_DOMAIN", + description = "Tests if a string refers to a valid domain name. Domain names are evaluated according" + + " to the standards RFC1034 section 3, and RFC1123 section 2.1.", + params = { + "address - The string to test" + }, + returns = "True if the string refers to a valid domain name and false if otherwise") + public static class IS_DOMAIN extends Predicate2StellarFunction { - public IS_DOMAIN() { - super(new DomainValidation()); + public IS_DOMAIN() { + super(new DomainValidation()); + } } - } - @Override - public Predicate getPredicate() { - return domain -> DomainValidator.getInstance().isValid(domain == null?null:domain.toString()); - } + @Override + public Predicate getPredicate() { + return domain -> DomainValidator.getInstance().isValid(domain == null ? null : domain.toString()); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/EmailValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/EmailValidation.java index 7fd487ec..0611d36b 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/EmailValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/EmailValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,30 +20,30 @@ package org.apache.metron.common.field.validation.network; +import java.util.function.Predicate; import org.apache.commons.validator.routines.EmailValidator; +import org.apache.metron.common.field.validation.SimpleValidation; import org.apache.metron.stellar.dsl.Predicate2StellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.common.field.validation.SimpleValidation; - -import java.util.function.Predicate; public class EmailValidation extends SimpleValidation { - @Stellar(name="IS_EMAIL" - ,description = "Tests if a string is a valid email address" - ,params = { - "address - The string to test" - } - , returns = "True if the string is a valid email address and false if otherwise.") - public static class IS_EMAIL extends Predicate2StellarFunction { + @SuppressWarnings("checkstyle:TypeName") + @Stellar(name = "IS_EMAIL", + description = "Tests if a string is a valid email address", + params = { + "address - The string to test" + }, + returns = "True if the string is a valid email address and false if otherwise.") + public static class IS_EMAIL extends Predicate2StellarFunction { - public IS_EMAIL() { - super(new EmailValidation()); + public IS_EMAIL() { + super(new EmailValidation()); + } } - } - @Override - public Predicate getPredicate() { - return email -> EmailValidator.getInstance().isValid(email == null?null:email.toString()); - } + @Override + public Predicate getPredicate() { + return email -> EmailValidator.getInstance().isValid(email == null ? null : email.toString()); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java index 35ec4ef6..32b6b04e 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/IPValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,126 +20,133 @@ package org.apache.metron.common.field.validation.network; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; import org.apache.commons.validator.routines.InetAddressValidator; +import org.apache.metron.common.field.validation.FieldValidation; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.Predicate2StellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.common.field.validation.FieldValidation; - -import java.util.*; -import java.util.function.Predicate; public class IPValidation implements FieldValidation, Predicate> { - @Stellar(name="IS_IP" - , description = "Determine if an string is an IP or not." - , params = { - "ip - An object which we wish to test is an ip" - ,"type (optional) - Object of string or collection type (e.g. list) one of IPV4 or IPV6 or both. The default is both IPV4 and IPV6." - } - , returns = "True if the string is an IP and false otherwise.") - public static class IS_IP extends Predicate2StellarFunction { - - public IS_IP() { - super(new IPValidation()); - } - } - - private enum IPType { - IPV4(ip -> InetAddressValidator.getInstance().isValidInet4Address(ip)) - ,IPV6(ip -> InetAddressValidator.getInstance().isValidInet6Address(ip)) - , DEFAULT(ip -> InetAddressValidator.getInstance().isValid(ip)); - Predicate validationPredicate; - IPType(Predicate validationPredicate) { - this.validationPredicate = validationPredicate; - } - public boolean isValid(String ip) { - return validationPredicate.test(ip); + @SuppressWarnings("checkstyle:TypeName") + @Stellar(name = "IS_IP", + description = "Determine if an string is an IP or not.", + params = { + "ip - An object which we wish to test is an ip", + + "type (optional) - Object of string or collection type (e.g. list) one of IPV4 or IPV6 or both. " + + "The default is both IPV4 and IPV6." + }, + returns = "True if the string is an IP and false otherwise.") + public static class IS_IP extends Predicate2StellarFunction { + + public IS_IP() { + super(new IPValidation()); + } } - public static IPType get(String type) { - if(type == null) { - return DEFAULT; - } - else { - try { - return IPType.valueOf(type); + + private enum IPType { + IPV4(ip -> InetAddressValidator.getInstance().isValidInet4Address(ip)), + IPV6(ip -> InetAddressValidator.getInstance().isValidInet6Address(ip)), + DEFAULT(ip -> InetAddressValidator.getInstance().isValid(ip)); + Predicate validationPredicate; + + IPType(Predicate validationPredicate) { + this.validationPredicate = validationPredicate; } - catch(Exception e) { - return DEFAULT; + + public boolean isValid(String ip) { + return validationPredicate.test(ip); + } + + public static IPType get(String type) { + if (type == null) { + return DEFAULT; + } else { + try { + return IPType.valueOf(type); + } catch (Exception e) { + return DEFAULT; + } + } } - } - } - } - private enum Config { - TYPE("type") - ; - String key; - Config(String key) { - this.key = key; - } - public List get(Map config ) { - Object o = config.get(key); - if(o == null) { - return Collections.singletonList("DEFAULT"); - } - if( o instanceof ArrayList){ - return (ArrayList)o; - } - return Collections.singletonList(o); - } - } - - /** - * Evaluates this predicate on the given argument. - * - * @param strings the input argument - * @return {@code true} if the input argument matches the predicate, - * otherwise {@code false} - */ - @Override - public boolean test(List strings) { - IPType type = IPType.DEFAULT; - if(strings.isEmpty()) { - return false; } - Object ip = strings.get(0); - if(ip == null) { - return false; + + private enum Config { + TYPE("type"); + String key; + + Config(String key) { + this.key = key; + } + + public List get(Map config) { + Object o = config.get(key); + if (o == null) { + return Collections.singletonList("DEFAULT"); + } + if (o instanceof ArrayList) { + return (ArrayList) o; + } + return Collections.singletonList(o); + } } - if(strings.size() >= 2) { - Object ipType = strings.get(1); - if(ipType != null ) - { - try { - type = IPType.get(ipType.toString()); - } catch (Exception e) { - type = IPType.DEFAULT; + + /** + * Evaluates this predicate on the given argument. + * + * @param strings the input argument + * @return {@code true} if the input argument matches the predicate, + * otherwise {@code false} + */ + @Override + public boolean test(List strings) { + IPType type = IPType.DEFAULT; + if (strings.isEmpty()) { + return false; + } + Object ip = strings.get(0); + if (ip == null) { + return false; + } + if (strings.size() >= 2) { + Object ipType = strings.get(1); + if (ipType != null) { + try { + type = IPType.get(ipType.toString()); + } catch (Exception e) { + type = IPType.DEFAULT; + } + } } - } + return type.isValid(ip.toString()); } - return type.isValid(ip.toString()); - } - - @Override - public boolean isValid( Map input - , Map validationConfig - , Map globalConfig - , Context context - ) { - List types = Config.TYPE.get(validationConfig); - - for(Object typeObject : types) { - IPType type = IPType.get(typeObject.toString()); - for (Object o : input.values()) { - if(o == null || type.isValid(o.toString())) { - return true; + + @Override + public boolean isValid(Map input, + Map validationConfig, + Map globalConfig, + Context context + ) { + List types = Config.TYPE.get(validationConfig); + + for (Object typeObject : types) { + IPType type = IPType.get(typeObject.toString()); + for (Object o : input.values()) { + if (o == null || type.isValid(o.toString())) { + return true; + } + } } - } + return false; } - return false; - } - @Override - public void initialize(Map validationConfig, Map globalConfig) { - } + @Override + public void initialize(Map validationConfig, Map globalConfig) { + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/URLValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/URLValidation.java index 33ffb3fa..ebbdc91c 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/URLValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/network/URLValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,31 +20,31 @@ package org.apache.metron.common.field.validation.network; +import java.util.function.Predicate; import org.apache.commons.validator.routines.UrlValidator; +import org.apache.metron.common.field.validation.SimpleValidation; import org.apache.metron.stellar.dsl.Predicate2StellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.common.field.validation.SimpleValidation; - -import java.util.function.Predicate; public class URLValidation extends SimpleValidation { - @Stellar(name="IS_URL" - ,description = "Tests if a string is a valid URL" - ,params = { - "url - The string to test" - } - , returns = "True if the string is a valid URL and false if otherwise." - ) - public static class IS_URL extends Predicate2StellarFunction { + @SuppressWarnings("checkstyle:TypeName") + @Stellar(name = "IS_URL", + description = "Tests if a string is a valid URL", + params = { + "url - The string to test" + }, + returns = "True if the string is a valid URL and false if otherwise." + ) + public static class IS_URL extends Predicate2StellarFunction { - public IS_URL() { - super(new URLValidation()); + public IS_URL() { + super(new URLValidation()); + } } - } - @Override - public Predicate getPredicate() { - return url -> UrlValidator.getInstance().isValid(url == null?null:url.toString()); - } + @Override + public Predicate getPredicate() { + return url -> UrlValidator.getInstance().isValid(url == null ? null : url.toString()); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/DateValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/DateValidation.java index 74d54642..8aef1f77 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/DateValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/DateValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,122 +20,119 @@ package org.apache.metron.common.field.validation.primitive; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.Predicate2StellarFunction; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.common.field.validation.FieldValidation; - import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; import java.util.Map; import java.util.function.Predicate; +import org.apache.metron.common.field.validation.FieldValidation; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.Predicate2StellarFunction; +import org.apache.metron.stellar.dsl.Stellar; public class DateValidation implements FieldValidation, Predicate> { - @Stellar(name="IS_DATE" - ,description = "Determines if the date contained in the string conforms to the specified format." - ,params = { - "date - The date in string form" - , "format - The format of the date" - } - ,returns = "True if the date is in the specified format and false if otherwise." - ) - public static class IS_DATE extends Predicate2StellarFunction { + @SuppressWarnings("checkstyle:TypeName") + @Stellar(name = "IS_DATE", + description = "Determines if the date contained in the string conforms to the specified format.", + params = { + "date - The date in string form", + "format - The format of the date" + }, + returns = "True if the date is in the specified format and false if otherwise." + ) + public static class IS_DATE extends Predicate2StellarFunction { - public IS_DATE() { - super(new DateValidation()); + public IS_DATE() { + super(new DateValidation()); + } } - } - /** - * Evaluates this predicate on the given argument. - * - * @param strings the input argument - * @return {@code true} if the input argument matches the predicate, - * otherwise {@code false} - */ - @Override - public boolean test(List strings) { - if(strings.isEmpty()) { - return false; - } - if(strings.size() >= 2) { - Object date = strings.get(0); - Object format = strings.get(1); - if(date == null || format == null) { - return false; - } - try { - SimpleDateFormat sdf = new SimpleDateFormat(format.toString()); - sdf.setLenient(false); - sdf.parse(date.toString()); - return true; - } - catch(ParseException pe) { - return false; - } - } - else { - return false; + /** + * Evaluates this predicate on the given argument. + * + * @param strings the input argument + * @return {@code true} if the input argument matches the predicate, + * otherwise {@code false} + */ + @Override + public boolean test(List strings) { + if (strings.isEmpty()) { + return false; + } + if (strings.size() >= 2) { + Object date = strings.get(0); + Object format = strings.get(1); + if (date == null || format == null) { + return false; + } + try { + SimpleDateFormat sdf = new SimpleDateFormat(format.toString()); + sdf.setLenient(false); + sdf.parse(date.toString()); + return true; + } catch (ParseException pe) { + return false; + } + } else { + return false; + } } - } - private enum Config { - FORMAT("format") - ; - String key; - Config(String key) { - this.key = key; - } - public T get(Map config, Class clazz) { - Object o = config.get(key); - if(o == null) { - return null; - } - return clazz.cast(o); - } - } - @Override - public boolean isValid( Map input - , Map validationConfig - , Map globalConfig - , Context context - ) - { - String format = Config.FORMAT.get(validationConfig, String.class); - if(format == null) { - return false; + private enum Config { + FORMAT("format"); + String key; + + Config(String key) { + this.key = key; + } + + public T get(Map config, Class clazz) { + Object o = config.get(key); + if (o == null) { + return null; + } + return clazz.cast(o); + } } - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setLenient(false); - for(Object o : input.values()) { - if(o == null) { + + @Override + public boolean isValid(Map input, + Map validationConfig, + Map globalConfig, + Context context) { + String format = Config.FORMAT.get(validationConfig, String.class); + if (format == null) { + return false; + } + SimpleDateFormat sdf = new SimpleDateFormat(format); + sdf.setLenient(false); + for (Object o : input.values()) { + if (o == null) { + return true; + } + try { + Date d = sdf.parse(o.toString()); + } catch (ParseException e) { + return false; + } + } return true; - } - try { - Date d = sdf.parse(o.toString()); - } catch (ParseException e) { - return false; - } } - return true; - } - @Override - public void initialize(Map validationConfig, Map globalConfig) { - String format = Config.FORMAT.get(validationConfig, String.class); - if(format == null) { - throw new IllegalStateException("You must specify '" + Config.FORMAT.key + "' in the config"); - } - SimpleDateFormat sdf = new SimpleDateFormat(format); - sdf.setLenient(false); - try { - sdf.format(new Date()); - } - catch(Exception e) { - throw new IllegalStateException("Invalid date format: " + format, e); + @Override + public void initialize(Map validationConfig, Map globalConfig) { + String format = Config.FORMAT.get(validationConfig, String.class); + if (format == null) { + throw new IllegalStateException("You must specify '" + Config.FORMAT.key + "' in the config"); + } + SimpleDateFormat sdf = new SimpleDateFormat(format); + sdf.setLenient(false); + try { + sdf.format(new Date()); + } catch (Exception e) { + throw new IllegalStateException("Invalid date format: " + format, e); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/IntegerValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/IntegerValidation.java index 1cebeaa2..6b887fdc 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/IntegerValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/IntegerValidation.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,29 +20,30 @@ package org.apache.metron.common.field.validation.primitive; +import java.util.function.Predicate; import org.apache.commons.validator.routines.LongValidator; +import org.apache.metron.common.field.validation.SimpleValidation; import org.apache.metron.stellar.dsl.Predicate2StellarFunction; import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.common.field.validation.SimpleValidation; -import java.util.function.Predicate; +public class IntegerValidation extends SimpleValidation { + @SuppressWarnings("checkstyle:TypeName") + @Stellar(name = "IS_INTEGER", + description = "Determines whether or not an object is an integer.", + params = { + "x - The object to test" + }, + returns = "True if the object can be converted to an integer and false if otherwise." + ) + public static class IS_INTEGER extends Predicate2StellarFunction { -public class IntegerValidation extends SimpleValidation{ - @Stellar(name="IS_INTEGER" - , description = "Determines whether or not an object is an integer." - , params = { - "x - The object to test" - } - , returns = "True if the object can be converted to an integer and false if otherwise." - ) - public static class IS_INTEGER extends Predicate2StellarFunction { + public IS_INTEGER() { + super(new IntegerValidation()); + } + } - public IS_INTEGER() { - super(new IntegerValidation()); + @Override + public Predicate getPredicate() { + return x -> LongValidator.getInstance().isValid(x == null ? null : x.toString()); } - } - @Override - public Predicate getPredicate() { - return x -> LongValidator.getInstance().isValid(x == null?null:x.toString()); - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/NotEmptyValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/NotEmptyValidation.java index 28ac002c..d59ca38b 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/NotEmptyValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/NotEmptyValidation.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,18 +20,17 @@ package org.apache.metron.common.field.validation.primitive; -import org.apache.metron.common.field.validation.SimpleValidation; - import java.util.function.Predicate; +import org.apache.metron.common.field.validation.SimpleValidation; public class NotEmptyValidation extends SimpleValidation { - @Override - public Predicate getPredicate() { - return s -> !(s == null || s.toString().trim().isEmpty()); - } + @Override + public Predicate getPredicate() { + return s -> !(s == null || s.toString().trim().isEmpty()); + } - @Override - protected boolean isNonExistentOk() { - return false; - } + @Override + protected boolean isNonExistentOk() { + return false; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/RegexValidation.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/RegexValidation.java index b67663d8..3dab7716 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/RegexValidation.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/field/validation/primitive/RegexValidation.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,53 +20,53 @@ package org.apache.metron.common.field.validation.primitive; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.common.field.validation.FieldValidation; - import java.util.Map; +import org.apache.metron.common.field.validation.FieldValidation; +import org.apache.metron.stellar.dsl.Context; public class RegexValidation implements FieldValidation { - private enum Config { - REGEX("pattern") - ; - String key; - Config(String key) { - this.key = key; - } - public T get(Map config, Class clazz) { - Object o = config.get(key); - if(o == null) { - return null; - } - return clazz.cast(o); - } - } + private enum Config { + REGEX("pattern"); + String key; - @Override - public boolean isValid( Map input - , Map validationConfig - , Map globalConfig - , Context context - ) { + Config(String key) { + this.key = key; + } - String regex = Config.REGEX.get(validationConfig, String.class); - if(regex == null) { - return false; + public T get(Map config, Class clazz) { + Object o = config.get(key); + if (o == null) { + return null; + } + return clazz.cast(o); + } } - for(Object o : input.values()) { - if(o != null && !o.toString().matches(regex)) { - return false; - } + + @Override + public boolean isValid(Map input, + Map validationConfig, + Map globalConfig, + Context context + ) { + + String regex = Config.REGEX.get(validationConfig, String.class); + if (regex == null) { + return false; + } + for (Object o : input.values()) { + if (o != null && !o.toString().matches(regex)) { + return false; + } + } + return true; } - return true; - } - @Override - public void initialize(Map validationConfig, Map globalConfig) { - String regex = Config.REGEX.get(validationConfig, String.class); - if(regex == null) { - throw new IllegalStateException("You must specify '" + Config.REGEX.key + "' in the config"); + @Override + public void initialize(Map validationConfig, Map globalConfig) { + String regex = Config.REGEX.get(validationConfig, String.class); + if (regex == null) { + throw new IllegalStateException("You must specify '" + Config.REGEX.key + "' in the config"); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java index 17f2fc75..fb7cf6b6 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,105 +36,105 @@ import org.apache.metron.common.utils.LazyLoggerFactory; public class SequenceFileIterable implements Iterable { - private static final LazyLogger LOGGER = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private List files; - private Configuration config; + private static final LazyLogger LOGGER = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final List files; + private final Configuration config; - public SequenceFileIterable(List files, Configuration config) { - this.files = files; - this.config = config; - } - - @Override - public Iterator iterator() { - return Iterators.concat(getIterators(files, config)); - } - - private Iterator[] getIterators(List files, Configuration config) { - return files.stream().map(f -> new SequenceFileIterator(f, config)).toArray(Iterator[]::new); - } + public SequenceFileIterable(List files, Configuration config) { + this.files = files; + this.config = config; + } - /** - * Cleans up all files read by this Iterable. - * - * @return true if success, false if any files were not deleted - * @throws IOException if there's an error cleaning up files - */ - public boolean cleanup() throws IOException { - FileSystem fileSystem = FileSystem.get(config); - boolean success = true; - for (Path file : files) { - success &= fileSystem.delete(file, false); + @Override + public Iterator iterator() { + return Iterators.concat(getIterators(files, config)); } - return success; - } - private static class SequenceFileIterator implements Iterator { - private Path path; - private Configuration config; - private SequenceFile.Reader reader; - private LongWritable key = new LongWritable(); - private BytesWritable value = new BytesWritable(); - private byte[] next; - private boolean finished = false; + private Iterator[] getIterators(List files, Configuration config) { + return files.stream().map(f -> new SequenceFileIterator(f, config)).toArray(Iterator[]::new); + } - public SequenceFileIterator(Path path, Configuration config) { - this.path = path; - this.config = config; + /** + * Cleans up all files read by this Iterable. + * + * @return true if success, false if any files were not deleted + * @throws IOException if there's an error cleaning up files + */ + public boolean cleanup() throws IOException { + FileSystem fileSystem = FileSystem.get(config); + boolean success = true; + for (Path file : files) { + success &= fileSystem.delete(file, false); + } + return success; } - @Override - public boolean hasNext() { - if (!finished && null == reader) { - try { - reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(path)); - LOGGER.debug("Writing file: {}", () -> path.toString()); - } catch (IOException e) { - throw new RuntimeException("Failed to get reader", e); + private static class SequenceFileIterator implements Iterator { + private final Path path; + private final Configuration config; + private SequenceFile.Reader reader; + private final LongWritable key = new LongWritable(); + private final BytesWritable value = new BytesWritable(); + private byte[] next; + private boolean finished = false; + + public SequenceFileIterator(Path path, Configuration config) { + this.path = path; + this.config = config; } - } else { - LOGGER.debug("finished={}, reader={}, next={}", finished, reader, next); - } - try { - //ensure hasnext is idempotent - if (!finished) { - if (null == next && reader.next(key, value)) { - next = value.copyBytes(); - } else if (null == next) { - close(); - } + + @Override + public boolean hasNext() { + if (!finished && null == reader) { + try { + reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(path)); + LOGGER.debug("Writing file: {}", () -> path.toString()); + } catch (IOException e) { + throw new RuntimeException("Failed to get reader", e); + } + } else { + LOGGER.debug("finished={}, reader={}, next={}", finished, reader, next); + } + try { + //ensure hasnext is idempotent + if (!finished) { + if (null == next && reader.next(key, value)) { + next = value.copyBytes(); + } else if (null == next) { + close(); + } + } + } catch (IOException e) { + close(); + throw new RuntimeException("Failed to get next record", e); + } + return (null != next); } - } catch (IOException e) { - close(); - throw new RuntimeException("Failed to get next record", e); - } - return (null != next); - } - private void close() { - LOGGER.debug("Closing file: {}", () -> path.toString()); - finished = true; - try { - if (reader != null) { - reader.close(); - reader = null; + private void close() { + LOGGER.debug("Closing file: {}", () -> path.toString()); + finished = true; + try { + if (reader != null) { + reader.close(); + reader = null; + } + } catch (IOException e) { + // ah well, we tried... + LOGGER.warn("Error closing file", e); + } } - } catch (IOException e) { - // ah well, we tried... - LOGGER.warn("Error closing file", e); - } - } - @Override - public byte[] next() { - byte[] ret = null; - if (hasNext()) { - ret = next; - next = null; //don't want same record more than once - } else { - throw new NoSuchElementException("No more records"); - } - return ret; + @Override + public byte[] next() { + byte[] ret = null; + if (hasNext()) { + ret = next; + next = null; //don't want same record more than once + } else { + throw new NoSuchElementException("No more records"); + } + return ret; + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/DefaultRawMessageStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/DefaultRawMessageStrategy.java index 780da968..31175bfa 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/DefaultRawMessageStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/DefaultRawMessageStrategy.java @@ -6,24 +6,26 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.message.metadata; +import java.lang.invoke.MethodHandles; +import java.util.Map; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.util.Map; - /** * The default implementation. Defines: *

      @@ -34,53 +36,57 @@ *
    */ public class DefaultRawMessageStrategy implements RawMessageStrategy { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - /** - * The default behavior is to use the raw data from kafka value as the message and the raw metadata as the metadata. - * - * @param rawMetadata The metadata read from kafka Key (e.g. the topic, index, etc.) - * @param rawMessage The raw message from the kafka value - * @param readMetadata True if we want to read read the metadata - * @param config The config for the RawMessageStrategy (See the rawMessageStrategyConfig in the SensorParserConfig) - * @return The {@link RawMessage} that includes metadata. - */ - @Override - public RawMessage get(Map rawMetadata, byte[] rawMessage, boolean readMetadata, Map config) { - return new RawMessage(rawMessage, rawMetadata); - } + /** + * The default behavior is to use the raw data from kafka value as the message and the raw metadata as the metadata. + * + * @param rawMetadata The metadata read from kafka Key (e.g. the topic, index, etc.) + * @param rawMessage The raw message from the kafka value + * @param readMetadata True if we want to read read the metadata + * @param config The config for the RawMessageStrategy (See the rawMessageStrategyConfig in the SensorParserConfig) + * @return The {@link RawMessage} that includes metadata. + */ + @Override + public RawMessage get(Map rawMetadata, byte[] rawMessage, boolean readMetadata, + Map config) { + return new RawMessage(rawMessage, rawMetadata); + } - /** - * Simple merging of metadata by adding the metadata into the message (if mergeMetadata is set to true). - * - * @param message The parsed message (note: prior to the field transformations) - * @param metadata The metadata passed along - * @param mergeMetadata Whether to merge the metadata or not - * @param config The config for the message strategy. - */ - @Override - public void mergeMetadata(JSONObject message, Map metadata, boolean mergeMetadata, Map config) { - if(mergeMetadata) { - message.putAll(metadata); + /** + * Simple merging of metadata by adding the metadata into the message (if mergeMetadata is set to true). + * + * @param message The parsed message (note: prior to the field transformations) + * @param metadata The metadata passed along + * @param mergeMetadata Whether to merge the metadata or not + * @param config The config for the message strategy. + */ + @Override + public void mergeMetadata(JSONObject message, Map metadata, boolean mergeMetadata, + Map config) { + if (mergeMetadata) { + message.putAll(metadata); + } } - } - /** - * The default mergeMetadata is false. - * @return false - */ - @Override - public boolean mergeMetadataDefault() { - return false; - } + /** + * The default mergeMetadata is false. + * + * @return false + */ + @Override + public boolean mergeMetadataDefault() { + return false; + } - /** - * The default readMetadata is false. - * @return false - */ - @Override - public boolean readMetadataDefault() { - return false; - } + /** + * The default readMetadata is false. + * + * @return false + */ + @Override + public boolean readMetadataDefault() { + return false; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/EnvelopedRawMessageStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/EnvelopedRawMessageStrategy.java index 3404e33a..01fe555a 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/EnvelopedRawMessageStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/EnvelopedRawMessageStrategy.java @@ -7,29 +7,30 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * http://www.apache.org/licenses/LICENSE-2.0 * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.message.metadata; +import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; - +import java.util.HashMap; +import java.util.Map; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.stellar.common.Constants; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.Map; - /** * An alternative strategy whereby *

      @@ -42,108 +43,116 @@ * Note, this strategy allows for parser chaining and for a fully worked example, check the parser chaining use-case. */ public class EnvelopedRawMessageStrategy implements RawMessageStrategy { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - /** - * The field from the rawMessageStrategyConfig in the SensorParserConfig that defines the field to use to - * define the data to be parsed. - */ - public static final String MESSAGE_FIELD_CONFIG = "messageField"; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + /** + * The field from the rawMessageStrategyConfig in the SensorParserConfig that defines the field to use to + * define the data to be parsed. + */ + public static final String MESSAGE_FIELD_CONFIG = "messageField"; - /** - * Retrieve the raw message by parsing the JSON Map in the kafka value and pulling the appropriate field. - * Also, augment the default metadata with the non-data fields in the JSON Map. - * - *

      Note: The data field in the JSON Map is not considered metadata. - * - * @param rawMetadata The metadata read from kafka Key (e.g. the topic, index, etc.) - * @param rawMessage The raw message from the kafka value - * @param readMetadata True if we want to read read the metadata - * @param config The config for the RawMessageStrategy (See the rawMessageStrategyConfig in the SensorParserConfig) - * @return The {@link RawMessage}, potentially including metadata - */ - @Override - public RawMessage get(Map rawMetadata, byte[] rawMessage, boolean readMetadata, Map config) { - String messageField = (String)config.get(MESSAGE_FIELD_CONFIG); - if(messageField == null) { - throw new IllegalStateException("You must specify a message field in the message supplier config. " + - "\"messageField\" field was expected but wasn't in the config."); - } - byte[] envelope = rawMessage; + /** + * Retrieve the raw message by parsing the JSON Map in the kafka value and pulling the appropriate field. + * Also, augment the default metadata with the non-data fields in the JSON Map. + * + *

      Note: The data field in the JSON Map is not considered metadata. + * + * @param rawMetadata The metadata read from kafka Key (e.g. the topic, index, etc.) + * @param rawMessage The raw message from the kafka value + * @param readMetadata True if we want to read read the metadata + * @param config The config for the RawMessageStrategy (See the rawMessageStrategyConfig in the SensorParserConfig) + * @return The {@link RawMessage}, potentially including metadata + */ + @Override + public RawMessage get(Map rawMetadata, byte[] rawMessage, boolean readMetadata, + Map config) { + String messageField = (String) config.get(MESSAGE_FIELD_CONFIG); + if (messageField == null) { + throw new IllegalStateException("You must specify a message field in the message supplier config. " + + "\"messageField\" field was expected but wasn't in the config."); + } + byte[] envelope = rawMessage; - try { - String prefix = MetadataUtil.INSTANCE.getMetadataPrefix(config); - Map extraMetadata = JSONUtils.INSTANCE.load(new String(envelope, - StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER); - String message = null; - if(extraMetadata != null) { - for(Map.Entry kv : extraMetadata.entrySet()) { - if(kv.getKey().equals(messageField)) { - message = (String)kv.getValue(); - } - rawMetadata.put(MetadataUtil.INSTANCE.prefixKey(prefix, kv.getKey()), kv.getValue()); + try { + String prefix = MetadataUtil.INSTANCE.getMetadataPrefix(config); + Map extraMetadata = JSONUtils.INSTANCE.load(new String(envelope, + StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER); + String message = null; + if (extraMetadata != null) { + for (Map.Entry kv : extraMetadata.entrySet()) { + if (kv.getKey().equals(messageField)) { + message = (String) kv.getValue(); + } + rawMetadata.put(MetadataUtil.INSTANCE.prefixKey(prefix, kv.getKey()), kv.getValue()); + } + } + if (message != null) { + if (!readMetadata) { + LOG.debug( + "Ignoring metadata; Message: " + message + " rawMetadata: " + rawMetadata + " and field = " + + messageField); + return new RawMessage(message.getBytes(StandardCharsets.UTF_8), new HashMap<>()); + } else { + //remove the message field from the metadata since it's data, not metadata. + rawMetadata.remove(MetadataUtil.INSTANCE.prefixKey(prefix, messageField)); + LOG.debug( + "Attaching metadata; Message: " + message + " rawMetadata: " + rawMetadata + " and field = " + + messageField); + return new RawMessage(message.getBytes(StandardCharsets.UTF_8), rawMetadata); + } + } + } catch (IOException e) { + throw new IllegalStateException("Expected a JSON Map as the envelope.", e); } - } - if(message != null) { - if(!readMetadata) { - LOG.debug("Ignoring metadata; Message: " + message + " rawMetadata: " + rawMetadata + " and field = " + messageField); - return new RawMessage(message.getBytes(StandardCharsets.UTF_8), new HashMap<>()); + return null; + } + + /** + * Merge the metadata into the original message. The strategy around duplicate keys is as follows: + *

        + *
      • If the string is the "original_string" field, then we choose the oldest original string
        • + *
      • For all other fields, the fields from the message hold precidence against metadata fields on collision.
        • + *
      + * + * @param message The parsed message (note: prior to the field transformations) + * @param metadata The metadata passed along + * @param mergeMetadata Whether to merge the metadata or not + * @param config The config for the message strategy. + */ + @Override + public void mergeMetadata(JSONObject message, Map metadata, boolean mergeMetadata, + Map config) { + //we want to ensure the original string from the metadata, if provided is used + String prefix = MetadataUtil.INSTANCE.getMetadataPrefix(config); + String originalStringFromMetadata = + (String) metadata.get(MetadataUtil.INSTANCE.prefixKey(prefix, Constants.Fields.ORIGINAL.getName())); + if (mergeMetadata) { + for (Map.Entry kv : metadata.entrySet()) { + //and that otherwise we prefer fields from the current message, not the metadata + message.putIfAbsent(kv.getKey(), kv.getValue()); + } } - else { - //remove the message field from the metadata since it's data, not metadata. - rawMetadata.remove(MetadataUtil.INSTANCE.prefixKey(prefix, messageField)); - LOG.debug("Attaching metadata; Message: " + message + " rawMetadata: " + rawMetadata + " and field = " + messageField); - return new RawMessage(message.getBytes(StandardCharsets.UTF_8), rawMetadata); + if (originalStringFromMetadata != null) { + message.put(Constants.Fields.ORIGINAL.getName(), originalStringFromMetadata); } - } - } catch (IOException e) { - throw new IllegalStateException("Expected a JSON Map as the envelope.", e); } - return null; - } - /** - * Merge the metadata into the original message. The strategy around duplicate keys is as follows: - *
        - *
      • If the string is the "original_string" field, then we choose the oldest original string
        • - *
      • For all other fields, the fields from the message hold precidence against metadata fields on collision.
        • - *
      - * @param message The parsed message (note: prior to the field transformations) - * @param metadata The metadata passed along - * @param mergeMetadata Whether to merge the metadata or not - * @param config The config for the message strategy. - */ - @Override - public void mergeMetadata(JSONObject message, Map metadata, boolean mergeMetadata, Map config) { - //we want to ensure the original string from the metadata, if provided is used - String prefix = MetadataUtil.INSTANCE.getMetadataPrefix(config); - String originalStringFromMetadata = (String)metadata.get(MetadataUtil.INSTANCE.prefixKey(prefix, Constants.Fields.ORIGINAL.getName())); - if(mergeMetadata) { - for (Map.Entry kv : metadata.entrySet()) { - //and that otherwise we prefer fields from the current message, not the metadata - message.putIfAbsent(kv.getKey(), kv.getValue()); - } - } - if(originalStringFromMetadata != null) { - message.put(Constants.Fields.ORIGINAL.getName(), originalStringFromMetadata); + /** + * By default merge metadata. + * + * @return true + */ + @Override + public boolean mergeMetadataDefault() { + return true; } - } - /** - * By default merge metadata. - * - * @return true - */ - @Override - public boolean mergeMetadataDefault() { - return true; - } - - /** - * By default read metadata. - * @return true - */ - @Override - public boolean readMetadataDefault() { - return true; - } + /** + * By default read metadata. + * + * @return true + */ + @Override + public boolean readMetadataDefault() { + return true; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/MetadataUtil.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/MetadataUtil.java index 628ef507..d67d3b0e 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/MetadataUtil.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/MetadataUtil.java @@ -7,66 +7,67 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.message.metadata; +import java.lang.invoke.MethodHandles; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.util.Map; - /** * Captures some common utility methods around metadata manipulation. */ public enum MetadataUtil { - INSTANCE; - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - /** - * The default metadata prefix. - */ - public static final String METADATA_PREFIX = "metron.metadata"; - /** - * The config key for defining the prefix. - */ - public static final String METADATA_PREFIX_CONFIG = "metadataPrefix"; + INSTANCE; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + /** + * The default metadata prefix. + */ + public static final String METADATA_PREFIX = "metron.metadata"; + /** + * The config key for defining the prefix. + */ + public static final String METADATA_PREFIX_CONFIG = "metadataPrefix"; - /** - * Return the prefix that we want to use for metadata keys. This comes from the config and is defaulted to - * 'metron.metadata'. - * - * @param config The rawMessageStrategyConfig - * @return the prefix for metadata keys - */ - public String getMetadataPrefix(Map config) { - String prefix = (String) config.getOrDefault(METADATA_PREFIX_CONFIG, METADATA_PREFIX); - if(StringUtils.isEmpty(prefix)) { - return null; + /** + * Return the prefix that we want to use for metadata keys. This comes from the config and is defaulted to + * 'metron.metadata'. + * + * @param config The rawMessageStrategyConfig + * @return the prefix for metadata keys + */ + public String getMetadataPrefix(Map config) { + String prefix = (String) config.getOrDefault(METADATA_PREFIX_CONFIG, METADATA_PREFIX); + if (StringUtils.isEmpty(prefix)) { + return null; + } + return prefix; } - return prefix; - } - /** - * Take a field and prefix it with the metadata key. - * - * @param prefix The metadata prefix to use (e.g. 'foo') - * @param key The key name (e.g. my_field) - * @return The prefixed key separated by a . (e.g. foo.my_field) - */ - public String prefixKey(String prefix, String key) { - if(StringUtils.isEmpty(prefix)) { - return key; - } - else { - return prefix + "." + key; + /** + * Take a field and prefix it with the metadata key. + * + * @param prefix The metadata prefix to use (e.g. 'foo') + * @param key The key name (e.g. my_field) + * @return The prefixed key separated by a . (e.g. foo.my_field) + */ + public String prefixKey(String prefix, String key) { + if (StringUtils.isEmpty(prefix)) { + return key; + } else { + return prefix + "." + key; + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessage.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessage.java index e5f80aa2..85a25c80 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessage.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessage.java @@ -6,15 +6,18 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.message.metadata; import java.util.Arrays; @@ -24,62 +27,70 @@ * A holder class for the message and metadata. */ public class RawMessage { - byte[] message; - Map metadata; + byte[] message; + Map metadata; - public RawMessage(byte[] message, Map metadata) { - this.message = message; - this.metadata = metadata; - } + public RawMessage(byte[] message, Map metadata) { + this.message = message; + this.metadata = metadata; + } - /** - * Get the data to be parsed. - * @return Raw bytes data of the message - */ - public byte[] getMessage() { - return message; - } + /** + * Get the data to be parsed. + * + * @return Raw bytes data of the message + */ + public byte[] getMessage() { + return message; + } - public void setMessage(byte[] message) { - this.message = message; - } + public void setMessage(byte[] message) { + this.message = message; + } - /** - * Get the metadata to use based on the RawMessageStrategy. - * @return Map of the metadata - */ - public Map getMetadata() { - return metadata; - } + /** + * Get the metadata to use based on the RawMessageStrategy. + * + * @return Map of the metadata + */ + public Map getMetadata() { + return metadata; + } - public void setMetadata(Map metadata) { - this.metadata = metadata; - } + public void setMetadata(Map metadata) { + this.metadata = metadata; + } - @Override - public String toString() { - return "RawMessage{" + - "message=" + Arrays.toString(message) + - ", metadata=" + metadata + - '}'; - } + @Override + public String toString() { + return "RawMessage{" + + "message=" + Arrays.toString(message) + + ", metadata=" + metadata + + '}'; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - RawMessage that = (RawMessage) o; + RawMessage that = (RawMessage) o; - if (!Arrays.equals(getMessage(), that.getMessage())) return false; - return getMetadata() != null ? getMetadata().equals(that.getMetadata()) : that.getMetadata() == null; + if (!Arrays.equals(getMessage(), that.getMessage())) { + return false; + } + return getMetadata() != null ? getMetadata().equals(that.getMetadata()) : that.getMetadata() == null; - } + } - @Override - public int hashCode() { - int result = Arrays.hashCode(getMessage()); - result = 31 * result + (getMetadata() != null ? getMetadata().hashCode() : 0); - return result; - } + @Override + public int hashCode() { + int result = Arrays.hashCode(getMessage()); + result = 31 * result + (getMetadata() != null ? getMetadata().hashCode() : 0); + return result; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessageStrategies.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessageStrategies.java index 50f85372..408d14f9 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessageStrategies.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessageStrategies.java @@ -7,75 +7,79 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.message.metadata; -import org.json.simple.JSONObject; +package org.apache.metron.common.message.metadata; import java.util.Map; +import org.json.simple.JSONObject; /** * The strategies which we can use to interpret data and metadata. This fits the normal enum pattern * that we use elsewhere for strategy pattern. */ public enum RawMessageStrategies implements RawMessageStrategy { - /** - * The default strategy. - */ - DEFAULT(new DefaultRawMessageStrategy()), - /** - * Enveloping strategy, used for parser chaining. - */ - ENVELOPE(new EnvelopedRawMessageStrategy()) - ; - RawMessageStrategy supplier; - RawMessageStrategies(RawMessageStrategy supplier) { - this.supplier = supplier; - } + /** + * The default strategy. + */ + DEFAULT(new DefaultRawMessageStrategy()), + /** + * Enveloping strategy, used for parser chaining. + */ + ENVELOPE(new EnvelopedRawMessageStrategy()); + RawMessageStrategy supplier; + + RawMessageStrategies(RawMessageStrategy supplier) { + this.supplier = supplier; + } - /** - * Retrieve the raw message given the strategy specified. Note the Javadocs for the individual strategy for more info. - * - * @param rawMetadata The metadata read from kafka Key (e.g. the topic, index, etc.) - * @param originalMessage The raw bytes of the original message - * @param readMetadata True if we want to read read the metadata - * @param config The config for the RawMessageStrategy (See the rawMessageStrategyConfig in the SensorParserConfig) - * @return a {@link RawMessage} based on the given strategy. - */ - @Override - public RawMessage get(Map rawMetadata, byte[] originalMessage, boolean readMetadata, Map config) { - return this.supplier.get(rawMetadata, originalMessage, readMetadata, config); - } + /** + * Retrieve the raw message given the strategy specified. Note the Javadocs for the individual strategy for more info. + * + * @param rawMetadata The metadata read from kafka Key (e.g. the topic, index, etc.) + * @param originalMessage The raw bytes of the original message + * @param readMetadata True if we want to read read the metadata + * @param config The config for the RawMessageStrategy (See the rawMessageStrategyConfig in the SensorParserConfig) + * @return a {@link RawMessage} based on the given strategy. + */ + @Override + public RawMessage get(Map rawMetadata, byte[] originalMessage, boolean readMetadata, + Map config) { + return this.supplier.get(rawMetadata, originalMessage, readMetadata, config); + } - /** - * Merge metadata given the strategy specified. Note the Javadocs for the individual strategy for more info. - * - * @param message The parsed message (note: prior to the field transformations) - * @param metadata The metadata passed along - * @param mergeMetadata Whether to merge the metadata or not - * @param config The config for the message strategy. - */ - @Override - public void mergeMetadata(JSONObject message, Map metadata, boolean mergeMetadata, Map config) { - this.supplier.mergeMetadata(message, metadata, mergeMetadata, config); - } + /** + * Merge metadata given the strategy specified. Note the Javadocs for the individual strategy for more info. + * + * @param message The parsed message (note: prior to the field transformations) + * @param metadata The metadata passed along + * @param mergeMetadata Whether to merge the metadata or not + * @param config The config for the message strategy. + */ + @Override + public void mergeMetadata(JSONObject message, Map metadata, boolean mergeMetadata, + Map config) { + this.supplier.mergeMetadata(message, metadata, mergeMetadata, config); + } - @Override - public boolean mergeMetadataDefault() { - return this.supplier.mergeMetadataDefault(); - } + @Override + public boolean mergeMetadataDefault() { + return this.supplier.mergeMetadataDefault(); + } - @Override - public boolean readMetadataDefault() { - return this.supplier.readMetadataDefault(); - } + @Override + public boolean readMetadataDefault() { + return this.supplier.readMetadataDefault(); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessageStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessageStrategy.java index ccd7768b..4e76d918 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessageStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/message/metadata/RawMessageStrategy.java @@ -6,21 +6,23 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.message.metadata; -import org.json.simple.JSONObject; +package org.apache.metron.common.message.metadata; import java.io.Serializable; import java.util.Map; +import org.json.simple.JSONObject; /** * This is a strategy which defines how parsers: @@ -32,46 +34,48 @@ * metadata is read or merged. */ public interface RawMessageStrategy extends Serializable { - /** - * Retrieve the RawMessage (e.g. the data and metadata) given raw data and raw metadata read from kafka. - * Note that the base metadata from kafka key and tuples, etc. along with prefixing is handled in the MetadataUtil. - * This is intended for individual strategies to append OTHER metadata. - * - * @param rawMetadata The metadata read from kafka Key (e.g. the topic, index, etc.) - * @param rawMessage The raw message from the kafka value - * @param readMetadata True if we want to read read the metadata - * @param config The config for the RawMessageStrategy (See the rawMessageStrategyConfig in the SensorParserConfig) - * @return The RawMessage, which defines the data and metadata - */ - RawMessage get( Map rawMetadata - , byte[] rawMessage - , boolean readMetadata - , Map config - ); + /** + * Retrieve the RawMessage (e.g. the data and metadata) given raw data and raw metadata read from kafka. + * Note that the base metadata from kafka key and tuples, etc. along with prefixing is handled in the MetadataUtil. + * This is intended for individual strategies to append OTHER metadata. + * + * @param rawMetadata The metadata read from kafka Key (e.g. the topic, index, etc.) + * @param rawMessage The raw message from the kafka value + * @param readMetadata True if we want to read read the metadata + * @param config The config for the RawMessageStrategy (See the rawMessageStrategyConfig in the SensorParserConfig) + * @return The RawMessage, which defines the data and metadata + */ + RawMessage get(Map rawMetadata, + byte[] rawMessage, + boolean readMetadata, + Map config + ); - /** - * Merge the metadata into the message. Note: Each strategy may merge differently based on their own config. - * - * @param message The parsed message (note: prior to the field transformations) - * @param metadata The metadata passed along - * @param mergeMetadata Whether to merge the metadata or not - * @param config The config for the message strategy. - */ - void mergeMetadata( JSONObject message - , Map metadata - , boolean mergeMetadata - , Map config - ); + /** + * Merge the metadata into the message. Note: Each strategy may merge differently based on their own config. + * + * @param message The parsed message (note: prior to the field transformations) + * @param metadata The metadata passed along + * @param mergeMetadata Whether to merge the metadata or not + * @param config The config for the message strategy. + */ + void mergeMetadata(JSONObject message, + Map metadata, + boolean mergeMetadata, + Map config + ); - /** - * The default value for merging metadata. - * @return true if default to merge, false otherwise - */ - boolean mergeMetadataDefault(); + /** + * The default value for merging metadata. + * + * @return true if default to merge, false otherwise + */ + boolean mergeMetadataDefault(); - /** - * The default value for reading metadata. - * @return true if default to read, false otherwise - */ - boolean readMetadataDefault(); + /** + * The default value for reading metadata. + * + * @return true if default to read, false otherwise + */ + boolean readMetadataDefault(); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/PerformanceLogger.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/PerformanceLogger.java index b5e8646b..e85232c9 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/PerformanceLogger.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/PerformanceLogger.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,169 +30,168 @@ public class PerformanceLogger { - private static final String LOG_PERCENT = "performance.logging.percent.records"; - private static final Integer LOG_PERCENT_DEFAULT = 1; - private Supplier> configSupplier; - private ThresholdCalculator thresholdCalc; - private Timing timing; - private Logger logger; - - /** - * Minimum constructor for this class. - *

      - * Options: - *

      - *
        - *
      • performance.logging.percent.records = Integer value 1-100 indicating percent probability - * that a logging statement should trigger writing timing info. - *
        • - *
      - * - * @param configSupplier provides configuration for the logger as a Map<String, Object> - * @param loggerName name for the underlying logger. This name is used when enabling/disabling - * the logger. - */ - public PerformanceLogger(Supplier> configSupplier, String loggerName) { - this(configSupplier, LoggerFactory.getLogger(loggerName), new ThresholdCalculator(), - new Timing()); - } - - /** - * Constructor that allows more fine grained control. - * - * @param configSupplier provides configuration for the logger as a Map<String, Object> - * @param logger The actual logger to be used - * @param thresholdCalc The {@link ThresholdCalculator} to use - * @param timing The {@link Timing} to use - */ - public PerformanceLogger(Supplier> configSupplier, Logger logger, - ThresholdCalculator thresholdCalc, Timing timing) { - this.configSupplier = configSupplier; - this.thresholdCalc = thresholdCalc; - this.timing = timing; - this.logger = logger; - this.logger.info("{} set to {}", LOG_PERCENT, getPercentThreshold()); - } - - /** - * Marks a timer start. Works in conjunction with the log methods. Calling log after - * calling mark will log elapsed time for the provided markName. - * - * @param markName The name of the mark to use - */ - public void mark(String markName) { - timing.mark(markName); - } - - /** - * Log a message at DEBUG level for the given markName. - * Warns when logging for a markName that hasn't been set. - * - *

      This form avoids superfluous string concatenation when the logger - * is disabled for the DEBUG level.

      - * - * @param markName name of the marked timer to log elapsed time for - */ - public void log(String markName) { - if (okToLog()) { - log(markName, ""); + private static final String LOG_PERCENT = "performance.logging.percent.records"; + private static final Integer LOG_PERCENT_DEFAULT = 1; + private final Supplier> configSupplier; + private final ThresholdCalculator thresholdCalc; + private final Timing timing; + private final Logger logger; + + /** + * Minimum constructor for this class. + *

      + * Options: + *

      + *
        + *
      • performance.logging.percent.records = Integer value 1-100 indicating percent probability + * that a logging statement should trigger writing timing info. + *
        • + *
      + * + * @param configSupplier provides configuration for the logger as a Map<String, Object> + * @param loggerName name for the underlying logger. This name is used when enabling/disabling + * the logger. + */ + public PerformanceLogger(Supplier> configSupplier, String loggerName) { + this(configSupplier, LoggerFactory.getLogger(loggerName), new ThresholdCalculator(), + new Timing()); } - } - - /** - * Log a message at DEBUG level for the given markName according to the specified message. - * Warns when logging for a markName that hasn't been set. - * - *

      This form avoids superfluous string concatenation when the logger - * is disabled for the DEBUG level.

      - * - * @param markName name of the marked timer to log elapsed time for - * @param message message to log - */ - public void log(String markName, String message) { - if (okToLog()) { - if (timing.exists(markName)) { - logger.debug("markName={},time(ns)={},message={}", markName, timing.getElapsed(markName), - message); - } else { - logger.debug("markName={},time(ns)={},message={}", "WARNING - MARK NOT SET", - timing.getElapsed(markName), message); - } + + /** + * Constructor that allows more fine grained control. + * + * @param configSupplier provides configuration for the logger as a Map<String, Object> + * @param logger The actual logger to be used + * @param thresholdCalc The {@link ThresholdCalculator} to use + * @param timing The {@link Timing} to use + */ + public PerformanceLogger(Supplier> configSupplier, Logger logger, + ThresholdCalculator thresholdCalc, Timing timing) { + this.configSupplier = configSupplier; + this.thresholdCalc = thresholdCalc; + this.timing = timing; + this.logger = logger; + this.logger.info("{} set to {}", LOG_PERCENT, getPercentThreshold()); } - } - - private boolean okToLog() { - return logger.isDebugEnabled() && thresholdCalc.isPast(getPercentThreshold()); - } - - private Integer getPercentThreshold() { - return ConversionUtils.convert(getProperty(LOG_PERCENT, LOG_PERCENT_DEFAULT), Integer.class); - } - - private Object getProperty(String key, Object defaultValue) { - return configSupplier.get().getOrDefault(key, defaultValue); - } - - - /** - * Log a message at DEBUG level for the given markName according to the specified format - * and argument. Warns when logging for a markName that hasn't been set. - * - *

      This form avoids superfluous string concatenation when the logger - * is disabled for the DEBUG level.

      - * - * @param markName name of the marked timer to log elapsed time for - * @param format the format string - * @param arg argument to the format String - */ - public void log(String markName, String format, Object arg) { - if (okToLog()) { - FormattingTuple formattedMessage = MessageFormatter.format(format, arg); - log(markName, formattedMessage.getMessage()); + + /** + * Marks a timer start. Works in conjunction with the log methods. Calling log after + * calling mark will log elapsed time for the provided markName. + * + * @param markName The name of the mark to use + */ + public void mark(String markName) { + timing.mark(markName); } - } - - /** - * Log a message at DEBUG level according to the specified format and argument. - * - *

      This form avoids superfluous string concatenation when the logger - * is disabled for the DEBUG level.

      - * - * @param markName name of the marked timer to log elapsed time for - * @param format the format string - * @param arg1 first argument to the format String - * @param arg2 second argument to the format String - */ - public void log(String markName, String format, Object arg1, Object arg2) { - if (okToLog()) { - FormattingTuple formattedMessage = MessageFormatter.format(format, arg1, arg2); - log(markName, formattedMessage.getMessage()); + + /** + * Log a message at DEBUG level for the given markName. + * Warns when logging for a markName that hasn't been set. + * + *

      This form avoids superfluous string concatenation when the logger + * is disabled for the DEBUG level.

      + * + * @param markName name of the marked timer to log elapsed time for + */ + public void log(String markName) { + if (okToLog()) { + log(markName, ""); + } } - } - - /** - * Log a message at DEBUG level according to the specified format and arguments. - * - *

      This form avoids superfluous string concatenation when the logger - * is disabled for the DEBUG level. However, this variant incurs the hidden - * (and relatively small) cost of creating an Object[] before invoking the method, - * even if this logger is disabled for DEBUG. The variants taking - * {@link #log(String, String, Object) one} and {@link #log(String, String, Object, Object) two} - * arguments exist solely in order to avoid this hidden cost.

      - * - * @param markName name of the marked timer to log elapsed time for - * @param format the format string - * @param arguments a list of 3 or more arguments - */ - public void log(String markName, String format, Object... arguments) { - if (okToLog()) { - FormattingTuple formattedMessage = MessageFormatter.arrayFormat(format, arguments); - log(markName, formattedMessage.getMessage()); + + /** + * Log a message at DEBUG level for the given markName according to the specified message. + * Warns when logging for a markName that hasn't been set. + * + *

      This form avoids superfluous string concatenation when the logger + * is disabled for the DEBUG level.

      + * + * @param markName name of the marked timer to log elapsed time for + * @param message message to log + */ + public void log(String markName, String message) { + if (okToLog()) { + if (timing.exists(markName)) { + logger.debug("markName={},time(ns)={},message={}", markName, timing.getElapsed(markName), + message); + } else { + logger.debug("markName={},time(ns)={},message={}", "WARNING - MARK NOT SET", + timing.getElapsed(markName), message); + } + } } - } - public boolean isDebugEnabled() { - return logger.isDebugEnabled(); - } + /** + * Log a message at DEBUG level for the given markName according to the specified format + * and argument. Warns when logging for a markName that hasn't been set. + * + *

      This form avoids superfluous string concatenation when the logger + * is disabled for the DEBUG level.

      + * + * @param markName name of the marked timer to log elapsed time for + * @param format the format string + * @param arg argument to the format String + */ + public void log(String markName, String format, Object arg) { + if (okToLog()) { + FormattingTuple formattedMessage = MessageFormatter.format(format, arg); + log(markName, formattedMessage.getMessage()); + } + } + + /** + * Log a message at DEBUG level according to the specified format and argument. + * + *

      This form avoids superfluous string concatenation when the logger + * is disabled for the DEBUG level.

      + * + * @param markName name of the marked timer to log elapsed time for + * @param format the format string + * @param arg1 first argument to the format String + * @param arg2 second argument to the format String + */ + public void log(String markName, String format, Object arg1, Object arg2) { + if (okToLog()) { + FormattingTuple formattedMessage = MessageFormatter.format(format, arg1, arg2); + log(markName, formattedMessage.getMessage()); + } + } + + /** + * Log a message at DEBUG level according to the specified format and arguments. + * + *

      This form avoids superfluous string concatenation when the logger + * is disabled for the DEBUG level. However, this variant incurs the hidden + * (and relatively small) cost of creating an Object[] before invoking the method, + * even if this logger is disabled for DEBUG. The variants taking + * {@link #log(String, String, Object) one} and {@link #log(String, String, Object, Object) two} + * arguments exist solely in order to avoid this hidden cost.

      + * + * @param markName name of the marked timer to log elapsed time for + * @param format the format string + * @param arguments a list of 3 or more arguments + */ + public void log(String markName, String format, Object... arguments) { + if (okToLog()) { + FormattingTuple formattedMessage = MessageFormatter.arrayFormat(format, arguments); + log(markName, formattedMessage.getMessage()); + } + } + + private boolean okToLog() { + return logger.isDebugEnabled() && thresholdCalc.isPast(getPercentThreshold()); + } + + private Integer getPercentThreshold() { + return ConversionUtils.convert(getProperty(LOG_PERCENT, LOG_PERCENT_DEFAULT), Integer.class); + } + + private Object getProperty(String key, Object defaultValue) { + return configSupplier.get().getOrDefault(key, defaultValue); + } + + public boolean isDebugEnabled() { + return logger.isDebugEnabled(); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/ThresholdCalculator.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/ThresholdCalculator.java index 9cd592b7..dd00434c 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/ThresholdCalculator.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/ThresholdCalculator.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,22 +22,19 @@ public class ThresholdCalculator { - /** - * Returns true roughly the provided percent of the time, false 100%-'percent' of the time. - * - * @param percent Desired probability to return true - * @return true if the percent probability is true for this call. - */ - public boolean isPast(int percent) { - double rd = Math.random(); - if (rd <= toDecimal(percent)) { - return true; + /** + * Returns true roughly the provided percent of the time, false 100%-'percent' of the time. + * + * @param percent Desired probability to return true + * @return true if the percent probability is true for this call. + */ + public boolean isPast(int percent) { + double rd = Math.random(); + return rd <= toDecimal(percent); } - return false; - } - private double toDecimal(int percent) { - return percent / 100.0; - } + private double toDecimal(int percent) { + return percent / 100.0; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/Timing.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/Timing.java index 747ac816..6a4a0b82 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/Timing.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/performance/Timing.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,43 +25,43 @@ public class Timing { - Map startTimes; + Map startTimes; - public Timing() { - this.startTimes = new HashMap<>(); - } + public Timing() { + this.startTimes = new HashMap<>(); + } - /** - * Stores a starting time from current nanoTime with the provided name. - * - * @param name starting time mark name - */ - public void mark(String name) { - startTimes.put(name, System.nanoTime()); - } + /** + * Stores a starting time from current nanoTime with the provided name. + * + * @param name starting time mark name + */ + public void mark(String name) { + startTimes.put(name, System.nanoTime()); + } - /** - * Returns elapsed nanoTime given a stored marked time for the given name. Returns 0 for - * non-existent mark names. - * - * @param name mark name to get elapsed time for. - */ - public long getElapsed(String name) { - if (startTimes.containsKey(name)) { - return System.nanoTime() - startTimes.get(name); - } else { - return 0; + /** + * Returns elapsed nanoTime given a stored marked time for the given name. Returns 0 for + * non-existent mark names. + * + * @param name mark name to get elapsed time for. + */ + public long getElapsed(String name) { + if (startTimes.containsKey(name)) { + return System.nanoTime() - startTimes.get(name); + } else { + return 0; + } } - } - /** - * Checks existence of timing marker. - * - * @param name mark name to lookup. - * @return true if mark has been called with this name, false otherwise. - */ - public boolean exists(String name) { - return startTimes.containsKey(name); - } + /** + * Checks existence of timing marker. + * + * @param name mark name to lookup. + * @return true if mark has been called with this name, false otherwise. + */ + public boolean exists(String name) { + return startTimes.containsKey(name); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/Clock.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/Clock.java index 0dd36dd9..4d1eb85f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/Clock.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/Clock.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.system; import java.text.SimpleDateFormat; @@ -22,22 +25,22 @@ import java.util.TimeZone; public class Clock { - private final static String UTC = "UTC"; + private static final String UTC = "UTC"; - public long currentTimeMillis() { - return System.currentTimeMillis(); - } + public long currentTimeMillis() { + return System.currentTimeMillis(); + } - /** - * Retrieves the current time in a given format compatible with {@link SimpleDateFormat}. - * - * @param stdDateFormat The format to use - * @return String representation of the current time - */ - public String currentTimeFormatted(String stdDateFormat) { - SimpleDateFormat format = new SimpleDateFormat(stdDateFormat); - format.setTimeZone(TimeZone.getTimeZone(UTC)); - return format.format(new Date(currentTimeMillis())); - } + /** + * Retrieves the current time in a given format compatible with {@link SimpleDateFormat}. + * + * @param stdDateFormat The format to use + * @return String representation of the current time + */ + public String currentTimeFormatted(String stdDateFormat) { + SimpleDateFormat format = new SimpleDateFormat(stdDateFormat); + format.setTimeZone(TimeZone.getTimeZone(UTC)); + return format.format(new Date(currentTimeMillis())); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/Environment.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/Environment.java index 97faa040..6cff4ac8 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/Environment.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/Environment.java @@ -7,21 +7,24 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.system; /** * Useful so we can test mock dependency injection with environment variables. */ public class Environment { - public String get(String variable) { - return System.getenv().get(variable); - } + public String get(String variable) { + return System.getenv().get(variable); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/FakeClock.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/FakeClock.java index 1ba49efe..cb90c0f9 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/FakeClock.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/system/FakeClock.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.system; import java.util.concurrent.TimeUnit; @@ -34,128 +37,128 @@ * hence unchecked. */ public class FakeClock extends Clock { - private long now_ms = 0; - - @Override - public long currentTimeMillis() { - return now_ms; - } + private long nowMs = 0; - /** - * Advance the fake clock by a number of milliseconds. - * @param duration_ms The duration of the adjustment - * @throws IllegalArgumentClockNegative (unchecked) if you try to go backwards in time. - * This is not an allowed behavior, because most system clocks go to great - * effort to make sure it never happens, even with, e.g., anomalous events - * from a bad NTP server. - * If we really get a demand for this capability, we'll add methods that don't - * check for this. - * @throws IllegalArgumentClockOverflow (unchecked) if you try to add a duration - * that would overflow the Long value of {@code currentTimeMillis} - */ - public void elapseMillis(long duration_ms) { - long instant_ms = now_ms + duration_ms; - if (duration_ms < 0) { - throw new IllegalArgumentClockNegative(String.format( - "Attempted to move backward in time, by %d milliseconds." - , duration_ms)); - } - else if (instant_ms < 0) { - throw new IllegalArgumentClockOverflow(String.format( - "Attempted to advance beyond the edge of time, to epoch %d + %d." - , now_ms, duration_ms)); + @Override + public long currentTimeMillis() { + return nowMs; } - now_ms = instant_ms; - } - /** - * Advance the fake clock by a number of seconds. - * See {@code elapseMillis} for details. - * - * @param duration_secs The duration to elapse in seconds - */ - public void elapseSeconds(long duration_secs) { - elapseMillis(TimeUnit.SECONDS.toMillis(duration_secs)); - } + /** + * Advance the fake clock by a number of milliseconds. + * + * @param durationMs The duration of the adjustment + * @throws IllegalArgumentClockNegative (unchecked) if you try to go backwards in time. + * This is not an allowed behavior, because most system clocks go to great + * effort to make sure it never happens, even with, e.g., anomalous events + * from a bad NTP server. + * If we really get a demand for this capability, we'll add methods that don't + * check for this. + * @throws IllegalArgumentClockOverflow (unchecked) if you try to add a duration + * that would overflow the Long value of {@code currentTimeMillis} + */ + public void elapseMillis(long durationMs) { + long instantMs = nowMs + durationMs; + if (durationMs < 0) { + throw new IllegalArgumentClockNegative(String.format( + "Attempted to move backward in time, by %d milliseconds.", + durationMs)); + } else if (instantMs < 0) { + throw new IllegalArgumentClockOverflow(String.format( + "Attempted to advance beyond the edge of time, to epoch %d + %d.", + nowMs, durationMs)); + } + nowMs = instantMs; + } - /** - * Advance the fake clock to a point in time specified as milliseconds after 0. - * - * @param instant_ms - epoch time in milliseconds - * @throws IllegalArgumentClockNegative (unchecked) if you try to go backwards in time. - * This is not an allowed behavior, because most system clocks go to great - * effort to make sure it never happens, even with, e.g., anomalous events - * from a bad NTP server. - * If we really get a demand for this capability, we'll add methods that don't - * check for this. - * @throws IllegalArgumentClockZero (unchecked) if you try to "advance" the clock to the time it already is. - * Why? Because it implies your test code has lost track of previous increments, - * which might be problematic, so we do this in the spirit of "fail fast". - * If you *meant* to lose track, for instance if you were using random numbers of events, - * or whatever, you can always orient yourself in time by reading {@code currentTimeMillis}. - */ - public void advanceToMillis(long instant_ms) { - if (instant_ms < now_ms) { - throw new IllegalArgumentClockNegative(String.format( - "Attempted to move backward in time, from epoch %d to %d." - , now_ms, instant_ms)); + /** + * Advance the fake clock by a number of seconds. + * See {@code elapseMillis} for details. + * + * @param durationSecs The duration to elapse in seconds + */ + public void elapseSeconds(long durationSecs) { + elapseMillis(TimeUnit.SECONDS.toMillis(durationSecs)); } - if (instant_ms == now_ms) { - throw new IllegalArgumentClockZero(String.format( - "Time was set to current time, with null advance, at epoch %d." - , now_ms)); + + /** + * Advance the fake clock to a point in time specified as milliseconds after 0. + * + * @param instantMs - epoch time in milliseconds + * @throws IllegalArgumentClockNegative (unchecked) if you try to go backwards in time. + * This is not an allowed behavior, because most system clocks go to great + * effort to make sure it never happens, even with, e.g., anomalous events + * from a bad NTP server. + * If we really get a demand for this capability, we'll add methods that don't + * check for this. + * @throws IllegalArgumentClockZero (unchecked) if you try to "advance" the clock to the time it already is. + * Why? Because it implies your test code has lost track of previous increments, + * which might be problematic, so we do this in the spirit of "fail fast". + * If you *meant* to lose track, for instance if you were using random numbers of events, + * or whatever, you can always orient yourself in time by reading {@code currentTimeMillis}. + */ + public void advanceToMillis(long instantMs) { + if (instantMs < nowMs) { + throw new IllegalArgumentClockNegative(String.format( + "Attempted to move backward in time, from epoch %d to %d.", + nowMs, instantMs)); + } + if (instantMs == nowMs) { + throw new IllegalArgumentClockZero(String.format( + "Time was set to current time, with null advance, at epoch %d.", + nowMs)); + } + nowMs = instantMs; } - now_ms = instant_ms; - } - /** - * Advance the fake clock to a point in time specified as seconds after 0. - * See {@code advanceToMillis} for details. - * - * @param instant_secs - epoch time in seconds - */ - public void advanceToSeconds(long instant_secs) { - advanceToMillis(TimeUnit.SECONDS.toMillis(instant_secs)); - } + /** + * Advance the fake clock to a point in time specified as seconds after 0. + * See {@code advanceToMillis} for details. + * + * @param instantSecs - epoch time in seconds + */ + public void advanceToSeconds(long instantSecs) { + advanceToMillis(TimeUnit.SECONDS.toMillis(instantSecs)); + } - /** - * IllegalArgumentClockNegative (unchecked) is thrown if you try to go backwards in time. - * This is not an allowed behavior, because most system clocks go to great - * effort to make sure it never happens, even with, e.g., anomalous events - * from a bad NTP server. - * If we really get a demand for this capability, we'll add methods that don't - * check for this. - */ - public static class IllegalArgumentClockNegative extends IllegalArgumentException { - public IllegalArgumentClockNegative(String s) { - super(s); + /** + * IllegalArgumentClockNegative (unchecked) is thrown if you try to go backwards in time. + * This is not an allowed behavior, because most system clocks go to great + * effort to make sure it never happens, even with, e.g., anomalous events + * from a bad NTP server. + * If we really get a demand for this capability, we'll add methods that don't + * check for this. + */ + public static class IllegalArgumentClockNegative extends IllegalArgumentException { + public IllegalArgumentClockNegative(String s) { + super(s); + } } - } - /** - * IllegalArgumentClockZero (unchecked) is thrown if you try to "advance" the clock to the time it already is. - * Why? Because it implies your test code has lost track of previous increments, - * which might be problematic, so we do this in the spirit of "fail fast". - * If you *meant* to lose track, for instance if you were using random numbers of events, - * or whatever, you can always orient yourself in time by reading {@code currentTimeMillis}. - * - *

      Note that argument does not apply to ellapseMillis(0), so it does not throw - * this exception. - */ - public static class IllegalArgumentClockZero extends IllegalArgumentException { - public IllegalArgumentClockZero(String s) { - super(s); + /** + * IllegalArgumentClockZero (unchecked) is thrown if you try to "advance" the clock to the time it already is. + * Why? Because it implies your test code has lost track of previous increments, + * which might be problematic, so we do this in the spirit of "fail fast". + * If you *meant* to lose track, for instance if you were using random numbers of events, + * or whatever, you can always orient yourself in time by reading {@code currentTimeMillis}. + * + *

      Note that argument does not apply to ellapseMillis(0), so it does not throw + * this exception. + */ + public static class IllegalArgumentClockZero extends IllegalArgumentException { + public IllegalArgumentClockZero(String s) { + super(s); + } } - } - /** - * IllegalArgumentClockOverflow (unchecked) is thrown if you try to add a duration - * that would overflow the Long value of {@code currentTimeMillis}. - */ - public static class IllegalArgumentClockOverflow extends IllegalArgumentException { - public IllegalArgumentClockOverflow(String s) { - super(s); + /** + * IllegalArgumentClockOverflow (unchecked) is thrown if you try to add a duration + * that would overflow the Long value of {@code currentTimeMillis}. + */ + public static class IllegalArgumentClockOverflow extends IllegalArgumentException { + public IllegalArgumentClockOverflow(String s) { + super(s); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java index 871052d0..0010229c 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; @@ -24,18 +27,18 @@ * A typo strategy based on adding characters between ascii 97 and 123. */ public class AdditionStrategy implements TyposquattingStrategy { - @Override - public Set generateCandidates(String domain) { - Set ret = new HashSet<>(); - for(int i = 97;i < 123;++i) { - char c = Character.toChars(i)[0]; - ret.add(domain + c); + @Override + public Set generateCandidates(String domain) { + Set ret = new HashSet<>(); + for (int i = 97; i < 123; ++i) { + char c = Character.toChars(i)[0]; + ret.add(domain + c); + } + return ret; } - return ret; - } - @Override - public String name() { - return "Addition"; - } + @Override + public String name() { + return "Addition"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java index 32d4184b..83bbe702 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; @@ -24,38 +27,39 @@ * See http://dinaburg.org/bitsquatting.html for more */ public class BitsquattingStrategy implements TyposquattingStrategy { - public static int[] MASK = new int[] { 1, 2, 4, 8, 16, 32, 64, 128}; - @Override - public Set generateCandidates(String originalString) { - Set ret = new HashSet<>(); - char[] str = originalString.toCharArray(); - for(int i = 0;i < str.length;++i) { - char c = str[i]; - for(int j : MASK) { - int maskedNum = (int)c ^ j; - char maskedChar = (char)maskedNum; - if((maskedNum >= 48 && maskedNum <= 57) || (maskedNum >= 97 && maskedNum <= 122) || maskedNum == 45) { - ret.add(pasteTogether(str, i, maskedChar)); + public static int[] MASK = new int[] {1, 2, 4, 8, 16, 32, 64, 128}; + + @Override + public Set generateCandidates(String originalString) { + Set ret = new HashSet<>(); + char[] str = originalString.toCharArray(); + for (int i = 0; i < str.length; ++i) { + char c = str[i]; + for (int j : MASK) { + int maskedNum = (int) c ^ j; + char maskedChar = (char) maskedNum; + if ((maskedNum >= 48 && maskedNum <= 57) || (maskedNum >= 97 && maskedNum <= 122) || maskedNum == 45) { + ret.add(pasteTogether(str, i, maskedChar)); + } + } } - } + return ret; } - return ret; - } - - @Override - public String name() { - return "Bitsquatting"; - } - private static String pasteTogether(char[] str, int replacementPoint, char maskedChar) { - String ret = ""; - for(int i = 0;i < replacementPoint;++i) { - ret += str[i]; + @Override + public String name() { + return "Bitsquatting"; } - ret += maskedChar; - for(int i = replacementPoint+1;i < str.length;++i) { - ret += str[i]; + + private static String pasteTogether(char[] str, int replacementPoint, char maskedChar) { + String ret = ""; + for (int i = 0; i < replacementPoint; ++i) { + ret += str[i]; + } + ret += maskedChar; + for (int i = replacementPoint + 1; i < str.length; ++i) { + ret += str[i]; + } + return ret; } - return ret; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java index 19479de1..7f69c14e 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java @@ -7,109 +7,115 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import com.google.common.collect.ImmutableList; +import java.lang.invoke.MethodHandles; +import java.net.IDN; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.net.IDN; -import java.util.*; - /** - * Substituting characters for ascii or unicode analogues which are visually similar (e.g. latlmes.com for latimes.com) - * + * Substituting characters for ascii or unicode analogues which are visually similar (e.g. latlmes.com for latimes.com) */ -public class HomoglyphStrategy implements TyposquattingStrategy{ +public class HomoglyphStrategy implements TyposquattingStrategy { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final Map> glyphs = new HashMap>() - {{ - put('a', ImmutableList.of("à", "á", "â", "ã", "ä", "å", "ɑ", "а", "ạ", "ǎ", "ă", "ȧ", "ӓ")); - put('b', ImmutableList.of("d", "lb", "ib", "ʙ", "Ь", "b̔", "ɓ", "Б")); - put('c', ImmutableList.of("ϲ", "с", "ƈ", "ċ", "ć", "ç")); - put('d', ImmutableList.of("b", "cl", "dl", "di", "ԁ", "ժ", "ɗ", "đ")); - put('e', ImmutableList.of("é", "ê", "ë", "ē", "ĕ", "ě", "ė", "е", "ẹ", "ę", "є", "ϵ", "ҽ")); - put('f', ImmutableList.of("Ϝ", "ƒ", "Ғ")); - put('g', ImmutableList.of("q", "ɢ", "ɡ", "Ԍ", "Ԍ", "ġ", "ğ", "ց", "ǵ", "ģ")); - put('h', ImmutableList.of("lh", "ih", "һ", "հ", "Ꮒ", "н")); - put('i', ImmutableList.of("1", "l", "Ꭵ", "í", "ï", "ı", "ɩ", "ι", "ꙇ", "ǐ", "ĭ")); - put('j', ImmutableList.of("ј", "ʝ", "ϳ", "ɉ")); - put('k', ImmutableList.of("lk", "ik", "lc", "κ", "ⲕ", "κ")); - put('l', ImmutableList.of("1", "i", "ɫ", "ł")); - put('m', ImmutableList.of("n", "nn", "rn", "rr", "ṃ", "ᴍ", "м", "ɱ")); - put('n', ImmutableList.of("m", "r", "ń")); - put('o', ImmutableList.of("0", "Ο", "ο", "О", "о", "Օ", "ȯ", "ọ", "ỏ", "ơ", "ó", "ö", "ӧ")); - put('p', ImmutableList.of("ρ", "р", "ƿ", "Ϸ", "Þ")); - put('q', ImmutableList.of("g", "զ", "ԛ", "գ", "ʠ")); - put('r', ImmutableList.of("ʀ", "Г", "ᴦ", "ɼ", "ɽ")); - put('s', ImmutableList.of("Ⴝ", "Ꮪ", "ʂ", "ś", "ѕ")); - put('t', ImmutableList.of("τ", "т", "ţ")); - put('u', ImmutableList.of("μ", "υ", "Ս", "ս", "ц", "ᴜ", "ǔ", "ŭ")); - put('v', ImmutableList.of("ѵ", "ν", "v̇")); - put('w', ImmutableList.of("vv", "ѡ", "ա", "ԝ")); - put('x', ImmutableList.of("х", "ҳ", "ẋ")); - put('y', ImmutableList.of("ʏ", "γ", "у", "Ү", "ý")); - put('z', ImmutableList.of("ʐ", "ż", "ź", "ʐ", "ᴢ")); - }}; + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final Map> glyphs = new HashMap>() { + { + put('a', ImmutableList.of("à", "á", "â", "ã", "ä", "å", "ɑ", "а", "ạ", "ǎ", "ă", "ȧ", "ӓ")); + put('b', ImmutableList.of("d", "lb", "ib", "ʙ", "Ь", "b̔", "ɓ", "Б")); + put('c', ImmutableList.of("ϲ", "с", "ƈ", "ċ", "ć", "ç")); + put('d', ImmutableList.of("b", "cl", "dl", "di", "ԁ", "ժ", "ɗ", "đ")); + put('e', ImmutableList.of("é", "ê", "ë", "ē", "ĕ", "ě", "ė", "е", "ẹ", "ę", "є", "ϵ", "ҽ")); + put('f', ImmutableList.of("Ϝ", "ƒ", "Ғ")); + put('g', ImmutableList.of("q", "ɢ", "ɡ", "Ԍ", "Ԍ", "ġ", "ğ", "ց", "ǵ", "ģ")); + put('h', ImmutableList.of("lh", "ih", "һ", "հ", "Ꮒ", "н")); + put('i', ImmutableList.of("1", "l", "Ꭵ", "í", "ï", "ı", "ɩ", "ι", "ꙇ", "ǐ", "ĭ")); + put('j', ImmutableList.of("ј", "ʝ", "ϳ", "ɉ")); + put('k', ImmutableList.of("lk", "ik", "lc", "κ", "ⲕ", "κ")); + put('l', ImmutableList.of("1", "i", "ɫ", "ł")); + put('m', ImmutableList.of("n", "nn", "rn", "rr", "ṃ", "ᴍ", "м", "ɱ")); + put('n', ImmutableList.of("m", "r", "ń")); + put('o', ImmutableList.of("0", "Ο", "ο", "О", "о", "Օ", "ȯ", "ọ", "ỏ", "ơ", "ó", "ö", "ӧ")); + put('p', ImmutableList.of("ρ", "р", "ƿ", "Ϸ", "Þ")); + put('q', ImmutableList.of("g", "զ", "ԛ", "գ", "ʠ")); + put('r', ImmutableList.of("ʀ", "Г", "ᴦ", "ɼ", "ɽ")); + put('s', ImmutableList.of("Ⴝ", "Ꮪ", "ʂ", "ś", "ѕ")); + put('t', ImmutableList.of("τ", "т", "ţ")); + put('u', ImmutableList.of("μ", "υ", "Ս", "ս", "ц", "ᴜ", "ǔ", "ŭ")); + put('v', ImmutableList.of("ѵ", "ν", "v̇")); + put('w', ImmutableList.of("vv", "ѡ", "ա", "ԝ")); + put('x', ImmutableList.of("х", "ҳ", "ẋ")); + put('y', ImmutableList.of("ʏ", "γ", "у", "Ү", "ý")); + put('z', ImmutableList.of("ʐ", "ż", "ź", "ʐ", "ᴢ")); + } + }; - @Override - public Set generateCandidates(String originalString) { - Set result = new HashSet<>(); - String domain = originalString; - if(StringUtils.isEmpty(domain)) { - return result; - } - if(isAce(domain)) { - //this is an ace domain. - domain = IDN.toUnicode(domain); - } - for(int ws = 0;ws < domain.length();ws++) { - for(int i = 0;i < domain.length() - ws + 1;++i) { - String win = domain.substring(i, i+ws); - for(int j = 0;j < ws;j++) { - char c = win.charAt(j); - if( glyphs.containsKey(c)) { - for( String g : glyphs.get(c)) { - String winNew = win.replaceAll("" + c, g); - String d = domain.substring(0, i) + winNew + domain.substring(i + ws); - result.add(d); - if(!isAce(d)) { - try { - String dAscii = IDN.toASCII(d, IDN.ALLOW_UNASSIGNED); - if (!d.equals(dAscii)) { - result.add(dAscii); - } - } - catch(IllegalArgumentException iae) { - LOG.debug("Unable to parse " + d + ": " + iae.getMessage(), iae); + @SuppressWarnings("checkstyle:LocalVariableName") + @Override + public Set generateCandidates(String originalString) { + Set result = new HashSet<>(); + String domain = originalString; + if (StringUtils.isEmpty(domain)) { + return result; + } + if (isAce(domain)) { + //this is an ace domain. + domain = IDN.toUnicode(domain); + } + for (int ws = 0; ws < domain.length(); ws++) { + for (int i = 0; i < domain.length() - ws + 1; ++i) { + String win = domain.substring(i, i + ws); + for (int j = 0; j < ws; j++) { + char c = win.charAt(j); + if (glyphs.containsKey(c)) { + for (String g : glyphs.get(c)) { + String winNew = win.replaceAll("" + c, g); + String d = domain.substring(0, i) + winNew + domain.substring(i + ws); + result.add(d); + if (!isAce(d)) { + try { + String dAscii = IDN.toASCII(d, IDN.ALLOW_UNASSIGNED); + if (!d.equals(dAscii)) { + result.add(dAscii); + } + } catch (IllegalArgumentException iae) { + LOG.debug("Unable to parse " + d + ": " + iae.getMessage(), iae); + } + } + } + } } - } } - } } - } + return result; } - return result; - } - public static boolean isAce(String domainRaw) { - String domain = domainRaw.toLowerCase(); - return domain.startsWith("xn--") || domain.contains(".xn--"); - } + public static boolean isAce(String domainRaw) { + String domain = domainRaw.toLowerCase(); + return domain.startsWith("xn--") || domain.contains(".xn--"); + } - @Override - public String name() { - return "Homoglyph"; - } + @Override + public String name() { + return "Homoglyph"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java index ca6ce8f0..10d3be54 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; @@ -24,18 +27,18 @@ * Typos based on random hyphenation (e.g. am-azon.com vs amazon.com) */ public class HyphenationStrategy implements TyposquattingStrategy { - @Override - public Set generateCandidates(String originalString) { - Set ret = new HashSet<>(); - for(int i = 1;i < originalString.length();++i) { - ret.add(originalString.substring(0, i) + "-" + originalString.substring(i)); + @Override + public Set generateCandidates(String originalString) { + Set ret = new HashSet<>(); + for (int i = 1; i < originalString.length(); ++i) { + ret.add(originalString.substring(0, i) + "-" + originalString.substring(i)); + } + return ret; } - return ret; - } - @Override - public String name() { - return "Hyphenation"; - } + @Override + public String name() { + return "Hyphenation"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java index b601a359..da0916d7 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; @@ -24,33 +27,33 @@ * Typo strategy based on random insertion of common typos based on keyboard layout proximity. */ public class InsertionStrategy implements TyposquattingStrategy { - @Override - public Set generateCandidates(String domain) { - Set ret = new HashSet<>(); - for(int i = 1;i < domain.length() - 1;++i) { - for(Keyboards keyboard : Keyboards.values()) { - String mapping = keyboard.getMapping().get(domain.charAt(i)); - if(mapping != null) { - for(Character c : mapping.toCharArray()) { - ret.add(domain.substring(0, i) - + c - + domain.charAt(i) - + domain.substring(i+1, domain.length()) - ); - ret.add(domain.substring(0, i) - + domain.charAt(i) - + c - + domain.substring(i+1, domain.length()) - ); - } + @Override + public Set generateCandidates(String domain) { + Set ret = new HashSet<>(); + for (int i = 1; i < domain.length() - 1; ++i) { + for (Keyboards keyboard : Keyboards.values()) { + String mapping = keyboard.getMapping().get(domain.charAt(i)); + if (mapping != null) { + for (Character c : mapping.toCharArray()) { + ret.add(domain.substring(0, i) + + c + + domain.charAt(i) + + domain.substring(i + 1) + ); + ret.add(domain.substring(0, i) + + domain.charAt(i) + + c + + domain.substring(i + 1) + ); + } + } + } } - } + return ret; } - return ret; - } - @Override - public String name() { - return "Insertion"; - } + @Override + public String name() { + return "Insertion"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java index 6fbd5ecf..12011b8f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashMap; @@ -25,127 +28,133 @@ * typos. */ public enum Keyboards { - QWERTY(new HashMap() - {{ - put('j', "ikmnhu"); - put('w', "3esaq2"); - put('v', "cfgb"); - put('l', "kop"); - put('y', "7uhgt6"); - put('x', "zsdc"); - put('r', "5tfde4"); - put('u', "8ijhy7"); - put('a', "qwsz"); - put('q', "12wa"); - put('c', "xdfv"); - put('b', "vghn"); - put('e', "4rdsw3"); - put('d', "rfcxse"); - put('g', "yhbvft"); - put('p', "lo0"); - put('i', "9okju8"); - put('h', "ujnbgy"); - put('k', "olmji"); - put('f', "tgvcdr"); - put('m', "njk"); - put('s', "edxzaw"); - put('o', "0plki9"); - put('n', "bhjm"); - put('1', "2q"); - put('0', "po9"); - put('3', "4ew2"); - put('2', "3wq1"); - put('5', "6tr4"); - put('4', "5re3"); - put('7', "8uy6"); - put('6', "7yt5"); - put('9', "0oi8"); - put('8', "9iu7"); - put('z', "asx"); - put('t', "6ygfr5"); - }}), - QWERTZ(new HashMap() {{ - put('j', "ikmnhu"); - put('w', "3esaq2"); - put('v', "cfgb"); - put('l', "kop"); - put('y', "asx"); - put('x', "ysdc"); - put('r', "5tfde4"); - put('u', "8ijhz7"); - put('a', "qwsy"); - put('q', "12wa"); - put('c', "xdfv"); - put('b', "vghn"); - put('e', "4rdsw3"); - put('d', "rfcxse"); - put('g', "zhbvft"); - put('p', "lo0"); - put('i', "9okju8"); - put('h', "ujnbgz"); - put('k', "olmji"); - put('f', "tgvcdr"); - put('m', "njk"); - put('s', "edxyaw"); - put('o', "0plki9"); - put('n', "bhjm"); - put('1', "2q"); - put('0', "po9"); - put('3', "4ew2"); - put('2', "3wq1"); - put('5', "6tr4"); - put('4', "5re3"); - put('7', "8uz6"); - put('6', "7zt5"); - put('9', "0oi8"); - put('8', "9iu7"); - put('z', "7uhgt6"); - put('t', "6zgfr5"); - }}), - AZERTY(new HashMap() {{ - put('j', "iknhu"); - put('w', "sxq"); - put('v', "cfgb"); - put('l', "kopm"); - put('y', "7uhgt6"); - put('x', "zsdc"); - put('r', "5tfde4"); - put('u', "8ijhy7"); - put('a', "2zq1"); - put('q', "zswa"); - put('c', "xdfv"); - put('b', "vghn"); - put('e', "4rdsz3"); - put('d', "rfcxse"); - put('g', "yhbvft"); - put('p', "lo0m"); - put('i', "9okju8"); - put('h', "ujnbgy"); - put('k', "olji"); - put('f', "tgvcdr"); - put('m', "lp"); - put('s', "edxwqz"); - put('o', "0plki9"); - put('n', "bhj"); - put('1', "2a"); - put('0', "po9"); - put('3', "4ez2"); - put('2', "3za1"); - put('5', "6tr4"); - put('4', "5re3"); - put('7', "8uy6"); - put('6', "7yt5"); - put('9', "0oi8"); - put('8', "9iu7"); - put('z', "3esqa2"); - put('t', "6ygfr5"); - }}) - ; - private Map mapping; - Keyboards(Map mapping) { - this.mapping = mapping; - } - public Map getMapping() { - return mapping; - } + QWERTY(new HashMap() { + { + put('j', "ikmnhu"); + put('w', "3esaq2"); + put('v', "cfgb"); + put('l', "kop"); + put('y', "7uhgt6"); + put('x', "zsdc"); + put('r', "5tfde4"); + put('u', "8ijhy7"); + put('a', "qwsz"); + put('q', "12wa"); + put('c', "xdfv"); + put('b', "vghn"); + put('e', "4rdsw3"); + put('d', "rfcxse"); + put('g', "yhbvft"); + put('p', "lo0"); + put('i', "9okju8"); + put('h', "ujnbgy"); + put('k', "olmji"); + put('f', "tgvcdr"); + put('m', "njk"); + put('s', "edxzaw"); + put('o', "0plki9"); + put('n', "bhjm"); + put('1', "2q"); + put('0', "po9"); + put('3', "4ew2"); + put('2', "3wq1"); + put('5', "6tr4"); + put('4', "5re3"); + put('7', "8uy6"); + put('6', "7yt5"); + put('9', "0oi8"); + put('8', "9iu7"); + put('z', "asx"); + put('t', "6ygfr5"); + } + }), + QWERTZ(new HashMap() { + { + put('j', "ikmnhu"); + put('w', "3esaq2"); + put('v', "cfgb"); + put('l', "kop"); + put('y', "asx"); + put('x', "ysdc"); + put('r', "5tfde4"); + put('u', "8ijhz7"); + put('a', "qwsy"); + put('q', "12wa"); + put('c', "xdfv"); + put('b', "vghn"); + put('e', "4rdsw3"); + put('d', "rfcxse"); + put('g', "zhbvft"); + put('p', "lo0"); + put('i', "9okju8"); + put('h', "ujnbgz"); + put('k', "olmji"); + put('f', "tgvcdr"); + put('m', "njk"); + put('s', "edxyaw"); + put('o', "0plki9"); + put('n', "bhjm"); + put('1', "2q"); + put('0', "po9"); + put('3', "4ew2"); + put('2', "3wq1"); + put('5', "6tr4"); + put('4', "5re3"); + put('7', "8uz6"); + put('6', "7zt5"); + put('9', "0oi8"); + put('8', "9iu7"); + put('z', "7uhgt6"); + put('t', "6zgfr5"); + } + }), + AZERTY(new HashMap() { + { + put('j', "iknhu"); + put('w', "sxq"); + put('v', "cfgb"); + put('l', "kopm"); + put('y', "7uhgt6"); + put('x', "zsdc"); + put('r', "5tfde4"); + put('u', "8ijhy7"); + put('a', "2zq1"); + put('q', "zswa"); + put('c', "xdfv"); + put('b', "vghn"); + put('e', "4rdsz3"); + put('d', "rfcxse"); + put('g', "yhbvft"); + put('p', "lo0m"); + put('i', "9okju8"); + put('h', "ujnbgy"); + put('k', "olji"); + put('f', "tgvcdr"); + put('m', "lp"); + put('s', "edxwqz"); + put('o', "0plki9"); + put('n', "bhj"); + put('1', "2a"); + put('0', "po9"); + put('3', "4ez2"); + put('2', "3za1"); + put('5', "6tr4"); + put('4', "5re3"); + put('7', "8uy6"); + put('6', "7yt5"); + put('9', "0oi8"); + put('8', "9iu7"); + put('z', "3esqa2"); + put('t', "6ygfr5"); + } + }); + private final Map mapping; + + Keyboards(Map mapping) { + this.mapping = mapping; + } + + public Map getMapping() { + return mapping; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java index c49dbb07..b22dfe9b 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; @@ -25,17 +28,17 @@ */ public class OmissionStrategy implements TyposquattingStrategy { - @Override - public Set generateCandidates(String domain) { - HashSet ret = new HashSet<>(); - for(int i = 0;i < domain.length();++i) { - ret.add(domain.substring(0, i) + domain.substring(i+1, domain.length())); + @Override + public Set generateCandidates(String domain) { + HashSet ret = new HashSet<>(); + for (int i = 0; i < domain.length(); ++i) { + ret.add(domain.substring(0, i) + domain.substring(i + 1)); + } + return ret; } - return ret; - } - @Override - public String name() { - return "Omission"; - } + @Override + public String name() { + return "Omission"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java index fa1b0aa1..1108cede 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; @@ -24,25 +27,25 @@ * Typo strategy around repeating existing characters. */ public class RepetitionStrategy implements TyposquattingStrategy { - @Override - public Set generateCandidates(String domain) { - Set ret = new HashSet<>(); - for(int i = 0;i < domain.length();++i) { - Character c = domain.charAt(i); - if(Character.isLetter(c)) { - ret.add( domain.substring(0, i) - + c - + c - + domain.substring(i+1, domain.length()) - ); - } + @Override + public Set generateCandidates(String domain) { + Set ret = new HashSet<>(); + for (int i = 0; i < domain.length(); ++i) { + Character c = domain.charAt(i); + if (Character.isLetter(c)) { + ret.add(domain.substring(0, i) + + c + + c + + domain.substring(i + 1) + ); + } + } + return ret; } - return ret; - } - @Override - public String name() { - return "Repetition"; - } + @Override + public String name() { + return "Repetition"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java index 78254d97..abc80953 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java @@ -7,43 +7,46 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; import java.util.Set; public class ReplacementStrategy implements TyposquattingStrategy { - @Override - public Set generateCandidates(String domain) { - Set ret = new HashSet<>(); - for(int i = 0;i < domain.length();++i) { - for(Keyboards keyboard : Keyboards.values()) { - String mapping = keyboard.getMapping().get(domain.charAt(i)); - if(mapping != null) { - for(Character c : mapping.toCharArray()) { - ret.add( domain.substring(0, i) - + c - + domain.substring(i+1, domain.length()) - ); - } + @Override + public Set generateCandidates(String domain) { + Set ret = new HashSet<>(); + for (int i = 0; i < domain.length(); ++i) { + for (Keyboards keyboard : Keyboards.values()) { + String mapping = keyboard.getMapping().get(domain.charAt(i)); + if (mapping != null) { + for (Character c : mapping.toCharArray()) { + ret.add(domain.substring(0, i) + + c + + domain.substring(i + 1) + ); + } + } + } } - } + return ret; } - return ret; - } - @Override - public String name() { - return "Replacement"; - } + @Override + public String name() { + return "Replacement"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java index 2dc06e4e..b2bf194d 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; @@ -23,26 +26,25 @@ /** * A typo strategy around random subdomains (e.g. am.azon.com vs amazon.com) */ -public class SubdomainStrategy implements TyposquattingStrategy{ +public class SubdomainStrategy implements TyposquattingStrategy { - @Override - public Set generateCandidates(String domain) { - Set ret = new HashSet<>(); - for(int i = 1;i < domain.length();++i) { - Character c = domain.charAt(i); - Character prevC = domain.charAt(i-1); - if(c != '-' && c != '.' - && prevC != '-' && prevC != '.' - ) - { - ret.add(domain.substring(0, i) + "." + domain.substring(i, domain.length())); - } + @Override + public Set generateCandidates(String domain) { + Set ret = new HashSet<>(); + for (int i = 1; i < domain.length(); ++i) { + Character c = domain.charAt(i); + Character prevC = domain.charAt(i - 1); + if (c != '-' && c != '.' + && prevC != '-' && prevC != '.' + ) { + ret.add(domain.substring(0, i) + "." + domain.substring(i)); + } + } + return ret; } - return ret; - } - @Override - public String name() { - return "Subdomain"; - } + @Override + public String name() { + return "Subdomain"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java index 7962553e..e50447d0 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java @@ -7,40 +7,43 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; import java.util.Set; public class TranspositionStrategy implements TyposquattingStrategy { - @Override - public Set generateCandidates(String domain) { + @Override + public Set generateCandidates(String domain) { - Set ret = new HashSet<>(); - for(int i = 0; i < domain.length()-1;++i) { - char nextC = domain.charAt(i+1); - char c = domain.charAt(i); - if(nextC != c) { - ret.add( domain.substring(0, i) - + nextC - + c - + domain.substring(i+2) - ); - } + Set ret = new HashSet<>(); + for (int i = 0; i < domain.length() - 1; ++i) { + char nextC = domain.charAt(i + 1); + char c = domain.charAt(i); + if (nextC != c) { + ret.add(domain.substring(0, i) + + nextC + + c + + domain.substring(i + 2) + ); + } + } + return ret; } - return ret; - } - @Override - public String name() { - return "Transposition"; - } + @Override + public String name() { + return "Transposition"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java index c8a89d9a..5cef1364 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java @@ -7,115 +7,118 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - /** * This is a set of strategies for generating likely typos from domains. */ public enum TyposquattingStrategies implements TyposquattingStrategy { - ADDITION(new AdditionStrategy()), - BITSQUATTING(new BitsquattingStrategy()), - HOMOGLYPH(new HomoglyphStrategy()), - HYPHENATION(new HyphenationStrategy()), - INSERTION(new InsertionStrategy()), - OMISSION(new OmissionStrategy()), - REPETITION(new RepetitionStrategy()), - REPLACEMENT(new ReplacementStrategy()), - SUBDOMAIN(new SubdomainStrategy()), - TRANSPOSITION(new TranspositionStrategy()), - VOWELSWAP(new VowelSwapStrategy()), - ; - TyposquattingStrategy strategy; - TyposquattingStrategies(TyposquattingStrategy strategy) { - this.strategy = strategy; - } - - @Override - public Set generateCandidates(String originalString) { - Set candidates = strategy.generateCandidates(originalString); - candidates.remove(originalString); - return candidates; - } + ADDITION(new AdditionStrategy()), + BITSQUATTING(new BitsquattingStrategy()), + HOMOGLYPH(new HomoglyphStrategy()), + HYPHENATION(new HyphenationStrategy()), + INSERTION(new InsertionStrategy()), + OMISSION(new OmissionStrategy()), + REPETITION(new RepetitionStrategy()), + REPLACEMENT(new ReplacementStrategy()), + SUBDOMAIN(new SubdomainStrategy()), + TRANSPOSITION(new TranspositionStrategy()), + VOWELSWAP(new VowelSwapStrategy()), + ; + TyposquattingStrategy strategy; - /** - * Generates all typosquatting candidates. - * - * @param originalString The original string to generate candidates for - * @return A set of candidates for typosquatting - */ - public static Set generateAllCandidates(String originalString) { - Set ret = new HashSet<>(); - for(TyposquattingStrategy s : values() ) { - ret.addAll(s.generateCandidates(originalString)); + TyposquattingStrategies(TyposquattingStrategy strategy) { + this.strategy = strategy; } - return ret; - } - /** - * Retrieves a strategy by name. - * - * @param name The name of the strategy - * @return The associated {@link TyposquattingStrategy} - */ - public static TyposquattingStrategies byName(String name) { - for(TyposquattingStrategies s : values()) { - if(s.strategy.name().equals(name)) { - return s; - } + @Override + public Set generateCandidates(String originalString) { + Set candidates = strategy.generateCandidates(originalString); + candidates.remove(originalString); + return candidates; } - return null; - } - @Stellar(namespace="DOMAIN" - ,name="TYPOSQUAT" - ,description="Generate potential typosquatted domain from a passed domain. Strategies largely match https://github.com/elceef/dnstwist" - ,params = { - "domain - Domain (without subdomains or TLD) to generate typosquatted domains for." - } - ,returns = "A set of potential typosquatted domains." - ) - public static class Generate implements StellarFunction{ + /** + * Generates all typosquatting candidates. + * + * @param originalString The original string to generate candidates for + * @return A set of candidates for typosquatting + */ + public static Set generateAllCandidates(String originalString) { + Set ret = new HashSet<>(); + for (TyposquattingStrategy s : values()) { + ret.addAll(s.generateCandidates(originalString)); + } + return ret; + } - @Override - public Object apply(List args, Context context) throws ParseException { - if(args.size() == 0) { + /** + * Retrieves a strategy by name. + * + * @param name The name of the strategy + * @return The associated {@link TyposquattingStrategy} + */ + public static TyposquattingStrategies byName(String name) { + for (TyposquattingStrategies s : values()) { + if (s.strategy.name().equals(name)) { + return s; + } + } return null; - } - Object dnObj = args.get(0); - if(dnObj == null) { - return null; - } - if(!(dnObj instanceof String)) { - throw new IllegalStateException("Expected a domain without subdomains or a TLD, but got " + dnObj); - } - return TyposquattingStrategies.generateAllCandidates((String)dnObj); } - @Override - public void initialize(Context context) { + @Stellar(namespace = "DOMAIN", + name = "TYPOSQUAT", + description = "Generate potential typosquatted domain from a passed domain. Strategies largely match https://github.com/elceef/dnstwist", + params = { + "domain - Domain (without subdomains or TLD) to generate typosquatted domains for." + }, + returns = "A set of potential typosquatted domains." + ) + public static class Generate implements StellarFunction { - } + @Override + public Object apply(List args, Context context) throws ParseException { + if (args.size() == 0) { + return null; + } + Object dnObj = args.get(0); + if (dnObj == null) { + return null; + } + if (!(dnObj instanceof String)) { + throw new IllegalStateException("Expected a domain without subdomains or a TLD, but got " + dnObj); + } + return TyposquattingStrategies.generateAllCandidates((String) dnObj); + } - @Override - public boolean isInitialized() { - return true; + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java index 37e3f161..f189f580 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java @@ -7,19 +7,23 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.Set; public interface TyposquattingStrategy { - Set generateCandidates(String domain); - String name(); + Set generateCandidates(String domain); + + String name(); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java index 0a3388d1..99a6bbfd 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.typosquat; import java.util.HashSet; @@ -24,34 +27,36 @@ * A typo strategy around swapping vowels (e.g. omazon.com vs amazon.com) */ public class VowelSwapStrategy implements TyposquattingStrategy { - private static Set VOWELS = new HashSet() {{ - add('a'); - add('e'); - add('i'); - add('o'); - add('u'); - }}; + private static final Set VOWELS = new HashSet() { + { + add('a'); + add('e'); + add('i'); + add('o'); + add('u'); + } + }; - @Override - public Set generateCandidates(String domain) { + @Override + public Set generateCandidates(String domain) { - HashSet ret = new HashSet<>(); - for(int i = 0;i < domain.length();++i) { - char c = domain.charAt(i); - for(char vowel : VOWELS) { - if(VOWELS.contains(c)) { - ret.add( domain.substring(0, i) - + vowel - + domain.substring(i+1) - ); + HashSet ret = new HashSet<>(); + for (int i = 0; i < domain.length(); ++i) { + char c = domain.charAt(i); + for (char vowel : VOWELS) { + if (VOWELS.contains(c)) { + ret.add(domain.substring(0, i) + + vowel + + domain.substring(i + 1) + ); + } + } } - } + return ret; } - return ret; - } - @Override - public String name() { - return "Vowel-swap"; - } + @Override + public String name() { + return "Vowel-swap"; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java index f9c53c85..bcb200a1 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,69 +34,69 @@ */ public enum CompressionStrategies implements CompressionStrategy { - GZIP(new CompressionStrategy() { + GZIP(new CompressionStrategy() { + @Override + public void compress(File inFile, File outFile) throws IOException { + try (FileInputStream fis = new FileInputStream(inFile); + FileOutputStream fos = new FileOutputStream(outFile); + GZIPOutputStream gzipOS = new GZIPOutputStream(fos)) { + byte[] buffer = new byte[1024]; + int len; + while ((len = fis.read(buffer)) != -1) { + gzipOS.write(buffer, 0, len); + } + } + } + + @Override + public void decompress(File inFile, File outFile) throws IOException { + try (FileInputStream fis = new FileInputStream(inFile); + GZIPInputStream gis = new GZIPInputStream(fis); + FileOutputStream fos = new FileOutputStream(outFile)) { + byte[] buffer = new byte[1024]; + int len; + while ((len = gis.read(buffer)) != -1) { + fos.write(buffer, 0, len); + } + } + + } + + @Override + public boolean test(File gzipFile) { + try (FileInputStream fis = new FileInputStream(gzipFile); + GZIPInputStream gis = new GZIPInputStream(fis)) { + byte[] buffer = new byte[1024]; + // this will throw an exception on malformed file + gis.read(buffer); + } catch (ZipException | EOFException e) { + return false; + } catch (IOException e) { + throw new IllegalStateException("Error occurred while attempting to validate gzip file", e); + } + return true; + } + }); + + private final CompressionStrategy strategy; + + CompressionStrategies(CompressionStrategy strategy) { + this.strategy = strategy; + } + @Override public void compress(File inFile, File outFile) throws IOException { - try (FileInputStream fis = new FileInputStream(inFile); - FileOutputStream fos = new FileOutputStream(outFile); - GZIPOutputStream gzipOS = new GZIPOutputStream(fos)) { - byte[] buffer = new byte[1024]; - int len; - while ((len = fis.read(buffer)) != -1) { - gzipOS.write(buffer, 0, len); - } - } + strategy.compress(inFile, outFile); } @Override public void decompress(File inFile, File outFile) throws IOException { - try (FileInputStream fis = new FileInputStream(inFile); - GZIPInputStream gis = new GZIPInputStream(fis); - FileOutputStream fos = new FileOutputStream(outFile)) { - byte[] buffer = new byte[1024]; - int len; - while ((len = gis.read(buffer)) != -1) { - fos.write(buffer, 0, len); - } - } - + strategy.decompress(inFile, outFile); } @Override public boolean test(File gzipFile) { - try (FileInputStream fis = new FileInputStream(gzipFile); - GZIPInputStream gis = new GZIPInputStream(fis)) { - byte[] buffer = new byte[1024]; - // this will throw an exception on malformed file - gis.read(buffer); - } catch (ZipException | EOFException e) { - return false; - } catch (IOException e) { - throw new IllegalStateException("Error occurred while attempting to validate gzip file", e); - } - return true; + return strategy.test(gzipFile); } - }); - - private CompressionStrategy strategy; - - CompressionStrategies(CompressionStrategy strategy) { - this.strategy = strategy; - } - - @Override - public void compress(File inFile, File outFile) throws IOException { - strategy.compress(inFile, outFile); - } - - @Override - public void decompress(File inFile, File outFile) throws IOException { - strategy.decompress(inFile, outFile); - } - - @Override - public boolean test(File gzipFile) { - return strategy.test(gzipFile); - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategy.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategy.java index 6a98c95a..03a9a703 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategy.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategy.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,30 +25,30 @@ public interface CompressionStrategy { - /** - * Compress infile. - * - * @param inFile file to compress - * @param outFile destination path for compressed output - * @throws IOException Any IO error - */ - void compress(File inFile, File outFile) throws IOException; + /** + * Compress infile. + * + * @param inFile file to compress + * @param outFile destination path for compressed output + * @throws IOException Any IO error + */ + void compress(File inFile, File outFile) throws IOException; - /** - * Decompress infile. - * - * @param inFile file to decompress - * @param outFile destination path for decompressed output - * @throws IOException Any IO error - */ - void decompress(File inFile, File outFile) throws IOException; + /** + * Decompress infile. + * + * @param inFile file to decompress + * @param outFile destination path for decompressed output + * @throws IOException Any IO error + */ + void decompress(File inFile, File outFile) throws IOException; - /** - * Test if file is proper gzip format. True if valid, false otherwise. - * - * @param gzipFile file to check for gzip compression - * @return true if file is a gzip format, false otherwise. - */ - boolean test(File gzipFile); + /** + * Test if file is proper gzip format. True if valid, false otherwise. + * + * @param gzipFile file to check for gzip compression + * @return true if file is a gzip format, false otherwise. + */ + boolean test(File gzipFile); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/HDFSUtils.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/HDFSUtils.java index d1697f4e..2f1136cb 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/HDFSUtils.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/HDFSUtils.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,75 +31,75 @@ public class HDFSUtils { - public static byte[] readBytes(String path) throws IOException { - return readBytes(new Path(path)); - } + public static byte[] readBytes(String path) throws IOException { + return readBytes(new Path(path)); + } - /** - * Reads a provided path as raw bytes. - * - * @param inPath The path to be read - * @return The raw bytes of the contents - * @throws IOException If an error occurs during reading the path - */ - public static byte[] readBytes(Path inPath) throws IOException { - FileSystem fs = FileSystem.get(inPath.toUri(), new Configuration()); - try (FSDataInputStream inputStream = fs.open(inPath)) { - return IOUtils.toByteArray(inputStream); + /** + * Reads a provided path as raw bytes. + * + * @param inPath The path to be read + * @return The raw bytes of the contents + * @throws IOException If an error occurs during reading the path + */ + public static byte[] readBytes(Path inPath) throws IOException { + FileSystem fs = FileSystem.get(inPath.toUri(), new Configuration()); + try (FSDataInputStream inputStream = fs.open(inPath)) { + return IOUtils.toByteArray(inputStream); + } } - } - /** - * Reads full file contents into a List of Strings. Reads from local FS if file:/// used as the - * scheme. Initializes file system with default configuration. - * Automatically handles file system translation for the provided path's scheme. - * Opens and closes the file system on each call. Never null. - * Null/empty scheme defaults to default configured FS. - * - * @param path path to file - * @return file contents as a String - * @throws IOException If an error occurs during reading the path - */ - public static List readFile(String path) throws IOException { - return readFile(new Configuration(), path); - } + /** + * Reads full file contents into a List of Strings. Reads from local FS if file:/// used as the + * scheme. Initializes file system with default configuration. + * Automatically handles file system translation for the provided path's scheme. + * Opens and closes the file system on each call. Never null. + * Null/empty scheme defaults to default configured FS. + * + * @param path path to file + * @return file contents as a String + * @throws IOException If an error occurs during reading the path + */ + public static List readFile(String path) throws IOException { + return readFile(new Configuration(), path); + } - /** - * Reads full file contents into a String. Reads from local FS if file:/// used as the scheme. - * Opens and closes the file system on each call. - * Never null. Automatically handles file system translation for the provided path's scheme. - * Null/empty scheme defaults to default configured FS. - * - * @param config Hadoop configuration - * @param path path to file - * @return file contents as a String - * @throws IOException If an error occurs during reading the path - */ - public static List readFile(Configuration config, String path) throws IOException { - Path inPath = new Path(path); - FileSystem fs = FileSystem.get(inPath.toUri(), config); - try (FSDataInputStream inputStream = fs.open(inPath)) { - return IOUtils.readLines(inputStream, "UTF-8"); + /** + * Reads full file contents into a String. Reads from local FS if file:/// used as the scheme. + * Opens and closes the file system on each call. + * Never null. Automatically handles file system translation for the provided path's scheme. + * Null/empty scheme defaults to default configured FS. + * + * @param config Hadoop configuration + * @param path path to file + * @return file contents as a String + * @throws IOException If an error occurs during reading the path + */ + public static List readFile(Configuration config, String path) throws IOException { + Path inPath = new Path(path); + FileSystem fs = FileSystem.get(inPath.toUri(), config); + try (FSDataInputStream inputStream = fs.open(inPath)) { + return IOUtils.readLines(inputStream, "UTF-8"); + } } - } - /** - * Write file to HDFS. Writes to local FS if file:/// used as the scheme. - * Automatically handles file system translation for the provided path's scheme. - * Null/empty scheme defaults to default configured FS. - * - * @param config filesystem configuration - * @param bytes bytes to write - * @param path output path - * @throws IOException This is the generic Hadoop "everything is an IOException." - */ - public static void write(Configuration config, byte[] bytes, String path) throws IOException { - Path outPath = new Path(path); - FileSystem fs = FileSystem.get(outPath.toUri(), config); - fs.mkdirs(outPath.getParent()); - try (FSDataOutputStream outputStream = fs.create(outPath)) { - outputStream.write(bytes); - outputStream.flush(); + /** + * Write file to HDFS. Writes to local FS if file:/// used as the scheme. + * Automatically handles file system translation for the provided path's scheme. + * Null/empty scheme defaults to default configured FS. + * + * @param config filesystem configuration + * @param bytes bytes to write + * @param path output path + * @throws IOException This is the generic Hadoop "everything is an IOException." + */ + public static void write(Configuration config, byte[] bytes, String path) throws IOException { + Path outPath = new Path(path); + FileSystem fs = FileSystem.get(outPath.toUri(), config); + fs.mkdirs(outPath.getParent()); + try (FSDataOutputStream outputStream = fs.create(outPath)) { + outputStream.write(bytes); + outputStream.flush(); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/HashUtils.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/HashUtils.java index 2048a91b..0c35f791 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/HashUtils.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/HashUtils.java @@ -7,24 +7,26 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils; -import org.apache.commons.codec.digest.DigestUtils; -import org.json.simple.JSONObject; +import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; - -import static java.nio.charset.StandardCharsets.UTF_8; +import org.apache.commons.codec.digest.DigestUtils; +import org.json.simple.JSONObject; /** * A set of utilities for converted a message to a SHA-256 hex-encoded hash string. Can be applied @@ -32,24 +34,24 @@ */ public class HashUtils { - /** - * Produces a hash of a JSON message from a subset of the fields. - * - * @param message The JSON message to be hashed - * @param hashFields The fields to hash - * @return A string containing the resulting hash - */ - public static String getMessageHash(JSONObject message, Collection hashFields) { - List hashElements = hashFields.stream().map(errorField -> - String.format("%s-%s", errorField, message.get(errorField))).collect(Collectors.toList()); - return DigestUtils.sha256Hex(String.join("|", hashElements).getBytes(UTF_8)); - } + /** + * Produces a hash of a JSON message from a subset of the fields. + * + * @param message The JSON message to be hashed + * @param hashFields The fields to hash + * @return A string containing the resulting hash + */ + public static String getMessageHash(JSONObject message, Collection hashFields) { + List hashElements = hashFields.stream().map(errorField -> + String.format("%s-%s", errorField, message.get(errorField))).collect(Collectors.toList()); + return DigestUtils.sha256Hex(String.join("|", hashElements).getBytes(UTF_8)); + } - public static String getMessageHash(JSONObject message) { - return DigestUtils.sha256Hex(message.toJSONString().getBytes(UTF_8)); - } + public static String getMessageHash(JSONObject message) { + return DigestUtils.sha256Hex(message.toJSONString().getBytes(UTF_8)); + } - public static String getMessageHash(byte[] bytes) { - return DigestUtils.sha256Hex(bytes); - } + public static String getMessageHash(byte[] bytes) { + return DigestUtils.sha256Hex(bytes); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java index 93eee8a3..b91f2a03 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java @@ -25,10 +25,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.zjsonpatch.JsonPatch; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; - import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -39,200 +35,207 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; public enum JSONUtils { - INSTANCE; - - public static class ReferenceSupplier implements Supplier> { - Type type; - protected ReferenceSupplier() { - Type superClass = this.getClass().getGenericSuperclass(); - if(superClass instanceof Class) { - throw new IllegalArgumentException("Internal error: ReferenceSupplier constructed without actual type information"); - } else { - this.type = ((ParameterizedType)superClass).getActualTypeArguments()[0]; - } - } - - @Override - public TypeReference get() { - return new TypeReference() { + INSTANCE; + + public static class ReferenceSupplier implements Supplier> { + Type type; + + protected ReferenceSupplier() { + Type superClass = this.getClass().getGenericSuperclass(); + if (superClass instanceof Class) { + throw new IllegalArgumentException( + "Internal error: ReferenceSupplier constructed without actual type information"); + } else { + this.type = ((ParameterizedType) superClass).getActualTypeArguments()[0]; + } + } + @Override - public Type getType() { - return type; + public TypeReference get() { + return new TypeReference() { + @Override + public Type getType() { + return type; + } + }; + } + } + + public static final ReferenceSupplier> MAP_SUPPLIER = + new ReferenceSupplier>() { + }; + public static final ReferenceSupplier> LIST_SUPPLIER = new ReferenceSupplier>() { + }; + + private static final ThreadLocal _parser = ThreadLocal.withInitial(() -> + new JSONParser()); + + private static final ThreadLocal _mapper = ThreadLocal.withInitial(() -> + new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL) + .configure(JsonParser.Feature.ALLOW_COMMENTS, true)); + + public T convert(Object original, Class targetClass) { + return _mapper.get().convertValue(original, targetClass); + } + + public ObjectMapper getMapper() { + return _mapper.get(); + } + + + public T load(InputStream is, ReferenceSupplier ref) throws IOException { + return _mapper.get().readValue(is, ref.get()); + } + + public T load(String is, ReferenceSupplier ref) throws IOException { + return _mapper.get().readValue(is, ref.get()); + } + + /** + * Loads JSON from a file and ensures it's located in the specified class. + * + * @param f The file to read from + * @param ref A {@link ReferenceSupplier} for the class to be loaded into. + * @param The type parameter of the class + * @return The JSON loaded into the provided class + * @throws IOException If there's an issue loading the JSON + */ + public T load(File f, ReferenceSupplier ref) throws IOException { + try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { + return _mapper.get().readValue(is, ref.get()); + } + } + + public T load(InputStream is, Class clazz) throws IOException { + return _mapper.get().readValue(is, clazz); + } + + /** + * Loads JSON from a file and ensures it's located in the provided class. + * + * @param f The file to read from + * @param clazz The class to read into + * @param The type parameter of the class + * @return The JSON loaded into the provided class + * @throws IOException If there's an issue loading the JSON + */ + public T load(File f, Class clazz) throws IOException { + try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { + return _mapper.get().readValue(is, clazz); } - }; - } - } - - public final static ReferenceSupplier> MAP_SUPPLIER = new ReferenceSupplier>() {}; - public final static ReferenceSupplier> LIST_SUPPLIER = new ReferenceSupplier>(){}; - - private static ThreadLocal _parser = ThreadLocal.withInitial(() -> - new JSONParser()); - - private static ThreadLocal _mapper = ThreadLocal.withInitial(() -> - new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL) - .configure(JsonParser.Feature.ALLOW_COMMENTS, true)); - - public T convert(Object original, Class targetClass) { - return _mapper.get().convertValue(original, targetClass); - } - - public ObjectMapper getMapper() { - return _mapper.get(); - } - - - public T load(InputStream is, ReferenceSupplier ref) throws IOException { - return _mapper.get().readValue(is, (TypeReference)ref.get()); - } - - public T load(String is, ReferenceSupplier ref) throws IOException { - return _mapper.get().readValue(is, (TypeReference)ref.get()); - } - - /** - * Loads JSON from a file and ensures it's located in the specified class. - * - * @param f The file to read from - * @param ref A {@link ReferenceSupplier} for the class to be loaded into. - * @param The type parameter of the class - * @return The JSON loaded into the provided class - * @throws IOException If there's an issue loading the JSON - */ - public T load(File f, ReferenceSupplier ref) throws IOException { - try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { - return _mapper.get().readValue(is, (TypeReference)ref.get()); - } - } - - public T load(InputStream is, Class clazz) throws IOException { - return _mapper.get().readValue(is, clazz); - } - - /** - * Loads JSON from a file and ensures it's located in the provided class. - * - * @param f The file to read from - * @param clazz The class to read into - * @param The type parameter of the class - * @return The JSON loaded into the provided class - * @throws IOException If there's an issue loading the JSON - */ - public T load(File f, Class clazz) throws IOException { - try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { - return _mapper.get().readValue(is, clazz); - } - } - - public T load(String is, Class clazz) throws IOException { - return _mapper.get().readValue(is, clazz); - } - - /** - * Converts an object to a JSON string. Can be a pretty string - * - * @param o The object to convert - * @param pretty Pretty formatted string if true, otherwise not pretty formatted. - * @return A JSON string representation of the object - * @throws JsonProcessingException If there's an issue converting to JSON. - */ - public String toJSON(Object o, boolean pretty) throws JsonProcessingException { - if (pretty) { - return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsString(o); - } else { - return _mapper.get().writeValueAsString(o); - } - } - - public byte[] toJSONPretty(String config) throws IOException { - return toJSONPretty(readTree(config)); - } - - public byte[] toJSONPretty(Object config) throws JsonProcessingException { - return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsBytes(config); - } - - /** - * Transforms a bean (aka POJO) to a JSONObject. - */ - public JSONObject toJSONObject(Object o) throws JsonProcessingException, ParseException { - return (JSONObject) _parser.get().parse(toJSON(o, false)); - } - - /** - * Reads a JSON string into a JsonNode Object. - * - * @param json JSON value to deserialize - * @return deserialized JsonNode Object - */ - JsonNode readTree(String json) throws IOException { - return _mapper.get().readTree(json); - } - - /** - * Reads a JSON byte array into a JsonNode Object. - * - * @param json JSON value to deserialize - * @return deserialized JsonNode Object - */ - JsonNode readTree(byte[] json) throws IOException { - return _mapper.get().readTree(json); - } - - /** - * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902) - * Operations: - *

        - *
      • add
        • - *
      • remove
        • - *
      • replace
        • - *
      • move
        • - *
      • copy
        • - *
      • test
        • - *
      - * - * @param patch Array of JSON patches, e.g. [{ "op": "move", "from": "/a", "path": "/c" }] - * @param source Source JSON to apply patch to - * @return new json after applying the patch - */ - public byte[] applyPatch(String patch, String source) throws IOException { - JsonNode patchNode = readTree(patch); - JsonNode sourceNode = readTree(source); - return toJSONPretty(JsonPatch.apply(patchNode, sourceNode)); - } - - /** - * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902) - * - * @param patch Array of JSON patches in raw bytes - * @param source Source JSON in raw bytes to apply patch to - * @return new json after applying the patch - * - * @see JSONUtils#applyPatch(String, String) - */ - public byte[] applyPatch(byte[] patch, byte[] source) throws IOException { - JsonNode patchNode = readTree(patch); - JsonNode sourceNode = readTree(source); - return toJSONPretty(JsonPatch.apply(patchNode, sourceNode)); - } - - - /** - * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902) - * - * @param patch List of JSON patches in map form - * @param source Source JSON in map form to apply patch to - * @return new json after applying the patch - * - * @see JSONUtils#applyPatch(String, String) - */ - public Map applyPatch(List> patch, Map source) { - JsonNode originalNode = convert(source, JsonNode.class); - JsonNode patchNode = convert(patch, JsonNode.class); - JsonNode patched = JsonPatch.apply(patchNode, originalNode); - return _mapper.get().convertValue(patched, new TypeReference>() { }); - } + } + + public T load(String is, Class clazz) throws IOException { + return _mapper.get().readValue(is, clazz); + } + + /** + * Converts an object to a JSON string. Can be a pretty string + * + * @param o The object to convert + * @param pretty Pretty formatted string if true, otherwise not pretty formatted. + * @return A JSON string representation of the object + * @throws JsonProcessingException If there's an issue converting to JSON. + */ + public String toJSON(Object o, boolean pretty) throws JsonProcessingException { + if (pretty) { + return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsString(o); + } else { + return _mapper.get().writeValueAsString(o); + } + } + + public byte[] toJSONPretty(String config) throws IOException { + return toJSONPretty(readTree(config)); + } + + public byte[] toJSONPretty(Object config) throws JsonProcessingException { + return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsBytes(config); + } + + /** + * Transforms a bean (aka POJO) to a JSONObject. + */ + public JSONObject toJSONObject(Object o) throws JsonProcessingException, ParseException { + return (JSONObject) _parser.get().parse(toJSON(o, false)); + } + + /** + * Reads a JSON string into a JsonNode Object. + * + * @param json JSON value to deserialize + * @return deserialized JsonNode Object + */ + JsonNode readTree(String json) throws IOException { + return _mapper.get().readTree(json); + } + + /** + * Reads a JSON byte array into a JsonNode Object. + * + * @param json JSON value to deserialize + * @return deserialized JsonNode Object + */ + JsonNode readTree(byte[] json) throws IOException { + return _mapper.get().readTree(json); + } + + /** + * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902) + * Operations: + *
        + *
      • add
        • + *
      • remove
        • + *
      • replace
        • + *
      • move
        • + *
      • copy
        • + *
      • test
        • + *
      + * + * @param patch Array of JSON patches, e.g. [{ "op": "move", "from": "/a", "path": "/c" }] + * @param source Source JSON to apply patch to + * @return new json after applying the patch + */ + public byte[] applyPatch(String patch, String source) throws IOException { + JsonNode patchNode = readTree(patch); + JsonNode sourceNode = readTree(source); + return toJSONPretty(JsonPatch.apply(patchNode, sourceNode)); + } + + /** + * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902) + * + * @param patch Array of JSON patches in raw bytes + * @param source Source JSON in raw bytes to apply patch to + * @return new json after applying the patch + * @see JSONUtils#applyPatch(String, String) + */ + public byte[] applyPatch(byte[] patch, byte[] source) throws IOException { + JsonNode patchNode = readTree(patch); + JsonNode sourceNode = readTree(source); + return toJSONPretty(JsonPatch.apply(patchNode, sourceNode)); + } + + + /** + * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902) + * + * @param patch List of JSON patches in map form + * @param source Source JSON in map form to apply patch to + * @return new json after applying the patch + * @see JSONUtils#applyPatch(String, String) + */ + public Map applyPatch(List> patch, Map source) { + JsonNode originalNode = convert(source, JsonNode.class); + JsonNode patchNode = convert(patch, JsonNode.class); + JsonNode patched = JsonPatch.apply(patchNode, originalNode); + return _mapper.get().convertValue(patched, new TypeReference>() { + }); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/KeyUtil.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/KeyUtil.java index 11fce4de..312feffb 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/KeyUtil.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/KeyUtil.java @@ -7,57 +7,61 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils; import com.google.common.hash.HashFunction; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; +@SuppressWarnings("checkstyle:MemberName") public enum KeyUtil { - INSTANCE; - private static final int SEED = 0xDEADBEEF; - public static final int HASH_PREFIX_SIZE=16; - ThreadLocal hFunction= new ThreadLocal() { - @Override - protected HashFunction initialValue() { - return Hashing.murmur3_128(SEED); - } - }; + INSTANCE; + private static final int SEED = 0xDEADBEEF; + public static final int HASH_PREFIX_SIZE = 16; + ThreadLocal hFunction = new ThreadLocal() { + @Override + protected HashFunction initialValue() { + return Hashing.murmur3_128(SEED); + } + }; - /** - * Determines the prefix, given a key. - * - * @param key The key to get a prefix for - * @return A 16 byte hashed prefix based on the key. - */ - public byte[] getPrefix(byte[] key) { - Hasher hasher = hFunction.get().newHasher(); - hasher.putBytes(key); - return hasher.hash().asBytes(); - } + /** + * Determines the prefix, given a key. + * + * @param key The key to get a prefix for + * @return A 16 byte hashed prefix based on the key. + */ + public byte[] getPrefix(byte[] key) { + Hasher hasher = hFunction.get().newHasher(); + hasher.putBytes(key); + return hasher.hash().asBytes(); + } - /** - * Merges a prefix and a key into a single byte array. Simple concatenation. - * - * @param prefix A byte array with the prefix - * @param key A byte array with the key - * @return A byte array with the prefix and key concatenated. - */ - public byte[] merge(byte[] prefix, byte[] key) { - byte[] val = new byte[key.length + prefix.length]; - int offset = 0; - System.arraycopy(prefix, 0, val, offset, prefix.length); - offset += prefix.length; - System.arraycopy(key, 0, val, offset, key.length); - return val; - } + /** + * Merges a prefix and a key into a single byte array. Simple concatenation. + * + * @param prefix A byte array with the prefix + * @param key A byte array with the key + * @return A byte array with the prefix and key concatenated. + */ + public byte[] merge(byte[] prefix, byte[] key) { + byte[] val = new byte[key.length + prefix.length]; + int offset = 0; + System.arraycopy(prefix, 0, val, offset, prefix.length); + offset += prefix.length; + System.arraycopy(key, 0, val, offset, key.length); + return val; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLogger.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLogger.java index 8a984183..31275925 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLogger.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLogger.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,80 +17,81 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.Marker; -import java.util.function.Supplier; - /** * Extension of a Logger interface that supports lazy argument evaluation * Useful when some arguments are costly to evaluate, and you only want to incur * that cost if the logging level is actually enabled. * + *

      * Please note this interface is a temporary measure until the move to use SLF4J 2.x */ public interface LazyLogger extends Logger { - Logger getLogger(); + Logger getLogger(); - void trace(String format, Supplier arg); + void trace(String format, Supplier arg); - void trace(String format, Supplier arg1, Supplier arg2); + void trace(String format, Supplier arg1, Supplier arg2); - void trace(String format, Supplier... arguments); + void trace(String format, Supplier... arguments); - void trace(Marker marker, String format, Supplier arg); + void trace(Marker marker, String format, Supplier arg); - void trace(Marker marker, String format, Supplier arg1, Supplier arg2); + void trace(Marker marker, String format, Supplier arg1, Supplier arg2); - void trace(Marker marker, String format, Supplier... arguments); + void trace(Marker marker, String format, Supplier... arguments); - void debug(String format, Supplier arg); + void debug(String format, Supplier arg); - void debug(String format, Supplier arg1, Supplier arg2); + void debug(String format, Supplier arg1, Supplier arg2); - void debug(String format, Supplier... arguments); + void debug(String format, Supplier... arguments); - void debug(Marker marker, String format, Supplier arg); + void debug(Marker marker, String format, Supplier arg); - void debug(Marker marker, String format, Supplier arg1, Supplier arg2); + void debug(Marker marker, String format, Supplier arg1, Supplier arg2); - void debug(Marker marker, String format, Supplier... arguments); + void debug(Marker marker, String format, Supplier... arguments); - void info(String format, Supplier arg); + void info(String format, Supplier arg); - void info(String format, Supplier arg1, Supplier arg2); + void info(String format, Supplier arg1, Supplier arg2); - void info(String format, Supplier... arguments); + void info(String format, Supplier... arguments); - void info(Marker marker, String format, Supplier arg); + void info(Marker marker, String format, Supplier arg); - void info(Marker marker, String format, Supplier arg1, Supplier arg2); + void info(Marker marker, String format, Supplier arg1, Supplier arg2); - void info(Marker marker, String format, Supplier... arguments); + void info(Marker marker, String format, Supplier... arguments); - void warn(String format, Supplier arg); + void warn(String format, Supplier arg); - void warn(String format, Supplier arg1, Supplier arg2); + void warn(String format, Supplier arg1, Supplier arg2); - void warn(String format, Supplier... arguments); + void warn(String format, Supplier... arguments); - void warn(Marker marker, String format, Supplier arg); + void warn(Marker marker, String format, Supplier arg); - void warn(Marker marker, String format, Supplier arg1, Supplier arg2); + void warn(Marker marker, String format, Supplier arg1, Supplier arg2); - void warn(Marker marker, String format, Supplier... arguments); + void warn(Marker marker, String format, Supplier... arguments); - void error(String format, Supplier arg); + void error(String format, Supplier arg); - void error(String format, Supplier arg1, Supplier arg2); + void error(String format, Supplier arg1, Supplier arg2); - void error(String format, Supplier... arguments); + void error(String format, Supplier... arguments); - void error(Marker marker, String format, Supplier arg); + void error(Marker marker, String format, Supplier arg); - void error(Marker marker, String format, Supplier arg1, Supplier arg2); + void error(Marker marker, String format, Supplier arg1, Supplier arg2); - void error(Marker marker, String format, Supplier... arguments); + void error(Marker marker, String format, Supplier... arguments); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLoggerFactory.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLoggerFactory.java index d536cdee..e5bc37fb 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLoggerFactory.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLoggerFactory.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,6 +17,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils; import org.slf4j.ILoggerFactory; @@ -26,46 +29,46 @@ * that cost if the logging level is actually enabled. */ public class LazyLoggerFactory { - /** - * Return a logger named according to the name parameter using the statically - * bound {@link ILoggerFactory} instance. - * - * @param name The name of the logger. - * @return logger - */ - public static LazyLogger getLogger(String name) { - final Logger logger = org.slf4j.LoggerFactory.getLogger(name); - if (logger != null) { // explicit NP check to remove guava dependency - return new LazyLoggerImpl(logger); - } else { - throw new NullPointerException(String.format("Logger not returned for class %s", - name == null ? "Null String" : name)); + /** + * Return a logger named according to the name parameter using the statically + * bound {@link ILoggerFactory} instance. + * + * @param name The name of the logger. + * @return logger + */ + public static LazyLogger getLogger(String name) { + final Logger logger = org.slf4j.LoggerFactory.getLogger(name); + if (logger != null) { // explicit NP check to remove guava dependency + return new LazyLoggerImpl(logger); + } else { + throw new NullPointerException(String.format("Logger not returned for class %s", + name == null ? "Null String" : name)); + } } - } - /** - * Return a logger named corresponding to the class passed as parameter, using - * the statically bound {@link ILoggerFactory} instance. - * - * @param clazz the returned logger will be named after clazz - * @return logger - */ - public static LazyLogger getLogger(Class clazz) { - return getLogger(clazz.getName()); - } + /** + * Return a logger named corresponding to the class passed as parameter, using + * the statically bound {@link ILoggerFactory} instance. + * + * @param clazz the returned logger will be named after clazz + * @return logger + */ + public static LazyLogger getLogger(Class clazz) { + return getLogger(clazz.getName()); + } - /** - * Return a lazylogger wrapping the passed Logger instance - * - * @param logger the returned logger will be named after clazz - * @return logger - */ - public static LazyLogger getLogger(Logger logger) { - if (logger != null) { - return new LazyLoggerImpl(logger); - } else { - throw new NullPointerException("Null logger passed"); + /** + * Return a lazylogger wrapping the passed Logger instance. + * + * @param logger the returned logger will be named after clazz + * @return logger + */ + public static LazyLogger getLogger(Logger logger) { + if (logger != null) { + return new LazyLoggerImpl(logger); + } else { + throw new NullPointerException("Null logger passed"); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLoggerImpl.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLoggerImpl.java index e7d57db1..c127b28a 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLoggerImpl.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/LazyLoggerImpl.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,13 +17,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.utils; -import org.slf4j.Logger; -import org.slf4j.Marker; +package org.apache.metron.common.utils; import java.util.Arrays; import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.Marker; /** * Wrapper class around a slf4j Logger interface that adds lazy evaluated method arguments @@ -30,543 +32,543 @@ * We can tag parameterised varargs arguments as safe here because we only assume they are Objects */ public class LazyLoggerImpl implements LazyLogger { - private final Logger logger; - - LazyLoggerImpl(final Logger logger) { - if (logger != null) { // Explicitly NP check to remove Guava as a dependency - this.logger = logger; - } else { - throw new NullPointerException("Null logger passed"); - } - } - - @Override - public Logger getLogger() { - return logger; - } - - @Override - public String getName() { - return logger.getName(); - } - - @Override - public boolean isTraceEnabled() { - return logger.isTraceEnabled(); - } - - @Override - public void trace(String msg) { - logger.trace(msg); - } - - @Override - public void trace(String format, Object arg) { - logger.trace(format, arg); - } - - @Override - public void trace(String format, Supplier arg) { - if (logger.isTraceEnabled()) { - logger.trace(format, arg.get()); - } - } - - @Override - public void trace(String format, Object arg1, Object arg2) { - logger.trace(format, arg1, arg2); - } - - @Override - public void trace(String format, Supplier arg1, Supplier arg2) { - if (logger.isTraceEnabled()) { - logger.trace(format, arg1.get(), arg2.get()); - } - } - - @Override - public void trace(String format, Object... arguments) { - logger.trace(format, arguments); - } - - @Override - @SafeVarargs - public final void trace(String format, Supplier... arguments) { - if (logger.isTraceEnabled()) { - logger.trace(format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void trace(String msg, Throwable t) { - logger.trace(msg, t); - } - - @Override - public boolean isTraceEnabled(Marker marker) { - return logger.isTraceEnabled(marker); - } - - @Override - public void trace(Marker marker, String msg) { - logger.trace(marker, msg); - } - - @Override - public void trace(Marker marker, String format, Object arg) { - logger.trace(marker, format, arg); - } - - @Override - public void trace(Marker marker, String format, Supplier arg) { - if (logger.isTraceEnabled(marker)) { - logger.trace(marker, format, arg.get()); - } - } - - @Override - public void trace(Marker marker, String format, Object arg1, Object arg2) { - logger.trace(marker, format, arg1, arg2); - } - - @Override - public void trace(Marker marker, String format, Supplier arg1, Supplier arg2) { - if (logger.isTraceEnabled(marker)) { - logger.trace(marker, format, arg1.get(), arg2.get()); - } - } - - @Override - public void trace(Marker marker, String format, Object... arguments) { - logger.trace(marker, format, arguments); - } - - @Override - @SafeVarargs - public final void trace(Marker marker, String format, Supplier... arguments) { - if (logger.isTraceEnabled(marker)) { - logger.trace(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void trace(Marker marker, String msg, Throwable t) { - logger.trace(marker, msg, t); - } - - @Override - public boolean isDebugEnabled() { - return logger.isDebugEnabled(); - } - - @Override - public void debug(String msg) { - logger.debug(msg); - } - - @Override - public void debug(String format, Object arg) { - logger.debug(format, arg); - } - - @Override - public void debug(String format, Supplier arg) { - if (logger.isDebugEnabled()) { - logger.debug(format, arg.get()); - } - } - - @Override - public void debug(String format, Object arg1, Object arg2) { - logger.debug(format, arg1, arg2); - } - - @Override - public void debug(String format, Supplier arg1, Supplier arg2) { - if (logger.isDebugEnabled()) { - logger.debug(format, arg1.get(), arg2.get()); - } - } - - @Override - public void debug(String format, Object... arguments) { - logger.debug(format, arguments); - } - - @Override - @SafeVarargs - public final void debug(String format, Supplier... arguments) { - if (logger.isDebugEnabled()) { - logger.debug(format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void debug(String msg, Throwable t) { - logger.debug(msg, t); - } - - @Override - public boolean isDebugEnabled(Marker marker) { - return logger.isDebugEnabled(marker); - } - - @Override - public void debug(Marker marker, String msg) { - logger.debug(marker, msg); - } - - @Override - public void debug(Marker marker, String format, Object arg) { - logger.debug(marker, format, arg); - } - - @Override - public void debug(Marker marker, String format, Supplier arg) { - if (logger.isDebugEnabled(marker)) { - logger.debug(marker, format, arg.get()); - } - } - - @Override - public void debug(Marker marker, String format, Object arg1, Object arg2) { - logger.debug(marker, format, arg1, arg2); - } - - @Override - public void debug(Marker marker, String format, Supplier arg1, Supplier arg2) { - if (logger.isDebugEnabled(marker)) { - logger.debug(marker, format, arg1.get(), arg2.get()); - } - } - - @Override - public void debug(Marker marker, String format, Object... arguments) { - logger.debug(marker, format, arguments); - } - - @Override - @SafeVarargs - public final void debug(Marker marker, String format, Supplier... arguments) { - if (logger.isDebugEnabled(marker)) { - logger.debug(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void debug(Marker marker, String msg, Throwable t) { - logger.debug(marker, msg, t); - } - - @Override - public boolean isInfoEnabled() { - return logger.isInfoEnabled(); - } - - @Override - public void info(String msg) { - logger.info(msg); - } - - @Override - public void info(String format, Object arg) { - logger.info(format, arg); - } - - @Override - public void info(String format, Supplier arg) { - if (logger.isInfoEnabled()) { - logger.info(format, arg.get()); - } - } - - @Override - public void info(String format, Object arg1, Object arg2) { - logger.info(format, arg1, arg2); - } - - @Override - public void info(String format, Supplier arg1, Supplier arg2) { - if (logger.isInfoEnabled()) { - logger.info(format, arg1.get(), arg2.get()); - } - } - - @Override - public void info(String format, Object... arguments) { - logger.info(format, arguments); - } - - @Override - @SafeVarargs - public final void info(String format, Supplier... arguments) { - if (logger.isInfoEnabled()) { - logger.info(format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void info(String msg, Throwable t) { - logger.info(msg, t); - } - - @Override - public boolean isInfoEnabled(Marker marker) { - return logger.isInfoEnabled(marker); - } - - @Override - public void info(Marker marker, String msg) { - logger.info(marker, msg); - } - - @Override - public void info(Marker marker, String format, Object arg) { - logger.info(marker, format, arg); - } - - @Override - public void info(Marker marker, String format, Supplier arg) { - if (logger.isInfoEnabled(marker)) { - logger.info(marker, format, arg.get()); - } - } - - @Override - public void info(Marker marker, String format, Object arg1, Object arg2) { - logger.info(marker, format, arg1, arg2); - } - - @Override - public void info(Marker marker, String format, Supplier arg1, Supplier arg2) { - if (logger.isInfoEnabled(marker)) { - logger.info(marker, format, arg1.get(), arg2.get()); - } - } - - @Override - public void info(Marker marker, String format, Object... arguments) { - logger.info(marker, format, arguments); - } - - @Override - @SafeVarargs - public final void info(Marker marker, String format, Supplier... arguments) { - if (logger.isInfoEnabled(marker)) { - logger.info(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void info(Marker marker, String msg, Throwable t) { - logger.info(marker, msg, t); - } - - @Override - public boolean isWarnEnabled() { - return logger.isWarnEnabled(); - } - - @Override - public void warn(String msg) { - logger.warn(msg); - } - - @Override - public void warn(String format, Object arg) { - logger.warn(format, arg); - } - - @Override - public void warn(String format, Supplier arg) { - if (logger.isWarnEnabled()) { - logger.warn(format, arg.get()); - } - } - - @Override - public void warn(String format, Object... arguments) { - logger.warn(format, arguments); - } - - @Override - @SafeVarargs - public final void warn(String format, Supplier... arguments) { - if (logger.isWarnEnabled()) { - logger.warn(format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void warn(String format, Object arg1, Object arg2) { - logger.warn(format, arg1, arg2); - } - - @Override - public void warn(String format, Supplier arg1, Supplier arg2) { - if (logger.isWarnEnabled()) { - logger.warn(format, arg1.get(), arg2.get()); - } - } - - @Override - public void warn(String msg, Throwable t) { - logger.warn(msg, t); - } - - @Override - public boolean isWarnEnabled(Marker marker) { - return logger.isWarnEnabled(marker); - } - - @Override - public void warn(Marker marker, String msg) { - logger.warn(marker, msg); - } - - @Override - public void warn(Marker marker, String format, Object arg) { - logger.warn(marker, format, arg); - } - - @Override - public void warn(Marker marker, String format, Supplier arg) { - if (logger.isWarnEnabled(marker)) { - logger.warn(marker, format, arg.get()); - } - } - - @Override - public void warn(Marker marker, String format, Object arg1, Object arg2) { - logger.warn(marker, format, arg1, arg2); - } - - @Override - public void warn(Marker marker, String format, Supplier arg1, Supplier arg2) { - if (logger.isWarnEnabled(marker)) { - logger.warn(marker, format, arg1.get(), arg2.get()); - } - } - - @Override - public void warn(Marker marker, String format, Object... arguments) { - logger.warn(marker, format, arguments); - } - - @Override - @SafeVarargs - public final void warn(Marker marker, String format, Supplier... arguments) { - if (logger.isWarnEnabled(marker)) { - logger.warn(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void warn(Marker marker, String msg, Throwable t) { - logger.warn(marker, msg, t); - } - - @Override - public boolean isErrorEnabled() { - return logger.isErrorEnabled(); - } - - @Override - public void error(String msg) { - logger.error(msg); - } - - @Override - public void error(String format, Object arg) { - logger.error(format, arg); - } - - @Override - public void error(String format, Supplier arg) { - if (logger.isErrorEnabled()) { - logger.error(format, arg.get()); - } - } - - @Override - public void error(String format, Object arg1, Object arg2) { - logger.error(format, arg1, arg2); - } - - @Override - public void error(String format, Supplier arg1, Supplier arg2) { - if (logger.isErrorEnabled()) { - logger.error(format, arg1.get(), arg2.get()); - } - } - - @Override - public void error(String format, Object... arguments) { - logger.error(format, arguments); - } - - @Override - @SafeVarargs - public final void error(String format, Supplier... arguments) { - if (logger.isErrorEnabled()) { - logger.error(format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void error(String msg, Throwable t) { - logger.error(msg, t); - } - - @Override - public boolean isErrorEnabled(Marker marker) { - return logger.isErrorEnabled(marker); - } - - @Override - public void error(Marker marker, String msg) { - logger.error(marker, msg); - } - - @Override - public void error(Marker marker, String format, Object arg) { - logger.error(marker, format, arg); - } - - @Override - public void error(Marker marker, String format, Supplier arg) { - if (logger.isErrorEnabled(marker)) { - logger.error(marker, format, arg.get()); - } - } - - @Override - public void error(Marker marker, String format, Object arg1, Object arg2) { - logger.error(marker, format, arg1, arg2); - } - - @Override - public final void error(Marker marker, String format, Supplier arg1, Supplier arg2) { - if (logger.isErrorEnabled(marker)) { - logger.error(marker, format, arg1.get(), arg2.get()); - } - } - - @Override - public void error(Marker marker, String format, Object... arguments) { - logger.error(marker, format, arguments); - } - - @Override - @SafeVarargs - public final void error(Marker marker, String format, Supplier... arguments) { - if (logger.isErrorEnabled(marker)) { - logger.error(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); - } - } - - @Override - public void error(Marker marker, String msg, Throwable t) { - logger.error(marker, msg, t); - } + private final Logger logger; + + LazyLoggerImpl(final Logger logger) { + if (logger != null) { // Explicitly NP check to remove Guava as a dependency + this.logger = logger; + } else { + throw new NullPointerException("Null logger passed"); + } + } + + @Override + public Logger getLogger() { + return logger; + } + + @Override + public String getName() { + return logger.getName(); + } + + @Override + public boolean isTraceEnabled() { + return logger.isTraceEnabled(); + } + + @Override + public boolean isTraceEnabled(Marker marker) { + return logger.isTraceEnabled(marker); + } + + @Override + public void trace(String msg) { + logger.trace(msg); + } + + @Override + public void trace(String format, Object arg) { + logger.trace(format, arg); + } + + @Override + public void trace(String format, Supplier arg) { + if (logger.isTraceEnabled()) { + logger.trace(format, arg.get()); + } + } + + @Override + public void trace(String format, Object arg1, Object arg2) { + logger.trace(format, arg1, arg2); + } + + @Override + public void trace(String format, Supplier arg1, Supplier arg2) { + if (logger.isTraceEnabled()) { + logger.trace(format, arg1.get(), arg2.get()); + } + } + + @Override + public void trace(String format, Object... arguments) { + logger.trace(format, arguments); + } + + @Override + @SafeVarargs + public final void trace(String format, Supplier... arguments) { + if (logger.isTraceEnabled()) { + logger.trace(format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void trace(String msg, Throwable t) { + logger.trace(msg, t); + } + + @Override + public void trace(Marker marker, String msg) { + logger.trace(marker, msg); + } + + @Override + public void trace(Marker marker, String format, Object arg) { + logger.trace(marker, format, arg); + } + + @Override + public void trace(Marker marker, String format, Supplier arg) { + if (logger.isTraceEnabled(marker)) { + logger.trace(marker, format, arg.get()); + } + } + + @Override + public void trace(Marker marker, String format, Object arg1, Object arg2) { + logger.trace(marker, format, arg1, arg2); + } + + @Override + public void trace(Marker marker, String format, Supplier arg1, Supplier arg2) { + if (logger.isTraceEnabled(marker)) { + logger.trace(marker, format, arg1.get(), arg2.get()); + } + } + + @Override + public void trace(Marker marker, String format, Object... arguments) { + logger.trace(marker, format, arguments); + } + + @Override + @SafeVarargs + public final void trace(Marker marker, String format, Supplier... arguments) { + if (logger.isTraceEnabled(marker)) { + logger.trace(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void trace(Marker marker, String msg, Throwable t) { + logger.trace(marker, msg, t); + } + + @Override + public boolean isDebugEnabled() { + return logger.isDebugEnabled(); + } + + @Override + public boolean isDebugEnabled(Marker marker) { + return logger.isDebugEnabled(marker); + } + + @Override + public void debug(String msg) { + logger.debug(msg); + } + + @Override + public void debug(String format, Object arg) { + logger.debug(format, arg); + } + + @Override + public void debug(String format, Supplier arg) { + if (logger.isDebugEnabled()) { + logger.debug(format, arg.get()); + } + } + + @Override + public void debug(String format, Object arg1, Object arg2) { + logger.debug(format, arg1, arg2); + } + + @Override + public void debug(String format, Supplier arg1, Supplier arg2) { + if (logger.isDebugEnabled()) { + logger.debug(format, arg1.get(), arg2.get()); + } + } + + @Override + public void debug(String format, Object... arguments) { + logger.debug(format, arguments); + } + + @Override + @SafeVarargs + public final void debug(String format, Supplier... arguments) { + if (logger.isDebugEnabled()) { + logger.debug(format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void debug(String msg, Throwable t) { + logger.debug(msg, t); + } + + @Override + public void debug(Marker marker, String msg) { + logger.debug(marker, msg); + } + + @Override + public void debug(Marker marker, String format, Object arg) { + logger.debug(marker, format, arg); + } + + @Override + public void debug(Marker marker, String format, Supplier arg) { + if (logger.isDebugEnabled(marker)) { + logger.debug(marker, format, arg.get()); + } + } + + @Override + public void debug(Marker marker, String format, Object arg1, Object arg2) { + logger.debug(marker, format, arg1, arg2); + } + + @Override + public void debug(Marker marker, String format, Supplier arg1, Supplier arg2) { + if (logger.isDebugEnabled(marker)) { + logger.debug(marker, format, arg1.get(), arg2.get()); + } + } + + @Override + public void debug(Marker marker, String format, Object... arguments) { + logger.debug(marker, format, arguments); + } + + @Override + @SafeVarargs + public final void debug(Marker marker, String format, Supplier... arguments) { + if (logger.isDebugEnabled(marker)) { + logger.debug(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void debug(Marker marker, String msg, Throwable t) { + logger.debug(marker, msg, t); + } + + @Override + public boolean isInfoEnabled() { + return logger.isInfoEnabled(); + } + + @Override + public boolean isInfoEnabled(Marker marker) { + return logger.isInfoEnabled(marker); + } + + @Override + public void info(String msg) { + logger.info(msg); + } + + @Override + public void info(String format, Object arg) { + logger.info(format, arg); + } + + @Override + public void info(String format, Supplier arg) { + if (logger.isInfoEnabled()) { + logger.info(format, arg.get()); + } + } + + @Override + public void info(String format, Object arg1, Object arg2) { + logger.info(format, arg1, arg2); + } + + @Override + public void info(String format, Supplier arg1, Supplier arg2) { + if (logger.isInfoEnabled()) { + logger.info(format, arg1.get(), arg2.get()); + } + } + + @Override + public void info(String format, Object... arguments) { + logger.info(format, arguments); + } + + @Override + @SafeVarargs + public final void info(String format, Supplier... arguments) { + if (logger.isInfoEnabled()) { + logger.info(format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void info(String msg, Throwable t) { + logger.info(msg, t); + } + + @Override + public void info(Marker marker, String msg) { + logger.info(marker, msg); + } + + @Override + public void info(Marker marker, String format, Object arg) { + logger.info(marker, format, arg); + } + + @Override + public void info(Marker marker, String format, Supplier arg) { + if (logger.isInfoEnabled(marker)) { + logger.info(marker, format, arg.get()); + } + } + + @Override + public void info(Marker marker, String format, Object arg1, Object arg2) { + logger.info(marker, format, arg1, arg2); + } + + @Override + public void info(Marker marker, String format, Supplier arg1, Supplier arg2) { + if (logger.isInfoEnabled(marker)) { + logger.info(marker, format, arg1.get(), arg2.get()); + } + } + + @Override + public void info(Marker marker, String format, Object... arguments) { + logger.info(marker, format, arguments); + } + + @Override + @SafeVarargs + public final void info(Marker marker, String format, Supplier... arguments) { + if (logger.isInfoEnabled(marker)) { + logger.info(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void info(Marker marker, String msg, Throwable t) { + logger.info(marker, msg, t); + } + + @Override + public boolean isWarnEnabled() { + return logger.isWarnEnabled(); + } + + @Override + public boolean isWarnEnabled(Marker marker) { + return logger.isWarnEnabled(marker); + } + + @Override + public void warn(String msg) { + logger.warn(msg); + } + + @Override + public void warn(String format, Object arg) { + logger.warn(format, arg); + } + + @Override + public void warn(String format, Supplier arg) { + if (logger.isWarnEnabled()) { + logger.warn(format, arg.get()); + } + } + + @Override + public void warn(String format, Object... arguments) { + logger.warn(format, arguments); + } + + @Override + @SafeVarargs + public final void warn(String format, Supplier... arguments) { + if (logger.isWarnEnabled()) { + logger.warn(format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void warn(String format, Object arg1, Object arg2) { + logger.warn(format, arg1, arg2); + } + + @Override + public void warn(String format, Supplier arg1, Supplier arg2) { + if (logger.isWarnEnabled()) { + logger.warn(format, arg1.get(), arg2.get()); + } + } + + @Override + public void warn(String msg, Throwable t) { + logger.warn(msg, t); + } + + @Override + public void warn(Marker marker, String msg) { + logger.warn(marker, msg); + } + + @Override + public void warn(Marker marker, String format, Object arg) { + logger.warn(marker, format, arg); + } + + @Override + public void warn(Marker marker, String format, Supplier arg) { + if (logger.isWarnEnabled(marker)) { + logger.warn(marker, format, arg.get()); + } + } + + @Override + public void warn(Marker marker, String format, Object arg1, Object arg2) { + logger.warn(marker, format, arg1, arg2); + } + + @Override + public void warn(Marker marker, String format, Supplier arg1, Supplier arg2) { + if (logger.isWarnEnabled(marker)) { + logger.warn(marker, format, arg1.get(), arg2.get()); + } + } + + @Override + public void warn(Marker marker, String format, Object... arguments) { + logger.warn(marker, format, arguments); + } + + @Override + @SafeVarargs + public final void warn(Marker marker, String format, Supplier... arguments) { + if (logger.isWarnEnabled(marker)) { + logger.warn(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void warn(Marker marker, String msg, Throwable t) { + logger.warn(marker, msg, t); + } + + @Override + public boolean isErrorEnabled() { + return logger.isErrorEnabled(); + } + + @Override + public boolean isErrorEnabled(Marker marker) { + return logger.isErrorEnabled(marker); + } + + @Override + public void error(String msg) { + logger.error(msg); + } + + @Override + public void error(String format, Object arg) { + logger.error(format, arg); + } + + @Override + public void error(String format, Supplier arg) { + if (logger.isErrorEnabled()) { + logger.error(format, arg.get()); + } + } + + @Override + public void error(String format, Object arg1, Object arg2) { + logger.error(format, arg1, arg2); + } + + @Override + public void error(String format, Supplier arg1, Supplier arg2) { + if (logger.isErrorEnabled()) { + logger.error(format, arg1.get(), arg2.get()); + } + } + + @Override + public void error(String format, Object... arguments) { + logger.error(format, arguments); + } + + @Override + @SafeVarargs + public final void error(String format, Supplier... arguments) { + if (logger.isErrorEnabled()) { + logger.error(format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void error(String msg, Throwable t) { + logger.error(msg, t); + } + + @Override + public void error(Marker marker, String msg) { + logger.error(marker, msg); + } + + @Override + public void error(Marker marker, String format, Object arg) { + logger.error(marker, format, arg); + } + + @Override + public void error(Marker marker, String format, Supplier arg) { + if (logger.isErrorEnabled(marker)) { + logger.error(marker, format, arg.get()); + } + } + + @Override + public void error(Marker marker, String format, Object arg1, Object arg2) { + logger.error(marker, format, arg1, arg2); + } + + @Override + public final void error(Marker marker, String format, Supplier arg1, Supplier arg2) { + if (logger.isErrorEnabled(marker)) { + logger.error(marker, format, arg1.get(), arg2.get()); + } + } + + @Override + public void error(Marker marker, String format, Object... arguments) { + logger.error(marker, format, arguments); + } + + @Override + @SafeVarargs + public final void error(Marker marker, String format, Supplier... arguments) { + if (logger.isErrorEnabled(marker)) { + logger.error(marker, format, Arrays.stream(arguments).map(Supplier::get).toArray()); + } + } + + @Override + public void error(Marker marker, String msg, Throwable t) { + logger.error(marker, msg, t); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/MessageUtils.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/MessageUtils.java index d736ff95..869e2188 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/MessageUtils.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/MessageUtils.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils; import org.apache.metron.stellar.common.Constants; @@ -22,11 +25,11 @@ public class MessageUtils { - public static String getGuid(JSONObject message) { - return (String) message.get(Constants.GUID); - } + public static String getGuid(JSONObject message) { + return (String) message.get(Constants.GUID); + } - public static String getSensorType(JSONObject message) { - return (String) message.get(Constants.SENSOR_TYPE); - } + public static String getSensorType(JSONObject message) { + return (String) message.get(Constants.SENSOR_TYPE); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/ReflectionUtils.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/ReflectionUtils.java index 4689014d..1e468759 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/ReflectionUtils.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/ReflectionUtils.java @@ -7,117 +7,120 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils; import java.lang.reflect.InvocationTargetException; public class ReflectionUtils { - /** - * Creates an instance of a provided class. - * - * @param className The name of the class to create an instance of - * @param defaultClass The default if the class name is null/empty or is an inner class - * @param The type parameter of the default class - * @return An instance of the provided class. - */ - public static T createInstance(String className, T defaultClass) { - T instance; - if (className == null || className.length() == 0 || className.charAt(0) == '$') { - return defaultClass; - } else { - instance = createInstance(className); + /** + * Creates an instance of a provided class. + * + * @param className The name of the class to create an instance of + * @param defaultClass The default if the class name is null/empty or is an inner class + * @param The type parameter of the default class + * @return An instance of the provided class. + */ + public static T createInstance(String className, T defaultClass) { + T instance; + if (className == null || className.length() == 0 || className.charAt(0) == '$') { + return defaultClass; + } else { + instance = createInstance(className); + } + return instance; } - return instance; - } - /** - * Attempts to create instance from specified class name. No-arg constructor assumed. - * - * @param className fully qualified name of class to instantiate. e.g. foo.bar.Baz - * @param Instance created from passed class - * @return Object of type T - */ - public static T createInstance(String className) { - T instance; - try { - Class clazz = (Class) Class.forName(className); - instance = createInstance(clazz); - } catch (ClassNotFoundException e) { - throw new IllegalStateException("Unable to instantiate connector: class not found", e); + /** + * Attempts to create instance from specified class name. No-arg constructor assumed. + * + * @param className fully qualified name of class to instantiate. e.g. foo.bar.Baz + * @param Instance created from passed class + * @return Object of type T + */ + public static T createInstance(String className) { + T instance; + try { + Class clazz = (Class) Class.forName(className); + instance = createInstance(clazz); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Unable to instantiate connector: class not found", e); + } + return instance; } - return instance; - } - /** - * Create instance from no-args constructor. - * - * @param clazz Class to create instance from - * @param Instance created from passed class - * @return Object of type T - */ - public static T createInstance(Class clazz) { - return createInstance(clazz, null, null); - } + /** + * Create instance from no-args constructor. + * + * @param clazz Class to create instance from + * @param Instance created from passed class + * @return Object of type T + */ + public static T createInstance(Class clazz) { + return createInstance(clazz, null, null); + } - /** - * Create instance from passed class with specified parameter types and arguments. If parameter - * types is null, defaults to attempting to instantiate the no-arg constructor. - * - * @param clazz Class to create instance from - * @param parameterTypes parameter types to use for looking up the desired constructor - * @param parameters arguments to pass into the constructor when instantiating the object. - * @param Instance created from passed class - * @return Object of type T - */ - public static T createInstance(Class clazz, Class[] parameterTypes, Object[] parameters) { - T instance; - try { - if (parameterTypes != null) { - instance = clazz.getConstructor(parameterTypes).newInstance(parameters); - } else { - instance = clazz.getConstructor().newInstance(); - } - } catch (InstantiationException e) { - throw new IllegalStateException("Unable to instantiate connector.", e); - } catch (IllegalAccessException e) { - throw new IllegalStateException("Unable to instantiate connector: illegal access", e); - } catch (InvocationTargetException e) { - throw new IllegalStateException("Unable to instantiate connector", e); - } catch (NoSuchMethodException e) { - throw new IllegalStateException("Unable to instantiate connector: no such method", e); + /** + * Create instance from passed class with specified parameter types and arguments. If parameter + * types is null, defaults to attempting to instantiate the no-arg constructor. + * + * @param clazz Class to create instance from + * @param parameterTypes parameter types to use for looking up the desired constructor + * @param parameters arguments to pass into the constructor when instantiating the object. + * @param Instance created from passed class + * @return Object of type T + */ + public static T createInstance(Class clazz, Class[] parameterTypes, Object[] parameters) { + T instance; + try { + if (parameterTypes != null) { + instance = clazz.getConstructor(parameterTypes).newInstance(parameters); + } else { + instance = clazz.getConstructor().newInstance(); + } + } catch (InstantiationException e) { + throw new IllegalStateException("Unable to instantiate connector.", e); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Unable to instantiate connector: illegal access", e); + } catch (InvocationTargetException e) { + throw new IllegalStateException("Unable to instantiate connector", e); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to instantiate connector: no such method", e); + } + return instance; } - return instance; - } - /** - * Create instance from passed class name with specified parameter types and arguments. If parameter - * types is null, defaults to attempting to instantiate the no-arg constructor. - * - * @param className Class to create instance from - * @param parameterTypes parameter types to use for looking up the desired constructor - * @param parameters arguments to pass into the constructor when instantiating the object. - * @param Instance created from passed class - * @return Object of type T - */ - public static T createInstance(String className, Class[] parameterTypes, Object[] parameters) { - T instance; - try { - Class clazz = (Class) Class.forName(className); - instance = createInstance(clazz, parameterTypes, parameters); - } catch (ClassNotFoundException e) { - throw new IllegalStateException("Unable to instantiate connector: class not found", e); + /** + * Create instance from passed class name with specified parameter types and arguments. If parameter + * types is null, defaults to attempting to instantiate the no-arg constructor. + * + * @param className Class to create instance from + * @param parameterTypes parameter types to use for looking up the desired constructor + * @param parameters arguments to pass into the constructor when instantiating the object. + * @param Instance created from passed class + * @return Object of type T + */ + public static T createInstance(String className, Class[] parameterTypes, Object[] parameters) { + T instance; + try { + Class clazz = (Class) Class.forName(className); + instance = createInstance(clazz, parameterTypes, parameters); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Unable to instantiate connector: class not found", e); + } + return instance; } - return instance; - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/RuntimeErrors.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/RuntimeErrors.java index 266c67c6..4102f035 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/RuntimeErrors.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/RuntimeErrors.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,62 +20,61 @@ package org.apache.metron.common.utils; -import org.apache.commons.lang3.tuple.Pair; +import static java.lang.String.format; import java.util.Optional; import java.util.function.Function; - -import static java.lang.String.format; +import org.apache.commons.lang3.tuple.Pair; public enum RuntimeErrors { - ILLEGAL_ARG(t -> new IllegalArgumentException(formatReason(t), t.getRight().orElse(null))), - ILLEGAL_STATE(t -> new IllegalStateException(formatReason(t), t.getRight().orElse(null))); + ILLEGAL_ARG(t -> new IllegalArgumentException(formatReason(t), t.getRight().orElse(null))), + ILLEGAL_STATE(t -> new IllegalStateException(formatReason(t), t.getRight().orElse(null))); - Function>, RuntimeException> func; + Function>, RuntimeException> func; - RuntimeErrors(Function>, RuntimeException> func) { - this.func = func; - } + RuntimeErrors(Function>, RuntimeException> func) { + this.func = func; + } - /** - * Throw runtime exception with "reason". - * - * @param reason Message to include in exception - */ - public void throwRuntime(String reason) { - throwRuntime(reason, Optional.empty()); - } + /** + * Throw runtime exception with "reason". + * + * @param reason Message to include in exception + */ + public void throwRuntime(String reason) { + throwRuntime(reason, Optional.empty()); + } - /** - * Throw runtime exception with format "reason + cause message + cause Throwable". - * - * @param reason Message to include in exception - * @param t Wrapped exception - */ - public void throwRuntime(String reason, Throwable t) { - throwRuntime(reason, Optional.of(t)); - } + /** + * Throw runtime exception with format "reason + cause message + cause Throwable". + * + * @param reason Message to include in exception + * @param t Wrapped exception + */ + public void throwRuntime(String reason, Throwable t) { + throwRuntime(reason, Optional.of(t)); + } - /** - * Throw runtime exception with format "reason + cause message + cause Throwable". - * If the optional Throwable is empty/null, the exception will only include "reason". - * - * @param reason Message to include in exception - * @param t Optional wrapped exception - */ - public void throwRuntime(String reason, Optional t) { - throw func.apply(Pair.of(reason, t)); - } + /** + * Throw runtime exception with format "reason + cause message + cause Throwable". + * If the optional Throwable is empty/null, the exception will only include "reason". + * + * @param reason Message to include in exception + * @param t Optional wrapped exception + */ + public void throwRuntime(String reason, Optional t) { + throw func.apply(Pair.of(reason, t)); + } - private static String formatReason(Pair> p) { - return formatReason(p.getLeft(), p.getRight()); - } + private static String formatReason(Pair> p) { + return formatReason(p.getLeft(), p.getRight()); + } - private static String formatReason(String reason, Optional t) { - if (t.isPresent()) { - return format("%s - reason:%s", reason, t.get()); - } else { - return format("%s", reason); + private static String formatReason(String reason, Optional t) { + if (t.isPresent()) { + return format("%s - reason:%s", reason, t.get()); + } else { + return format("%s", reason); + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java index d08f70a7..f29701ae 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,31 +26,32 @@ public class StringUtils { - /** - * Joins string optionals potentially containing string with a delimiter. - * - * @param delim The delimiter - * @param parts The string optionals to join - * @return The joined string, without any missing optionals. - */ - public static String join(String delim, Optional... parts) { - return Joiner.on(delim).join( - Arrays.asList(parts).stream().filter( - part -> part.isPresent() - ).map( part -> part.get()) - .toArray() - ); - } + /** + * Joins string optionals potentially containing string with a delimiter. + * + * @param delim The delimiter + * @param parts The string optionals to join + * @return The joined string, without any missing optionals. + */ + public static String join(String delim, Optional... parts) { + return Joiner.on(delim).join( + Arrays.asList(parts).stream().filter( + part -> part.isPresent() + ).map(part -> part.get()) + .toArray() + ); + } - /** - * Strips specified number of lines from beginning for String val. - * @param val The input string - * @param numLines The number of lines to remove - */ - public static String stripLines(String val, int numLines) { - int start = org.apache.commons.lang3.StringUtils.ordinalIndexOf(val, System.lineSeparator(), numLines); - start = start >= 0 ? start : 0; - return val.substring(start); - } + /** + * Strips specified number of lines from beginning for String val. + * + * @param val The input string + * @param numLines The number of lines to remove + */ + public static String stripLines(String val, int numLines) { + int start = org.apache.commons.lang3.StringUtils.ordinalIndexOf(val, System.lineSeparator(), numLines); + start = start >= 0 ? start : 0; + return val.substring(start); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/cli/CLIOptions.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/cli/CLIOptions.java index c43b30bf..8c7d4430 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/cli/CLIOptions.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/cli/CLIOptions.java @@ -7,25 +7,29 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils.cli; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; +@SuppressWarnings("checkstyle:InterfaceTypeParameterName") public interface CLIOptions & CLIOptions> { - Option getOption(); + Option getOption(); - boolean has(CommandLine cli); + boolean has(CommandLine cli); - String get(CommandLine cli); + String get(CommandLine cli); - OptionHandler getHandler(); + OptionHandler getHandler(); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/cli/OptionHandler.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/cli/OptionHandler.java index 61fe06b5..66b24d59 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/cli/OptionHandler.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/cli/OptionHandler.java @@ -7,93 +7,101 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils.cli; import com.google.common.base.Function; import com.google.common.base.Joiner; -import org.apache.commons.cli.*; - import java.util.EnumMap; import java.util.Optional; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; -public abstract class OptionHandler & CLIOptions> implements Function -{ - public Optional getValue(OPT_T option, CommandLine cli) { - return Optional.empty(); - } +@SuppressWarnings("checkstyle:MethodTypeParameterName") +public abstract class OptionHandler & CLIOptions> implements Function { + public Optional getValue(OPT_T option, CommandLine cli) { + return Optional.empty(); + } - public abstract String getShortCode(); + public abstract String getShortCode(); - /** - * Returns options based on the {@link CLIOptions} provided. - * - * @param values the values to produce options with - * @return The resulting options - */ - public static Options getOptions(CLIOptions[] values) { - Options ret = new Options(); - for(CLIOptions o : values) { - ret.addOption(o.getOption()); + /** + * Returns options based on the {@link CLIOptions} provided. + * + * @param values the values to produce options with + * @return The resulting options + */ + public static Options getOptions(CLIOptions[] values) { + Options ret = new Options(); + for (CLIOptions o : values) { + ret.addOption(o.getOption()); + } + return ret; } - return ret; - } - public static void printHelp(String name, CLIOptions[] values) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp( name, getOptions(values)); - } + public static void printHelp(String name, CLIOptions[] values) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp(name, getOptions(values)); + } - /** - * Creates a config based on the provided command line options, allowing user to get an enum - * based view of the options map. - * - * @param cli The command line with the parsed options. - * @param values The values of the options. - * @param clazz The class of the enum representing the option keys. - * @param The type parameter of the enum - * @return A Map of enum key -> option value - */ - public static & CLIOptions> - EnumMap > createConfig(CommandLine cli, OPT_T[] values, Class clazz) { - EnumMap > ret = new EnumMap<>(clazz); - for(OPT_T option : values) { - ret.put(option, option.getHandler().getValue(option, cli)); + /** + * Creates a config based on the provided command line options, allowing user to get an enum + * based view of the options map. + * + * @param cli The command line with the parsed options. + * @param values The values of the options. + * @param clazz The class of the enum representing the option keys. + * @param The type parameter of the enum + * @return A Map of enum key -> option value + */ + public static & CLIOptions> + EnumMap> createConfig(CommandLine cli, OPT_T[] values, Class clazz) { + EnumMap> ret = new EnumMap<>(clazz); + for (OPT_T option : values) { + ret.put(option, option.getHandler().getValue(option, cli)); + } + return ret; } - return ret; - } - /** - * Parses options of a program and returns a command line representing them. - * - * @param name Name of the program - * @param parser The command line parser to be used - * @param args The arguments to the program - * @param values The cli option values - * @param helpOption The cli help option - * @return A command line representing the parsed options. - */ - public static CommandLine parse(String name, CommandLineParser parser, String[] args, CLIOptions[] values, CLIOptions helpOption) { - try { - CommandLine cli = parser.parse(getOptions(values), args); - if(helpOption.has(cli)) { - printHelp(name, values); - System.exit(0); - } - return cli; - } catch (ParseException e) { - System.err.println("Unable to parse args: " + Joiner.on(' ').join(args)); - e.printStackTrace(System.err); - printHelp(name, values); - System.exit(-1); - return null; + /** + * Parses options of a program and returns a command line representing them. + * + * @param name Name of the program + * @param parser The command line parser to be used + * @param args The arguments to the program + * @param values The cli option values + * @param helpOption The cli help option + * @return A command line representing the parsed options. + */ + public static CommandLine parse(String name, CommandLineParser parser, String[] args, CLIOptions[] values, + CLIOptions helpOption) { + try { + CommandLine cli = parser.parse(getOptions(values), args); + if (helpOption.has(cli)) { + printHelp(name, values); + System.exit(0); + } + return cli; + } catch (ParseException e) { + System.err.println("Unable to parse args: " + Joiner.on(' ').join(args)); + e.printStackTrace(System.err); + printHelp(name, values); + System.exit(-1); + return null; + } } - } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/file/ReaderSpliterator.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/file/ReaderSpliterator.java index 908c51a2..b387964f 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/file/ReaderSpliterator.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/file/ReaderSpliterator.java @@ -7,16 +7,21 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils.file; +import static java.util.Spliterators.spliterator; + import java.io.BufferedReader; import java.io.IOException; import java.io.UncheckedIOException; @@ -25,8 +30,6 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; -import static java.util.Spliterators.spliterator; - /** * A Spliterator which works well on sequential streams by constructing a * fixed batch size split rather than inheriting the spliterator from BufferedReader.lines() @@ -38,107 +41,109 @@ * described at http://bytefish.de/blog/jdk8_files_lines_parallel_stream/ */ public class ReaderSpliterator implements Spliterator { - private static int characteristics = NONNULL | ORDERED | IMMUTABLE; - private int batchSize ; - private BufferedReader reader; - public ReaderSpliterator(BufferedReader reader) { - this(reader, 128); - } + private static final int characteristics = NONNULL | ORDERED | IMMUTABLE; + private final int batchSize; + private final BufferedReader reader; - public ReaderSpliterator(BufferedReader reader, int batchSize) { - this.batchSize = batchSize; - this.reader = reader; - } - - @Override - public void forEachRemaining(Consumer action) { - if (action == null) { - throw new NullPointerException(); + public ReaderSpliterator(BufferedReader reader) { + this(reader, 128); } - try { - for (String line = null; (line = reader.readLine()) != null;) { - action.accept(line); - } - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new IllegalStateException(e); + + public ReaderSpliterator(BufferedReader reader, int batchSize) { + this.batchSize = batchSize; + this.reader = reader; } - } - @Override - public boolean tryAdvance(Consumer action) { - if (action == null) { - throw new NullPointerException(); + @Override + public void forEachRemaining(Consumer action) { + if (action == null) { + throw new NullPointerException(); + } + try { + for (String line = null; (line = reader.readLine()) != null; ) { + action.accept(line); + } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); + } } - try { - final String line = reader.readLine(); - if (line == null) { - return false; - } - action.accept(line); - return true; - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new IllegalStateException(e); + + @Override + public boolean tryAdvance(Consumer action) { + if (action == null) { + throw new NullPointerException(); + } + try { + final String line = reader.readLine(); + if (line == null) { + return false; + } + action.accept(line); + return true; + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); + } } - } - @Override - public Spliterator trySplit() { - final ConsumerWithLookback holder = new ConsumerWithLookback(); - if (!tryAdvance(holder)) { - return null; + @SuppressWarnings("checkstyle:RightCurly") + @Override + public Spliterator trySplit() { + final ConsumerWithLookback holder = new ConsumerWithLookback(); + if (!tryAdvance(holder)) { + return null; + } + final String[] batch = new String[batchSize]; + int j = 0; + do { + batch[j] = holder.value; + } + while (++j < batchSize && tryAdvance(holder)); + return spliterator(batch, 0, j, characteristics() | SIZED); } - final String[] batch = new String[batchSize]; - int j = 0; - do { - batch[j] = holder.value; + + @Override + public long estimateSize() { + return Long.MAX_VALUE; } - while (++j < batchSize && tryAdvance(holder)); - return spliterator(batch, 0, j, characteristics() | SIZED); - } - @Override - public long estimateSize() { - return Long.MAX_VALUE; - } + @Override + public int characteristics() { + return characteristics; + } - @Override - public int characteristics() { - return characteristics; - } + static class ConsumerWithLookback implements Consumer { + String value; - static class ConsumerWithLookback implements Consumer { - String value; - @Override - public void accept(String string) { - this.value = string; + @Override + public void accept(String string) { + this.value = string; + } } - } - public static Stream lineStream(BufferedReader in, int batchSize) { - return lineStream(in, batchSize, false); - } + public static Stream lineStream(BufferedReader in, int batchSize) { + return lineStream(in, batchSize, false); + } - /** - * Creates a {@link Stream} with a ReaderSpliterator underlying it. - * - * @param in the input for creating the stream - * @param batchSize The batch size to be used by the spliterator - * @param isParallel True if stream should be parallel, false otherwise - * @return The created {@link Stream} - */ - public static Stream lineStream(BufferedReader in, int batchSize, boolean isParallel) { - return StreamSupport.stream(new ReaderSpliterator(in, batchSize), isParallel) - .onClose(() -> { - try { - in.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - ); - } + /** + * Creates a {@link Stream} with a ReaderSpliterator underlying it. + * + * @param in the input for creating the stream + * @param batchSize The batch size to be used by the spliterator + * @param isParallel True if stream should be parallel, false otherwise + * @return The created {@link Stream} + */ + public static Stream lineStream(BufferedReader in, int batchSize, boolean isParallel) { + return StreamSupport.stream(new ReaderSpliterator(in, batchSize), isParallel) + .onClose(() -> { + try { + in.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverter.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverter.java index a02e31e9..78161a76 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverter.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverter.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,5 +23,5 @@ import java.io.Serializable; public interface TimestampConverter extends Serializable { - public long toNanoseconds(long input); + long toNanoseconds(long input); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverters.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverters.java index a6dc2a9a..8c8f4580 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverters.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverters.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,33 +23,32 @@ import com.google.common.base.Joiner; -public enum TimestampConverters implements TimestampConverter{ - MILLISECONDS(tsMilli -> tsMilli*1000000L) - ,MICROSECONDS(tsMicro -> tsMicro*1000L) - ,NANOSECONDS(tsNano -> tsNano); - TimestampConverter converter; - TimestampConverters(TimestampConverter converter) { - this.converter = converter; - } +public enum TimestampConverters implements TimestampConverter { + MILLISECONDS(tsMilli -> tsMilli * 1000000L), MICROSECONDS(tsMicro -> tsMicro * 1000L), + NANOSECONDS(tsNano -> tsNano); + TimestampConverter converter; - /** - * Gets a {@link TimestampConverter} by name. - * - * @param converter The name of the converter to get - * @return The timestamp converter - */ - public static TimestampConverter getConverter(String converter) { - if(converter != null) { - return TimestampConverters.valueOf(converter.toUpperCase()).converter; + TimestampConverters(TimestampConverter converter) { + this.converter = converter; } - else { - throw new IllegalStateException(converter + " is not a valid timestamp converter: " - + Joiner.on(",").join(TimestampConverters.values())); + + /** + * Gets a {@link TimestampConverter} by name. + * + * @param converter The name of the converter to get + * @return The timestamp converter + */ + public static TimestampConverter getConverter(String converter) { + if (converter != null) { + return TimestampConverters.valueOf(converter.toUpperCase()).converter; + } else { + throw new IllegalStateException(converter + " is not a valid timestamp converter: " + + Joiner.on(",").join(TimestampConverters.values())); + } } - } - @Override - public long toNanoseconds(long in) { - return converter.toNanoseconds(in); - } + @Override + public long toNanoseconds(long in) { + return converter.toNanoseconds(in); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessage.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessage.java index a00f9b88..4c13bbb1 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessage.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessage.java @@ -7,52 +7,59 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.writer; import java.util.Objects; public class BulkMessage { - private MessageId id; - private MESSAGE_T message; + private final MessageId id; + private final MESSAGE_T message; - public BulkMessage(MessageId id, MESSAGE_T message) { - this.id = id; - this.message = message; - } + public BulkMessage(MessageId id, MESSAGE_T message) { + this.id = id; + this.message = message; + } - public BulkMessage(String id, MESSAGE_T message) { - this(new MessageId(id), message); - } + public BulkMessage(String id, MESSAGE_T message) { + this(new MessageId(id), message); + } - public MessageId getId() { - return id; - } + public MessageId getId() { + return id; + } - public MESSAGE_T getMessage() { - return message; - } + public MESSAGE_T getMessage() { + return message; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - BulkMessage that = (BulkMessage) o; - return Objects.equals(id, that.id) && - Objects.equals(message, that.message); - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + BulkMessage that = (BulkMessage) o; + return Objects.equals(id, that.id) + && Objects.equals(message, that.message); + } - @Override - public int hashCode() { + @Override + public int hashCode() { - return Objects.hash(id, message); - } + return Objects.hash(id, message); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java index 51bc599c..81daeb43 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkMessageWriter.java @@ -7,38 +7,42 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.writer; -import org.apache.metron.common.configuration.writer.WriterConfiguration; +package org.apache.metron.common.writer; import java.io.Serializable; import java.util.List; import java.util.Map; +import org.apache.metron.common.configuration.writer.WriterConfiguration; +@SuppressWarnings("checkstyle:InterfaceTypeParameterName") public interface BulkMessageWriter extends AutoCloseable, Serializable { - void init(Map stormConf, WriterConfiguration config) throws Exception; + void init(Map stormConf, WriterConfiguration config) throws Exception; - /** - * Writes the messages to a particular output (e.g. Elasticsearch). A response is returned with successful and failed message ids. - * @param sensorType The type of sensor generating the messages - * @param configurations Configurations that should be passed to the writer (e.g. index and - * @param messages A list of messages to be written. Message ids are used in the response to report successes/failures. - * @return A response containing successes and failures within the batch. - * @throws Exception If an unrecoverable error is made, an Exception is thrown which should be treated as a full-batch failure (e.g. target system is down). - */ - BulkWriterResponse write(String sensorType - , WriterConfiguration configurations - , List> messages - ) throws Exception; + /** + * Writes the messages to a particular output (e.g. Elasticsearch). A response is returned with successful and failed message ids. + * + * @param sensorType The type of sensor generating the messages + * @param configurations Configurations that should be passed to the writer (e.g. index and + * @param messages A list of messages to be written. Message ids are used in the response to report successes/failures. + * @return A response containing successes and failures within the batch. + * @throws Exception If an unrecoverable error is made, an Exception is thrown which should be treated as a full-batch failure + * (e.g. target system is down). + */ + BulkWriterResponse write(String sensorType, + WriterConfiguration configurations, + List> messages) throws Exception; - String getName(); + String getName(); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java index 20fa70c7..56dc2e51 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterResponse.java @@ -21,12 +21,11 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; -import org.apache.metron.common.configuration.writer.WriterConfiguration; - import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.metron.common.configuration.writer.WriterConfiguration; /** * This class contains the results of a {@link org.apache.metron.common.writer.BulkMessageWriter#write(String, WriterConfiguration, List)} @@ -34,21 +33,21 @@ * {@link org.apache.metron.common.writer.MessageId}. */ public class BulkWriterResponse { - private Multimap errors = ArrayListMultimap.create(); - private List successes = new ArrayList<>(); + private final Multimap errors = ArrayListMultimap.create(); + private final List successes = new ArrayList<>(); public void addError(Throwable error, MessageId id) { errors.put(error, id); } - /** - * Adds provided errors and associated tuples. - * - * @param error The error to add - * @param ids Iterable of all messages with the error - */ + /** + * Adds provided errors and associated tuples. + * + * @param error The error to add + * @param ids Iterable of all messages with the error + */ public void addAllErrors(Throwable error, Iterable ids) { - if(ids != null) { + if (ids != null) { errors.putAll(error, ids); } } @@ -61,13 +60,13 @@ public void addSuccess(MessageId success) { successes.add(success); } - /** - * Adds all provided successes. - * - * @param allSuccesses Successes to add - */ + /** + * Adds all provided successes. + * + * @param allSuccesses Successes to add + */ public void addAllSuccesses(Iterable allSuccesses) { - if(allSuccesses != null) { + if (allSuccesses != null) { Iterables.addAll(successes, allSuccesses); } } @@ -82,12 +81,18 @@ public List getSuccesses() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } BulkWriterResponse that = (BulkWriterResponse) o; - if (!errors.equals(that.errors)) return false; + if (!errors.equals(that.errors)) { + return false; + } return successes.equals(that.successes); } @@ -101,9 +106,9 @@ public int hashCode() { @Override public String toString() { - return "BulkWriterResponse{" + - "errors=" + errors + - ", successes=" + successes + - '}'; + return "BulkWriterResponse{" + + "errors=" + errors + + ", successes=" + successes + + '}'; } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/MessageId.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/MessageId.java index 5227a045..2f95bfa7 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/MessageId.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/MessageId.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,36 +24,40 @@ public class MessageId { - private String id; - - public MessageId(String id) { - this.id = id; - } - - public String getId() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - MessageId messageId = (MessageId) o; - return Objects.equals(id, messageId.id); - } - - @Override - public int hashCode() { - - return Objects.hash(id); - } - - @Override - public String toString() { - return "MessageId{" + - "id='" + id + '\'' + - '}'; - } + private final String id; + + public MessageId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MessageId messageId = (MessageId) o; + return Objects.equals(id, messageId.id); + } + + @Override + public int hashCode() { + + return Objects.hash(id); + } + + @Override + public String toString() { + return "MessageId{" + + "id='" + id + '\'' + + '}'; + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java index 65f64844..48f63804 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/writer/MessageWriter.java @@ -7,23 +7,27 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.writer; -import org.apache.metron.common.configuration.writer.WriterConfiguration; +package org.apache.metron.common.writer; import java.io.Serializable; +import org.apache.metron.common.configuration.writer.WriterConfiguration; public interface MessageWriter extends AutoCloseable, Serializable { - void init(); - void write(String sensorType, WriterConfiguration configurations, BulkMessage message) throws Exception; - String getName(); + void init(); + + void write(String sensorType, WriterConfiguration configurations, BulkMessage message) throws Exception; + + String getName(); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/ConfigurationsCache.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/ConfigurationsCache.java index 462d7540..564310c0 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/ConfigurationsCache.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/ConfigurationsCache.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.zookeeper; import org.apache.metron.common.configuration.Configurations; @@ -23,22 +26,23 @@ * A cache for multiple Configurations object. This cache is generally kept in * sync with zookeeper changes. */ -public interface ConfigurationsCache extends AutoCloseable{ - /** - * Return the Configurations object given the specific type of Configurations object. - * @param configClass The Configurations class to return - * @param The type of Configurations class. - * @return The Configurations object - */ - T get(Class configClass); +public interface ConfigurationsCache extends AutoCloseable { + /** + * Return the Configurations object given the specific type of Configurations object. + * + * @param configClass The Configurations class to return + * @param The type of Configurations class. + * @return The Configurations object + */ + T get(Class configClass); - /** - * Reset the cache and reload from zookeeper. - */ - void reset(); + /** + * Reset the cache and reload from zookeeper. + */ + void reset(); - /** - * Start the cache. - */ - void start(); + /** + * Start the cache. + */ + void start(); } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ConfigurationsUpdater.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ConfigurationsUpdater.java index da0c6edc..39cbcdd3 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ConfigurationsUpdater.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ConfigurationsUpdater.java @@ -7,28 +7,30 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.zookeeper.configurations; +import java.io.IOException; +import java.io.Serializable; +import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; +import java.util.function.Supplier; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.Configurations; import org.apache.metron.common.utils.LazyLogger; import org.apache.metron.common.utils.LazyLoggerFactory; -import java.io.IOException; -import java.io.Serializable; -import java.lang.invoke.MethodHandles; -import java.util.function.Supplier; - /** * Handles update for an underlying Configurations object. This is the base abstract implementation. * You will find system-specific implementations (e.g. IndexingUpdater, ParserUpdater, etc.) which @@ -37,119 +39,125 @@ * @param the Type of Configuration */ public abstract class ConfigurationsUpdater implements Serializable { - protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private Reloadable reloadable; - private Supplier configSupplier; + protected static final LazyLogger LOG = LazyLoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final Reloadable reloadable; + private final Supplier configSupplier; - /** - * Construct a ConfigurationsUpdater. - * @param reloadable A callback which gets called whenever a reload happens - * @param configSupplier A Supplier which creates the Configurations object. - */ - public ConfigurationsUpdater(Reloadable reloadable - , Supplier configSupplier - ) - { - this.reloadable = reloadable; - this.configSupplier = configSupplier; - } - - /** - * Update callback, this is called whenever a path is updated in zookeeper which we are monitoring. - * - * @param client The CuratorFramework - * @param path The zookeeper path - * @param data The change. - * @throws IOException When update is impossible. - */ - public void update(CuratorFramework client, String path, byte[] data) throws IOException { - if (data.length != 0) { - String name = path.substring(path.lastIndexOf("/") + 1); - if (path.startsWith(getType().getZookeeperRoot())) { - LOG.debug("Updating the {} config: {} -> {}", () -> getType().name(), () -> name, - () -> new String(data == null?"".getBytes(StandardCharsets.UTF_8):data, StandardCharsets.UTF_8)); - update(name, data); - reloadCallback(name, getType()); - } else if (ConfigurationType.GLOBAL.getZookeeperRoot().equals(path)) { - LOG.debug("Updating the global config: {}", - () -> new String(data == null?"".getBytes(StandardCharsets.UTF_8):data, StandardCharsets.UTF_8)); - getConfigurations().updateGlobalConfig(data); - reloadCallback(name, ConfigurationType.GLOBAL); - } + /** + * Construct a ConfigurationsUpdater. + * + * @param reloadable A callback which gets called whenever a reload happens + * @param configSupplier A Supplier which creates the Configurations object. + */ + public ConfigurationsUpdater(Reloadable reloadable, + Supplier configSupplier) { + this.reloadable = reloadable; + this.configSupplier = configSupplier; } - } - /** - * Delete callback, this is called whenever a path is deleted in zookeeper which we are monitoring. - * - * @param client The CuratorFramework - * @param path The zookeeper path - * @param data The change. - * @throws IOException When update is impossible. - */ - public void delete(CuratorFramework client, String path, byte[] data) throws IOException { - String name = path.substring(path.lastIndexOf("/") + 1); - if (path.startsWith(getType().getZookeeperRoot())) { - LOG.debug("Deleting {} {} config from internal cache", getType().name(), name); - delete(name); - reloadCallback(name, getType()); - } else if (ConfigurationType.GLOBAL.getZookeeperRoot().equals(path)) { - LOG.debug("Deleting global config from internal cache"); - getConfigurations().deleteGlobalConfig(); - reloadCallback(name, ConfigurationType.GLOBAL); + /** + * Update callback, this is called whenever a path is updated in zookeeper which we are monitoring. + * + * @param client The CuratorFramework + * @param path The zookeeper path + * @param data The change. + * @throws IOException When update is impossible. + */ + public void update(CuratorFramework client, String path, byte[] data) throws IOException { + if (data.length != 0) { + String name = path.substring(path.lastIndexOf("/") + 1); + if (path.startsWith(getType().getZookeeperRoot())) { + LOG.debug("Updating the {} config: {} -> {}", () -> getType().name(), () -> name, + () -> new String(data == null ? "".getBytes(StandardCharsets.UTF_8) : data, + StandardCharsets.UTF_8)); + update(name, data); + reloadCallback(name, getType()); + } else if (ConfigurationType.GLOBAL.getZookeeperRoot().equals(path)) { + LOG.debug("Updating the global config: {}", + () -> new String(data == null ? "".getBytes(StandardCharsets.UTF_8) : data, + StandardCharsets.UTF_8)); + getConfigurations().updateGlobalConfig(data); + reloadCallback(name, ConfigurationType.GLOBAL); + } + } } - } - /** - * The ConfigurationsType that we're monitoring. - * @return The ConfigurationsType enum - */ - public abstract ConfigurationType getType(); + /** + * The simple update. This differs from the full update elsewhere in that + * this is ONLY called on updates to path to the zookeeper nodes which correspond + * to your configurations type (rather than all configurations type). + * + * @param name The path + * @param data The data updated + * @throws IOException when update is unable to happen + */ + public abstract void update(String name, byte[] data) throws IOException; - /** - * The simple update. This differs from the full update elsewhere in that - * this is ONLY called on updates to path to the zookeeper nodes which correspond - * to your configurations type (rather than all configurations type). - * @param name The path - * @param data The data updated - * @throws IOException when update is unable to happen - */ - public abstract void update(String name, byte[] data) throws IOException; + /** + * Delete callback, this is called whenever a path is deleted in zookeeper which we are monitoring. + * + * @param client The CuratorFramework + * @param path The zookeeper path + * @param data The change. + * @throws IOException When update is impossible. + */ + public void delete(CuratorFramework client, String path, byte[] data) throws IOException { + String name = path.substring(path.lastIndexOf("/") + 1); + if (path.startsWith(getType().getZookeeperRoot())) { + LOG.debug("Deleting {} {} config from internal cache", getType().name(), name); + delete(name); + reloadCallback(name, getType()); + } else if (ConfigurationType.GLOBAL.getZookeeperRoot().equals(path)) { + LOG.debug("Deleting global config from internal cache"); + getConfigurations().deleteGlobalConfig(); + reloadCallback(name, ConfigurationType.GLOBAL); + } + } - /** - * The simple delete. This differs from the full delete elsewhere in that - * this is ONLY called on deletes to path to the zookeeper nodes which correspond - * to your configurations type (rather than all configurations type). - * @param name the path - */ - public abstract void delete(String name); + /** + * The simple delete. This differs from the full delete elsewhere in that + * this is ONLY called on deletes to path to the zookeeper nodes which correspond + * to your configurations type (rather than all configurations type). + * + * @param name the path + */ + public abstract void delete(String name); - /** - * Gets the class for the {@link Configurations} type. - * - * @return The class - */ - public abstract Class getConfigurationClass(); + /** + * The ConfigurationsType that we're monitoring. + * + * @return The ConfigurationsType enum + */ + public abstract ConfigurationType getType(); - /** - * This pulls the configuration from zookeeper and updates the cache. It represents the initial state. - * Force update is called when the zookeeper cache is initialized to ensure that the caches are updated. - * @param client The ZK client interacting with configuration - */ - public abstract void forceUpdate(CuratorFramework client); + /** + * Gets the class for the {@link Configurations} type. + * + * @return The class + */ + public abstract Class getConfigurationClass(); - /** - * Create an empty {@link Configurations} object of type T. - * @return configurations - */ - public abstract T defaultConfigurations(); + /** + * This pulls the configuration from zookeeper and updates the cache. It represents the initial state. + * Force update is called when the zookeeper cache is initialized to ensure that the caches are updated. + * + * @param client The ZK client interacting with configuration + */ + public abstract void forceUpdate(CuratorFramework client); - protected void reloadCallback(String name, ConfigurationType type) { - reloadable.reloadCallback(name, type); - } + /** + * Create an empty {@link Configurations} object of type T. + * + * @return configurations + */ + public abstract T defaultConfigurations(); - protected T getConfigurations() { - return configSupplier.get(); - } + protected void reloadCallback(String name, ConfigurationType type) { + reloadable.reloadCallback(name, type); + } + + protected T getConfigurations() { + return configSupplier.get(); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/EnrichmentUpdater.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/EnrichmentUpdater.java index b1acf8e8..d9d623cb 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/EnrichmentUpdater.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/EnrichmentUpdater.java @@ -7,56 +7,58 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.zookeeper.configurations; +import java.io.IOException; +import java.util.function.Supplier; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.EnrichmentConfigurations; -import java.io.IOException; -import java.util.function.Supplier; - -public class EnrichmentUpdater extends ConfigurationsUpdater{ +public class EnrichmentUpdater extends ConfigurationsUpdater { - public EnrichmentUpdater(Reloadable reloadable, Supplier configSupplier) { - super(reloadable, configSupplier); - } + public EnrichmentUpdater(Reloadable reloadable, Supplier configSupplier) { + super(reloadable, configSupplier); + } - @Override - public Class getConfigurationClass() { - return EnrichmentConfigurations.class; - } + @Override + public Class getConfigurationClass() { + return EnrichmentConfigurations.class; + } - @Override - public void forceUpdate(CuratorFramework client) { - } + @Override + public void forceUpdate(CuratorFramework client) { + } - @Override - public EnrichmentConfigurations defaultConfigurations() { - return new EnrichmentConfigurations(); - } + @Override + public EnrichmentConfigurations defaultConfigurations() { + return new EnrichmentConfigurations(); + } - @Override - public ConfigurationType getType() { - return ConfigurationType.ENRICHMENT; - } + @Override + public ConfigurationType getType() { + return ConfigurationType.ENRICHMENT; + } - @Override - public void delete(String name) { - getConfigurations().delete(name); - } + @Override + public void delete(String name) { + getConfigurations().delete(name); + } - @Override - public void update(String name, byte[] data) throws IOException { - getConfigurations().updateSensorEnrichmentConfig(name, data); - } + @Override + public void update(String name, byte[] data) throws IOException { + getConfigurations().updateSensorEnrichmentConfig(name, data); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/IndexingUpdater.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/IndexingUpdater.java index 295b58f8..9a7e4252 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/IndexingUpdater.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/IndexingUpdater.java @@ -7,55 +7,57 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.zookeeper.configurations; +import java.io.IOException; +import java.util.function.Supplier; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.IndexingConfigurations; -import java.io.IOException; -import java.util.function.Supplier; - public class IndexingUpdater extends ConfigurationsUpdater { - public IndexingUpdater(Reloadable reloadable, Supplier configSupplier) { - super(reloadable, configSupplier); - } - - @Override - public Class getConfigurationClass() { - return IndexingConfigurations.class; - } - - @Override - public void forceUpdate(CuratorFramework client) { - } - - @Override - public IndexingConfigurations defaultConfigurations() { - return new IndexingConfigurations(); - } - - @Override - public ConfigurationType getType() { - return ConfigurationType.INDEXING; - } - - @Override - public void delete(String name) { - getConfigurations().delete(name); - } - - @Override - public void update(String name, byte[] data) throws IOException { - getConfigurations().updateSensorIndexingConfig(name, data); - } + public IndexingUpdater(Reloadable reloadable, Supplier configSupplier) { + super(reloadable, configSupplier); + } + + @Override + public Class getConfigurationClass() { + return IndexingConfigurations.class; + } + + @Override + public void forceUpdate(CuratorFramework client) { + } + + @Override + public IndexingConfigurations defaultConfigurations() { + return new IndexingConfigurations(); + } + + @Override + public ConfigurationType getType() { + return ConfigurationType.INDEXING; + } + + @Override + public void delete(String name) { + getConfigurations().delete(name); + } + + @Override + public void update(String name, byte[] data) throws IOException { + getConfigurations().updateSensorIndexingConfig(name, data); + } } diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ParserUpdater.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ParserUpdater.java index 9c0ff20a..c80c2467 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ParserUpdater.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ParserUpdater.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,15 +17,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.zookeeper.configurations; +import java.io.IOException; +import java.util.function.Supplier; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ParserConfigurations; -import java.io.IOException; -import java.util.function.Supplier; - public class ParserUpdater extends ConfigurationsUpdater { public ParserUpdater(Reloadable reloadable, Supplier configSupplier) { super(reloadable, configSupplier); diff --git a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/Reloadable.java b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/Reloadable.java index 308c74e8..412318a3 100644 --- a/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/Reloadable.java +++ b/flink-cyber/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/Reloadable.java @@ -7,21 +7,23 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.common.zookeeper.configurations; -import org.apache.metron.common.configuration.ConfigurationType; +package org.apache.metron.common.zookeeper.configurations; import java.io.Serializable; +import org.apache.metron.common.configuration.ConfigurationType; public interface Reloadable extends Serializable { - void reloadCallback(String name, ConfigurationType type); + void reloadCallback(String name, ConfigurationType type); } diff --git a/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/HashUtilsTest.java b/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/HashUtilsTest.java index 06ef635c..4741efb2 100644 --- a/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/HashUtilsTest.java +++ b/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/HashUtilsTest.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,14 +19,13 @@ */ package org.apache.metron.common.utils; -import org.json.simple.JSONObject; -import org.junit.jupiter.api.Test; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Arrays; import java.util.Collection; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.jupiter.api.Assertions.assertEquals; +import org.json.simple.JSONObject; +import org.junit.jupiter.api.Test; @SuppressWarnings({"unchecked"}) public class HashUtilsTest { diff --git a/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/LazyLoggerImplTest.java b/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/LazyLoggerImplTest.java index b0e3dc42..20f76a8e 100644 --- a/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/LazyLoggerImplTest.java +++ b/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/LazyLoggerImplTest.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,10 +20,9 @@ package org.apache.metron.common.utils; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.Marker; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import java.util.Iterator; import java.util.List; @@ -29,8 +30,10 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; - -import static org.mockito.Mockito.*; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.Marker; /** * This set of mocking tests checks that diff --git a/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/TestConstants.java b/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/TestConstants.java index 8e884011..f785f015 100644 --- a/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/TestConstants.java +++ b/flink-cyber/metron-common/src/test/java/org/apache/metron/common/utils/TestConstants.java @@ -19,12 +19,12 @@ public class TestConstants { - public final static String SAMPLE_CONFIG_PATH = "../metron-integration-test/src/main/config/zookeeper/"; - public final static String PARSER_CONFIGS_PATH = "../metron-parsing/metron-parsers/src/main/config/zookeeper/"; - public final static String PARSER_COMMON_CONFIGS_PATH = "../metron-parsing/metron-parsers-common/src/main/config/zookeeper/"; - public final static String ENRICHMENTS_CONFIGS_PATH = "../metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/"; - public final static String SAMPLE_DATA_PATH = "../metron-integration-test/src/main/sample/data/"; - public final static String SAMPLE_DATA_INPUT_PATH = "../metron-integration-test/src/main/sample/data/yaf/raw/"; - public final static String SAMPLE_DATA_PARSED_PATH = "../metron-integration-test/src/main/sample/data/test/parsed/"; - public final static String SAMPLE_DATA_INDEXED_PATH = "../metron-integration-test/src/main/sample/data/test/indexed/"; + public static final String SAMPLE_CONFIG_PATH = "../metron-integration-test/src/main/config/zookeeper/"; + public static final String PARSER_CONFIGS_PATH = "../metron-parsing/metron-parsers/src/main/config/zookeeper/"; + public static final String PARSER_COMMON_CONFIGS_PATH = "../metron-parsing/metron-parsers-common/src/main/config/zookeeper/"; + public static final String ENRICHMENTS_CONFIGS_PATH = "../metron-enrichment/metron-enrichment-common/src/main/config/zookeeper/"; + public static final String SAMPLE_DATA_PATH = "../metron-integration-test/src/main/sample/data/"; + public static final String SAMPLE_DATA_INPUT_PATH = "../metron-integration-test/src/main/sample/data/yaf/raw/"; + public static final String SAMPLE_DATA_PARSED_PATH = "../metron-integration-test/src/main/sample/data/test/parsed/"; + public static final String SAMPLE_DATA_INDEXED_PATH = "../metron-integration-test/src/main/sample/data/test/indexed/"; } diff --git a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/ColumnList.java b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/ColumnList.java index 01a5cc69..ef64e09b 100644 --- a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/ColumnList.java +++ b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/ColumnList.java @@ -26,153 +26,160 @@ /** * Represents a list of HBase columns. * + *

      * There are two types of columns, standard and counter. * + *

      * Standard columns have column family (required), qualifier (optional), * timestamp (optional), and a value (optional) values. * + *

      * Counter columns have column family (required), qualifier (optional), * and an increment (optional, but recommended) values. * + *

      * Inserts/Updates can be added via the addColumn() and addCounter() * methods. * + *

      * Original code based on the Apache Storm project. See * https://github.com/apache/storm/tree/master/external/storm-hbase. */ public class ColumnList { - public static abstract class AbstractColumn { - byte[] family, qualifier; + public abstract static class AbstractColumn { + byte[] family; + byte[] qualifier; - AbstractColumn(byte[] family, byte[] qualifier){ - this.family = family; - this.qualifier = qualifier; + AbstractColumn(byte[] family, byte[] qualifier) { + this.family = family; + this.qualifier = qualifier; + } + + public byte[] getFamily() { + return family; + } + + public byte[] getQualifier() { + return qualifier; + } } - public byte[] getFamily() { - return family; + public static class Column extends AbstractColumn { + byte[] value; + long ts = -1; + + Column(byte[] family, byte[] qualifier, long ts, byte[] value) { + super(family, qualifier); + this.value = value; + this.ts = ts; + } + + public byte[] getValue() { + return value; + } + + public long getTs() { + return ts; + } } - public byte[] getQualifier() { - return qualifier; + public static class Counter extends AbstractColumn { + long incr = 0; + + Counter(byte[] family, byte[] qualifier, long incr) { + super(family, qualifier); + this.incr = incr; + } + + public long getIncrement() { + return incr; + } } - } - public static class Column extends AbstractColumn { - byte[] value; - long ts = -1; + private ArrayList columns; + private ArrayList counters; - Column(byte[] family, byte[] qualifier, long ts, byte[] value){ - super(family, qualifier); - this.value = value; - this.ts = ts; + private ArrayList columns() { + if (this.columns == null) { + this.columns = new ArrayList<>(); + } + return this.columns; } - public byte[] getValue() { - return value; + private ArrayList counters() { + if (this.counters == null) { + this.counters = new ArrayList<>(); + } + return this.counters; } - public long getTs() { - return ts; + /** + * Add a standard HBase column. + */ + public ColumnList addColumn(byte[] family, byte[] qualifier, long ts, byte[] value) { + columns().add(new Column(family, qualifier, ts, value)); + return this; } - } - public static class Counter extends AbstractColumn { - long incr = 0; - Counter(byte[] family, byte[] qualifier, long incr){ - super(family, qualifier); - this.incr = incr; + /** + * Add a standard HBase column. + */ + public ColumnList addColumn(byte[] family, byte[] qualifier, byte[] value) { + columns().add(new Column(family, qualifier, -1, value)); + return this; } - public long getIncrement() { - return incr; + /** + * Add a standard HBase column given an instance of a class that implements + * the IColumn interface. + */ + public ColumnList addColumn(IColumn column) { + return this.addColumn(column.family(), column.qualifier(), column.timestamp(), column.value()); } - } - private ArrayList columns; - private ArrayList counters; + /** + * Add an HBase counter column. + */ + public ColumnList addCounter(byte[] family, byte[] qualifier, long incr) { + counters().add(new Counter(family, qualifier, incr)); + return this; + } + + /** + * Add an HBase counter column given an instance of a class that implements the + * ICounter interface. + */ + public ColumnList addCounter(ICounter counter) { + return this.addCounter(counter.family(), counter.qualifier(), counter.increment()); + } + + + /** + * Query to determine if we have column definitions. + */ + public boolean hasColumns() { + return this.columns != null; + } + + /** + * Query to determine if we have counter definitions. + */ + public boolean hasCounters() { + return this.counters != null; + } - private ArrayList columns(){ - if(this.columns == null){ - this.columns = new ArrayList<>(); + /** + * Get the list of column definitions. + */ + public List getColumns() { + return this.columns; } - return this.columns; - } - private ArrayList counters(){ - if(this.counters == null){ - this.counters = new ArrayList<>(); + /** + * Get the list of counter definitions. + */ + public List getCounters() { + return this.counters; } - return this.counters; - } - - /** - * Add a standard HBase column. - */ - public ColumnList addColumn(byte[] family, byte[] qualifier, long ts, byte[] value){ - columns().add(new Column(family, qualifier, ts, value)); - return this; - } - - /** - * Add a standard HBase column - */ - public ColumnList addColumn(byte[] family, byte[] qualifier, byte[] value){ - columns().add(new Column(family, qualifier, -1, value)); - return this; - } - - /** - * Add a standard HBase column given an instance of a class that implements - * the IColumn interface. - */ - public ColumnList addColumn(IColumn column){ - return this.addColumn(column.family(), column.qualifier(), column.timestamp(), column.value()); - } - - /** - * Add an HBase counter column. - */ - public ColumnList addCounter(byte[] family, byte[] qualifier, long incr){ - counters().add(new Counter(family, qualifier, incr)); - return this; - } - - /** - * Add an HBase counter column given an instance of a class that implements the - * ICounter interface. - */ - public ColumnList addCounter(ICounter counter){ - return this.addCounter(counter.family(), counter.qualifier(), counter.increment()); - } - - - /** - * Query to determine if we have column definitions. - */ - public boolean hasColumns(){ - return this.columns != null; - } - - /** - * Query to determine if we have counter definitions. - */ - public boolean hasCounters(){ - return this.counters != null; - } - - /** - * Get the list of column definitions. - */ - public List getColumns(){ - return this.columns; - } - - /** - * Get the list of counter definitions. - */ - public List getCounters(){ - return this.counters; - } } \ No newline at end of file diff --git a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/HBaseProjectionCriteria.java b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/HBaseProjectionCriteria.java index 54b2f7ec..7051072c 100644 --- a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/HBaseProjectionCriteria.java +++ b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/HBaseProjectionCriteria.java @@ -21,7 +21,6 @@ package org.apache.metron.hbase; import com.google.common.collect.Lists; - import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.util.List; @@ -31,63 +30,64 @@ * If only columnFamily is specified all columns from that family will be returned. * If a column is specified only that column from that family will be returned. * + *

      * Original code based on the Apache Storm project. See * https://github.com/apache/storm/tree/master/external/storm-hbase. */ public class HBaseProjectionCriteria implements Serializable { - private List columnFamilies; - private List columns; + private final List columnFamilies; + private final List columns; - public static class ColumnMetaData implements Serializable { + public static class ColumnMetaData implements Serializable { - private byte[] columnFamily; - private byte[] qualifier; + private final byte[] columnFamily; + private final byte[] qualifier; - public ColumnMetaData(String columnFamily, String qualifier) { - this.columnFamily = columnFamily.getBytes(StandardCharsets.UTF_8); - this.qualifier = qualifier.getBytes(StandardCharsets.UTF_8); - } + public ColumnMetaData(String columnFamily, String qualifier) { + this.columnFamily = columnFamily.getBytes(StandardCharsets.UTF_8); + this.qualifier = qualifier.getBytes(StandardCharsets.UTF_8); + } - public byte[] getColumnFamily() { - return columnFamily; - } + public byte[] getColumnFamily() { + return columnFamily; + } - public byte[] getQualifier() { - return qualifier; + public byte[] getQualifier() { + return qualifier; + } } - } - public HBaseProjectionCriteria() { - columnFamilies = Lists.newArrayList(); - columns = Lists.newArrayList(); - } + public HBaseProjectionCriteria() { + columnFamilies = Lists.newArrayList(); + columns = Lists.newArrayList(); + } - /** - * all columns from this family will be included as result of HBase lookup. - * @param columnFamily - * @return column family along with all of its columns from an HBase lookup - */ - public HBaseProjectionCriteria addColumnFamily(String columnFamily) { - this.columnFamilies.add(columnFamily.getBytes(StandardCharsets.UTF_8)); - return this; - } + /** + * all columns from this family will be included as result of HBase lookup. + * + * @return column family along with all of its columns from an HBase lookup + */ + public HBaseProjectionCriteria addColumnFamily(String columnFamily) { + this.columnFamilies.add(columnFamily.getBytes(StandardCharsets.UTF_8)); + return this; + } - /** - * Only this column from the the columnFamily will be included as result of HBase lookup. - * @param column - * @return a particular column from a column family as part of an HBase lookup - */ - public HBaseProjectionCriteria addColumn(ColumnMetaData column) { - this.columns.add(column); - return this; - } + /** + * Only this column from the the columnFamily will be included as result of HBase lookup. + * + * @return a particular column from a column family as part of an HBase lookup + */ + public HBaseProjectionCriteria addColumn(ColumnMetaData column) { + this.columns.add(column); + return this; + } - public List getColumns() { - return columns; - } + public List getColumns() { + return columns; + } - public List getColumnFamilies() { - return columnFamilies; - } + public List getColumnFamilies() { + return columnFamilies; + } } diff --git a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/HTableProvider.java b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/HTableProvider.java index 201d067a..923b9352 100644 --- a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/HTableProvider.java +++ b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/HTableProvider.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.hbase; import java.io.IOException; @@ -28,39 +31,40 @@ public class HTableProvider implements TableProvider { - private static class RetryingConnection { + private static class RetryingConnection { - private Configuration config; - private Connection conn; + private final Configuration config; + private Connection conn; - RetryingConnection(Configuration config) { - this.config = config; - } + RetryingConnection(Configuration config) { + this.config = config; + } - public Connection getUnderlying() throws IOException { - if (conn == null || conn.isClosed()) { - conn = ConnectionFactory.createConnection(config); - } - return conn; + public Connection getUnderlying() throws IOException { + if (conn == null || conn.isClosed()) { + conn = ConnectionFactory.createConnection(config); + } + return conn; + } } - } - /** - * We have to handle serialization issues with Storm via indirections. Rather than re-implement - * the interface everywhere we touch HBase, we can use a lazy initialization scheme to encapsulate - * this within the HTableProvider. This is a sort of poor man's connection pool. - */ - private static Map> connMap = new ConcurrentHashMap<>(); + /** + * We have to handle serialization issues with Storm via indirections. Rather than re-implement + * the interface everywhere we touch HBase, we can use a lazy initialization scheme to encapsulate + * this within the HTableProvider. This is a sort of poor man's connection pool. + */ + private static final Map> connMap = new ConcurrentHashMap<>(); - @Override - public Table getTable(Configuration config, String tableName) - throws IOException { - return getConnection(config).getTable(TableName.valueOf(tableName)); - } + @Override + public Table getTable(Configuration config, String tableName) + throws IOException { + return getConnection(config).getTable(TableName.valueOf(tableName)); + } - private Connection getConnection(Configuration config) throws IOException { - ThreadLocal threadLocal = connMap.computeIfAbsent(config, c -> ThreadLocal.withInitial(() -> new RetryingConnection(config))); - return threadLocal.get().getUnderlying(); - } + private Connection getConnection(Configuration config) throws IOException { + ThreadLocal threadLocal = + connMap.computeIfAbsent(config, c -> ThreadLocal.withInitial(() -> new RetryingConnection(config))); + return threadLocal.get().getUnderlying(); + } } diff --git a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/IColumn.java b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/IColumn.java index b5255e35..2b289782 100644 --- a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/IColumn.java +++ b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/IColumn.java @@ -24,12 +24,16 @@ * Interface definition for classes that support being written to HBase as * a regular column. * + *

      * Original code based on the Apache Storm project. See * https://github.com/apache/storm/tree/master/external/storm-hbase. */ public interface IColumn { - byte[] family(); - byte[] qualifier(); - byte[] value(); - long timestamp(); + byte[] family(); + + byte[] qualifier(); + + byte[] value(); + + long timestamp(); } \ No newline at end of file diff --git a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/ICounter.java b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/ICounter.java index 73b65c0c..ae366ae9 100644 --- a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/ICounter.java +++ b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/ICounter.java @@ -24,11 +24,14 @@ * Interface definition for classes that support being written to HBase as * a counter column. * + *

      * Original code based on the Apache Storm project. See * https://github.com/apache/storm/tree/master/external/storm-hbase. */ public interface ICounter { - byte[] family(); - byte[] qualifier(); - long increment(); + byte[] family(); + + byte[] qualifier(); + + long increment(); } \ No newline at end of file diff --git a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/TableConfig.java b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/TableConfig.java index de2e929a..100ae61f 100644 --- a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/TableConfig.java +++ b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/TableConfig.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.hbase; import java.io.Serializable; @@ -23,6 +26,7 @@ import java.util.Set; +@SuppressWarnings("checkstyle:JavadocParagraph") public class TableConfig implements Serializable { static final long serialVersionUID = -1L; private String tableName; @@ -63,6 +67,8 @@ public String getConnectorImpl() { } /** + * batch getter. + * * @return Whether batch mode is enabled */ public boolean isBatch() { @@ -70,49 +76,56 @@ public boolean isBatch() { } /** - * @param batch - * Whether to enable HBase's client-side write buffer. - *

      - * When enabled your bolt will store put operations locally until the - * write buffer is full, so they can be sent to HBase in a single RPC - * call. When disabled each put operation is effectively an RPC and - * is sent straight to HBase. As your bolt can process thousands of - * values per second it is recommended that the write buffer is - * enabled. - *

      - * Enabled by default + * batch setter. + * + * @param batch Whether to enable HBase's client-side write buffer. + * + *

      When enabled your bolt will store put operations locally until the + * write buffer is full, so they can be sent to HBase in a single RPC + * call. When disabled each put operation is effectively an RPC and + * is sent straight to HBase. As your bolt can process thousands of + * values per second it is recommended that the write buffer is + * enabled. + * + *

      Enabled by default */ public void setBatch(boolean batch) { this.batch = batch; } + /** - * @param writeBufferSize - * Overrides the client-side write buffer size. - *

      - * By default the write buffer size is 2 MB (2097152 bytes). If you - * are storing larger data, you may want to consider increasing this - * value to allow your bolt to efficiently group together a larger - * number of records per RPC - *

      - * Overrides the write buffer size you have set in your - * hbase-site.xml e.g. hbase.client.write.buffer + * writeBufferSize setter. + * + * @param writeBufferSize Overrides the client-side write buffer size. + * + *

      By default the write buffer size is 2 MB (2097152 bytes). If you + * are storing larger data, you may want to consider increasing this + * value to allow your bolt to efficiently group together a larger + * number of records per RPC + * + *

      Overrides the write buffer size you have set in your + * hbase-site.xml e.g. hbase.client.write.buffer */ public void setWriteBufferSize(long writeBufferSize) { this.writeBufferSize = writeBufferSize; } /** + * writeBufferSize getter. + * * @return the writeBufferSize */ public long getWriteBufferSize() { return writeBufferSize; } + /** + * columnFamilies.keySet getter + * * @return A Set of configured column families */ public Set getColumnFamilies() { return this.columnFamilies.keySet(); } - } diff --git a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/TableProvider.java b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/TableProvider.java index f5b68d69..d24ba12a 100644 --- a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/TableProvider.java +++ b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/TableProvider.java @@ -7,14 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.hbase; import java.io.IOException; @@ -26,20 +29,22 @@ public interface TableProvider extends Serializable { - Table getTable(Configuration config, String tableName) throws IOException; + Table getTable(Configuration config, String tableName) throws IOException; - /** - * Factory method that creates TableProviders. - * - * @param impl attempt to create this type of TableProvider - * @param defaultSupplier provides default implementation if impl is null - * @return New table provider - */ - static TableProvider create(String impl, Supplier defaultSupplier) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { - if(impl == null) { - return defaultSupplier.get(); + /** + * Factory method that creates TableProviders. + * + * @param impl attempt to create this type of TableProvider + * @param defaultSupplier provides default implementation if impl is null + * @return New table provider + */ + static TableProvider create(String impl, Supplier defaultSupplier) + throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, + InstantiationException { + if (impl == null) { + return defaultSupplier.get(); + } + Class clazz = (Class) Class.forName(impl); + return clazz.getConstructor().newInstance(); } - Class clazz = (Class) Class.forName(impl); - return clazz.getConstructor().newInstance(); - } } diff --git a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/client/HBaseClient.java b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/client/HBaseClient.java index 8290add4..654da2aa 100644 --- a/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/client/HBaseClient.java +++ b/flink-cyber/metron-hbase-common/src/main/java/org/apache/metron/hbase/client/HBaseClient.java @@ -47,293 +47,296 @@ */ public class HBaseClient implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** - * The batch of queued Mutations. - */ - List mutations; - - /** - * The batch of queued Gets. - */ - List gets; - - /** - * The HBase table this client interacts with. - */ - private Table table; - - /** - * New hbase client. - */ - public HBaseClient(TableProvider provider, final Configuration configuration, final String tableName) { - this.mutations = new ArrayList<>(); - this.gets = new ArrayList<>(); - try { - LOG.info("connecting to table"); - this.table = provider.getTable(configuration, tableName); - LOG.info("connected to table"); - } catch (Exception e) { - String msg = String.format("Unable to open connection to HBase for table '%s'", tableName); - LOG.error(msg, e); - throw new RuntimeException(msg, e); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * The batch of queued Mutations. + */ + List mutations; + + /** + * The batch of queued Gets. + */ + List gets; + + /** + * The HBase table this client interacts with. + */ + private final Table table; + + /** + * New hbase client. + */ + public HBaseClient(TableProvider provider, final Configuration configuration, final String tableName) { + this.mutations = new ArrayList<>(); + this.gets = new ArrayList<>(); + try { + LOG.info("connecting to table"); + this.table = provider.getTable(configuration, tableName); + LOG.info("connected to table"); + } catch (Exception e) { + String msg = String.format("Unable to open connection to HBase for table '%s'", tableName); + LOG.error(msg, e); + throw new RuntimeException(msg, e); + } } - } - - /** - * Add a Mutation such as a Put or Increment to the batch. The Mutation is only queued for - * later execution. - * - * @param rowKey The row key of the Mutation. - * @param cols The columns affected by the Mutation. - * @param durability The durability of the mutation. - */ - public void addMutation(byte[] rowKey, ColumnList cols, Durability durability) { - - if (cols.hasColumns()) { - Put put = createPut(rowKey, cols, durability); - mutations.add(put); + + /** + * Add a Mutation such as a Put or Increment to the batch. The Mutation is only queued for + * later execution. + * + * @param rowKey The row key of the Mutation. + * @param cols The columns affected by the Mutation. + * @param durability The durability of the mutation. + */ + public void addMutation(byte[] rowKey, ColumnList cols, Durability durability) { + + if (cols.hasColumns()) { + Put put = createPut(rowKey, cols, durability); + mutations.add(put); + } + + if (cols.hasCounters()) { + Increment inc = createIncrement(rowKey, cols, durability); + mutations.add(inc); + } + + if (mutations.isEmpty()) { + mutations.add(new Put(rowKey)); + } } - if (cols.hasCounters()) { - Increment inc = createIncrement(rowKey, cols, durability); - mutations.add(inc); + /** + * Adds a Mutation such as a Put or Increment with a time to live. The Mutation is only queued + * for later execution. + * + * @param rowKey The row key of the Mutation. + * @param cols The columns affected by the Mutation. + * @param durability The durability of the mutation. + * @param timeToLiveMillis The time to live in milliseconds. + */ + public void addMutation(byte[] rowKey, ColumnList cols, Durability durability, Long timeToLiveMillis) { + + if (cols.hasColumns()) { + Put put = createPut(rowKey, cols, durability, timeToLiveMillis); + mutations.add(put); + } + + if (cols.hasCounters()) { + Increment inc = createIncrement(rowKey, cols, durability, timeToLiveMillis); + mutations.add(inc); + } + + if (mutations.isEmpty()) { + Put put = new Put(rowKey); + put.setTTL(timeToLiveMillis); + mutations.add(put); + } } - if (mutations.isEmpty()) { - mutations.add(new Put(rowKey)); + /** + * Remove all queued Mutations from the batch. + */ + public void clearMutations() { + mutations.clear(); } - } - - /** - * Adds a Mutation such as a Put or Increment with a time to live. The Mutation is only queued - * for later execution. - * - * @param rowKey The row key of the Mutation. - * @param cols The columns affected by the Mutation. - * @param durability The durability of the mutation. - * @param timeToLiveMillis The time to live in milliseconds. - */ - public void addMutation(byte[] rowKey, ColumnList cols, Durability durability, Long timeToLiveMillis) { - - if (cols.hasColumns()) { - Put put = createPut(rowKey, cols, durability, timeToLiveMillis); - mutations.add(put); + + /** + * Submits all queued Mutations. + * + * @return The number of mutation submitted. + */ + public int mutate() { + int mutationCount = mutations.size(); + Object[] result = new Object[mutationCount]; + try { + table.batch(mutations, result); + mutations.clear(); + } catch (Exception e) { + String msg = String.format("'%d' HBase write(s) failed on table '%s'", mutations.size(), tableName(table)); + LOG.error(msg, e); + throw new RuntimeException(msg, e); + } + + return mutationCount; } - if (cols.hasCounters()) { - Increment inc = createIncrement(rowKey, cols, durability, timeToLiveMillis); - mutations.add(inc); + /** + * Adds a Get to the batch. + * + * @param rowKey The row key of the Get + * @param criteria Defines the columns/families that will be retrieved. + */ + public void addGet(byte[] rowKey, HBaseProjectionCriteria criteria) { + Get get = new Get(rowKey); + + if (criteria != null) { + criteria.getColumnFamilies().forEach(cf -> get.addFamily(cf)); + criteria.getColumns().forEach(col -> get.addColumn(col.getColumnFamily(), col.getQualifier())); + } + + // queue the get + this.gets.add(get); } - if (mutations.isEmpty()) { - Put put = new Put(rowKey); - put.setTTL(timeToLiveMillis); - mutations.add(put); + /** + * Clears all queued Gets from the batch. + */ + public void clearGets() { + gets.clear(); } - } - - /** - * Remove all queued Mutations from the batch. - */ - public void clearMutations() { - mutations.clear(); - } - - /** - * Submits all queued Mutations. - * @return The number of mutation submitted. - */ - public int mutate() { - int mutationCount = mutations.size(); - Object[] result = new Object[mutationCount]; - try { - table.batch(mutations, result); - mutations.clear(); - } catch (Exception e) { - String msg = String.format("'%d' HBase write(s) failed on table '%s'", mutations.size(), tableName(table)); - LOG.error(msg, e); - throw new RuntimeException(msg, e); + + /** + * Submit all queued Gets. + * + * @return The Result of each queued Get. + */ + public Result[] getAll() { + try { + Result[] results = table.get(gets); + gets.clear(); + return results; + + } catch (Exception e) { + String msg = String.format("'%d' HBase read(s) failed on table '%s'", gets.size(), tableName(table)); + LOG.error(msg, e); + throw new RuntimeException(msg, e); + } } - return mutationCount; - } - - /** - * Adds a Get to the batch. - * - * @param rowKey The row key of the Get - * @param criteria Defines the columns/families that will be retrieved. - */ - public void addGet(byte[] rowKey, HBaseProjectionCriteria criteria) { - Get get = new Get(rowKey); - - if (criteria != null) { - criteria.getColumnFamilies().forEach(cf -> get.addFamily(cf)); - criteria.getColumns().forEach(col -> get.addColumn(col.getColumnFamily(), col.getQualifier())); + /** + * Close the table. + */ + @Override + public void close() throws IOException { + if (table != null) { + table.close(); + } } - // queue the get - this.gets.add(get); - } - - /** - * Clears all queued Gets from the batch. - */ - public void clearGets() { - gets.clear(); - } - - /** - * Submit all queued Gets. - * - * @return The Result of each queued Get. - */ - public Result[] getAll() { - try { - Result[] results = table.get(gets); - gets.clear(); - return results; - - } catch (Exception e) { - String msg = String.format("'%d' HBase read(s) failed on table '%s'", gets.size(), tableName(table)); - LOG.error(msg, e); - throw new RuntimeException(msg, e); + /** + * Creates an HBase Put. + * + * @param rowKey The row key. + * @param cols The columns to put. + * @param durability The durability of the put. + */ + private Put createPut(byte[] rowKey, ColumnList cols, Durability durability) { + Put put = new Put(rowKey); + put.setDurability(durability); + addColumns(cols, put); + return put; } - } - - /** - * Close the table. - */ - @Override - public void close() throws IOException { - if(table != null) { - table.close(); + + /** + * Creates an HBase Put. + * + * @param rowKey The row key. + * @param cols The columns to put. + * @param durability The durability of the put. + * @param timeToLiveMillis The TTL in milliseconds. + */ + private Put createPut(byte[] rowKey, ColumnList cols, Durability durability, long timeToLiveMillis) { + Put put = new Put(rowKey); + put.setDurability(durability); + put.setTTL(timeToLiveMillis); + addColumns(cols, put); + return put; } - } - - /** - * Creates an HBase Put. - * - * @param rowKey The row key. - * @param cols The columns to put. - * @param durability The durability of the put. - */ - private Put createPut(byte[] rowKey, ColumnList cols, Durability durability) { - Put put = new Put(rowKey); - put.setDurability(durability); - addColumns(cols, put); - return put; - } - - /** - * Creates an HBase Put. - * - * @param rowKey The row key. - * @param cols The columns to put. - * @param durability The durability of the put. - * @param timeToLiveMillis The TTL in milliseconds. - */ - private Put createPut(byte[] rowKey, ColumnList cols, Durability durability, long timeToLiveMillis) { - Put put = new Put(rowKey); - put.setDurability(durability); - put.setTTL(timeToLiveMillis); - addColumns(cols, put); - return put; - } - - /** - * Adds the columns to the Put - * - * @param cols The columns to add. - * @param put The Put. - */ - private void addColumns(ColumnList cols, Put put) { - for (ColumnList.Column col : cols.getColumns()) { - - if (col.getTs() > 0) { - put.addColumn(col.getFamily(), col.getQualifier(), col.getTs(), col.getValue()); - - } else { - put.addColumn(col.getFamily(), col.getQualifier(), col.getValue()); - } + + /** + * Adds the columns to the Put. + * + * @param cols The columns to add. + * @param put The Put. + */ + private void addColumns(ColumnList cols, Put put) { + for (ColumnList.Column col : cols.getColumns()) { + + if (col.getTs() > 0) { + put.addColumn(col.getFamily(), col.getQualifier(), col.getTs(), col.getValue()); + + } else { + put.addColumn(col.getFamily(), col.getQualifier(), col.getValue()); + } + } } - } - - /** - * Creates an HBase Increment for a counter. - * - * @param rowKey The row key. - * @param cols The columns to include. - * @param durability The durability of the increment. - */ - private Increment createIncrement(byte[] rowKey, ColumnList cols, Durability durability) { - Increment inc = new Increment(rowKey); - inc.setDurability(durability); - cols.getCounters().forEach(cnt -> inc.addColumn(cnt.getFamily(), cnt.getQualifier(), cnt.getIncrement())); - return inc; - } - - /** - * Creates an HBase Increment for a counter. - * - * @param rowKey The row key. - * @param cols The columns to include. - * @param durability The durability of the increment. - */ - private Increment createIncrement(byte[] rowKey, ColumnList cols, Durability durability, long timeToLiveMillis) { - Increment inc = new Increment(rowKey); - inc.setDurability(durability); - inc.setTTL(timeToLiveMillis); - cols.getCounters().forEach(cnt -> inc.addColumn(cnt.getFamily(), cnt.getQualifier(), cnt.getIncrement())); - return inc; - } - - /** - * Returns the name of the HBase table. - *

      Attempts to avoid any null pointers that might be encountered along the way. - * @param table The table to retrieve the name of. - * @return The name of the table - */ - private static String tableName(Table table) { - String tableName = "null"; - if(table != null) { - if(table.getName() != null) { - tableName = table.getName().getNameAsString(); - } + + /** + * Creates an HBase Increment for a counter. + * + * @param rowKey The row key. + * @param cols The columns to include. + * @param durability The durability of the increment. + */ + private Increment createIncrement(byte[] rowKey, ColumnList cols, Durability durability) { + Increment inc = new Increment(rowKey); + inc.setDurability(durability); + cols.getCounters().forEach(cnt -> inc.addColumn(cnt.getFamily(), cnt.getQualifier(), cnt.getIncrement())); + return inc; } - return tableName; - } - - /** - * Puts a record into the configured HBase table synchronously (not batched). - */ - public void put(String rowKey, String columnFamily, String columnQualifier, String value) - throws IOException { - Put put = new Put(Bytes.toBytes(rowKey)); - put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columnQualifier), - Bytes.toBytes(value)); - table.put(put); - } - - /** - * Scans an entire table returning all row keys as a List of Strings. - * - *

      - * **WARNING**: Do not use this method unless you're absolutely crystal clear about the performance - * impact. Doing full table scans in HBase can adversely impact performance. - * - * @return List of all row keys as Strings for this table. - */ - public List readRecords() throws IOException { - Scan scan = new Scan(); - ResultScanner scanner = table.getScanner(scan); - List rows = new ArrayList<>(); - for (Result r = scanner.next(); r != null; r = scanner.next()) { - rows.add(Bytes.toString(r.getRow())); + + /** + * Creates an HBase Increment for a counter. + * + * @param rowKey The row key. + * @param cols The columns to include. + * @param durability The durability of the increment. + */ + private Increment createIncrement(byte[] rowKey, ColumnList cols, Durability durability, long timeToLiveMillis) { + Increment inc = new Increment(rowKey); + inc.setDurability(durability); + inc.setTTL(timeToLiveMillis); + cols.getCounters().forEach(cnt -> inc.addColumn(cnt.getFamily(), cnt.getQualifier(), cnt.getIncrement())); + return inc; + } + + /** + * Returns the name of the HBase table. + * + *

      Attempts to avoid any null pointers that might be encountered along the way. + * + * @param table The table to retrieve the name of. + * @return The name of the table + */ + private static String tableName(Table table) { + String tableName = "null"; + if (table != null) { + if (table.getName() != null) { + tableName = table.getName().getNameAsString(); + } + } + return tableName; + } + + /** + * Puts a record into the configured HBase table synchronously (not batched). + */ + public void put(String rowKey, String columnFamily, String columnQualifier, String value) + throws IOException { + Put put = new Put(Bytes.toBytes(rowKey)); + put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columnQualifier), + Bytes.toBytes(value)); + table.put(put); + } + + /** + * Scans an entire table returning all row keys as a List of Strings. + * + *

      + * **WARNING**: Do not use this method unless you're absolutely crystal clear about the performance + * impact. Doing full table scans in HBase can adversely impact performance. + * + * @return List of all row keys as Strings for this table. + */ + public List readRecords() throws IOException { + Scan scan = new Scan(); + ResultScanner scanner = table.getScanner(scan); + List rows = new ArrayList<>(); + for (Result r = scanner.next(); r != null; r = scanner.next()) { + rows.add(Bytes.toString(r.getRow())); + } + return rows; } - return rows; - } } diff --git a/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java b/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java index 6e17ae48..409c7356 100644 --- a/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java +++ b/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,7 +25,6 @@ import com.google.protobuf.Message; import com.google.protobuf.Service; import com.google.protobuf.ServiceException; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -37,7 +38,6 @@ import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -45,7 +45,21 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.filter.CompareFilter; @@ -266,9 +280,9 @@ public Result get(Get get) throws IOException { for (byte[] qualifier : qualifiers) { if (qualifier == null) qualifier = "".getBytes(StandardCharsets.UTF_8); - if (!data.get(row).containsKey(family) || - !data.get(row).get(family).containsKey(qualifier) || - data.get(row).get(family).get(qualifier).isEmpty()) + if (!data.get(row).containsKey(family) + || !data.get(row).get(family).containsKey(qualifier) + || data.get(row).get(family).get(qualifier).isEmpty()) continue; Map.Entry timestampAndValue = data.get(row).get(family).get(qualifier).lastEntry(); kvs.add(new KeyValue(row, family, qualifier, timestampAndValue.getKey(), timestampAndValue.getValue())); @@ -330,15 +344,15 @@ public ResultScanner getScanner(Scan scan) throws IOException { // if row is equal to startRow emit it. When startRow (inclusive) and // stopRow (exclusive) is the same, it should not be excluded which would // happen w/o this control. - if (st != null && st.length > 0 && - Bytes.BYTES_COMPARATOR.compare(st, row) != 0) { + if (st != null && st.length > 0 + && Bytes.BYTES_COMPARATOR.compare(st, row) != 0) { // if row is before startRow do not emit, pass to next row - if (st != null && st.length > 0 && - Bytes.BYTES_COMPARATOR.compare(st, row) > 0) + if (st != null && st.length > 0 + && Bytes.BYTES_COMPARATOR.compare(st, row) > 0) continue; // if row is equal to stopRow or after it do not emit, stop iteration - if (sp != null && sp.length > 0 && - Bytes.BYTES_COMPARATOR.compare(sp, row) <= 0) + if (sp != null && sp.length > 0 + && Bytes.BYTES_COMPARATOR.compare(sp, row) <= 0) break; } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/Application.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/Application.java index 518e0585..48bf5f65 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/Application.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/Application.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,19 +31,19 @@ @SpringBootApplication public class Application { - public static void main(String[] args) { - SpringApplication.run(Application.class, args); - } + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } - /** - * This method added to get around an incompatibility between Spring Hateoas and Springfox Swagger - * with spring-plugin-core. - * https://github.com/springfox/springfox/issues/2932 - */ - @Bean - public PluginRegistry discoverers( - OrderAwarePluginRegistry relProviderPluginRegistry) { - return relProviderPluginRegistry; - } + /** + * This method added to get around an incompatibility between Spring Hateoas and Springfox Swagger + * with spring-plugin-core. + * https://github.com/springfox/springfox/issues/2932 + */ + @Bean + public PluginRegistry discoverers( + OrderAwarePluginRegistry relProviderPluginRegistry) { + return relProviderPluginRegistry; + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java index 4f350756..22729dec 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,13 +21,12 @@ package com.cloudera.parserchains.queryservice; import com.cloudera.parserchains.queryservice.config.AppProperties; +import java.nio.file.Files; +import java.nio.file.Paths; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; -import java.nio.file.Files; -import java.nio.file.Paths; - /** * This class is being used to do initialization for the REST application. * e.g. creating folder hierarchies needed by the application. @@ -34,17 +35,17 @@ @Slf4j public class StartupComponent implements CommandLineRunner { - private final AppProperties appProperties; + private final AppProperties appProperties; - public StartupComponent(AppProperties appProperties) { - this.appProperties = appProperties; - } + public StartupComponent(AppProperties appProperties) { + this.appProperties = appProperties; + } - @Override - public void run(String... args) throws Exception { - log.info("Running startup stuff"); - log.info("Creating the config dir: {}", appProperties.getConfigPath()); - Files.createDirectories(Paths.get(appProperties.getConfigPath())); - log.info("Done creating the config dir"); - } + @Override + public void run(String... args) throws Exception { + log.info("Running startup stuff"); + log.info("Creating the config dir: {}", appProperties.getConfigPath()); + Files.createDirectories(Paths.get(appProperties.getConfigPath())); + log.info("Done creating the config dir"); + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/ApplicationConstants.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/ApplicationConstants.java index d9e7e3fe..196324b6 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/ApplicationConstants.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/ApplicationConstants.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,40 +25,40 @@ public class ApplicationConstants { - public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; - // root - public static final String API_BASE_URL = "/api/v1"; + public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; + // root + public static final String API_BASE_URL = "/api/v1"; - // parser-config endpoint constants - public static final String PARSER_CONFIG_BASE_URL = API_BASE_URL + "/parserconfig"; - // pipeline controller constants - public static final String PIPELINE_BASE_URL = API_BASE_URL + "/pipeline"; - // Endpoint names - public static final String API_CHAINS = "/chains"; + // parser-config endpoint constants + public static final String PARSER_CONFIG_BASE_URL = API_BASE_URL + "/parserconfig"; + // pipeline controller constants + public static final String PIPELINE_BASE_URL = API_BASE_URL + "/pipeline"; + // Endpoint names + public static final String API_CHAINS = "/chains"; - public static final String API_CLUSTERS = "/clusters"; - public static final String API_JOBS = API_CLUSTERS + "/{clusterId}/jobs"; - public static final String API_INDEXING = "/indexing"; - public static final String API_PARSER_FORM_CONFIG = "/parser-form-configuration"; - public static final String API_PARSER_TYPES = "/parser-types"; - public static final String API_PARSER_TEST = "/tests"; - public static final String API_PARSER_TEST_SAMPLES = API_PARSER_TEST + "/samples"; - // URLs - public static final String API_CHAINS_URL = PARSER_CONFIG_BASE_URL + API_CHAINS; - public static final String API_CHAINS_CREATE_URL = API_CHAINS_URL; - public static final String API_CHAINS_READ_URL = API_CHAINS_URL + "/{id}"; - public static final String API_CHAINS_UPDATE_URL = API_CHAINS_READ_URL; - public static final String API_CHAINS_DELETE_URL = API_CHAINS_READ_URL; - public static final String API_PARSER_FORM_CONFIG_URL = PARSER_CONFIG_BASE_URL + API_PARSER_FORM_CONFIG; - public static final String API_PARSER_TYPES_URL = PARSER_CONFIG_BASE_URL + API_PARSER_TYPES; - public static final String API_PARSER_TEST_URL = PARSER_CONFIG_BASE_URL + API_PARSER_TEST; + public static final String API_CLUSTERS = "/clusters"; + public static final String API_JOBS = API_CLUSTERS + "/{clusterId}/jobs"; + public static final String API_INDEXING = "/indexing"; + public static final String API_PARSER_FORM_CONFIG = "/parser-form-configuration"; + public static final String API_PARSER_TYPES = "/parser-types"; + public static final String API_PARSER_TEST = "/tests"; + public static final String API_PARSER_TEST_SAMPLES = API_PARSER_TEST + "/samples"; + // URLs + public static final String API_CHAINS_URL = PARSER_CONFIG_BASE_URL + API_CHAINS; + public static final String API_CHAINS_CREATE_URL = API_CHAINS_URL; + public static final String API_CHAINS_READ_URL = API_CHAINS_URL + "/{id}"; + public static final String API_CHAINS_UPDATE_URL = API_CHAINS_READ_URL; + public static final String API_CHAINS_DELETE_URL = API_CHAINS_READ_URL; + public static final String API_PARSER_FORM_CONFIG_URL = PARSER_CONFIG_BASE_URL + API_PARSER_FORM_CONFIG; + public static final String API_PARSER_TYPES_URL = PARSER_CONFIG_BASE_URL + API_PARSER_TYPES; + public static final String API_PARSER_TEST_URL = PARSER_CONFIG_BASE_URL + API_PARSER_TEST; - //Param names - public static final String TEST_RUN_PARAM = "testRun"; - public static final String PIPELINE_NAME_PARAM = "pipelineName"; - public static final String CHAIN_PARAM = "chain"; - public static final String ID_PARAM = "id"; - public static final String BODY_PARAM = "body"; - public static final String HEADERS_PARAM = "headers"; - public static final String STATUS_PARAM = "status"; + //Param names + public static final String TEST_RUN_PARAM = "testRun"; + public static final String PIPELINE_NAME_PARAM = "pipelineName"; + public static final String CHAIN_PARAM = "chain"; + public static final String ID_PARAM = "id"; + public static final String BODY_PARAM = "body"; + public static final String HEADERS_PARAM = "headers"; + public static final String STATUS_PARAM = "status"; } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedAllClusterReponseException.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedAllClusterReponseException.java index 31c4796c..b2772d55 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedAllClusterReponseException.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedAllClusterReponseException.java @@ -1,12 +1,11 @@ package com.cloudera.parserchains.queryservice.common.exception; import com.cloudera.service.common.response.ResponseBody; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; -import java.util.List; - @Getter @Setter @AllArgsConstructor diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedJobAction.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedJobAction.java index bc2f8eb0..958ecfd0 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedJobAction.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/FailedJobAction.java @@ -6,18 +6,21 @@ @ResponseStatus(value = HttpStatus.BAD_REQUEST) public class FailedJobAction extends RuntimeException { public FailedJobAction() { - this(null,null); + this(null, null); } + public FailedJobAction(String message) { - this(message,null); + this(message, null); } + public FailedJobAction(Throwable cause) { - this(null,cause); + this(null, cause); } + public FailedJobAction(String message, Throwable cause) { - super(message); - if (cause != null) { - this.initCause(cause); - } + super(message); + if (cause != null) { + this.initCause(cause); + } } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/JobValidationException.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/JobValidationException.java index cf9ea527..3bf553a5 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/JobValidationException.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/JobValidationException.java @@ -6,18 +6,21 @@ @ResponseStatus(value = HttpStatus.BAD_REQUEST) public class JobValidationException extends RuntimeException { public JobValidationException() { - this(null,null); + this(null, null); } + public JobValidationException(String message) { - this(message,null); + this(message, null); } + public JobValidationException(Throwable cause) { - this(null,cause); + this(null, cause); } + public JobValidationException(String message, Throwable cause) { - super(message); - if (cause != null) { - this.initCause(cause); - } + super(message); + if (cause != null) { + this.initCause(cause); + } } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaClusterNotFound.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaClusterNotFound.java index 74f1ac32..aa855101 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaClusterNotFound.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaClusterNotFound.java @@ -6,18 +6,21 @@ @ResponseStatus(value = HttpStatus.NOT_FOUND) public class KafkaClusterNotFound extends RuntimeException { public KafkaClusterNotFound() { - this(null,null); + this(null, null); } + public KafkaClusterNotFound(String message) { - this(message,null); + this(message, null); } + public KafkaClusterNotFound(Throwable cause) { - this(null,cause); + this(null, cause); } + public KafkaClusterNotFound(String message, Throwable cause) { - super(message); - if (cause != null) { - this.initCause(cause); - } + super(message); + if (cause != null) { + this.initCause(cause); + } } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaException.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaException.java index f37b558b..d0cf61f4 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaException.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/KafkaException.java @@ -6,18 +6,21 @@ @ResponseStatus(value = HttpStatus.BAD_REQUEST) public class KafkaException extends RuntimeException { public KafkaException() { - this(null,null); + this(null, null); } + public KafkaException(String message) { - this(message,null); + this(message, null); } + public KafkaException(Throwable cause) { - this(null,cause); + this(null, cause); } + public KafkaException(String message, Throwable cause) { - super(message); - if (cause != null) { - this.initCause(cause); - } + super(message); + if (cause != null) { + this.initCause(cause); + } } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/CollectionsUtils.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/CollectionsUtils.java index 5218cd02..5ac53ac0 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/CollectionsUtils.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/CollectionsUtils.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,28 +25,28 @@ public class CollectionsUtils { - public static Map toMap(K key1, V value1) { - Map map = new HashMap(); - map.put(key1, value1); - return map; - } + public static Map toMap(K key1, V value1) { + Map map = new HashMap(); + map.put(key1, value1); + return map; + } - public static Map toMap(K key1, V value1, K key2, V value2) { - Map map = toMap(key1, value1); - map.put(key2, value2); - return map; - } + public static Map toMap(K key1, V value1, K key2, V value2) { + Map map = toMap(key1, value1); + map.put(key2, value2); + return map; + } - public static Map toMap(K key1, V value1, K key2, V value2, K key3, V value3) { - Map map = toMap(key1, value1, key2, value2); - map.put(key3, value3); - return map; - } + public static Map toMap(K key1, V value1, K key2, V value2, K key3, V value3) { + Map map = toMap(key1, value1, key2, value2); + map.put(key3, value3); + return map; + } - public static Map toMap(K key1, V value1, K key2, V value2, K key3, V value3, K key4, - V value4) { - Map map = toMap(key1, value1, key2, value2, key3, value3); - map.put(key4, value4); - return map; - } + public static Map toMap(K key1, V value1, K key2, V value2, K key3, V value3, K key4, + V value4) { + Map map = toMap(key1, value1, key2, value2, key3, value3); + map.put(key4, value4); + return map; + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/IDGenerator.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/IDGenerator.java index 408b85d0..a327b396 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/IDGenerator.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/IDGenerator.java @@ -14,5 +14,5 @@ public interface IDGenerator { - T incrementAndGet(); + T incrementAndGet(); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/UniqueIDGenerator.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/UniqueIDGenerator.java index 77526cd4..2ec52493 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/UniqueIDGenerator.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/UniqueIDGenerator.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,42 +29,42 @@ public class UniqueIDGenerator implements IDGenerator { - public static final String ID_FILENAME = "idgenerator"; - private final long seed; - private final Path idFile; - private final Path idGenSourceDir; + public static final String ID_FILENAME = "idgenerator"; + private final long seed; + private final Path idFile; + private final Path idGenSourceDir; - public UniqueIDGenerator(Path idGenSourceDir) { - this(idGenSourceDir, 0L); - } + public UniqueIDGenerator(Path idGenSourceDir) { + this(idGenSourceDir, 0L); + } - public UniqueIDGenerator(Path idGenSourceDir, long seed) { - this.idGenSourceDir = idGenSourceDir; - this.idFile = idGenSourceDir.resolve(Paths.get(ID_FILENAME)); - this.seed = seed; - } + public UniqueIDGenerator(Path idGenSourceDir, long seed) { + this.idGenSourceDir = idGenSourceDir; + this.idFile = idGenSourceDir.resolve(Paths.get(ID_FILENAME)); + this.seed = seed; + } - @Override - public Long incrementAndGet() { - synchronized (UniqueIDGenerator.class) { - if (Files.exists(idFile)) { - try { - List lines = Files.readAllLines(idFile); - long id = Long.parseLong(lines.get(0)); - Files.write(idFile, Long.toString(++id).getBytes(), StandardOpenOption.TRUNCATE_EXISTING); - return id; - } catch (IOException e) { - throw new RuntimeException("Unable to find and increment id", e); - } - } else { - try { - long id = seed; - Files.write(idFile, Long.toString(++id).getBytes()); - return id; - } catch (IOException e) { - throw new RuntimeException("Unable to find and increment id", e); + @Override + public Long incrementAndGet() { + synchronized (UniqueIDGenerator.class) { + if (Files.exists(idFile)) { + try { + List lines = Files.readAllLines(idFile); + long id = Long.parseLong(lines.get(0)); + Files.write(idFile, Long.toString(++id).getBytes(), StandardOpenOption.TRUNCATE_EXISTING); + return id; + } catch (IOException e) { + throw new RuntimeException("Unable to find and increment id", e); + } + } else { + try { + long id = seed; + Files.write(idFile, Long.toString(++id).getBytes()); + return id; + } catch (IOException e) { + throw new RuntimeException("Unable to find and increment id", e); + } + } } - } } - } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AngularForwardingConfig.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AngularForwardingConfig.java index 6236b77d..55110a41 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AngularForwardingConfig.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AngularForwardingConfig.java @@ -12,6 +12,7 @@ package com.cloudera.parserchains.queryservice.config; +import java.io.IOException; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.ClassPathResource; @@ -20,8 +21,6 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import org.springframework.web.servlet.resource.PathResourceResolver; -import java.io.IOException; - @Configuration public class AngularForwardingConfig implements WebMvcConfigurer { @@ -39,8 +38,8 @@ protected Resource getResource(String resourcePath, Resource location) throws IO Resource requestedResource = location.createRelative(resourcePath); return requestedResource.exists() && requestedResource.isReadable() - ? requestedResource - : new ClassPathResource("/static/index.html"); + ? requestedResource + : new ClassPathResource("/static/index.html"); } }); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AppProperties.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AppProperties.java index ce6227c9..5c426920 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AppProperties.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AppProperties.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,58 +24,58 @@ import org.springframework.core.env.Environment; /** - * Application properties should be in lowercase! + * Application properties should be in lowercase!. */ public class AppProperties { - private enum Options implements ConfigOption { - CONFIG_PATH("."), - PIPELINES_PATH("../../.."), - SAMPLE_FOLDER_PATH("../samples/"), - INDEX_PATH("../../index/conf/mapping-config.json"); + private enum Options implements ConfigOption { + CONFIG_PATH("."), + PIPELINES_PATH("../../.."), + SAMPLE_FOLDER_PATH("../samples/"), + INDEX_PATH("../../index/conf/mapping-config.json"); - @Override - public String get(Environment source) { - return source.getProperty(optionKey, defaultValue); - } + @Override + public String get(Environment source) { + return source.getProperty(optionKey, defaultValue); + } - private String optionKey; - private String defaultValue; + private final String optionKey; + private final String defaultValue; - Options(String defaultValue) { - this.optionKey = normalizeProperty(name()); - this.defaultValue = defaultValue; - } + Options(String defaultValue) { + this.optionKey = normalizeProperty(name()); + this.defaultValue = defaultValue; + } - private String normalizeProperty(String name) { - return name.replace('_', '.').toLowerCase(); + private String normalizeProperty(String name) { + return name.replace('_', '.').toLowerCase(); + } } - } - @Autowired - private Environment environment; + @Autowired + private Environment environment; - public AppProperties() { - } + public AppProperties() { + } - @Autowired - AppProperties(Environment environment) { - this.environment = environment; - } + @Autowired + AppProperties(Environment environment) { + this.environment = environment; + } - public String getConfigPath() { - return Options.CONFIG_PATH.get(environment); - } + public String getConfigPath() { + return Options.CONFIG_PATH.get(environment); + } - public String getPipelinesPath() { - return Options.PIPELINES_PATH.get(environment); - } + public String getPipelinesPath() { + return Options.PIPELINES_PATH.get(environment); + } - public String getIndexPath() { - return Options.INDEX_PATH.get(environment); - } + public String getIndexPath() { + return Options.INDEX_PATH.get(environment); + } - public String getSampleFolderPath() { - return Options.SAMPLE_FOLDER_PATH.get(environment); - } + public String getSampleFolderPath() { + return Options.SAMPLE_FOLDER_PATH.get(environment); + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/CachingConfig.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/CachingConfig.java index 1f34b9c5..64ce9713 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/CachingConfig.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/CachingConfig.java @@ -13,8 +13,9 @@ package com.cloudera.parserchains.queryservice.config; import com.google.common.cache.CacheBuilder; -import lombok.RequiredArgsConstructor; -import org.springframework.beans.factory.annotation.Value; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; @@ -22,16 +23,7 @@ import org.springframework.cache.support.SimpleCacheManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; -import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -import org.springframework.web.servlet.resource.PathResourceResolver; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; @EnableCaching @Configuration @@ -48,9 +40,9 @@ public CacheManager cacheManager() { private Cache createExpiringCache(String name, long expireAfter, TimeUnit timeUnit) { return new ConcurrentMapCache(name, - CacheBuilder.newBuilder() - .expireAfterWrite(expireAfter, timeUnit) - .build() - .asMap(), true); + CacheBuilder.newBuilder() + .expireAfterWrite(expireAfter, timeUnit) + .build() + .asMap(), true); } } \ No newline at end of file diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/ConfigOption.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/ConfigOption.java index a2d902f6..1191c686 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/ConfigOption.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/ConfigOption.java @@ -14,6 +14,6 @@ public interface ConfigOption { - T get(V source); + T get(V source); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/MainConfiguration.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/MainConfiguration.java index 70d64833..6b4960c4 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/MainConfiguration.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/MainConfiguration.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,43 +41,43 @@ @Configuration public class MainConfiguration { - @Bean - public com.fasterxml.jackson.databind.ObjectMapper objectMapper() { - return JSONUtils.INSTANCE.getMapper(); - } + @Bean + public com.fasterxml.jackson.databind.ObjectMapper objectMapper() { + return JSONUtils.INSTANCE.getMapper(); + } - @Bean - public AppProperties appProperties() { - return new AppProperties(); - } + @Bean + public AppProperties appProperties() { + return new AppProperties(); + } - @Bean - public IDGenerator idGenerator() { - return new UUIDGenerator(); - } + @Bean + public IDGenerator idGenerator() { + return new UUIDGenerator(); + } - @Bean - public ParserCatalog parserCatalog() { - return new ClassIndexParserCatalog(); - } + @Bean + public ParserCatalog parserCatalog() { + return new ClassIndexParserCatalog(); + } - @Bean - public ParserBuilder parserBuilder() { - return new ReflectiveParserBuilder(); - } + @Bean + public ParserBuilder parserBuilder() { + return new ReflectiveParserBuilder(); + } - @Bean - public ObjectMapper mapper() { - return new ParserSummaryMapper(); - } + @Bean + public ObjectMapper mapper() { + return new ParserSummaryMapper(); + } - @Bean - public ChainRunner chainRunner() { - return new DefaultChainRunner(); - } + @Bean + public ChainRunner chainRunner() { + return new DefaultChainRunner(); + } - @Bean - public ChainBuilder chainBuilder() { - return new DefaultChainBuilder(parserBuilder(), parserCatalog()); - } + @Bean + public ChainBuilder chainBuilder() { + return new DefaultChainBuilder(parserBuilder(), parserCatalog()); + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/ClouderaKafkaProperties.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/ClouderaKafkaProperties.java index 0f54af59..14274547 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/ClouderaKafkaProperties.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/ClouderaKafkaProperties.java @@ -8,8 +8,8 @@ @Setter public class ClouderaKafkaProperties extends KafkaProperties { - private String replyTopic; + private String replyTopic; - private String requestTopic; + private String requestTopic; } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/ClouderaReplyingKafkaTemplate.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/ClouderaReplyingKafkaTemplate.java index 0a4ab211..88377b2e 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/ClouderaReplyingKafkaTemplate.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/ClouderaReplyingKafkaTemplate.java @@ -5,30 +5,31 @@ import org.springframework.kafka.listener.GenericMessageListenerContainer; import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; -public class ClouderaReplyingKafkaTemplate extends ReplyingKafkaTemplate { +public class ClouderaReplyingKafkaTemplate extends ReplyingKafkaTemplate { - @Getter - private String requestTopic; + @Getter + private String requestTopic; - public ClouderaReplyingKafkaTemplate(ProducerFactory producerFactory, - GenericMessageListenerContainer replyContainer, String requestTopic) { - this(producerFactory, replyContainer); - this.requestTopic = requestTopic; - } + public ClouderaReplyingKafkaTemplate(ProducerFactory producerFactory, + GenericMessageListenerContainer replyContainer, String requestTopic) { + this(producerFactory, replyContainer); + this.requestTopic = requestTopic; + } - public ClouderaReplyingKafkaTemplate(ProducerFactory producerFactory, - GenericMessageListenerContainer replyContainer, String requestTopic, boolean autoFlush) { - this(producerFactory, replyContainer, autoFlush); - this.requestTopic = requestTopic; - } + public ClouderaReplyingKafkaTemplate(ProducerFactory producerFactory, + GenericMessageListenerContainer replyContainer, String requestTopic, + boolean autoFlush) { + this(producerFactory, replyContainer, autoFlush); + this.requestTopic = requestTopic; + } - private ClouderaReplyingKafkaTemplate(ProducerFactory producerFactory, - GenericMessageListenerContainer replyContainer) { - super(producerFactory, replyContainer); - } + private ClouderaReplyingKafkaTemplate(ProducerFactory producerFactory, + GenericMessageListenerContainer replyContainer) { + super(producerFactory, replyContainer); + } - private ClouderaReplyingKafkaTemplate(ProducerFactory producerFactory, - GenericMessageListenerContainer replyContainer, boolean autoFlush) { - super(producerFactory, replyContainer, autoFlush); - } + private ClouderaReplyingKafkaTemplate(ProducerFactory producerFactory, + GenericMessageListenerContainer replyContainer, boolean autoFlush) { + super(producerFactory, replyContainer, autoFlush); + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java index 1f5bd907..b7b95b45 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java @@ -3,13 +3,16 @@ import com.cloudera.service.common.config.kafka.ClouderaKafkaProperties; import com.cloudera.service.common.request.RequestBody; import com.cloudera.service.common.response.ResponseBody; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -27,13 +30,6 @@ import org.springframework.kafka.support.serializer.JsonDeserializer; import org.springframework.kafka.support.serializer.JsonSerializer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - @Slf4j @EnableKafka @Configuration @@ -41,82 +37,91 @@ @EnableConfigurationProperties({ClouderaKafkaProperties.class}) public class KafkaConfig { - /*** - * Provides the default kafka properties for the consumers - * @return Default kafka properties for the consumers - */ - @Bean - @Primary - public ClouderaKafkaProperties kafkaProperties() { - return new ClouderaKafkaProperties(); - } - - /*** - * Provides a map with key=clusterId and value=ClouderaKafkaProperties - * @return Map with key of clusterId and value - kafkaProperties. This map is a mapping between clusterId and connection details for that cluster - */ - @Bean(name = "kafka-external-cluster-map") - @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) - @ConfigurationProperties("spring.kafka.external-clusters") - public Map replyKafkaPropertiesMap() { - return new HashMap<>(); - } - - /** - * Map of KafkaTemplates that with key=clusterId, which allows to send requests over Kafka to different clusters and - * get responses. - * - * @param replyKafkaPropertiesMap - replyKafkaPropertiesMap bean with properties for each cluster, injected by Spring - * @return Map with key of clusterId and value - kafkaTemplate that allows to send requests over Kafka to different clusters and get - * responses. - */ - @Bean(name = "kafkaTemplatePool") - @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) - public Map> kafkaTemplatePool( - @Qualifier("kafka-external-cluster-map") Map replyKafkaPropertiesMap) { - final Map> templatePool = new HashMap<>(); - - replyKafkaPropertiesMap.forEach((clusterId, kafkaProperties) -> { - final ProducerFactory producerFactory = producerFactory(kafkaProperties); - final ConsumerFactory consumerFactory = consumerFactory(kafkaProperties); - final GenericMessageListenerContainer replyContainer = replyContainer(consumerFactory, - kafkaProperties.getReplyTopic()); - - final ClouderaReplyingKafkaTemplate kafkaTemplate = replyingKafkaTemplate(producerFactory, - replyContainer, kafkaProperties.getRequestTopic()); - kafkaTemplate.start(); - - templatePool.put(clusterId, kafkaTemplate); - }); - - return Collections.unmodifiableMap(templatePool); - } - - @Bean - @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) - public Set kafkaClustersSet(@Qualifier("kafka-external-cluster-map") Map replyKafkaPropertiesMap) { - return Collections.unmodifiableSet(replyKafkaPropertiesMap.keySet()); - } - - - private ProducerFactory producerFactory(ClouderaKafkaProperties kafkaProperties) { - return new DefaultKafkaProducerFactory<>(kafkaProperties.buildConsumerProperties(), new StringSerializer(), new JsonSerializer<>()); - } - - private ConsumerFactory consumerFactory(ClouderaKafkaProperties kafkaProperties) { - return new DefaultKafkaConsumerFactory<>(kafkaProperties.buildConsumerProperties(), new StringDeserializer(), new JsonDeserializer<>(ResponseBody.class)); - } - - private static ClouderaReplyingKafkaTemplate replyingKafkaTemplate( - ProducerFactory producerFactory, GenericMessageListenerContainer replyContainer, - String requestTopic) { - return new ClouderaReplyingKafkaTemplate<>(producerFactory, replyContainer, requestTopic); - } - - private static GenericMessageListenerContainer replyContainer( - ConsumerFactory consumerFactory, String replyTopic) { - ContainerProperties containerProperties = new ContainerProperties(replyTopic); - return new ConcurrentMessageListenerContainer<>(consumerFactory, containerProperties); - } + /** + * Provides the default kafka properties for the consumers. + * + * @return Default kafka properties for the consumers + */ + @Bean + @Primary + public ClouderaKafkaProperties kafkaProperties() { + return new ClouderaKafkaProperties(); + } + + /** + * Provides a map with key=clusterId and value=ClouderaKafkaProperties. + * + * @return Map with key of clusterId and value - kafkaProperties. + * This map is a mapping between clusterId and connection details for that cluster + */ + @Bean(name = "kafka-external-cluster-map") + @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) + @ConfigurationProperties("spring.kafka.external-clusters") + public Map replyKafkaPropertiesMap() { + return new HashMap<>(); + } + + /** + * Map of KafkaTemplates that with key=clusterId, which allows to send requests over Kafka to different clusters and + * get responses. + * + * @param replyKafkaPropertiesMap - replyKafkaPropertiesMap bean with properties for each cluster, injected by Spring + * @return Map with key of clusterId and value - kafkaTemplate that allows to send requests over Kafka to different clusters and get + * responses. + */ + @Bean(name = "kafkaTemplatePool") + @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) + public Map> kafkaTemplatePool( + @Qualifier("kafka-external-cluster-map") Map replyKafkaPropertiesMap) { + final Map> templatePool = + new HashMap<>(); + + replyKafkaPropertiesMap.forEach((clusterId, kafkaProperties) -> { + final ProducerFactory producerFactory = producerFactory(kafkaProperties); + final ConsumerFactory consumerFactory = consumerFactory(kafkaProperties); + final GenericMessageListenerContainer replyContainer = replyContainer(consumerFactory, + kafkaProperties.getReplyTopic()); + + final ClouderaReplyingKafkaTemplate kafkaTemplate = + replyingKafkaTemplate(producerFactory, + replyContainer, kafkaProperties.getRequestTopic()); + kafkaTemplate.start(); + + templatePool.put(clusterId, kafkaTemplate); + }); + + return Collections.unmodifiableMap(templatePool); + } + + @Bean + @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) + public Set kafkaClustersSet( + @Qualifier("kafka-external-cluster-map") Map replyKafkaPropertiesMap) { + return Collections.unmodifiableSet(replyKafkaPropertiesMap.keySet()); + } + + + private ProducerFactory producerFactory(ClouderaKafkaProperties kafkaProperties) { + return new DefaultKafkaProducerFactory<>(kafkaProperties.buildConsumerProperties(), new StringSerializer(), + new JsonSerializer<>()); + } + + private ConsumerFactory consumerFactory(ClouderaKafkaProperties kafkaProperties) { + return new DefaultKafkaConsumerFactory<>(kafkaProperties.buildConsumerProperties(), new StringDeserializer(), + new JsonDeserializer<>(ResponseBody.class)); + } + + private static ClouderaReplyingKafkaTemplate replyingKafkaTemplate( + ProducerFactory producerFactory, + GenericMessageListenerContainer replyContainer, + String requestTopic) { + return new ClouderaReplyingKafkaTemplate<>(producerFactory, replyContainer, requestTopic); + } + + private static GenericMessageListenerContainer replyContainer( + ConsumerFactory consumerFactory, String replyTopic) { + ContainerProperties containerProperties = new ContainerProperties(replyTopic); + return new ConcurrentMessageListenerContainer<>(consumerFactory, containerProperties); + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java index c2a44480..40119b10 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java @@ -12,6 +12,12 @@ package com.cloudera.parserchains.queryservice.controller; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_CHAINS; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_CHAINS_READ_URL; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_INDEXING; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TEST; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL; + import com.cloudera.parserchains.core.ChainLink; import com.cloudera.parserchains.core.InvalidParserException; import com.cloudera.parserchains.core.model.define.ParserChainSchema; @@ -35,6 +41,14 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.flink.core.fs.Path; @@ -50,21 +64,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; - -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_CHAINS; -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_CHAINS_READ_URL; -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_INDEXING; -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TEST; -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL; - /** * The controller responsible for operations on parser chains. */ @@ -93,15 +92,15 @@ public class ChainController { @Operation(summary = "Retrieves all available parser chains.", - responses = { - @ApiResponse(responseCode = "200", description = "A list of all parser chains.", content = @Content( - mediaType = "application/json", - array = @ArraySchema(schema = @Schema(implementation = ParserChainSummary.class)))) - }) + responses = { + @ApiResponse(responseCode = "200", description = "A list of all parser chains.", content = @Content( + mediaType = "application/json", + array = @ArraySchema(schema = @Schema(implementation = ParserChainSummary.class)))) + }) @GetMapping(value = API_CHAINS) public ResponseEntity> findAll( - @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") - @RequestParam(name = "pipelineName", required = false) String pipelineName + @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") + @RequestParam(name = "pipelineName", required = false) String pipelineName ) throws IOException { String configPath = getConfigPath(pipelineName); @@ -110,18 +109,18 @@ public ResponseEntity> findAll( } @Operation(summary = "Creates a new parser chain.", - responses = { - @ApiResponse(responseCode = "200", description = "The parser chain was created.", content = @Content( - mediaType = "application/json", - schema = @Schema(implementation = ParserChainSchema.class))), - @ApiResponse(responseCode = "404", description = "Unable to create a new parser chain.") - }) + responses = { + @ApiResponse(responseCode = "200", description = "The parser chain was created.", content = @Content( + mediaType = "application/json", + schema = @Schema(implementation = ParserChainSchema.class))), + @ApiResponse(responseCode = "404", description = "Unable to create a new parser chain.") + }) @PostMapping(value = API_CHAINS) public ResponseEntity create( - @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") - @RequestParam(name = "pipelineName", required = false) String pipelineName, - @Parameter(name = "parserChain", description = "The parser chain to create.", required = true) - @RequestBody ParserChainSchema chain) throws IOException { + @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") + @RequestParam(name = "pipelineName", required = false) String pipelineName, + @Parameter(name = "parserChain", description = "The parser chain to create.", required = true) + @RequestBody ParserChainSchema chain) throws IOException { String configPath = getConfigPath(pipelineName); ParserChainSchema createdChain = chainPersistenceService.create(chain, Paths.get(configPath)); @@ -129,25 +128,25 @@ public ResponseEntity create( return ResponseEntity.notFound().build(); } else { return ResponseEntity - .created(URI.create(API_CHAINS_READ_URL.replace("{id}", createdChain.getId()))) - .body(createdChain); + .created(URI.create(API_CHAINS_READ_URL.replace("{id}", createdChain.getId()))) + .body(createdChain); } } @Operation(summary = "Retrieves an existing parser chain.", - responses = { - @ApiResponse(responseCode = "200", description = "The parser chain with the given ID.", content = @Content( - mediaType = "application/json", - schema = @Schema(implementation = ParserChainSchema.class))), - @ApiResponse(responseCode = "404", description = "The parser chain does not exist.") - }) + responses = { + @ApiResponse(responseCode = "200", description = "The parser chain with the given ID.", content = @Content( + mediaType = "application/json", + schema = @Schema(implementation = ParserChainSchema.class))), + @ApiResponse(responseCode = "404", description = "The parser chain does not exist.") + }) @GetMapping(value = API_CHAINS + "/{id}") public ResponseEntity read( - @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") - @RequestParam(name = "pipelineName", required = false) String pipelineName, - @Parameter(name = "id", description = "The ID of the parser chain to retrieve.", required = true) - @PathVariable String id) throws IOException { + @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") + @RequestParam(name = "pipelineName", required = false) String pipelineName, + @Parameter(name = "id", description = "The ID of the parser chain to retrieve.", required = true) + @PathVariable String id) throws IOException { String configPath = getConfigPath(pipelineName); ParserChainSchema chain = chainPersistenceService.read(id, Paths.get(configPath)); @@ -160,18 +159,18 @@ public ResponseEntity read( @Operation(summary = "Updates an existing parser chain.", - responses = { - @ApiResponse(responseCode = "204", description = "The parser chain was updated."), - @ApiResponse(responseCode = "404", description = "The parser chain does not exist.") - }) + responses = { + @ApiResponse(responseCode = "204", description = "The parser chain was updated."), + @ApiResponse(responseCode = "404", description = "The parser chain does not exist.") + }) @PutMapping(value = API_CHAINS + "/{id}") public ResponseEntity update( - @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") - @RequestParam(name = "pipelineName", required = false) String pipelineName, - @Parameter(name = "parserChain", description = "The new parser chain definition.", required = true) - @RequestBody ParserChainSchema chain, - @Parameter(name = "id", description = "The ID of the parser chain to update.") - @PathVariable String id) throws IOException { + @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") + @RequestParam(name = "pipelineName", required = false) String pipelineName, + @Parameter(name = "parserChain", description = "The new parser chain definition.", required = true) + @RequestBody ParserChainSchema chain, + @Parameter(name = "id", description = "The ID of the parser chain to update.") + @PathVariable String id) throws IOException { String configPath = getConfigPath(pipelineName); try { ParserChainSchema updatedChain = chainPersistenceService.update(id, chain, Paths.get(configPath)); @@ -186,17 +185,16 @@ public ResponseEntity update( } } - @Operation(summary = "Deletes a parser chain." - , responses = { - @ApiResponse(responseCode = "204", description = "The parser chain was deleted."), - @ApiResponse(responseCode = "404", description = "The parser chain does not exist.") + @Operation(summary = "Deletes a parser chain.", responses = { + @ApiResponse(responseCode = "204", description = "The parser chain was deleted."), + @ApiResponse(responseCode = "404", description = "The parser chain does not exist.") }) @DeleteMapping(value = API_CHAINS + "/{id}") public ResponseEntity delete( - @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") - @RequestParam(name = "pipelineName", required = false) String pipelineName, - @Parameter(name = "id", description = "The ID of the parser chain to delete.", required = true) - @PathVariable String id) throws IOException { + @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") + @RequestParam(name = "pipelineName", required = false) String pipelineName, + @Parameter(name = "id", description = "The ID of the parser chain to delete.", required = true) + @PathVariable String id) throws IOException { String configPath = getConfigPath(pipelineName); if (chainPersistenceService.delete(id, Paths.get(configPath))) { @@ -208,14 +206,14 @@ public ResponseEntity delete( @Operation(summary = "Loads table mappings for the indexing job.", - responses = { - @ApiResponse(responseCode = "200", description = "The mapping file parsed successfully."), - }) + responses = { + @ApiResponse(responseCode = "200", description = "The mapping file parsed successfully."), + }) @PostMapping(value = API_INDEXING) public ResponseEntity> getMappingsFromPath( - @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") - @RequestParam(name = "pipelineName", required = false) String pipelineName, - @RequestBody IndexMappingDescriptor body) throws IOException { + @Parameter(name = "pipelineName", description = "The pipeline to execute request in.") + @RequestParam(name = "pipelineName", required = false) String pipelineName, + @RequestBody IndexMappingDescriptor body) throws IOException { final String indexPath = getIndexingPath(body.getFilePath(), pipelineName); try { @@ -236,22 +234,22 @@ public ResponseEntity> getMappingsFromPath( @Operation(summary = "Executes a parser chain to parse sample data.", - responses = { - @ApiResponse(responseCode = "200", description = "The result of parsing the message.", content = @Content( - mediaType = "application/json", - schema = @Schema(implementation = ChainTestResponse.class))) - }) + responses = { + @ApiResponse(responseCode = "200", description = "The result of parsing the message.", content = @Content( + mediaType = "application/json", + schema = @Schema(implementation = ChainTestResponse.class))) + }) @PostMapping(value = API_PARSER_TEST) public ResponseEntity test( - @Parameter(name = "testRun", description = "Describes the parser chain test to run.", required = true) - @RequestBody ChainTestRequest testRun) throws IOException { + @Parameter(name = "testRun", description = "Describes the parser chain test to run.", required = true) + @RequestBody ChainTestRequest testRun) throws IOException { ParserChainSchema chain = testRun.getParserChainSchema(); ChainTestResponse results = new ChainTestResponse(); testRun.getSampleData().getSource() - .stream() - .limit(MAX_SAMPLES_PER_TEST) - .map(sample -> doTest(chain, sample)) - .forEach(results::addResult); + .stream() + .limit(MAX_SAMPLES_PER_TEST) + .map(sample -> doTest(chain, sample)) + .forEach(results::addResult); return ResponseEntity.ok(results); } @@ -271,11 +269,11 @@ private ParserResult doTest(ParserChainSchema schema, String textToParse) { } catch (InvalidParserException e) { log.info("The parser chain is invalid as constructed.", e); ResultLog log = ResultLogBuilder.error() - .parserId(e.getBadParser().getLabel()) - .exception(e) - .build(); + .parserId(e.getBadParser().getLabel()) + .exception(e) + .build(); result = new ParserResult() - .setLog(log); + .setLog(log); } return result; } @@ -297,9 +295,9 @@ private String getPipelinePath(String pipelineName, Supplier defaultPath } return Optional.ofNullable(pipelineService.findAll()) - .map(pipelineMap -> pipelineMap.get(pipelineName)) - .map(PipelineResult::getPath) - .map(Path::getPath) - .orElseGet(defaultPathSupplier); + .map(pipelineMap -> pipelineMap.get(pipelineName)) + .map(PipelineResult::getPath) + .map(Path::getPath) + .orElseGet(defaultPathSupplier); } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java index e74685b1..7beac4f8 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java @@ -9,14 +9,13 @@ import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; +import java.util.List; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.util.List; - /** * The controller responsible for operations with cluster to run and stop cyber jobs on the clusters. */ @@ -30,7 +29,7 @@ public class ClusterController { @Operation(description = "Retrieves information about all cluster services.") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "A list of all clusters.") + @ApiResponse(responseCode = "200", description = "A list of all clusters.") }) @GetMapping public List getAllServices() throws FailedAllClusterReponseException { @@ -39,14 +38,14 @@ public List getAllServices() throws FailedAllClusterReponseExcepti @Operation(description = "Retrieves information about a cluster with specified id.") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "A response with cluster information."), - @ApiResponse(responseCode = "404", description = "The cluster does not exist.") + @ApiResponse(responseCode = "200", description = "A response with cluster information."), + @ApiResponse(responseCode = "404", description = "The cluster does not exist.") }) @GetMapping(value = "/{id}") public ResponseBody getClusterService( - @Parameter(name = "id", description = "The ID of the cluster to retrieve.", required = true) - @PathVariable("id") String clusterId) throws FailedClusterReponseException { + @Parameter(name = "id", description = "The ID of the cluster to retrieve.", required = true) + @PathVariable("id") String clusterId) throws FailedClusterReponseException { return clusterService.getClusterInfo(clusterId); } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/GlobalExceptionAdviceController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/GlobalExceptionAdviceController.java index 476bbbcb..36b7551d 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/GlobalExceptionAdviceController.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/GlobalExceptionAdviceController.java @@ -3,13 +3,12 @@ import com.cloudera.parserchains.queryservice.common.exception.FailedAllClusterReponseException; import com.cloudera.parserchains.queryservice.common.exception.FailedClusterReponseException; import com.cloudera.service.common.response.ResponseBody; +import java.util.List; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.bind.annotation.ExceptionHandler; -import java.util.List; - @ControllerAdvice public class GlobalExceptionAdviceController { diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/JobController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/JobController.java index b9d7c10e..0eec3698 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/JobController.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/JobController.java @@ -10,6 +10,8 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; +import java.io.IOException; +import java.util.Base64; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.PathVariable; @@ -20,9 +22,6 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; -import java.util.Base64; - /** * The controller responsible for operations with cluster to run and stop cyber jobs on the clusters. */ @@ -36,40 +35,51 @@ public class JobController { @Operation(description = "Retrieves information about all cluster services.") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "A list of all clusters.") + @ApiResponse(responseCode = "200", description = "A list of all clusters.") }) @PostMapping("/{action}") - public ResponseBody executeAction(@Parameter(name = "requestBody", description = "The new parser chain definition.", required = true) - @RequestBody com.cloudera.service.common.request.RequestBody body, - @Parameter(name = "clusterId", description = "The ID of the cluster to retrieve.", required = true) - @PathVariable("clusterId") String clusterId, - @Schema(name = "action", description = "Jobs action for start stop restart Job", allowableValues = {JobActions.Constants.START_VALUE, JobActions.Constants.RESTART_VALUE, JobActions.Constants.STOP_VALUE, JobActions.Constants.STATUS_VALUE, JobActions.Constants.GET_CONFIG_VALUE, JobActions.Constants.UPDATE_CONFIG_VALUE}, required = true) - @PathVariable("action") String action) throws FailedClusterReponseException { + public ResponseBody executeAction( + @Parameter(name = "requestBody", description = "The new parser chain definition.", required = true) + @RequestBody com.cloudera.service.common.request.RequestBody body, + @Parameter(name = "clusterId", description = "The ID of the cluster to retrieve.", required = true) + @PathVariable("clusterId") String clusterId, + @Schema(name = "action", description = "Jobs action for start stop restart Job", allowableValues = { + JobActions.Constants.START_VALUE, JobActions.Constants.RESTART_VALUE, JobActions.Constants.STOP_VALUE, + JobActions.Constants.STATUS_VALUE, JobActions.Constants.GET_CONFIG_VALUE, + JobActions.Constants.UPDATE_CONFIG_VALUE}, required = true) + @PathVariable("action") String action) throws FailedClusterReponseException { return jobService.makeRequest(clusterId, body, action); } @PostMapping("/config/{pipeline}/{jobIdHex}") - public ResponseBody updateJobConfig(@Parameter(name = "clusterId", description = "The ID of the cluster to update config on.", required = true) - @PathVariable("clusterId") String clusterId, - @Schema(name = "pipeline", description = "Pipeline in which the job is running", allowableValues = {JobActions.Constants.START_VALUE, JobActions.Constants.RESTART_VALUE, JobActions.Constants.STOP_VALUE, JobActions.Constants.STATUS_VALUE, JobActions.Constants.GET_CONFIG_VALUE, JobActions.Constants.UPDATE_CONFIG_VALUE}, required = true) - @PathVariable("pipeline") String pipeline, - @Schema(name = "jobIdHex", description = "Job ID to update config of", allowableValues = {JobActions.Constants.START_VALUE, JobActions.Constants.RESTART_VALUE, JobActions.Constants.STOP_VALUE, JobActions.Constants.STATUS_VALUE, JobActions.Constants.GET_CONFIG_VALUE, JobActions.Constants.UPDATE_CONFIG_VALUE}, required = true) - @PathVariable("jobIdHex") String jobIdHex, - @RequestParam("config") MultipartFile config) throws FailedClusterReponseException, IOException { + public ResponseBody updateJobConfig( + @Parameter(name = "clusterId", description = "The ID of the cluster to update config on.", required = true) + @PathVariable("clusterId") String clusterId, + @Schema(name = "pipeline", description = "Pipeline in which the job is running", allowableValues = { + JobActions.Constants.START_VALUE, JobActions.Constants.RESTART_VALUE, JobActions.Constants.STOP_VALUE, + JobActions.Constants.STATUS_VALUE, JobActions.Constants.GET_CONFIG_VALUE, + JobActions.Constants.UPDATE_CONFIG_VALUE}, required = true) + @PathVariable("pipeline") String pipeline, + @Schema(name = "jobIdHex", description = "Job ID to update config of", allowableValues = { + JobActions.Constants.START_VALUE, JobActions.Constants.RESTART_VALUE, JobActions.Constants.STOP_VALUE, + JobActions.Constants.STATUS_VALUE, JobActions.Constants.GET_CONFIG_VALUE, + JobActions.Constants.UPDATE_CONFIG_VALUE}, required = true) + @PathVariable("jobIdHex") String jobIdHex, + @RequestParam("config") MultipartFile config) throws FailedClusterReponseException, IOException { final String filename = config.getOriginalFilename(); - if (filename == null || !filename.endsWith(".tar.gz")){ + if (filename == null || !filename.endsWith(".tar.gz")) { throw new RuntimeException("You should provide config as a .tar.gz archive"); } //Kafka message body should be less than 1Mb, but we should fit a base64 encoded file + other JSON fields. - if (Math.ceil(config.getSize() / 3d) * 4 >= uploadFileMaxSize){ + if (Math.ceil(config.getSize() / 3d) * 4 >= uploadFileMaxSize) { throw new RuntimeException("Provided file should be less than ~750Kb in size."); } final com.cloudera.service.common.request.RequestBody requestBody = - com.cloudera.service.common.request.RequestBody.builder() - .clusterServiceId(clusterId) - .jobIdHex(jobIdHex) - .payload(Base64.getEncoder().encode(config.getBytes())) - .build(); + com.cloudera.service.common.request.RequestBody.builder() + .clusterServiceId(clusterId) + .jobIdHex(jobIdHex) + .payload(Base64.getEncoder().encode(config.getBytes())) + .build(); return jobService.makeRequest(clusterId, requestBody, JobActions.Constants.UPDATE_CONFIG_VALUE); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java index 6fa6d0ad..b1f1c214 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java @@ -3,8 +3,10 @@ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the * License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -13,6 +15,10 @@ package com.cloudera.parserchains.queryservice.controller; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_FORM_CONFIG; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TYPES; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL; + import com.cloudera.parserchains.core.model.define.ParserID; import com.cloudera.parserchains.queryservice.model.describe.ParserDescriptor; import com.cloudera.parserchains.queryservice.model.summary.ParserSummary; @@ -22,21 +28,15 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import java.io.IOException; +import java.util.List; +import java.util.Map; import lombok.RequiredArgsConstructor; -import org.apache.commons.collections4.MapUtils; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_FORM_CONFIG; -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TYPES; -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL; - /** * The controller responsible for operations on parsers. */ @@ -49,13 +49,12 @@ public class ParserController { private final ParserDiscoveryService parserDiscoveryService; - @Operation(summary = "Retrieves all available parsers.", - responses = { - @ApiResponse(responseCode = "200", description = "A list of all parser types.", content = @Content( - mediaType = "application/json", - array = @ArraySchema(schema = @Schema(implementation = ParserSummary.class)))) - }) + responses = { + @ApiResponse(responseCode = "200", description = "A list of all parser types.", content = @Content( + mediaType = "application/json", + array = @ArraySchema(schema = @Schema(implementation = ParserSummary.class)))) + }) @GetMapping(value = API_PARSER_TYPES) public ResponseEntity> findAll() throws IOException { List types = parserDiscoveryService.findAll(); @@ -64,10 +63,10 @@ public ResponseEntity> findAll() throws IOException { @Operation(summary = "Describes the configuration parameters for all available parsers.", - responses = { - @ApiResponse(responseCode = "200", description = "A map of parser types and their associated configuration parameters.", content = @Content( - mediaType = "application/json")), - @ApiResponse(responseCode = "404", description = "Unable to retrieve.")}) + responses = { + @ApiResponse(responseCode = "200", description = "A map of parser types and their associated configuration parameters.", + content = @Content(mediaType = "application/json")), + @ApiResponse(responseCode = "404", description = "Unable to retrieve.")}) @GetMapping(value = API_PARSER_FORM_CONFIG) public ResponseEntity> describeAll() { Map configs = parserDiscoveryService.describeAll(); diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java index 86dbac76..76b79e91 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -25,20 +27,16 @@ import com.cloudera.parserchains.queryservice.model.describe.SampleFolderDescriptor; import com.cloudera.parserchains.queryservice.model.sample.ParserSample; import com.cloudera.parserchains.queryservice.service.ParserSampleService; - -import java.io.IOException; -import java.net.URI; -import java.util.List; -import lombok.RequiredArgsConstructor; - - import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; -import org.springframework.beans.factory.annotation.Autowired; +import java.io.IOException; +import java.net.URI; +import java.util.List; +import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.PathVariable; @@ -61,13 +59,14 @@ public class ParserSampleController { private final AppProperties appProperties; @Operation(summary = "Retrieves all parser samples for the specified chain.", - responses = { - @ApiResponse(responseCode = "200", description = "A list of all parser samples for the specified chain.") - }) + responses = { + @ApiResponse(responseCode = "200", description = "A list of all parser samples for the specified chain.") + }) @PostMapping(value = API_PARSER_TEST_SAMPLES + "/{id}") - public ResponseEntity> findAllById(@Parameter(name = "id", description = "The ID of the parser chain to retrieve samples for.", required = true) - @PathVariable String id, - @RequestBody SampleFolderDescriptor body) throws IOException { + public ResponseEntity> findAllById( + @Parameter(name = "id", description = "The ID of the parser chain to retrieve samples for.", required = true) + @PathVariable String id, + @RequestBody SampleFolderDescriptor body) throws IOException { String sampleFolderPath = getSampleFolderPath(body); List types = parserSampleService.findAllById(sampleFolderPath, id); if (types == null) { @@ -78,25 +77,27 @@ public ResponseEntity> findAllById(@Parameter(name = "id", de @Operation(summary = "Create or replace parser chain sample list.", - responses = { - @ApiResponse(responseCode = "204", description = "The parser chain list was created/replaced.", content = @Content(array = @ArraySchema(schema = @Schema(implementation = ParserSample.class)))), - @ApiResponse(responseCode = "404", description = "The parser chain does not exist.") - }) + responses = { + @ApiResponse(responseCode = "204", description = "The parser chain list was created/replaced.", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ParserSample.class)))), + @ApiResponse(responseCode = "404", description = "The parser chain does not exist.") + }) @PutMapping(value = API_PARSER_TEST_SAMPLES + "/{id}") public ResponseEntity> update( - @Parameter(name = "sampleList", description = "The new sample definition list.", required = true) - @RequestBody SampleFolderDescriptor body, - @Parameter(name = "id", description = "The ID of the parser chain sample to update.") - @PathVariable String id) { + @Parameter(name = "sampleList", description = "The new sample definition list.", required = true) + @RequestBody SampleFolderDescriptor body, + @Parameter(name = "id", description = "The ID of the parser chain sample to update.") + @PathVariable String id) { String sampleFolderPath = getSampleFolderPath(body); try { - List createdSampleList = parserSampleService.update(sampleFolderPath, id, body.getSampleList()); + List createdSampleList = + parserSampleService.update(sampleFolderPath, id, body.getSampleList()); if (null == createdSampleList) { return ResponseEntity.notFound().build(); } return ResponseEntity - .created(URI.create(API_PARSER_TEST_SAMPLES + "/" + id)) - .body(createdSampleList); + .created(URI.create(API_PARSER_TEST_SAMPLES + "/" + id)) + .body(createdSampleList); // TODO: fix this exception handling } catch (IOException ioe) { throw new RuntimeException("Unable to create parser chain samples with id=" + id); @@ -105,8 +106,8 @@ public ResponseEntity> update( private String getSampleFolderPath(SampleFolderDescriptor body) { String sampleFolderPath = StringUtils.hasText(body.getFolderPath()) - ? body.getFolderPath() - : appProperties.getSampleFolderPath(); + ? body.getFolderPath() + : appProperties.getSampleFolderPath(); if (!sampleFolderPath.endsWith("/")) { sampleFolderPath = sampleFolderPath.concat("/"); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/PipelineController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/PipelineController.java index d8eef055..fe8b6ca3 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/PipelineController.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/PipelineController.java @@ -12,11 +12,16 @@ package com.cloudera.parserchains.queryservice.controller; +import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PIPELINE_BASE_URL; + import com.cloudera.parserchains.queryservice.model.exec.PipelineResult; import com.cloudera.parserchains.queryservice.service.PipelineService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; +import java.io.IOException; +import java.util.Map; +import java.util.Set; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; @@ -28,12 +33,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.io.IOException; -import java.util.Map; -import java.util.Set; - -import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PIPELINE_BASE_URL; - /** * The controller responsible for operations on pipelines. */ @@ -46,8 +45,8 @@ public class PipelineController { @Operation(description = "Finds and returns all available pipelines.") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "A list of all pipelines."), - @ApiResponse(responseCode = "404", description = "No valid pipelines found.") + @ApiResponse(responseCode = "200", description = "A list of all pipelines."), + @ApiResponse(responseCode = "404", description = "No valid pipelines found.") }) @GetMapping public ResponseEntity> findAll() throws IOException { @@ -60,7 +59,7 @@ public ResponseEntity> findAll() throws IOException { @Operation(description = "Allows to create a new pipeline.") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "A new list of all pipelines.") + @ApiResponse(responseCode = "200", description = "A new list of all pipelines.") }) @PostMapping("/{pipelineName}") public ResponseEntity> createPipeline(@PathVariable String pipelineName) throws IOException { @@ -73,7 +72,7 @@ public ResponseEntity> createPipeline(@PathVariable String pipelineN @Operation(description = "Allows to rename existing pipeline.") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "A new list of all pipelines.") + @ApiResponse(responseCode = "200", description = "A new list of all pipelines.") }) @PutMapping("/{pipelineName}") public ResponseEntity> renamePipeline(@PathVariable String pipelineName, @@ -87,7 +86,7 @@ public ResponseEntity> renamePipeline(@PathVariable String pipelineN @Operation(description = "Allows to delete existing pipeline.") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "A new list of all pipelines.") + @ApiResponse(responseCode = "200", description = "A new list of all pipelines.") }) @DeleteMapping("/{pipelineName}") public ResponseEntity> deletePipeline(@PathVariable String pipelineName) throws IOException { diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/DocumentationPluginsManagerBootAdapter.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/DocumentationPluginsManagerBootAdapter.java index bfbd17bc..8fffdb51 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/DocumentationPluginsManagerBootAdapter.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/DocumentationPluginsManagerBootAdapter.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/TypeNameExtractorBootAdapter.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/TypeNameExtractorBootAdapter.java index 66ce66f3..ad222db9 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/TypeNameExtractorBootAdapter.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/TypeNameExtractorBootAdapter.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ConfigParamDescriptor.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ConfigParamDescriptor.java index 87187f77..acbddcd4 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ConfigParamDescriptor.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ConfigParamDescriptor.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,8 +22,7 @@ import com.cloudera.parserchains.core.catalog.WidgetType; import com.cloudera.parserchains.core.model.define.ConfigValueSchema; - -import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -30,201 +31,202 @@ */ public class ConfigParamDescriptor { - /** - * The unique name of the configuration parameter. This value - * is used only to identify the parameter and is not displayed - * to the user. - */ - private String name; - - /** - * Defines the type of widget presented to the user when - * configuring the parameter. - */ - private WidgetType type; - private static final WidgetType DEFAULT_WIDGET_TYPE = WidgetType.TEXT; - - /** - * A label for the parameter that is displayed to the user. - */ - private String label; - - /** - * A description of the parameter that is displayed to the user. - */ - private String description; - - /** - * Defines whether the user is required to define a value for this configuration parameter. - */ - private boolean required; - - /** - * Defines whether the user is able to enter multiple values per parameter. - */ - private boolean multipleValues; - - /** - * Specifies if the parameter specifies the output field name. - * If true, the selection will be provided with possible field names. - *

      This value is optional. - */ - private boolean isOutputName; - - /** - * Defines a path that allows the UI to organize the parameters. - * - *

      In cases where multiple, associated values are accepted, like a field rename with a - * "from" and "to" value, the associated values should use the same path like "config.fieldToRename". - * - *

      The UI expects the root of all paths to be "config". - */ - private String path; - private static final String DEFAULT_PATH = "config"; - - /** - * Should the user be allowed to enter multiple values for this parameter? - */ - private boolean multiple; - private static final boolean DEFAULT_MULTIPLE = false; - - /** - * The default value used if none other is provided. - */ - private List defaultValue; - - public ConfigParamDescriptor() { - this.path = DEFAULT_PATH; - this.multiple = DEFAULT_MULTIPLE; - this.type = DEFAULT_WIDGET_TYPE; - } - - public String getName() { - return name; - } - - public ConfigParamDescriptor setName(String name) { - this.name = name; - return this; - } - - public WidgetType getType() { - return type; - } - - public ConfigParamDescriptor setType(WidgetType type) { - this.type = type; - return this; - } - - public String getLabel() { - return label; - } - - public ConfigParamDescriptor setLabel(String label) { - this.label = label; - return this; - } - - public String getDescription() { - return description; - } - - public ConfigParamDescriptor setDescription(String description) { - this.description = description; - return this; - } - - public boolean isRequired() { - return required; - } - - public ConfigParamDescriptor setRequired(boolean required) { - this.required = required; - return this; - } - - public boolean isMultipleValues() { - return multipleValues; - } - - public ConfigParamDescriptor setMultipleValues(boolean multipleValues) { - this.multipleValues = multipleValues; - return this; - } - - public boolean isOutputName() { - return isOutputName; - } - - public ConfigParamDescriptor setOutputName(boolean outputName) { - isOutputName = outputName; - return this; - } - - public String getPath() { - return path; - } - - public ConfigParamDescriptor setPath(String path) { - this.path = path; - return this; - } - - public boolean isMultiple() { - return multiple; - } - - public ConfigParamDescriptor setMultiple(boolean multiple) { - this.multiple = multiple; - return this; - } - - public List getDefaultValue() { - return defaultValue; - } - - public ConfigParamDescriptor addDefaultValue(String name, String value) { - // the front-end requires a list of values. the list will only ever contain a single, default value - this.defaultValue = Arrays.asList(new ConfigValueSchema().addValue(name, value)); - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConfigParamDescriptor that = (ConfigParamDescriptor) o; - return required == that.required && - multiple == that.multiple && - Objects.equals(name, that.name) && - Objects.equals(type, that.type) && - Objects.equals(label, that.label) && - Objects.equals(description, that.description) && - Objects.equals(path, that.path) && - Objects.equals(defaultValue, that.defaultValue); - } - - @Override - public int hashCode() { - return Objects.hash(name, type, label, description, required, path, multiple, defaultValue); - } - - @Override - public String toString() { - return "ConfigParamDescriptor{" + - "name='" + name + '\'' + - ", type='" + type + '\'' + - ", label='" + label + '\'' + - ", description='" + description + '\'' + - ", required=" + required + - ", path='" + path + '\'' + - ", multiple=" + multiple + - ", defaultValue='" + defaultValue + '\'' + - '}'; - } + /** + * The unique name of the configuration parameter. This value + * is used only to identify the parameter and is not displayed + * to the user. + */ + private String name; + + /** + * Defines the type of widget presented to the user when + * configuring the parameter. + */ + private WidgetType type; + private static final WidgetType DEFAULT_WIDGET_TYPE = WidgetType.TEXT; + + /** + * A label for the parameter that is displayed to the user. + */ + private String label; + + /** + * A description of the parameter that is displayed to the user. + */ + private String description; + + /** + * Defines whether the user is required to define a value for this configuration parameter. + */ + private boolean required; + + /** + * Defines whether the user is able to enter multiple values per parameter. + */ + private boolean multipleValues; + + /** + * Specifies if the parameter specifies the output field name. + * If true, the selection will be provided with possible field names. + * + *

      This value is optional. + */ + private boolean isOutputName; + + /** + * Defines a path that allows the UI to organize the parameters. + * + *

      In cases where multiple, associated values are accepted, like a field rename with a + * "from" and "to" value, the associated values should use the same path like "config.fieldToRename". + * + *

      The UI expects the root of all paths to be "config". + */ + private String path; + private static final String DEFAULT_PATH = "config"; + + /** + * Should the user be allowed to enter multiple values for this parameter?. + */ + private boolean multiple; + private static final boolean DEFAULT_MULTIPLE = false; + + /** + * The default value used if none other is provided. + */ + private List defaultValue; + + public ConfigParamDescriptor() { + this.path = DEFAULT_PATH; + this.multiple = DEFAULT_MULTIPLE; + this.type = DEFAULT_WIDGET_TYPE; + } + + public String getName() { + return name; + } + + public ConfigParamDescriptor setName(String name) { + this.name = name; + return this; + } + + public WidgetType getType() { + return type; + } + + public ConfigParamDescriptor setType(WidgetType type) { + this.type = type; + return this; + } + + public String getLabel() { + return label; + } + + public ConfigParamDescriptor setLabel(String label) { + this.label = label; + return this; + } + + public String getDescription() { + return description; + } + + public ConfigParamDescriptor setDescription(String description) { + this.description = description; + return this; + } + + public boolean isRequired() { + return required; + } + + public ConfigParamDescriptor setRequired(boolean required) { + this.required = required; + return this; + } + + public boolean isMultipleValues() { + return multipleValues; + } + + public ConfigParamDescriptor setMultipleValues(boolean multipleValues) { + this.multipleValues = multipleValues; + return this; + } + + public boolean isOutputName() { + return isOutputName; + } + + public ConfigParamDescriptor setOutputName(boolean outputName) { + isOutputName = outputName; + return this; + } + + public String getPath() { + return path; + } + + public ConfigParamDescriptor setPath(String path) { + this.path = path; + return this; + } + + public boolean isMultiple() { + return multiple; + } + + public ConfigParamDescriptor setMultiple(boolean multiple) { + this.multiple = multiple; + return this; + } + + public List getDefaultValue() { + return defaultValue; + } + + public ConfigParamDescriptor addDefaultValue(String name, String value) { + // the front-end requires a list of values. the list will only ever contain a single, default value + this.defaultValue = Collections.singletonList(new ConfigValueSchema().addValue(name, value)); + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ConfigParamDescriptor that = (ConfigParamDescriptor) o; + return required == that.required + && multiple == that.multiple + && Objects.equals(name, that.name) + && Objects.equals(type, that.type) + && Objects.equals(label, that.label) + && Objects.equals(description, that.description) + && Objects.equals(path, that.path) + && Objects.equals(defaultValue, that.defaultValue); + } + + @Override + public int hashCode() { + return Objects.hash(name, type, label, description, required, path, multiple, defaultValue); + } + + @Override + public String toString() { + return "ConfigParamDescriptor{" + + "name='" + name + '\'' + + ", type='" + type + '\'' + + ", label='" + label + '\'' + + ", description='" + description + '\'' + + ", required=" + required + + ", path='" + path + '\'' + + ", multiple=" + multiple + + ", defaultValue='" + defaultValue + '\'' + + '}'; + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ParserDescriptor.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ParserDescriptor.java index 226787b0..0441360f 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ParserDescriptor.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ParserDescriptor.java @@ -6,9 +6,11 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + *

      + * http://www.apache.org/licenses/LICENSE-2.0 + * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,11 +24,10 @@ import com.cloudera.parserchains.core.model.define.ParserName; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; - import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; /** * Describes features of a parser including its configuration parameters. @@ -37,86 +38,86 @@ @JsonPropertyOrder({"id", "name", "schemaItems"}) public class ParserDescriptor { - /** - * The id of the parser. - * - *

      This id should be unique amongst all of the available parsers. This is - * derived from the name of the class implementing the parser. - */ - @JsonProperty("id") - private ParserID parserID; - - /** - * The name of the parser. This is a descriptive name provided by - * the parser author and is not guaranteed to be unique. - */ - @JsonProperty("name") - private ParserName parserName; - - /** - * Describes the configuration parameters accepted by this parser. - */ - @JsonProperty("schemaItems") - private List configurations; - - public ParserDescriptor() { - configurations = new ArrayList<>(); - } - - public ParserID getParserID() { - return parserID; - } - - public ParserDescriptor setParserID(ParserID parserID) { - this.parserID = parserID; - return this; - } - - public ParserName getParserName() { - return parserName; - } - - public ParserDescriptor setParserName(ParserName parserName) { - this.parserName = parserName; - return this; - } - - public List getConfigurations() { - return configurations; - } - - public ParserDescriptor setConfigurations(List configurations) { - this.configurations = configurations; - return this; - } - - public ParserDescriptor addConfiguration(ConfigParamDescriptor item) { - configurations.add(item); - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; + /** + * The id of the parser. + * + *

      This id should be unique amongst all of the available parsers. This is + * derived from the name of the class implementing the parser. + */ + @JsonProperty("id") + private ParserID parserID; + + /** + * The name of the parser. This is a descriptive name provided by + * the parser author and is not guaranteed to be unique. + */ + @JsonProperty("name") + private ParserName parserName; + + /** + * Describes the configuration parameters accepted by this parser. + */ + @JsonProperty("schemaItems") + private List configurations; + + public ParserDescriptor() { + configurations = new ArrayList<>(); + } + + public ParserID getParserID() { + return parserID; + } + + public ParserDescriptor setParserID(ParserID parserID) { + this.parserID = parserID; + return this; + } + + public ParserName getParserName() { + return parserName; + } + + public ParserDescriptor setParserName(ParserName parserName) { + this.parserName = parserName; + return this; + } + + public List getConfigurations() { + return configurations; + } + + public ParserDescriptor setConfigurations(List configurations) { + this.configurations = configurations; + return this; + } + + public ParserDescriptor addConfiguration(ConfigParamDescriptor item) { + configurations.add(item); + return this; } - if (o == null || getClass() != o.getClass()) { - return false; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ParserDescriptor that = (ParserDescriptor) o; + return new EqualsBuilder() + .append(parserID, that.parserID) + .append(parserName, that.parserName) + .append(configurations, that.configurations) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(parserID) + .append(parserName) + .append(configurations) + .toHashCode(); } - ParserDescriptor that = (ParserDescriptor) o; - return new EqualsBuilder() - .append(parserID, that.parserID) - .append(parserName, that.parserName) - .append(configurations, that.configurations) - .isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(parserID) - .append(parserName) - .append(configurations) - .toHashCode(); - } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/SampleFolderDescriptor.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/SampleFolderDescriptor.java index 5a12957c..c08c6c91 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/SampleFolderDescriptor.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/SampleFolderDescriptor.java @@ -1,11 +1,10 @@ package com.cloudera.parserchains.queryservice.model.describe; import com.cloudera.parserchains.queryservice.model.sample.ParserSample; +import java.util.List; import lombok.Getter; import lombok.Setter; -import java.util.List; - @Getter @Setter public class SampleFolderDescriptor { diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/JobActions.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/JobActions.java index acbc984c..afa03ccc 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/JobActions.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/JobActions.java @@ -3,7 +3,9 @@ import org.apache.commons.lang3.StringUtils; public enum JobActions { - START(Constants.START_VALUE), STOP(Constants.STOP_VALUE), RESTART(Constants.RESTART_VALUE), UPDATE_CONFIG(Constants.UPDATE_CONFIG_VALUE), GET_CONFIG(Constants.GET_CONFIG_VALUE), STATUS(Constants.STATUS_VALUE); + START(Constants.START_VALUE), STOP(Constants.STOP_VALUE), RESTART(Constants.RESTART_VALUE), + UPDATE_CONFIG(Constants.UPDATE_CONFIG_VALUE), GET_CONFIG(Constants.GET_CONFIG_VALUE), + STATUS(Constants.STATUS_VALUE); public final String action; diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/KafkaMessageType.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/KafkaMessageType.java index 263c9bce..d4710708 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/KafkaMessageType.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/enums/KafkaMessageType.java @@ -3,31 +3,31 @@ import java.util.Arrays; public enum KafkaMessageType { - //ChainController messages - CHAIN_FIND_ALL, - CHAIN_CREATE, - CHAIN_READ, - CHAIN_UPDATE, - CHAIN_DELETE, - CHAIN_INDEXING_GET, - CHAIN_TEST, + //ChainController messages + CHAIN_FIND_ALL, + CHAIN_CREATE, + CHAIN_READ, + CHAIN_UPDATE, + CHAIN_DELETE, + CHAIN_INDEXING_GET, + CHAIN_TEST, - //ParserController messages - PARSER_FIND_ALL, - PARSER_DESCRIBE_ALL, + //ParserController messages + PARSER_FIND_ALL, + PARSER_DESCRIBE_ALL, - //PipelineController - PIPELINES_FIND_ALL, + //PipelineController + PIPELINES_FIND_ALL, - UNKNOWN; + UNKNOWN; - public static KafkaMessageType fromString(String name){ - if (name == null){ - return UNKNOWN; + public static KafkaMessageType fromString(String name) { + if (name == null) { + return UNKNOWN; + } + return Arrays.stream(KafkaMessageType.values()) + .filter(value -> value.name().equals(name)) + .findFirst() + .orElse(UNKNOWN); } - return Arrays.stream(KafkaMessageType.values()) - .filter(value -> value.name().equals(name)) - .findFirst() - .orElse(UNKNOWN); - } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ChainTestRequest.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ChainTestRequest.java index c3e2e130..537e34c4 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ChainTestRequest.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ChainTestRequest.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ChainTestResponse.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ChainTestResponse.java index 76679f95..6cb774b8 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ChainTestResponse.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ChainTestResponse.java @@ -12,11 +12,10 @@ package com.cloudera.parserchains.queryservice.model.exec; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; - import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; /** * Defines the data model for the response to a parser chain @@ -74,21 +73,21 @@ public boolean equals(Object o) { } ChainTestResponse that = (ChainTestResponse) o; return new EqualsBuilder() - .append(results, that.results) - .isEquals(); + .append(results, that.results) + .isEquals(); } @Override public int hashCode() { return new HashCodeBuilder(17, 37) - .append(results) - .toHashCode(); + .append(results) + .toHashCode(); } @Override public String toString() { - return "ParserResults{" + - "results=" + results + - '}'; + return "ParserResults{" + + "results=" + results + + '}'; } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ParserResult.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ParserResult.java index 68fff0ef..ea697d01 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ParserResult.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ParserResult.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,8 +21,11 @@ package com.cloudera.parserchains.queryservice.model.exec; import com.fasterxml.jackson.annotation.JsonInclude; - -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; /** * The result of parsing a message with a parser chain or parser. @@ -43,6 +48,7 @@ public class ParserResult { /** * The individual parser results; one for each parser in the chain. + * *

      If no results, this field should not be shown when serialized to JSON. */ @JsonInclude(JsonInclude.Include.NON_EMPTY) @@ -113,10 +119,10 @@ public boolean equals(Object o) { return false; } ParserResult that = (ParserResult) o; - return Objects.equals(input, that.input) && - Objects.equals(output, that.output) && - Objects.equals(log, that.log) && - Objects.equals(parserResults, that.parserResults); + return Objects.equals(input, that.input) + && Objects.equals(output, that.output) + && Objects.equals(log, that.log) + && Objects.equals(parserResults, that.parserResults); } @Override @@ -126,11 +132,11 @@ public int hashCode() { @Override public String toString() { - return "ParserResult{" + - "input=" + input + - ", output=" + output + - ", log=" + log + - ", chainResults=" + parserResults + - '}'; + return "ParserResult{" + + "input=" + input + + ", output=" + output + + ", log=" + log + + ", chainResults=" + parserResults + + '}'; } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/PipelineResult.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/PipelineResult.java index 7229fa0c..ed579bf4 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/PipelineResult.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/PipelineResult.java @@ -12,7 +12,7 @@ @AllArgsConstructor public class PipelineResult { - private String name; - private Path path; + private String name; + private Path path; } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ResultLog.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ResultLog.java index 9026ea52..f64a7865 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ResultLog.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/ResultLog.java @@ -63,7 +63,7 @@ public String getMessage() { } public String getParserId() { - return parserId; + return parserId; } public String getParserName() { @@ -84,31 +84,31 @@ public boolean equals(Object o) { } ResultLog resultLog = (ResultLog) o; return new EqualsBuilder() - .append(type, resultLog.type) - .append(message, resultLog.message) - .append(parserId, resultLog.parserId) - .append(parserName, resultLog.parserName) - .isEquals(); + .append(type, resultLog.type) + .append(message, resultLog.message) + .append(parserId, resultLog.parserId) + .append(parserName, resultLog.parserName) + .isEquals(); } @Override public int hashCode() { return new HashCodeBuilder(17, 37) - .append(type) - .append(message) - .append(parserId) - .append(parserName) - .toHashCode(); + .append(type) + .append(message) + .append(parserId) + .append(parserName) + .toHashCode(); } @Override public String toString() { - return "ResultLog{" + - "type='" + type + '\'' + - ", message='" + message + '\'' + - ", parserId='" + parserId + '\'' + - ", parserName='" + parserName + '\'' + - '}'; + return "ResultLog{" + + "type='" + type + '\'' + + ", message='" + message + '\'' + + ", parserId='" + parserId + '\'' + + ", parserName='" + parserName + '\'' + + '}'; } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/SampleData.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/SampleData.java index 699b8f43..3b0648bd 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/SampleData.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/exec/SampleData.java @@ -19,8 +19,8 @@ * Defines the sample data that is received by the front-end * when a test execution of a parser chain is requested. * - *

      See also {@link ChainTestRequest} which is the top-level class for the - * data model used for the "Live View" feature. + *

      See also {@link ChainTestRequest} which is the top-level class for the + * data model used for the "Live View" feature. */ public class SampleData { diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/sample/ParserSample.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/sample/ParserSample.java index acf3fb99..b09b3f60 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/sample/ParserSample.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/sample/ParserSample.java @@ -6,8 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at + * *

      * http://www.apache.org/licenses/LICENSE-2.0 + * *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,7 +24,7 @@ /** * Describes a parser source sample that's used to store and retrieve - * samples for Live View from the storage + * samples for Live View from the storage. */ public class ParserSample { @@ -95,8 +97,8 @@ public boolean equals(Object o) { return false; } ParserSample that = (ParserSample) o; - return Objects.equals(id, that.id) && - Objects.equals(name, that.name); + return Objects.equals(id, that.id) + && Objects.equals(name, that.name); } @Override @@ -106,12 +108,12 @@ public int hashCode() { @Override public String toString() { - return "ParserSample{" + - "id=" + id + - ", name='" + name + '\'' + - ", description='" + description + '\'' + - ", source='" + source + '\'' + - ", expectedResult='" + expectedResult + '\'' + - '}'; + return "ParserSample{" + + "id=" + id + + ", name='" + name + '\'' + + ", description='" + description + '\'' + + ", source='" + source + '\'' + + ", expectedResult='" + expectedResult + '\'' + + '}'; } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ObjectMapper.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ObjectMapper.java index f06de1c6..62524027 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ObjectMapper.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ObjectMapper.java @@ -12,7 +12,7 @@ package com.cloudera.parserchains.queryservice.model.summary; -public interface ObjectMapper { +public interface ObjectMapper { T reform(R source); diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserChainSummary.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserChainSummary.java index d5193b0c..197f3e2b 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserChainSummary.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserChainSummary.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,63 +21,62 @@ package com.cloudera.parserchains.queryservice.model.summary; import com.cloudera.parserchains.core.model.define.ParserChainSchema; - import java.util.Objects; public class ParserChainSummary { - private String id; - private String name; - - public ParserChainSummary() { - } + private String id; + private String name; - public ParserChainSummary(ParserChainSchema chain) { - this.id = chain.getId(); - this.name = chain.getName(); - } + public ParserChainSummary() { + } - public String getId() { - return id; - } + public ParserChainSummary(ParserChainSchema chain) { + this.id = chain.getId(); + this.name = chain.getName(); + } - public ParserChainSummary setId(String id) { - this.id = id; - return this; - } + public String getId() { + return id; + } - public String getName() { - return name; - } + public ParserChainSummary setId(String id) { + this.id = id; + return this; + } - public ParserChainSummary setName(String name) { - this.name = name; - return this; - } + public String getName() { + return name; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; + public ParserChainSummary setName(String name) { + this.name = name; + return this; } - if (o == null || getClass() != o.getClass()) { - return false; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ParserChainSummary that = (ParserChainSummary) o; + return Objects.equals(id, that.id) + && Objects.equals(name, that.name); } - ParserChainSummary that = (ParserChainSummary) o; - return Objects.equals(id, that.id) && - Objects.equals(name, that.name); - } - @Override - public int hashCode() { - return Objects.hash(id, name); - } + @Override + public int hashCode() { + return Objects.hash(id, name); + } - @Override - public String toString() { - return "ParserChainSummary{" + - "id='" + id + '\'' + - ", name='" + name + '\'' + - '}'; - } + @Override + public String toString() { + return "ParserChainSummary{" + + "id='" + id + '\'' + + ", name='" + name + '\'' + + '}'; + } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserSummary.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserSummary.java index ad0f8515..2ffd3d07 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserSummary.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserSummary.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      + * http://www.apache.org/licenses/LICENSE-2.0 * + *

      * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +24,6 @@ import com.cloudera.parserchains.core.model.define.ParserName; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; /** @@ -31,69 +32,69 @@ */ public class ParserSummary { - @JsonProperty("id") - private ParserID id; + @JsonProperty("id") + private ParserID id; - @JsonProperty("name") - private ParserName name; + @JsonProperty("name") + private ParserName name; - @JsonIgnore - private String description; + @JsonIgnore + private String description; - public String getDescription() { - return description; - } + public String getDescription() { + return description; + } - public ParserSummary setDescription(String description) { - this.description = description; - return this; - } + public ParserSummary setDescription(String description) { + this.description = description; + return this; + } - public ParserID getId() { - return id; - } + public ParserID getId() { + return id; + } - public ParserSummary setId(ParserID id) { - this.id = id; - return this; - } + public ParserSummary setId(ParserID id) { + this.id = id; + return this; + } - public ParserName getName() { - return name; - } + public ParserName getName() { + return name; + } - public ParserSummary setName(ParserName name) { - this.name = name; - return this; - } + public ParserSummary setName(ParserName name) { + this.name = name; + return this; + } - public ParserSummary setName(String name) { - return setName(ParserName.of(name)); - } + public ParserSummary setName(String name) { + return setName(ParserName.of(name)); + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ParserSummary that = (ParserSummary) o; + return Objects.equals(id, that.id) + && Objects.equals(name, that.name); } - if (o == null || getClass() != o.getClass()) { - return false; + + @Override + public int hashCode() { + return Objects.hash(id, name); + } + + @Override + public String toString() { + return "ParserType{" + + "id=" + id + + ", name=" + name + + '}'; } - ParserSummary that = (ParserSummary) o; - return Objects.equals(id, that.id) && - Objects.equals(name, that.name); - } - - @Override - public int hashCode() { - return Objects.hash(id, name); - } - - @Override - public String toString() { - return "ParserType{" + - "id=" + id + - ", name=" + name + - '}'; - } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserSummaryMapper.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserSummaryMapper.java index fbf3ac2e..46bce3d8 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserSummaryMapper.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/summary/ParserSummaryMapper.java @@ -21,9 +21,9 @@ public class ParserSummaryMapper implements ObjectMapper clazz; try { clazz = Class.forName(clazzName); - } catch(ClassNotFoundException e) { + } catch (ClassNotFoundException e) { String msg = String.format("Parser class not found; class=%s", clazzName); throw new IllegalArgumentException(msg, e); } - if(Parser.class.isAssignableFrom(clazz)) { + if (Parser.class.isAssignableFrom(clazz)) { // the cast is guaranteed to be safe because of the 'if' condition above @SuppressWarnings("unchecked") Class parserClass = (Class) clazz; return ParserInfo.builder() - .parserClass(parserClass) - .name(source.getName().getName()) - .description(source.getDescription()) - .build(); + .parserClass(parserClass) + .name(source.getName().getName()) + .description(source.getDescription()) + .build(); } else { String msg = String.format("Parser class is not a valid parser; class=%s", clazzName); diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainBuilderService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainBuilderService.java index d52bb447..fce0f763 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainBuilderService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainBuilderService.java @@ -24,6 +24,7 @@ public interface ChainBuilderService { /** * Builds a parser chain from a {@link ParserChainSchema} which defines * what a parser chain looks like. + * * @param chainSchema The blueprint for building the parser chain. * @return The first link in the parser chain. * @throws InvalidParserException If the user has defined an invalid parser chain. diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainExecutorService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainExecutorService.java index e22bbd4e..86f2a293 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainExecutorService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainExecutorService.java @@ -22,7 +22,8 @@ public interface ChainExecutorService { /** * Executes a parser chain by parsing a message. - * @param chain The parser chain to execute. + * + * @param chain The parser chain to execute. * @param textToParse The text to parse. * @return The result of parsing the text with the parser chain. */ diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainPersistenceService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainPersistenceService.java index e8f64b52..d6e95219 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainPersistenceService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ChainPersistenceService.java @@ -14,7 +14,6 @@ import com.cloudera.parserchains.core.model.define.ParserChainSchema; import com.cloudera.parserchains.queryservice.model.summary.ParserChainSummary; - import java.io.IOException; import java.nio.file.Path; import java.util.List; @@ -24,42 +23,39 @@ */ public interface ChainPersistenceService { - List findAll(Path path) throws IOException; - - /** - * Creates and persists a new parser chain. - * @param chain The parser chain to save. - * @param path The path to save the parser chain to. - * @return - * @throws IOException - */ - ParserChainSchema create(ParserChainSchema chain, Path path) throws IOException; - - /** - * Retrieves a persisted parser chain. - * @param id The id of the parser chain. - * @param path The path to save the parser chain to. - * @return The parser chain with the given id. - * @throws IOException - */ - ParserChainSchema read(String id, Path path) throws IOException; - - /** - * Updates an existing parser chain. - * @param id The id of the parser chain. - * @param chain The new definition of the parser chain. - * @param path The path where the parser chain is persisted. - * @return - * @throws IOException - */ - ParserChainSchema update(String id, ParserChainSchema chain, Path path) throws IOException; - - /** - * Deletes a persisted parser chain. - * @param id The id of the parser chain. - * @param path The path where the parser chain is persisted. - * @return - * @throws IOException - */ - boolean delete(String id, Path path) throws IOException; + List findAll(Path path) throws IOException; + + /** + * Creates and persists a new parser chain. + * + * @param chain The parser chain to save. + * @param path The path to save the parser chain to. + */ + ParserChainSchema create(ParserChainSchema chain, Path path) throws IOException; + + /** + * Retrieves a persisted parser chain. + * + * @param id The id of the parser chain. + * @param path The path to save the parser chain to. + * @return The parser chain with the given id. + */ + ParserChainSchema read(String id, Path path) throws IOException; + + /** + * Updates an existing parser chain. + * + * @param id The id of the parser chain. + * @param chain The new definition of the parser chain. + * @param path The path where the parser chain is persisted. + */ + ParserChainSchema update(String id, ParserChainSchema chain, Path path) throws IOException; + + /** + * Deletes a persisted parser chain. + * + * @param id The id of the parser chain. + * @param path The path where the parser chain is persisted. + */ + boolean delete(String id, Path path) throws IOException; } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java index c1e3c3d3..af3ef8ec 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java @@ -6,14 +6,13 @@ import com.cloudera.service.common.request.RequestType; import com.cloudera.service.common.response.ResponseBody; import com.cloudera.service.common.response.ResponseType; +import java.util.List; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Service; -import java.util.List; -import java.util.stream.Collectors; - @Slf4j @Service @RequiredArgsConstructor @@ -21,11 +20,14 @@ public class ClusterService { private final KafkaService kafkaService; public List getAllClusterInfo() throws FailedAllClusterReponseException { - List> response = kafkaService.sendWithReply(RequestType.GET_ALL_CLUSTERS_SERVICE_REQUEST, RequestBody.builder().build()); + List> response = + kafkaService.sendWithReply(RequestType.GET_ALL_CLUSTERS_SERVICE_REQUEST, RequestBody.builder().build()); List failedResponses = response.stream() - .filter(pair -> pair.getKey() != null && ResponseType.GET_ALL_CLUSTERS_SERVICE_RESPONSE != pair.getKey()) - .map(Pair::getValue) - .collect(Collectors.toList()); + .filter(pair -> pair.getKey() != null + && ResponseType.GET_ALL_CLUSTERS_SERVICE_RESPONSE + != pair.getKey()) + .map(Pair::getValue) + .collect(Collectors.toList()); if (!failedResponses.isEmpty()) { throw new FailedAllClusterReponseException(failedResponses); } @@ -37,7 +39,8 @@ public ResponseBody getClusterInfo(String clusterId) throws FailedClusterReponse } public ResponseBody getClusterInfo(String clusterId, RequestBody body) throws FailedClusterReponseException { - Pair response = kafkaService.sendWithReply(RequestType.GET_CLUSTER_SERVICE_REQUEST, clusterId, body); + Pair response = + kafkaService.sendWithReply(RequestType.GET_CLUSTER_SERVICE_REQUEST, clusterId, body); if (response.getKey() != ResponseType.GET_CLUSTER_SERVICE_RESPONSE) { throw new FailedClusterReponseException(response.getValue()); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/IndexingService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/IndexingService.java index 791b3089..cf7f7f76 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/IndexingService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/IndexingService.java @@ -2,6 +2,9 @@ import com.cloudera.cyber.indexing.MappingDto; import com.cloudera.parserchains.core.utils.JSONUtils; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; @@ -10,10 +13,6 @@ import org.apache.flink.core.fs.Path; import org.springframework.stereotype.Service; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Map; - @Slf4j @Service @RequiredArgsConstructor @@ -27,8 +26,9 @@ public Object getMappingsFromPath(String path) throws IOException { } try (FSDataInputStream fsDataInputStream = fileSystem.open(indexingPath)) { final String chainString = IOUtils.toString(fsDataInputStream, StandardCharsets.UTF_8); - final JSONUtils.ReferenceSupplier> ref = new JSONUtils.ReferenceSupplier>() { - }; + final JSONUtils.ReferenceSupplier> ref = + new JSONUtils.ReferenceSupplier>() { + }; //validate the json value is the valid mapping json JSONUtils.INSTANCE.load(chainString, ref); //Converting to map so that we ignore all the overwritten getters diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/JobService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/JobService.java index e194d452..8695baf8 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/JobService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/JobService.java @@ -23,7 +23,8 @@ public class JobService { private final KafkaService kafkaService; private final ClusterService clusterService; - public ResponseBody makeRequest(String clusterId, RequestBody body, String actionString) throws FailedClusterReponseException { + public ResponseBody makeRequest(String clusterId, RequestBody body, String actionString) + throws FailedClusterReponseException { JobActions action = Utils.getEnumFromString(actionString, JobActions.class, JobActions::getAction); switch (action) { case START: @@ -36,10 +37,12 @@ public ResponseBody makeRequest(String clusterId, RequestBody body, String actio return sendAction(clusterId, body, RequestType.RESTART_JOB_REQUEST, ResponseType.RESTART_JOB_RESPONSE); case UPDATE_CONFIG: validateJobId(body, action.getAction()); - return sendAction(clusterId, body, RequestType.UPDATE_JOB_CONFIG_REQUEST, ResponseType.UPDATE_JOB_CONFIG_RESPONSE); + return sendAction(clusterId, body, RequestType.UPDATE_JOB_CONFIG_REQUEST, + ResponseType.UPDATE_JOB_CONFIG_RESPONSE); case GET_CONFIG: validateJobId(body, action.getAction()); - return sendAction(clusterId, body, RequestType.GET_JOB_CONFIG_REQUEST, ResponseType.GET_JOB_CONFIG_RESPONSE); + return sendAction(clusterId, body, RequestType.GET_JOB_CONFIG_REQUEST, + ResponseType.GET_JOB_CONFIG_RESPONSE); case STATUS: validateJobId(body, action.getAction()); return clusterService.getClusterInfo(clusterId, body); @@ -48,10 +51,11 @@ public ResponseBody makeRequest(String clusterId, RequestBody body, String actio } } - private ResponseBody sendAction(String clusterId, RequestBody body, RequestType requestType, ResponseType responseType) { + private ResponseBody sendAction(String clusterId, RequestBody body, RequestType requestType, + ResponseType responseType) { Pair response = kafkaService.sendWithReply(requestType, clusterId, body); if (response.getKey() != responseType) { - throw new FailedJobAction("Failed to " + requestType.name() +" a job: " + response.getValue()); + throw new FailedJobAction("Failed to " + requestType.name() + " a job: " + response.getValue()); } return response.getValue(); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java index 2e20f6e8..9b82a84e 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java @@ -9,18 +9,6 @@ import com.cloudera.service.common.response.ClusterMeta; import com.cloudera.service.common.response.ResponseBody; import com.cloudera.service.common.response.ResponseType; -import lombok.AllArgsConstructor; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.kafka.core.KafkaOperations; -import org.springframework.kafka.requestreply.RequestReplyFuture; -import org.springframework.stereotype.Service; - import java.time.Duration; import java.util.List; import java.util.Map; @@ -29,6 +17,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.requestreply.RequestReplyFuture; +import org.springframework.stereotype.Service; @Slf4j @Service @@ -38,7 +34,8 @@ public class KafkaService { private final Long kafkaTemplateTimeout; - public KafkaService(@Qualifier("kafkaTemplatePool") Map> kafkaTemplatePool, + public KafkaService(@Qualifier("kafkaTemplatePool") + Map> kafkaTemplatePool, @Value("${kafka.reply.future.timeout:45}") Long replyFutureTimeout, @Value("${kafka.reply.timeout:45}") Long kafkaTemplateTimeout) { this.kafkaTemplatePool = kafkaTemplatePool; @@ -48,7 +45,8 @@ public KafkaService(@Qualifier("kafkaTemplatePool") Map sendWithReply(RequestType requestType, String clusterId, RequestBody body) { - final ClouderaReplyingKafkaTemplate kafkaTemplate = kafkaTemplatePool.get(clusterId); + final ClouderaReplyingKafkaTemplate kafkaTemplate = + kafkaTemplatePool.get(clusterId); if (kafkaTemplate == null) { log.error("Cluster not found with cluster id: '{}'", clusterId); throw new KafkaClusterNotFound("Cluster not found! with cluster id '" + clusterId + "'"); @@ -58,31 +56,41 @@ public Pair sendWithReply(RequestType requestType, S public List> sendWithReply(RequestType requestType, RequestBody body) { return kafkaTemplatePool.entrySet().stream() - .map(entry -> send(requestType, body, entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); + .map(entry -> send(requestType, body, entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); } - private Pair send(RequestType requestType, RequestBody body, String clusterId, ClouderaReplyingKafkaTemplate kafkaTemplate) { - ProducerRecord producerRecord = new ProducerRecord<>(kafkaTemplate.getRequestTopic(), requestType.name(), body); - RequestReplyFuture replyFuture = kafkaTemplate.sendAndReceive(producerRecord, Duration.ofSeconds(kafkaTemplateTimeout)); + private Pair send(RequestType requestType, RequestBody body, String clusterId, + ClouderaReplyingKafkaTemplate kafkaTemplate) { + ProducerRecord producerRecord = + new ProducerRecord<>(kafkaTemplate.getRequestTopic(), requestType.name(), body); + RequestReplyFuture replyFuture = + kafkaTemplate.sendAndReceive(producerRecord, Duration.ofSeconds(kafkaTemplateTimeout)); try { ConsumerRecord consumerRecord = replyFuture.get(replyFutureTimeout, TimeUnit.SECONDS); if (consumerRecord == null) { throw new KafkaException("Got no reply from kafka"); } ResponseBody responseBody = consumerRecord.value(); - return Pair.of(Utils.getEnumFromString(consumerRecord.key(), ResponseType.class, ResponseType::name), responseBody); + return Pair.of(Utils.getEnumFromString(consumerRecord.key(), ResponseType.class, ResponseType::name), + responseBody); } catch (ExecutionException e) { // Handle exception - log.error("Exception thrown when attempting to retrieve the information from kafka. Message: '{}' \n Cause: '{}'", e.getMessage(), e.getCause().getMessage()); - return Pair.of(null, ResponseBody.builder().clusterMeta(ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build()); + log.error( + "Exception thrown when attempting to retrieve the information from kafka. Message: '{}' \n Cause: '{}'", + e.getMessage(), e.getCause().getMessage()); + return Pair.of(null, ResponseBody.builder().clusterMeta( + ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build()); } catch (TimeoutException e) { // Handle exception log.error("Timeout while waiting for reply"); - return Pair.of(null, ResponseBody.builder().clusterMeta(ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build()); + return Pair.of(null, ResponseBody.builder().clusterMeta( + ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build()); } catch (InterruptedException e) { - log.warn("Kafka throws interruption exception. Message: '{}' Cause: '{}'", e.getMessage(), Optional.ofNullable(e.getCause()).map(Throwable::getMessage).orElse("Cause is empty")); - return Pair.of(null, ResponseBody.builder().clusterMeta(ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build()); + log.warn("Kafka throws interruption exception. Message: '{}' Cause: '{}'", e.getMessage(), + Optional.ofNullable(e.getCause()).map(Throwable::getMessage).orElse("Cause is empty")); + return Pair.of(null, ResponseBody.builder().clusterMeta( + ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build()); } } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ParserDiscoveryService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ParserDiscoveryService.java index 97daedb5..d3734553 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ParserDiscoveryService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ParserDiscoveryService.java @@ -24,26 +24,26 @@ */ public interface ParserDiscoveryService { - /** - * Finds all the parser types available for the user to build parser chains with. - * - * @return list of ParserSummaries - * @throws IOException in case there are issues reading parsers. - */ - List findAll() throws IOException; + /** + * Finds all the parser types available for the user to build parser chains with. + * + * @return list of ParserSummaries + * @throws IOException in case there are issues reading parsers. + */ + List findAll() throws IOException; - /** - * Describes the configuration parameters available for a given parser. - * - * @param name The parser name. - * @return ParserDescriptor based on its ID. - */ - ParserDescriptor describe(ParserID name); + /** + * Describes the configuration parameters available for a given parser. + * + * @param name The parser name. + * @return ParserDescriptor based on its ID. + */ + ParserDescriptor describe(ParserID name); - /** - * Describes the configuration parameters for all available parser types. - * - * @return ParserDescriptors with their IDs. - */ - Map describeAll(); + /** + * Describes the configuration parameters for all available parser types. + * + * @return ParserDescriptors with their IDs. + */ + Map describeAll(); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ParserSampleService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ParserSampleService.java index ee86edaa..ed4e9da1 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ParserSampleService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ParserSampleService.java @@ -2,6 +2,11 @@ import com.cloudera.parserchains.core.utils.JSONUtils; import com.cloudera.parserchains.queryservice.model.sample.ParserSample; +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; @@ -11,12 +16,6 @@ import org.apache.flink.core.fs.Path; import org.springframework.stereotype.Service; -import java.io.BufferedOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.List; - @Slf4j @Service @RequiredArgsConstructor @@ -31,25 +30,29 @@ public List findAllById(String sampleFolderPath, String id) throws } try (FSDataInputStream fsDataInputStream = fileSystem.open(samplePath)) { final String chainString = IOUtils.toString(fsDataInputStream, StandardCharsets.UTF_8); - final JSONUtils.ReferenceSupplier> ref = new JSONUtils.ReferenceSupplier>() { - }; + final JSONUtils.ReferenceSupplier> ref = + new JSONUtils.ReferenceSupplier>() { + }; return JSONUtils.INSTANCE.load(chainString, ref); } catch (Exception e) { - throw new RuntimeException(String.format("Wasn't able to read the chain sample file [%s]!", samplePathStr), e); + throw new RuntimeException(String.format("Wasn't able to read the chain sample file [%s]!", samplePathStr), + e); } } - public List update(String sampleFolderPath, String id, List sampleList) throws IOException { + public List update(String sampleFolderPath, String id, List sampleList) + throws IOException { final String samplePathStr = sampleFolderPath.concat(id).concat(".json"); final Path samplePath = new Path(samplePathStr); FileSystem fileSystem = samplePath.getFileSystem(); try (FSDataOutputStream fsDataOutputStream = fileSystem.create(samplePath, FileSystem.WriteMode.OVERWRITE); - DataOutputStream dataOutputStream = new DataOutputStream(new BufferedOutputStream(fsDataOutputStream))) { + DataOutputStream dataOutputStream = new DataOutputStream(new BufferedOutputStream(fsDataOutputStream))) { final String json = JSONUtils.INSTANCE.toJSON(sampleList, true); dataOutputStream.write(json.getBytes(StandardCharsets.UTF_8)); } catch (Exception e) { - throw new RuntimeException(String.format("Wasn't able to create the chain sample file [%s]!", samplePathStr), e); + throw new RuntimeException( + String.format("Wasn't able to create the chain sample file [%s]!", samplePathStr), e); } return findAllById(sampleFolderPath, id); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/PipelineService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/PipelineService.java index 2264a6d2..b2a59d3a 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/PipelineService.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/PipelineService.java @@ -14,6 +14,9 @@ import com.cloudera.parserchains.queryservice.config.AppProperties; import com.cloudera.parserchains.queryservice.model.exec.PipelineResult; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.flink.core.fs.FileStatus; @@ -23,130 +26,126 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - @Slf4j @Service @RequiredArgsConstructor public class PipelineService { - private final AppProperties appProperties; - private final CacheManager cacheManager; + private final AppProperties appProperties; + private final CacheManager cacheManager; - @Cacheable("pipelinePathMap") - public Map findAll() throws IOException { - final Path pipelineRootPath = getPipelineRootPath(); + @Cacheable("pipelinePathMap") + public Map findAll() throws IOException { + final Path pipelineRootPath = getPipelineRootPath(); - final FileSystem fileSystem = pipelineRootPath.getFileSystem(); - if (!fileSystem.exists(pipelineRootPath)) { - return null; - } - final FileStatus[] statusList = fileSystem.listStatus(pipelineRootPath); - if (statusList == null) { - return null; - } + final FileSystem fileSystem = pipelineRootPath.getFileSystem(); + if (!fileSystem.exists(pipelineRootPath)) { + return null; + } + final FileStatus[] statusList = fileSystem.listStatus(pipelineRootPath); + if (statusList == null) { + return null; + } - final Map pipelineMap = new HashMap<>(); - for (FileStatus fileStatus : statusList) { - if (fileStatus.isDir()) { - final Path pipelinePath = fileStatus.getPath(); - if (isValidPipeline(pipelinePath, fileSystem)) { - final String name = pipelinePath.getName(); - final PipelineResult pipeline = PipelineResult.builder() - .name(name) - .path(appendChainsPath(pipelinePath)) - .build(); - pipelineMap.put(name, pipeline); + final Map pipelineMap = new HashMap<>(); + for (FileStatus fileStatus : statusList) { + if (fileStatus.isDir()) { + final Path pipelinePath = fileStatus.getPath(); + if (isValidPipeline(pipelinePath, fileSystem)) { + final String name = pipelinePath.getName(); + final PipelineResult pipeline = PipelineResult.builder() + .name(name) + .path(appendChainsPath(pipelinePath)) + .build(); + pipelineMap.put(name, pipeline); + } + } } - } - } - return pipelineMap; - } - - public PipelineResult createPipeline(String pipelineName) throws IOException { - final Path pipelineRootPath = getPipelineRootPath(); - final FileSystem fileSystem = pipelineRootPath.getFileSystem(); - if (!fileSystem.exists(pipelineRootPath)) { - return null; + return pipelineMap; } - final Path pipelinePath = new Path(pipelineRootPath.getPath(), pipelineName); - final Path fullValidPath = new Path(pipelinePath, "parse/chains"); + public PipelineResult createPipeline(String pipelineName) throws IOException { + final Path pipelineRootPath = getPipelineRootPath(); + final FileSystem fileSystem = pipelineRootPath.getFileSystem(); + if (!fileSystem.exists(pipelineRootPath)) { + return null; + } - if (fileSystem.mkdirs(fullValidPath)) { - evictPipelineCache(); - return PipelineResult.builder() - .name(pipelineName) - .path(fullValidPath) - .build(); - } - return null; - } - - public PipelineResult renamePipeline(String pipelineName, String newName) throws IOException { - final Path pipelineRootPath = getPipelineRootPath(); - final FileSystem fileSystem = pipelineRootPath.getFileSystem(); - if (!fileSystem.exists(pipelineRootPath)) { - return null; - } + final Path pipelinePath = new Path(pipelineRootPath.getPath(), pipelineName); + final Path fullValidPath = new Path(pipelinePath, "parse/chains"); - final Path originalPath = new Path(pipelineRootPath, pipelineName); - if (!isValidPipeline(originalPath, fileSystem)) { - throw new RuntimeException("Provided original pipeline name doesn't correspond to a valid pipeline!"); + if (fileSystem.mkdirs(fullValidPath)) { + evictPipelineCache(); + return PipelineResult.builder() + .name(pipelineName) + .path(fullValidPath) + .build(); + } + return null; } - final Path newPath = new Path(pipelineRootPath, newName); - if (fileSystem.exists(newPath)) { - throw new RuntimeException("New pipeline name is already occupied!"); + public PipelineResult renamePipeline(String pipelineName, String newName) throws IOException { + final Path pipelineRootPath = getPipelineRootPath(); + final FileSystem fileSystem = pipelineRootPath.getFileSystem(); + if (!fileSystem.exists(pipelineRootPath)) { + return null; + } + + final Path originalPath = new Path(pipelineRootPath, pipelineName); + if (!isValidPipeline(originalPath, fileSystem)) { + throw new RuntimeException("Provided original pipeline name doesn't correspond to a valid pipeline!"); + } + final Path newPath = new Path(pipelineRootPath, newName); + + if (fileSystem.exists(newPath)) { + throw new RuntimeException("New pipeline name is already occupied!"); + } + + if (fileSystem.rename(originalPath, newPath)) { + evictPipelineCache(); + return PipelineResult.builder() + .name(newName) + .path(newPath) + .build(); + } + + return null; } - if (fileSystem.rename(originalPath, newPath)) { - evictPipelineCache(); - return PipelineResult.builder() - .name(newName) - .path(newPath) - .build(); + public boolean deletePipeline(String pipelineName) throws IOException { + final Path pipelineRootPath = getPipelineRootPath(); + final FileSystem fileSystem = pipelineRootPath.getFileSystem(); + if (!fileSystem.exists(pipelineRootPath)) { + return false; + } + + final Path pipelinePath = new Path(pipelineRootPath, pipelineName); + if (!isValidPipeline(pipelinePath, fileSystem)) { + throw new RuntimeException("Provided pipeline name doesn't correspond to a valid pipeline!"); + } + if (fileSystem.delete(pipelinePath, true)) { + evictPipelineCache(); + return true; + } + return false; } - return null; - } + private Path appendChainsPath(Path originalPath) { + return new Path(originalPath, "parse/chains"); + } - public boolean deletePipeline(String pipelineName) throws IOException { - final Path pipelineRootPath = getPipelineRootPath(); - final FileSystem fileSystem = pipelineRootPath.getFileSystem(); - if (!fileSystem.exists(pipelineRootPath)) { - return false; + private boolean isValidPipeline(Path originalPath, FileSystem fileSystem) throws IOException { + final Path fullPath = appendChainsPath(originalPath); + return fileSystem.exists(fullPath); } - final Path pipelinePath = new Path(pipelineRootPath, pipelineName); - if (!isValidPipeline(pipelinePath, fileSystem)) { - throw new RuntimeException("Provided pipeline name doesn't correspond to a valid pipeline!"); + private Path getPipelineRootPath() { + String pipelinePathStr = appProperties.getPipelinesPath(); + return new Path(pipelinePathStr); } - if (fileSystem.delete(pipelinePath, true)){ - evictPipelineCache(); - return true; + + private void evictPipelineCache() { + cacheManager.getCache("pipelinePathMap").clear(); } - return false; - } - - private Path appendChainsPath(Path originalPath){ - return new Path(originalPath, "parse/chains"); - } - - private boolean isValidPipeline(Path originalPath, FileSystem fileSystem) throws IOException { - final Path fullPath = appendChainsPath(originalPath); - return fileSystem.exists(fullPath); - } - - private Path getPipelineRootPath() { - String pipelinePathStr = appProperties.getPipelinesPath(); - return new Path(pipelinePathStr); - } - - private void evictPipelineCache() { - cacheManager.getCache("pipelinePathMap").clear(); - } } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ResultLogBuilder.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ResultLogBuilder.java index 8512bcb8..34b9c154 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ResultLogBuilder.java +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ResultLogBuilder.java @@ -13,18 +13,19 @@ package com.cloudera.parserchains.queryservice.service; import com.cloudera.parserchains.queryservice.model.exec.ResultLog; +import java.util.List; import org.apache.commons.lang3.RegExUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import java.util.List; - public class ResultLogBuilder { public static final String DEFAULT_SUCCESS_MESSAGE = "success"; public static final String INFO_TYPE = "info"; public static final String ERROR_TYPE = "error"; /** + * Returns new SuccessResultBuilder. + * * @return A builder that creates a result when a message is successfully parsed. */ public static SuccessResultBuilder success() { @@ -32,6 +33,8 @@ public static SuccessResultBuilder success() { } /** + * Returns new ErrorResultBuilder. + * * @return A builder that creates a result that indicates an error has occurred. */ public static ErrorResultBuilder error() { @@ -49,7 +52,7 @@ public static class ErrorResultBuilder { private String parserId; private String parserName; private String message; - private String type; + private final String type; private Throwable exception; public ErrorResultBuilder() { @@ -57,8 +60,8 @@ public ErrorResultBuilder() { } public ErrorResultBuilder parserId(String parserId) { - this.parserId = parserId; - return this; + this.parserId = parserId; + return this; } public ErrorResultBuilder parserName(String parserName) { @@ -78,14 +81,14 @@ public ErrorResultBuilder message(String message) { public ResultLog build() { ResultLog resultLog = new ResultLog() - .setParserId(parserId) - .setParserName(parserName) - .setType(type); - if(exception != null) { + .setParserId(parserId) + .setParserName(parserName) + .setType(type); + if (exception != null) { resultLog.setMessage(getUsefulMessage(exception)) .setStackTrace(ExceptionUtils.getStackTrace(exception)); } else { - resultLog.setMessage(message); + resultLog.setMessage(message); } return resultLog; } @@ -95,7 +98,7 @@ public static class SuccessResultBuilder { private String parserId; private String parserName; private String message; - private String type; + private final String type; public SuccessResultBuilder() { this.message = DEFAULT_SUCCESS_MESSAGE; @@ -119,10 +122,10 @@ public SuccessResultBuilder message(String message) { public ResultLog build() { return new ResultLog() - .setParserId(parserId) - .setParserName(parserName) - .setMessage(message) - .setType(type); + .setParserId(parserId) + .setParserName(parserName) + .setMessage(message) + .setType(type); } } @@ -132,21 +135,21 @@ public ResultLog build() { *

      The root cause exception does not always contain * a useful message. This traces backwards from the root * exception until it finds a useful error message. + * * @param t The exception. - * @return */ private static String getUsefulMessage(Throwable t) { List throwables = ExceptionUtils.getThrowableList(t); - for(int i=throwables.size()-1; i>=0; i--) { + for (int i = throwables.size() - 1; i >= 0; i--) { Throwable cause = throwables.get(i); String message = removeClassName(cause.getMessage()); - if(StringUtils.isNotBlank(message)) { + if (StringUtils.isNotBlank(message)) { return message; } } String message = t.getMessage(); - if(StringUtils.isNotBlank(message)) { + if (StringUtils.isNotBlank(message)) { return message; } else { return t.getClass().getCanonicalName(); @@ -160,12 +163,12 @@ private static String getUsefulMessage(Throwable t) { * is prepended to the error message. When returning the results to the user, * only the original error message, not the class name should be shown. * - * 

      +     * 
            * removeClassName("IllegalArgumentException: Field name is required.") = "Field name is required."
      * removeClassName("Field name is required.") = "Field name is required."
      * 
      
+     *
      * @param message The message to remove the class name from.
-     * @return
      */
     private static String removeClassName(String message) {
         return RegExUtils.replaceFirst(message, "^\\w+:\\s*", "");
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultChainBuilderService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultChainBuilderService.java
index c62fc80e..304c02ef 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultChainBuilderService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultChainBuilderService.java
@@ -21,7 +21,7 @@
 
 @Service
 public class DefaultChainBuilderService implements ChainBuilderService {
-    private ChainBuilder chainBuilder;
+    private final ChainBuilder chainBuilder;
 
     public DefaultChainBuilderService(ChainBuilder chainBuilder) {
         this.chainBuilder = chainBuilder;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultChainExecutorService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultChainExecutorService.java
index 93217b7a..21ba6de0 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultChainExecutorService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultChainExecutorService.java
@@ -12,6 +12,9 @@
 
 package com.cloudera.parserchains.queryservice.service.impl;
 
+import static com.cloudera.parserchains.queryservice.service.ResultLogBuilder.error;
+import static com.cloudera.parserchains.queryservice.service.ResultLogBuilder.success;
+
 import com.cloudera.parserchains.core.ChainLink;
 import com.cloudera.parserchains.core.ChainRunner;
 import com.cloudera.parserchains.core.Message;
@@ -19,20 +22,16 @@
 import com.cloudera.parserchains.queryservice.model.exec.ResultLog;
 import com.cloudera.parserchains.queryservice.service.ChainExecutorService;
 import com.cloudera.parserchains.queryservice.service.ResultLogBuilder;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
-
-import static com.cloudera.parserchains.queryservice.service.ResultLogBuilder.error;
-import static com.cloudera.parserchains.queryservice.service.ResultLogBuilder.success;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
 
 @Service
 @Slf4j
 public class DefaultChainExecutorService implements ChainExecutorService {
-    private ChainRunner chainRunner;
+    private final ChainRunner chainRunner;
 
     public DefaultChainExecutorService(ChainRunner chainRunner) {
         this.chainRunner = chainRunner;
@@ -48,13 +47,14 @@ public ParserResult execute(ChainLink chain, String textToParse) {
             } else {
                 return chainNotDefined(original);
             }
-        } catch(Throwable e) {
+        } catch (Throwable e) {
             return chainFailed(original, e);
         }
     }
 
     /**
      * Returns a {@link ParserResult} after a parser chain was executed.
+     *
      * @param messages The result of executing the parser chain.
      */
     private ParserResult chainExecuted(List messages) {
@@ -63,20 +63,20 @@ private ParserResult chainExecuted(List messages) {
         // define the input fields for the parser chain
         Message input = messages.get(0);
         result.setInput(input.getFields()
-                .entrySet()
-                .stream()
-                .collect(Collectors.toMap(
-                        e -> e.getKey().get(),
-                        e -> e.getValue().get())));
+                             .entrySet()
+                             .stream()
+                             .collect(Collectors.toMap(
+                                   e -> e.getKey().get(),
+                                   e -> e.getValue().get())));
 
         // define the fields output by the parser chain
-        Message output = messages.get(messages.size()-1);
+        Message output = messages.get(messages.size() - 1);
         result.setOutput(output.getFields()
-                .entrySet()
-                .stream()
-                .collect(Collectors.toMap(
-                        e -> e.getKey().get(),
-                        e -> e.getValue().get())));
+                               .entrySet()
+                               .stream()
+                               .collect(Collectors.toMap(
+                                     e -> e.getKey().get(),
+                                     e -> e.getValue().get())));
 
         // define the log section
         result.setLog(buildResultLog(output));
@@ -92,15 +92,15 @@ private ResultLog buildResultLog(Message output) {
         String parserId = output.getCreatedBy().getLinkName();
         String parserName = output.getCreatedBy().getParserName().getName();
         return output.getError()
-                    .map(e -> error()
-                            .parserId(parserId)
-                            .parserName(parserName)
-                            .exception(e)
-                            .build())
-                    .orElseGet(() -> success()
-                            .parserId(parserId)
-                            .parserName(parserName)
-                            .build());
+                     .map(e -> error()
+                           .parserId(parserId)
+                           .parserName(parserName)
+                           .exception(e)
+                           .build())
+                     .orElseGet(() -> success()
+                           .parserId(parserId)
+                           .parserName(parserName)
+                           .build());
     }
 
     private List buildParserByParserResults(List messages) {
@@ -112,20 +112,20 @@ private List buildParserByParserResults(List messages) {
             // define the input fields
             Message input = messages.get(i);
             result.setInput(input.getFields()
-                    .entrySet()
-                    .stream()
-                    .collect(Collectors.toMap(
-                            e -> e.getKey().get(),
-                            e -> e.getValue().get())));
+                                 .entrySet()
+                                 .stream()
+                                 .collect(Collectors.toMap(
+                                       e -> e.getKey().get(),
+                                       e -> e.getValue().get())));
 
             // define the output fields
             Message output = messages.get(i + 1);
             result.setOutput(output.getFields()
-                    .entrySet()
-                    .stream()
-                    .collect(Collectors.toMap(
-                            e -> e.getKey().get(),
-                            e -> e.getValue().get())));
+                                   .entrySet()
+                                   .stream()
+                                   .collect(Collectors.toMap(
+                                         e -> e.getKey().get(),
+                                         e -> e.getValue().get())));
 
             // define the log section
             ResultLog resultLog = buildResultLog(output);
@@ -139,6 +139,7 @@ private List buildParserByParserResults(List messages) {
     /**
      * Return a {@link ParserResult} indicating that an unexpected error occurred
      * while executing the parser chain.
+     *
      * @param original The original message to parse.
      */
     private ParserResult chainFailed(Message original, Throwable t) {
@@ -147,27 +148,29 @@ private ParserResult chainFailed(Message original, Throwable t) {
 
         // define the input fields
         result.setInput(original.getFields()
-                .entrySet()
-                .stream()
-                .collect(Collectors.toMap(
-                        e -> e.getKey().get(),
-                        e -> e.getValue().get())));
+                                .entrySet()
+                                .stream()
+                                .collect(Collectors.toMap(
+                                      e -> e.getKey().get(),
+                                      e -> e.getValue().get())));
 
         // there are no output fields
         // define the log section
         ResultLog log = ResultLogBuilder.error()
-                .parserId(original.getCreatedBy().getLinkName())
-                .parserName(original.getCreatedBy().getParserName().getName())
-                .exception(t)
-                .build();
+                                        .parserId(original.getCreatedBy().getLinkName())
+                                        .parserName(original.getCreatedBy().getParserName().getName())
+                                        .exception(t)
+                                        .build();
         return result.setLog(log);
     }
 
     /**
      * Return a {@link ParserResult} indicating that no parser chain has yet been
      * defined.  For example, there are no parsers in the chain.
+     *
      * 
      If a parser chain has not yet been defined by the user, the result returned
      * should indicate success even though we could not parse anything.
+     *
      * @param original The original message to parse.
      */
     private ParserResult chainNotDefined(Message original) {
@@ -175,19 +178,19 @@ private ParserResult chainNotDefined(Message original) {
 
         // define the input fields
         result.setInput(original.getFields()
-                .entrySet()
-                .stream()
-                .collect(Collectors.toMap(
-                        e -> e.getKey().get(),
-                        e -> e.getValue().get())));
+                                .entrySet()
+                                .stream()
+                                .collect(Collectors.toMap(
+                                      e -> e.getKey().get(),
+                                      e -> e.getValue().get())));
 
         // there are no output fields
         // define the log section
         ResultLog log = ResultLogBuilder.success()
-                .parserId(original.getCreatedBy().getLinkName())
-                .parserName(original.getCreatedBy().getParserName().getName())
-                .message("No parser chain defined.")
-                .build();
+                                        .parserId(original.getCreatedBy().getLinkName())
+                                        .parserName(original.getCreatedBy().getParserName().getName())
+                                        .message("No parser chain defined.")
+                                        .build();
         return result.setLog(log);
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultParserDiscoveryService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultParserDiscoveryService.java
index 477e671e..da62bdcc 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultParserDiscoveryService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultParserDiscoveryService.java
@@ -6,9 +6,11 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * 
+ * 
      
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,6 +22,7 @@
 
 import static com.cloudera.parserchains.core.utils.AnnotationUtils.getAnnotatedMethods;
 import static com.cloudera.parserchains.core.utils.AnnotationUtils.getAnnotatedParameters;
+
 import com.cloudera.parserchains.core.Parser;
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.Parameter;
@@ -48,123 +51,123 @@
 
 @Service
 public class DefaultParserDiscoveryService implements ParserDiscoveryService {
-  static final String DEFAULT_PATH_ROOT = "config";
-  static final String PATH_DELIMITER = ".";
-
-  @Autowired
-  private ParserCatalog catalog;
-
-  @Autowired
-  private ObjectMapper mapper;
-
-  @Autowired
-  public DefaultParserDiscoveryService(ParserCatalog catalog, ObjectMapper mapper) {
-    this.catalog = catalog;
-    this.mapper = mapper;
-  }
-
-  @Override
-  public List findAll() throws IOException {
-    return catalog.getParsers()
-            .stream()
-            .map(info -> mapper.reform(info))
-            .collect(Collectors.toList());
-  }
-
-  @Override
-  public ParserDescriptor describe(ParserID name) {
-    return describeAll().get(name);
-  }
-
-  @Override
-  public Map describeAll() {
-    return catalog.getParsers()
-            .stream()
-            .collect(Collectors.toMap(
-                    info -> ParserID.of(info.getParserClass()),
-                    info -> describeParser(info)));
-  }
-
-  private ParserDescriptor describeParser(ParserInfo parserInfo) {
-    List descriptors = describeParameters(parserInfo.getParserClass());
-
-    // sort by name for consistency
-    Comparator compareByName = Comparator.comparing(ConfigParamDescriptor::getName);
-    Collections.sort(descriptors, compareByName);
-
-    ParserID id = ParserID.of(parserInfo.getParserClass());
-    ParserName name = ParserName.of(parserInfo.getName());
-    return new ParserDescriptor()
-            .setParserID(id)
-            .setParserName(name)
-            .setConfigurations(descriptors);
-  }
-
-  private List describeParameters(Class parserClass) {
-    List results = new ArrayList<>();
-    Set> annotatedMethods = getAnnotatedMethods(parserClass).entrySet();
-    for(Map.Entry entry: annotatedMethods) {
-      Configurable configurable = entry.getKey();
-      Method method = entry.getValue();
-
-      List parameters = getAnnotatedParameters(method);
-      if(parameters.size() == 0) {
-        // there are no parameter annotations
-        ConfigParamDescriptor descriptor = describeByAnnotation(configurable, Optional.empty());
-        results.add(descriptor);
-
-      } else {
-        // there are parameter annotations
-        for (Parameter parameter : parameters) {
-          ConfigParamDescriptor descriptor = describeByAnnotation(configurable, Optional.of(parameter));
-          results.add(descriptor);
-        }
-      }
+    static final String DEFAULT_PATH_ROOT = "config";
+    static final String PATH_DELIMITER = ".";
+
+    @Autowired
+    private ParserCatalog catalog;
+
+    @Autowired
+    private ObjectMapper mapper;
+
+    @Autowired
+    public DefaultParserDiscoveryService(ParserCatalog catalog, ObjectMapper mapper) {
+        this.catalog = catalog;
+        this.mapper = mapper;
+    }
+
+    @Override
+    public List findAll() throws IOException {
+        return catalog.getParsers()
+                      .stream()
+                      .map(info -> mapper.reform(info))
+                      .collect(Collectors.toList());
+    }
+
+    @Override
+    public ParserDescriptor describe(ParserID name) {
+        return describeAll().get(name);
+    }
+
+    @Override
+    public Map describeAll() {
+        return catalog.getParsers()
+                      .stream()
+                      .collect(Collectors.toMap(
+                            info -> ParserID.of(info.getParserClass()),
+                            info -> describeParser(info)));
+    }
+
+    private ParserDescriptor describeParser(ParserInfo parserInfo) {
+        List descriptors = describeParameters(parserInfo.getParserClass());
+
+        // sort by name for consistency
+        Comparator compareByName = Comparator.comparing(ConfigParamDescriptor::getName);
+        Collections.sort(descriptors, compareByName);
+
+        ParserID id = ParserID.of(parserInfo.getParserClass());
+        ParserName name = ParserName.of(parserInfo.getName());
+        return new ParserDescriptor()
+              .setParserID(id)
+              .setParserName(name)
+              .setConfigurations(descriptors);
     }
-    return results;
-  }
-
-  private ConfigParamDescriptor describeByAnnotation(Configurable configurable, Optional parameter) {
-    /*
-     * If multiple=true, the front-end expects values to be contained within an array. if
-     * multiple=false, the value should NOT be wrapped in an array; just a single map.
-     * Currently, the backend always wraps values, even single values, in arrays.
-     *
-     * Having the backend adhere to what the front-end expects will take some additional
-     * work. As a work-around all configurations are marked as accepting multiple values,
-     * even those that do not.  The consequence of this is that all fields will show the blue,
-     * plus icon to add a field.
-     */
-    ConfigParamDescriptor paramDescriptor = new ConfigParamDescriptor()
-            .setPath(DEFAULT_PATH_ROOT + PATH_DELIMITER + configurable.key())
-            .setMultiple(true)
-            .setMultipleValues(configurable.multipleValues());
-    if (parameter.isPresent()) {
-      // use the parameter-level annotation
-      Parameter p = parameter.get();
-      paramDescriptor.setName(p.key())
-              .setLabel(p.label())
-              .setDescription(p.description())
-              .setRequired(p.required())
-              .setOutputName(p.isOutputName())
-              .setType(p.widgetType());
-      if (StringUtils.isNotBlank(p.defaultValue())) {
-        paramDescriptor.addDefaultValue(p.key(), p.defaultValue());
-      }
-
-    } else {
-      // use the method-level annotation
-      paramDescriptor.setName(configurable.key())
-              .setLabel(configurable.label())
-              .setDescription(configurable.description())
-              .setRequired(configurable.required())
-              .setOutputName(configurable.isOutputName())
-              .setType(configurable.widgetType());
-      if (StringUtils.isNotBlank(configurable.defaultValue())) {
-        paramDescriptor.addDefaultValue(configurable.key(), configurable.defaultValue());
-      }
+
+    private List describeParameters(Class parserClass) {
+        List results = new ArrayList<>();
+        Set> annotatedMethods = getAnnotatedMethods(parserClass).entrySet();
+        for (Map.Entry entry : annotatedMethods) {
+            Configurable configurable = entry.getKey();
+            Method method = entry.getValue();
+
+            List parameters = getAnnotatedParameters(method);
+            if (parameters.size() == 0) {
+                // there are no parameter annotations
+                ConfigParamDescriptor descriptor = describeByAnnotation(configurable, Optional.empty());
+                results.add(descriptor);
+
+            } else {
+                // there are parameter annotations
+                for (Parameter parameter : parameters) {
+                    ConfigParamDescriptor descriptor = describeByAnnotation(configurable, Optional.of(parameter));
+                    results.add(descriptor);
+                }
+            }
+        }
+        return results;
     }
 
-    return paramDescriptor;
-  }
+    private ConfigParamDescriptor describeByAnnotation(Configurable configurable, Optional parameter) {
+        /*
+         * If multiple=true, the front-end expects values to be contained within an array. if
+         * multiple=false, the value should NOT be wrapped in an array; just a single map.
+         * Currently, the backend always wraps values, even single values, in arrays.
+         *
+         * Having the backend adhere to what the front-end expects will take some additional
+         * work. As a work-around all configurations are marked as accepting multiple values,
+         * even those that do not.  The consequence of this is that all fields will show the blue,
+         * plus icon to add a field.
+         */
+        ConfigParamDescriptor paramDescriptor = new ConfigParamDescriptor()
+              .setPath(DEFAULT_PATH_ROOT + PATH_DELIMITER + configurable.key())
+              .setMultiple(true)
+              .setMultipleValues(configurable.multipleValues());
+        if (parameter.isPresent()) {
+            // use the parameter-level annotation
+            Parameter p = parameter.get();
+            paramDescriptor.setName(p.key())
+                           .setLabel(p.label())
+                           .setDescription(p.description())
+                           .setRequired(p.required())
+                           .setOutputName(p.isOutputName())
+                           .setType(p.widgetType());
+            if (StringUtils.isNotBlank(p.defaultValue())) {
+                paramDescriptor.addDefaultValue(p.key(), p.defaultValue());
+            }
+
+        } else {
+            // use the method-level annotation
+            paramDescriptor.setName(configurable.key())
+                           .setLabel(configurable.label())
+                           .setDescription(configurable.description())
+                           .setRequired(configurable.required())
+                           .setOutputName(configurable.isOutputName())
+                           .setType(configurable.widgetType());
+            if (StringUtils.isNotBlank(configurable.defaultValue())) {
+                paramDescriptor.addDefaultValue(configurable.key(), configurable.defaultValue());
+            }
+        }
+
+        return paramDescriptor;
+    }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/FileBasedChainPersistenceService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/FileBasedChainPersistenceService.java
index 18dcbeab..8decd296 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/FileBasedChainPersistenceService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/FileBasedChainPersistenceService.java
@@ -3,8 +3,10 @@
  * file distributed with this work for additional information regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
  * License.  You may obtain a copy of the License at
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -36,139 +38,142 @@
 @Slf4j
 public class FileBasedChainPersistenceService implements ChainPersistenceService {
 
-  @Autowired
-  private IDGenerator idGenerator;
+    @Autowired
+    private IDGenerator idGenerator;
+
+    /**
+     * Only return json files.
+     */
+    private final DirectoryStream.Filter fileFilter = new DirectoryStream.Filter() {
+        @Override
+        public boolean accept(Path entry) throws IOException {
+            PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:*.json");
+            // can't compare only the entry bc the globbing won't cross directory boundaries. We only care about the filename here, anyhow.
+            return matcher.matches(entry.getFileName());
+        }
+    };
+
+    @Autowired
+    public FileBasedChainPersistenceService(IDGenerator idGenerator) {
+        this.idGenerator = idGenerator;
+    }
 
-  /**
-   * Only return json files.
-   */
-  private DirectoryStream.Filter fileFilter = new DirectoryStream.Filter() {
     @Override
-    public boolean accept(Path entry) throws IOException {
-      PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:*.json");
-      // can't compare only the entry bc the globbing won't cross directory boundaries. We only care about the filename here, anyhow.
-      return matcher.matches(entry.getFileName());
+    public List findAll(Path path) throws IOException {
+        if (!Files.exists(path)) {
+            Files.createDirectories(path);
+        }
+        List inputs = getFilesSorted(path);
+        List summaries = new ArrayList<>();
+        for (Path file : inputs) {
+            try {
+                ParserChainSchema chain = JSONUtils.INSTANCE.load(file.toFile(), ParserChainSchema.class);
+                summaries.add(new ParserChainSummary(chain));
+            } catch (IOException ioe) {
+                log.warn(
+                      "Found a file in the config directory that was unable to be deserialized as a parser chain: '{}'",
+                      file, ioe);
+            }
+        }
+        return summaries;
     }
-  };
 
-  @Autowired
-  public FileBasedChainPersistenceService(IDGenerator idGenerator) {
-    this.idGenerator = idGenerator;
-  }
+    private List getFilesSorted(Path path) throws IOException {
+        List inputs = new ArrayList<>();
+        try (DirectoryStream files = Files.newDirectoryStream(path, fileFilter)) {
+            for (Path file : files) {
+                inputs.add(file);
+            }
+        }
+        sortByName(inputs);
+        return inputs;
+    }
 
-  @Override
-  public List findAll(Path path) throws IOException {
-    if (!Files.exists(path)) {
-      Files.createDirectories(path);
+    private void sortByName(List paths) {
+        paths.sort(Comparator.comparing(p -> p.getFileName().toString()));
     }
-    List inputs = getFilesSorted(path);
-    List summaries = new ArrayList<>();
-    for (Path file : inputs) {
-      try {
-        ParserChainSchema chain = JSONUtils.INSTANCE.load(file.toFile(), ParserChainSchema.class);
-        summaries.add(new ParserChainSummary(chain));
-      } catch (IOException ioe) {
-        log.warn(
-            "Found a file in the config directory that was unable to be deserialized as a parser chain: '{}'",
-            file, ioe);
-      }
+
+    @Override
+    public ParserChainSchema create(ParserChainSchema chain, Path path) throws IOException {
+        validateChain(chain, path);
+        String newId = String.valueOf(idGenerator.incrementAndGet());
+        chain.setId(newId);
+        writeChain(chain, path);
+        return chain;
     }
-    return summaries;
-  }
-
-  private List getFilesSorted(Path path) throws IOException {
-    List inputs = new ArrayList<>();
-    try (DirectoryStream files = Files.newDirectoryStream(path, fileFilter)) {
-      for (Path file : files) {
-        inputs.add(file);
-      }
+
+    private void validateChain(ParserChainSchema chain, Path path) throws IOException {
+        validateChain(null, chain, path);
     }
-    sortByName(inputs);
-    return inputs;
-  }
-
-  private void sortByName(List paths) {
-    paths.sort(Comparator.comparing(p -> p.getFileName().toString()));
-  }
-
-  @Override
-  public ParserChainSchema create(ParserChainSchema chain, Path path) throws IOException {
-    validateChain(chain, path);
-    String newId = String.valueOf(idGenerator.incrementAndGet());
-    chain.setId(newId);
-    writeChain(chain, path);
-    return chain;
-  }
-
-  private void validateChain(ParserChainSchema chain, Path path) throws IOException {
-    validateChain(null, chain, path);
-  }
-
-  private void validateChain(String id, ParserChainSchema chain, Path path) throws IOException {
-    if (chain == null) {
-      throw new RuntimeException("Provided chain can't be null!");
+
+    private void validateChain(String id, ParserChainSchema chain, Path path) throws IOException {
+        if (chain == null) {
+            throw new RuntimeException("Provided chain can't be null!");
+        }
+
+        final boolean duplicateName =
+              findAll(path).stream()
+                           //If we're changing the chain without changing its name, it won't be counted as a duplicate
+                           .anyMatch(chainSummary -> !chainSummary.getId().equals(id)
+                                                     && chainSummary.getName()
+                                                                    .equals(chain.getName()));
+        if (duplicateName) {
+            throw new RuntimeException("Duplicate chain names are restricted!");
+        }
+    }
+
+    private void writeChain(ParserChainSchema chain, Path outPath) throws IOException {
+        Path out = Paths.get(getFileName(chain.getId()));
+        out = outPath.resolve(out);
+        byte[] bytes = JSONUtils.INSTANCE.toJSONPretty(chain);
+        Files.write(out, bytes);
     }
 
-    final boolean duplicateName = findAll(path).stream()
-        //If we're changing the chain without changing its name, it won't be counted as a duplicate
-        .anyMatch(chainSummary -> !chainSummary.getId().equals(id) && chainSummary.getName().equals(chain.getName()));
-    if (duplicateName) {
-      throw new RuntimeException("Duplicate chain names are restricted!");
+    private String getFileName(String id) {
+        return id + ".json";
     }
-  }
-
-  private void writeChain(ParserChainSchema chain, Path outPath) throws IOException {
-    Path out = Paths.get(getFileName(chain.getId()));
-    out = outPath.resolve(out);
-    byte[] bytes = JSONUtils.INSTANCE.toJSONPretty(chain);
-    Files.write(out, bytes);
-  }
-
-  private String getFileName(String id) {
-    return id + ".json";
-  }
-
-  @Override
-  public ParserChainSchema read(String id, Path path) throws IOException {
-    Path inPath = findFile(id, path);
-    if (null == inPath) {
-      return null;
+
+    @Override
+    public ParserChainSchema read(String id, Path path) throws IOException {
+        Path inPath = findFile(id, path);
+        if (null == inPath) {
+            return null;
+        }
+        return JSONUtils.INSTANCE.load(inPath.toFile(), ParserChainSchema.class);
     }
-    return JSONUtils.INSTANCE.load(inPath.toFile(), ParserChainSchema.class);
-  }
-
-  private Path findFile(String id, Path root) throws IOException {
-    try (DirectoryStream files = Files.newDirectoryStream(root)) {
-      for (Path file : files) {
-        if (file.getFileName().toString().equals(getFileName(id))) {
-          return file;
+
+    private Path findFile(String id, Path root) throws IOException {
+        try (DirectoryStream files = Files.newDirectoryStream(root)) {
+            for (Path file : files) {
+                if (file.getFileName().toString().equals(getFileName(id))) {
+                    return file;
+                }
+            }
         }
-      }
+        return null;
     }
-    return null;
-  }
 
-  @Override
-  public ParserChainSchema update(String id, ParserChainSchema chain, Path path) throws IOException {
-    validateChain(id, chain, path);
+    @Override
+    public ParserChainSchema update(String id, ParserChainSchema chain, Path path) throws IOException {
+        validateChain(id, chain, path);
 
-    ParserChainSchema readChain = read(id, path);
-    if (null == readChain) {
-      return null;
+        ParserChainSchema readChain = read(id, path);
+        if (null == readChain) {
+            return null;
+        }
+        // enforce that the client cannot overwrite the chain ID
+        chain.setId(id);
+        writeChain(chain, path);
+        return read(id, path);
     }
-    // enforce that the client cannot overwrite the chain ID
-    chain.setId(id);
-    writeChain(chain, path);
-    return read(id, path);
-  }
-
-  @Override
-  public boolean delete(String id, Path path) throws IOException {
-    Path deletePath = findFile(id, path);
-    if (null == deletePath) {
-      return false;
+
+    @Override
+    public boolean delete(String id, Path path) throws IOException {
+        Path deletePath = findFile(id, path);
+        if (null == deletePath) {
+            return false;
+        }
+        Files.delete(deletePath);
+        return true;
     }
-    Files.delete(deletePath);
-    return true;
-  }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/FileBasedChainPersistenceServiceTest.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/FileBasedChainPersistenceServiceTest.java
index de9f4438..4ca19574 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/FileBasedChainPersistenceServiceTest.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/FileBasedChainPersistenceServiceTest.java
@@ -6,8 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
+ * 
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
+ * 
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,29 +20,28 @@
 
 package com.cloudera.parserchains.queryservice.service;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.lenient;
+
 import com.cloudera.parserchains.core.model.define.ParserChainSchema;
 import com.cloudera.parserchains.queryservice.common.utils.IDGenerator;
 import com.cloudera.parserchains.queryservice.model.summary.ParserChainSummary;
 import com.cloudera.parserchains.queryservice.service.impl.FileBasedChainPersistenceService;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
-import static org.mockito.Mockito.lenient;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
 
 @ExtendWith(MockitoExtension.class)
 public class FileBasedChainPersistenceServiceTest {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/ResultLogBuilderTest.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/ResultLogBuilderTest.java
index 686b3dec..4de19c55 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/ResultLogBuilderTest.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/test/java/com/cloudera/parserchains/queryservice/service/ResultLogBuilderTest.java
@@ -12,20 +12,20 @@
 
 package com.cloudera.parserchains.queryservice.service;
 
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.mockito.Mockito.mock;
+
 import com.cloudera.parserchains.core.InvalidParserException;
 import com.cloudera.parserchains.core.model.define.ParserSchema;
 import com.cloudera.parserchains.queryservice.model.exec.ResultLog;
 import org.junit.jupiter.api.Test;
 
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.mockito.Mockito.mock;
-
 public class ResultLogBuilderTest {
 
-    final static String parserId = "11c691cc-a141-43a0-b486-cb0e33bba820";
-    final static String parserName = "Some Test Parser";
-    final static String errorMessage = "This is the error to show to the user.";
+    static final String parserId = "11c691cc-a141-43a0-b486-cb0e33bba820";
+    static final String parserName = "Some Test Parser";
+    static final String errorMessage = "This is the error to show to the user.";
 
     @Test
     void success() {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainBuilder.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainBuilder.java
index 66017aa4..877fef11 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainBuilder.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainBuilder.java
@@ -22,6 +22,7 @@ public interface ChainBuilder {
     /**
      * Builds a parser chain from a {@link ParserChainSchema} which defines
      * what a parser chain looks like.
+     *
      * @param chainSchema The blueprint for building the parser chain.
      * @return The first link in the parser chain.
      * @throws InvalidParserException If the user has defined an invalid parser chain.
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainLink.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainLink.java
index bb61e1ee..8e1af298 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainLink.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainLink.java
@@ -21,7 +21,9 @@ public interface ChainLink {
 
     /**
      * Parses an message starting at this link in the chain.
+     *
      * 
      This involves all of the downstream links in this chain, not just this chain link.
+     *
      * @param input The input message to parse.
      * @return One {@link Message} for every link in the parser chain.
      */
@@ -29,7 +31,9 @@ public interface ChainLink {
 
     /**
      * Define the next link in the chain.
+     *
      * 
      If not defined, this is the last link in the parser chain.
+     *
      * @param nextLink The next chain link.
      */
     void setNext(ChainLink nextLink);
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainRunner.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainRunner.java
index debdd598..958de7a6 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainRunner.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ChainRunner.java
@@ -13,7 +13,6 @@
 package com.cloudera.parserchains.core;
 
 import com.cloudera.cyber.parser.MessageToParse;
-
 import java.util.List;
 
 /**
@@ -23,18 +22,23 @@ public interface ChainRunner {
 
     /**
      * Parses text input using a parser chain.
+     *
      * @param toParse The input to parse.
-     * @param chain The parser chain that parses each message.
+     * @param chain   The parser chain that parses each message.
      */
     List run(String toParse, ChainLink chain);
+
     List run(Message toParse, ChainLink chain, List results);
+
     List run(MessageToParse toParse, ChainLink chain);
 
 
     /**
      * The original message that is constructed for the parser chain.
+     *
      * @param toParse The text to parse.
      */
     Message originalMessage(String toParse);
+
     Message originalMessage(MessageToParse toParse);
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Constants.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Constants.java
index e5759ef1..465a5dba 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Constants.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Constants.java
@@ -15,6 +15,7 @@
 public class Constants {
     /**
      * The default input field assumed by many parsers.
+     *
      * 
      When a parser chain is executed the text to parse is added to a field by this name.
      */
     public static final String DEFAULT_INPUT_FIELD = "original_string";
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/DefaultChainBuilder.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/DefaultChainBuilder.java
index b20708fe..b257b8b3 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/DefaultChainBuilder.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/DefaultChainBuilder.java
@@ -18,15 +18,14 @@
 import com.cloudera.parserchains.core.model.define.ParserID;
 import com.cloudera.parserchains.core.model.define.ParserSchema;
 import com.cloudera.parserchains.core.model.define.RouteSchema;
-import lombok.extern.slf4j.Slf4j;
-
 import java.util.List;
 import java.util.Optional;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public class DefaultChainBuilder implements ChainBuilder {
-    private ParserBuilder parserBuilder;
-    private ParserCatalog parserCatalog;
+    private final ParserBuilder parserBuilder;
+    private final ParserCatalog parserCatalog;
 
     public DefaultChainBuilder(ParserBuilder parserBuilder, ParserCatalog parserCatalog) {
         this.parserBuilder = parserBuilder;
@@ -44,16 +43,16 @@ private ChainLink doBuild(ParserChainSchema chainSchema) throws InvalidParserExc
         // build the chain
         ChainLink head = null;
         ChainLink current = null;
-        for(ParserSchema parserSchema: chainSchema.getParsers()) {
+        for (ParserSchema parserSchema : chainSchema.getParsers()) {
             ChainLink next;
             boolean isRouter = ParserID.router().equals(parserSchema.getId());
-            if(isRouter) {
+            if (isRouter) {
                 next = buildRouter(parserSchema);
             } else {
                 next = buildLink(parserSchema, parserInfos);
             }
 
-            if(head == null) {
+            if (head == null) {
                 head = next;
                 current = next;
             } else {
@@ -67,18 +66,19 @@ private ChainLink doBuild(ParserChainSchema chainSchema) throws InvalidParserExc
 
     /**
      * Build a link in a parser chain.
+     *
      * @param parserSchema Defines a link in the chain.
-     * @param parserInfos The known set of parsers.
+     * @param parserInfos  The known set of parsers.
      * @return The next link in the chain.
      */
     private NextChainLink buildLink(ParserSchema parserSchema, List parserInfos)
-            throws InvalidParserException {
+          throws InvalidParserException {
         try {
             LinkName linkName = LinkName.of(parserSchema.getLabel(), parserSchema.getName());
             Parser parser = buildParser(parserSchema, parserInfos);
             return new NextChainLink(parser, linkName);
 
-        } catch(Exception e) {
+        } catch (Exception e) {
             // throw a checked exception so that we know which parser is the cause of the problem
             throw new InvalidParserException(parserSchema, e);
         }
@@ -86,6 +86,7 @@ private NextChainLink buildLink(ParserSchema parserSchema, List pars
 
     /**
      * Builds a router in a parser chain.
+     *
      * @param routerSchema Defines a router.
      * @return A router that is part of a parser chain.
      */
@@ -95,7 +96,7 @@ private RouterLink buildRouter(ParserSchema routerSchema) throws InvalidParserEx
             // build the router
             FieldName inputField = FieldName.of(routerSchema.getRouting().getMatchingField());
             routerLink = new RouterLink()
-                    .withInputField(inputField);
+                  .withInputField(inputField);
 
             // define the router's routes
             for (RouteSchema routeSchema : routerSchema.getRouting().getRoutes()) {
@@ -107,7 +108,7 @@ private RouterLink buildRouter(ParserSchema routerSchema) throws InvalidParserEx
                     routerLink.withRoute(regex, subChain);
                 }
             }
-        } catch(Exception e) {
+        } catch (Exception e) {
             // throw a checked exception so that we know which parser is the cause of the problem
             throw new InvalidParserException(routerSchema, e);
         }
@@ -116,18 +117,19 @@ private RouterLink buildRouter(ParserSchema routerSchema) throws InvalidParserEx
 
     /**
      * Builds a {@link Parser} given a {@link ParserSchema}.
+     *
      * @param parserSchema Defines the parser to build.
-     * @param parserInfos A list of information about all known parsers.
+     * @param parserInfos  A list of information about all known parsers.
      * @return A {@link Parser}.
      */
     private Parser buildParser(ParserSchema parserSchema,
                                List parserInfos) throws InvalidParserException {
         String className = parserSchema.getId().getId();
         Optional parserInfo = parserInfos
-                .stream()
-                .filter(info -> className.equals(info.getParserClass().getCanonicalName()))
-                .findFirst();
-        if(parserInfo.isPresent()) {
+              .stream()
+              .filter(info -> className.equals(info.getParserClass().getCanonicalName()))
+              .findFirst();
+        if (parserInfo.isPresent()) {
             return parserBuilder.build(parserInfo.get(), parserSchema);
 
         } else {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/DefaultChainRunner.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/DefaultChainRunner.java
index 2e8c5594..592c3772 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/DefaultChainRunner.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/DefaultChainRunner.java
@@ -14,11 +14,10 @@
 
 import com.cloudera.cyber.parser.MessageToParse;
 import com.cloudera.parserchains.core.model.define.ParserName;
-import lombok.extern.slf4j.Slf4j;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * Parses a {@link Message} using a parser chain.
@@ -33,6 +32,8 @@ public DefaultChainRunner() {
     }
 
     /**
+     * inputField setter.
+     *
      * @param inputField The name of the field that is initialized with the raw input.
      */
     public DefaultChainRunner withInputField(FieldName inputField) {
@@ -55,16 +56,22 @@ public List run(Message original, ChainLink chain, List result
         try {
             List chainResults = chain.process(original);
             results.addAll(chainResults);
-        } catch(Throwable t) {
-            String msg = "An unexpected error occurred while running a parser chain. " +
-                    "Ensure that no parser is throwing an unchecked exception. Parsers should " +
-                    "instead be reporting the error in the output message.";
+        } catch (Throwable t) {
+            String msg = "An unexpected error occurred while running a parser chain. "
+                         + "Ensure that no parser is throwing an unchecked exception. Parsers should "
+                         + "instead be reporting the error in the output message.";
             results = getErrorResult(original, msg);
             log.warn(msg, t);
         }
         return results;
     }
 
+    @Override
+    public List run(MessageToParse toParse, ChainLink chain) {
+        Message original = originalMessage(toParse);
+        return parseMessage(original, chain);
+    }
+
     private List parseMessage(Message original, ChainLink chain) {
         List results = new ArrayList<>();
         results.add(original);
@@ -76,33 +83,27 @@ private List parseMessage(Message original, ChainLink chain) {
         }
     }
 
-    @Override
-    public List run(MessageToParse toParse, ChainLink chain) {
-        Message original = originalMessage(toParse);
-        return parseMessage(original, chain);
-    }
-
     @Override
     public Message originalMessage(String toParse) {
         return Message.builder()
-                .addField(inputField, StringFieldValue.of(toParse))
-                .createdBy(ORIGINAL_MESSAGE_NAME)
-                .build();
+                      .addField(inputField, StringFieldValue.of(toParse))
+                      .createdBy(ORIGINAL_MESSAGE_NAME)
+                      .build();
     }
 
     @Override
     public Message originalMessage(MessageToParse toParse) {
         return Message.builder()
-                .addField(inputField, MessageToParseFieldValue.of(toParse))
-                .createdBy(ORIGINAL_MESSAGE_NAME)
-                .build();
+                      .addField(inputField, MessageToParseFieldValue.of(toParse))
+                      .createdBy(ORIGINAL_MESSAGE_NAME)
+                      .build();
     }
 
     private List getErrorResult(Message original, String errorMessage) {
         Message error = Message.builder()
-                .clone(original)
-                .withError(errorMessage)
-                .build();
+                               .clone(original)
+                               .withError(errorMessage)
+                               .build();
         return Collections.singletonList(error);
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/FieldName.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/FieldName.java
index 9829e500..88cac8ad 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/FieldName.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/FieldName.java
@@ -30,7 +30,7 @@ public static final FieldName of(String fieldName) {
      * Use {@link FieldName#of(String)}.
      */
     private FieldName(String fieldName) {
-        if(!validFieldName.matches(fieldName)) {
+        if (!validFieldName.matches(fieldName)) {
             throw new IllegalArgumentException(String.format("Invalid field name: '%s'", fieldName));
         }
         this.fieldName = fieldName;
@@ -46,15 +46,15 @@ public boolean equals(Object o) {
         }
         FieldName that = (FieldName) o;
         return new EqualsBuilder()
-                .append(fieldName, that.fieldName)
-                .isEquals();
+              .append(fieldName, that.fieldName)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(fieldName)
-                .toHashCode();
+              .append(fieldName)
+              .toHashCode();
     }
 
     public String get() {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/FieldValue.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/FieldValue.java
index f14805aa..9cbc7828 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/FieldValue.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/FieldValue.java
@@ -13,18 +13,14 @@
 package com.cloudera.parserchains.core;
 
 import com.cloudera.cyber.parser.MessageToParse;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
-import java.nio.charset.StandardCharsets;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
 
 /**
  * The value of a field contained within a {@link Message}.
  */
 public interface FieldValue {
     String get();
+
     byte[] toBytes();
+
     MessageToParse toMessageToParse();
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/InvalidParserException.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/InvalidParserException.java
index 60deb1a9..1580bda5 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/InvalidParserException.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/InvalidParserException.java
@@ -19,11 +19,13 @@
  * and is impossible to construct.
  */
 public class InvalidParserException extends Exception {
-    private ParserSchema badParser;
+    private final ParserSchema badParser;
 
     /**
+     * InvalidParserException constructor.
+     *
      * @param badParser The parser that caused this error.
-     * @param cause The root cause exception.
+     * @param cause     The root cause exception.
      */
     public InvalidParserException(ParserSchema badParser, Throwable cause) {
         super(cause);
@@ -31,8 +33,10 @@ public InvalidParserException(ParserSchema badParser, Throwable cause) {
     }
 
     /**
+     * InvalidParserException constructor.
+     *
      * @param badParser The parser that caused this error.
-     * @param message The error message.
+     * @param message   The error message.
      */
     public InvalidParserException(ParserSchema badParser, String message) {
         super(message);
@@ -40,9 +44,11 @@ public InvalidParserException(ParserSchema badParser, String message) {
     }
 
     /**
+     * InvalidParserException constructor.
+     *
      * @param badParser The parser that caused this error.
-     * @param message The error message.
-     * @param cause The root cause exception.
+     * @param message   The error message.
+     * @param cause     The root cause exception.
      */
     public InvalidParserException(ParserSchema badParser, String message, Throwable cause) {
         super(message, cause);
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/LinkName.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/LinkName.java
index 0ae15431..e1b0f7e7 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/LinkName.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/LinkName.java
@@ -31,7 +31,7 @@ public static final LinkName of(String linkName, ParserName parserName) {
      * Private constructor. Use {@link LinkName#of(String, ParserName)} instead.
      */
     private LinkName(String linkName, ParserName parserName) {
-        if(!validLinkName.matches(linkName)) {
+        if (!validLinkName.matches(linkName)) {
             throw new IllegalArgumentException(String.format("Invalid link name: '%s'", linkName));
         }
         this.linkName = linkName;
@@ -48,17 +48,17 @@ public boolean equals(Object o) {
         }
         LinkName that = (LinkName) o;
         return new EqualsBuilder()
-                .append(linkName, that.linkName)
-                .append(parserName, that.parserName)
-                .isEquals();
+              .append(linkName, that.linkName)
+              .append(parserName, that.parserName)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(linkName)
-                .append(parserName)
-                .toHashCode();
+              .append(linkName)
+              .append(parserName)
+              .toHashCode();
     }
 
     public String getLinkName() {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Message.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Message.java
index 142af234..9799d2c4 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Message.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Message.java
@@ -13,17 +13,23 @@
 package com.cloudera.parserchains.core;
 
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import lombok.EqualsAndHashCode;
 import lombok.ToString;
 
-import java.util.*;
-
 /**
  * A {@link Message} is consumed and parsed by a {@link Parser}.
  *
+ * 
      
  * A {@link Message} is composed of a collection of fields. The message fields
  * are represented as ({@link FieldName}, {@link FieldValue}) pairs.
  *
+ * 
      
  * A {@link Message} is immutable and a {@link Builder} should be used to
  * construct one.
  */
@@ -35,7 +41,7 @@ public class Message {
      * Constructs a {@link Message}.
      */
     public static class Builder {
-        private Map fields;
+        private final Map fields;
         private Throwable error;
         private LinkName createdBy;
         private boolean emit = true;
@@ -46,8 +52,8 @@ public Builder() {
 
         /**
          * Adds all fields from a {@link Message}.
+         *
          * @param message The message to copy fields from.
-         * @return
          */
         public Builder withFields(Message message) {
             Objects.requireNonNull(message, "A message is required.");
@@ -57,8 +63,8 @@ public Builder withFields(Message message) {
 
         /**
          * Clones a message by copying all underlying fields.
+         *
          * @param message The message to clone.
-         * @return
          */
         public Builder clone(Message message) {
             Objects.requireNonNull(message, "A message to clone is required.");
@@ -70,22 +76,22 @@ public Builder clone(Message message) {
 
         /**
          * Add a field to the message.
-         * @param name The name of the field to add.
+         *
+         * @param name  The name of the field to add.
          * @param value The value of the field to add.
-         * @return
          */
         public Builder addField(FieldName name, FieldValue value) {
             this.fields.put(
-                    Objects.requireNonNull(name, "A valid field name is required."),
-                    Objects.requireNonNull(value, "A valid field value is required."));
+                  Objects.requireNonNull(name, "A valid field name is required."),
+                  Objects.requireNonNull(value, "A valid field value is required."));
             return this;
         }
 
         /**
          * Add a field to the message.
-         * @param name The name of the field to add.
+         *
+         * @param name  The name of the field to add.
          * @param value The value of the field to add.
-         * @return
          */
         public Builder addField(FieldName name, String value) {
             return addField(name, StringFieldValue.of(value));
@@ -93,9 +99,9 @@ public Builder addField(FieldName name, String value) {
 
         /**
          * Add a field to the message.
-         * @param name The name of the field to add.
+         *
+         * @param name  The name of the field to add.
          * @param value The value of the field to add.
-         * @return
          */
         public Builder addField(String name, String value) {
             return addField(FieldName.of(name), StringFieldValue.of(value));
@@ -103,8 +109,8 @@ public Builder addField(String name, String value) {
 
         /**
          * Remove a field from the message.
+         *
          * @param name The name of the field to remove.
-         * @return
          */
         public Builder removeField(FieldName name) {
             this.fields.remove(Objects.requireNonNull(name, "The name of the field to remove is required."));
@@ -113,11 +119,11 @@ public Builder removeField(FieldName name) {
 
         /**
          * Removes multiple fields from the message.
+         *
          * @param fieldNames The name of the fields to remove.
-         * @return
          */
         public Builder removeFields(List fieldNames) {
-            for(FieldName fieldName: fieldNames) {
+            for (FieldName fieldName : fieldNames) {
                 this.fields.remove(Objects.requireNonNull(fieldName, "The name of the field to remove is required."));
             }
             return this;
@@ -126,12 +132,12 @@ public Builder removeFields(List fieldNames) {
         /**
          * Renames a field, if the field exists within the message. If the
          * field does not exist, no action taken.
+         *
          * @param from The original field name.
-         * @param to The new field name.
-         * @return
+         * @param to   The new field name.
          */
         public Builder renameField(FieldName from, FieldName to) {
-            if(fields.containsKey(from)) {
+            if (fields.containsKey(from)) {
                 FieldValue value = fields.remove(from);
                 fields.put(to, value);
             }
@@ -141,8 +147,8 @@ public Builder renameField(FieldName from, FieldName to) {
         /**
          * Adds an error to the message. This indicates that an error
          * occurred while parsing.
+         *
          * @param error The error that occurred.
-         * @return
          */
         public Builder withError(Throwable error) {
             this.error = Objects.requireNonNull(error, "An error is required.");
@@ -152,8 +158,8 @@ public Builder withError(Throwable error) {
         /**
          * Adds an error to the message. This indicates that an error
          * occurred while parsing.
+         *
          * @param message The error message.
-         * @return
          */
         public Builder withError(String message) {
             this.error = new IllegalStateException(Objects.requireNonNull(message, "An error message is required."));
@@ -163,9 +169,9 @@ public Builder withError(String message) {
         /**
          * Adds an error to the message to indicate that an error occurred
          * while parsing.
-         * @param message The error message.
+         *
+         * @param message   The error message.
          * @param rootCause The root cause exception.
-         * @return
          */
         public Builder withError(String message, Throwable rootCause) {
             this.error = new RuntimeException(message, rootCause);
@@ -175,8 +181,8 @@ public Builder withError(String message, Throwable rootCause) {
         /**
          * Assigns a {@link LinkName} to this message indicating which link in the
          * chain was responsible for creating the message.
+         *
          * @param createdBy The name of the link that created this message.
-         * @return
          */
         public Builder createdBy(LinkName createdBy) {
             this.createdBy = createdBy;
@@ -187,8 +193,10 @@ public Builder emit(boolean emit) {
             this.emit = emit;
             return this;
         }
+
         /**
          * Builds a {@link Message}.
+         *
          * @return The message.
          */
         public Message build() {
@@ -196,10 +204,10 @@ public Message build() {
         }
     }
 
-    private Map fields;
-    private Throwable error;
-    private LinkName createdBy;
-    private boolean emit;
+    private final Map fields;
+    private final Throwable error;
+    private final LinkName createdBy;
+    private final boolean emit;
 
     private Message(Builder builder) {
         this.fields = new HashMap<>();
@@ -210,6 +218,8 @@ private Message(Builder builder) {
     }
 
     /**
+     * Builder method.
+     *
      * @return A {@link Builder} that can be used to create a message.
      */
     public static Builder builder() {
@@ -218,11 +228,12 @@ public static Builder builder() {
 
     /**
      * Returns the value of a field within this message.
+     *
      * @param fieldName The name of the field.
      * @return The value of the field or Optional.empty if it does not exist.
      */
     public Optional getField(FieldName fieldName) {
-        if(fields.containsKey(fieldName)) {
+        if (fields.containsKey(fieldName)) {
             return Optional.of(fields.get(fieldName));
         } else {
             return Optional.empty();
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/MessageToParseFieldValue.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/MessageToParseFieldValue.java
index 1e367dc3..9f3a3aa1 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/MessageToParseFieldValue.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/MessageToParseFieldValue.java
@@ -13,9 +13,8 @@
 package com.cloudera.parserchains.core;
 
 import com.cloudera.cyber.parser.MessageToParse;
-import lombok.EqualsAndHashCode;
-
 import java.nio.charset.StandardCharsets;
+import lombok.EqualsAndHashCode;
 
 @EqualsAndHashCode
 public class MessageToParseFieldValue implements FieldValue {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/NextChainLink.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/NextChainLink.java
index a47f4392..81de5754 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/NextChainLink.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/NextChainLink.java
@@ -12,18 +12,23 @@
 
 package com.cloudera.parserchains.core;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A {@link ChainLink} that links directly to the next link in a chain.
  */
 public class NextChainLink implements ChainLink {
-    private Parser parser;
+    private final Parser parser;
     private Optional nextLink;
-    private LinkName linkName;
+    private final LinkName linkName;
 
     /**
-     * @param parser The parser at this link in the chain.
+     * NextChainLink constructor.
+     *
+     * @param parser   The parser at this link in the chain.
      * @param linkName The name of this link in the chain.
      */
     public NextChainLink(Parser parser, LinkName linkName) {
@@ -42,10 +47,10 @@ public List process(Message input) {
 
         // ensure the message is attributed to this link by name
         Message output = Message.builder()
-                .clone(parsed)
-                .createdBy(linkName)
-                .emit(emitMessage)
-                .build();
+                                .clone(parsed)
+                                .createdBy(linkName)
+                                .emit(emitMessage)
+                                .build();
         List results = new ArrayList<>();
         results.add(output);
 
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Parser.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Parser.java
index e272c64b..25148163 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Parser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Parser.java
@@ -14,7 +14,6 @@
 
 /**
  * Parses a {@link Message}.
- *
  */
 public interface Parser {
 
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ParserBuilder.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ParserBuilder.java
index 8080a70f..aba53a60 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ParserBuilder.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ParserBuilder.java
@@ -23,9 +23,9 @@ public interface ParserBuilder {
     /**
      * Constructs a {@link Parser} instance given the {@link ParserInfo} retrieved
      * from a {@link com.cloudera.parserchains.core.catalog.ParserCatalog}.
-     * @param parserInfo Describes the parser to build.
+     *
+     * @param parserInfo   Describes the parser to build.
      * @param parserSchema Describes how the parser should be configured.
-     * @return
      */
     Parser build(ParserInfo parserInfo, ParserSchema parserSchema) throws InvalidParserException;
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ReflectiveParserBuilder.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ReflectiveParserBuilder.java
index abbfda4a..6013a22c 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ReflectiveParserBuilder.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ReflectiveParserBuilder.java
@@ -14,6 +14,7 @@
 
 import static com.cloudera.parserchains.core.utils.AnnotationUtils.getAnnotatedMethodsInOrder;
 import static com.cloudera.parserchains.core.utils.AnnotationUtils.getAnnotatedParameters;
+
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.Parameter;
 import com.cloudera.parserchains.core.catalog.ParserInfo;
@@ -82,50 +83,51 @@ private void configure(Parser parser, ParserSchema parserSchema) throws InvalidP
                 invokeMethod(parser, parserSchema, annotationKey, method, value);
             } else if (key.required()) {
                 throw new InvalidParserException(parserSchema,
-                        String.format("Required field isn't provided: %s", annotationKey));
+                      String.format("Required field isn't provided: %s", annotationKey));
             }
         }
     }
 
     private void configureParams(ParserSchema parserSchema, List valuesSchema, Method method)
-        throws InvalidParserException {
+          throws InvalidParserException {
         final List paramsAnnotations = Arrays.stream(method.getParameterAnnotations())
-            .flatMap(Arrays::stream)
-            .filter(annotation -> annotation instanceof Parameter)
-            .map(annotation -> (Parameter) annotation)
-            .collect(Collectors.toList());
+                                                        .flatMap(Arrays::stream)
+                                                        .filter(annotation -> annotation instanceof Parameter)
+                                                        .map(annotation -> (Parameter) annotation)
+                                                        .collect(Collectors.toList());
 
         for (Parameter paramAnnotation : paramsAnnotations) {
             for (ConfigValueSchema value : valuesSchema) {
                 final Map valueMap = value.getValues();
                 final String annotationKey = paramAnnotation.key();
 
-                if (StringUtils.isNotBlank(paramAnnotation.defaultValue()) && valueMap.get(annotationKey) == null){
+                if (StringUtils.isNotBlank(paramAnnotation.defaultValue()) && valueMap.get(annotationKey) == null) {
                     valueMap.put(annotationKey, paramAnnotation.defaultValue());
                 }
-                if (paramAnnotation.required() && valueMap.get(annotationKey) == null){
+                if (paramAnnotation.required() && valueMap.get(annotationKey) == null) {
                     throw new InvalidParserException(parserSchema,
-                        String.format("Required parameter isn't provided: %s", annotationKey));
+                          String.format("Required parameter isn't provided: %s", annotationKey));
                 }
             }
         }
     }
 
-  private void invokeMethod(Parser parser,
+    private void invokeMethod(Parser parser,
                               ParserSchema parserSchema,
                               String configKey,
                               Method method,
                               Map configValues) throws InvalidParserException {
         List annotatedParams = getAnnotatedParameters(method);
         List methodArgs = buildMethodArgs(annotatedParams, configValues);
-        log.info(String.format("Invoking method %s(%s); key=%s, parser=%s", method.getName(), methodArgs, configKey, parser.getClass().getName()));
+        log.info(String.format("Invoking method %s(%s); key=%s, parser=%s", method.getName(), methodArgs, configKey,
+              parser.getClass().getName()));
 
         try {
             method.invoke(parser, methodArgs.toArray());
 
         } catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException e) {
             String message = String.format("Failed to invoke method %s(%s); key=%s, parser=%s",
-                    method.getName(), methodArgs, configKey, parser.getClass().getName());
+                  method.getName(), methodArgs, configKey, parser.getClass().getName());
             throw new InvalidParserException(parserSchema, message, e);
         }
     }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Regex.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Regex.java
index b1589167..73f1fa3a 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Regex.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Regex.java
@@ -12,11 +12,10 @@
 
 package com.cloudera.parserchains.core;
 
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.util.Objects;
 import java.util.regex.Pattern;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * A regular expression.
@@ -25,7 +24,7 @@ public final class Regex {
     private final String regex;
     private final Pattern pattern;
 
-    public static final Regex of(String regex) {
+    public static Regex of(String regex) {
         return new Regex(regex);
     }
 
@@ -39,6 +38,7 @@ private Regex(String regex) {
 
     /**
      * Tells whether a field value matches this regular expression.
+     *
      * @param fieldValue The value to match.
      * @return True if the field value matches the regular expression. Otherwise, false.
      */
@@ -48,6 +48,7 @@ public boolean matches(FieldValue fieldValue) {
 
     /**
      * Tells whether a field name matches this regular expression.
+     *
      * @param fieldName The value to match.
      * @return True if the field value matches the regular expression. Otherwise, false.
      */
@@ -57,11 +58,12 @@ public boolean matches(FieldName fieldName) {
 
     /**
      * Tells whether a string matches this regular expression.
+     *
      * @param value The value to match
      * @return True if the value matches the regular expression. Otherwise, false.
      */
     public boolean matches(String value) {
-        if(value == null) {
+        if (value == null) {
             return false;
         }
         return pattern.matcher(value).matches();
@@ -82,14 +84,14 @@ public boolean equals(Object o) {
         }
         Regex that = (Regex) o;
         return new EqualsBuilder()
-                .append(regex, that.regex)
-                .isEquals();
+              .append(regex, that.regex)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(regex)
-                .toHashCode();
+              .append(regex)
+              .toHashCode();
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/RouterLink.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/RouterLink.java
index ce635506..ffb5e5dd 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/RouterLink.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/RouterLink.java
@@ -40,7 +40,7 @@ public Route(Regex regex, ChainLink next) {
      * The name of the field whose value is used for routing.
      */
     private FieldName inputField;
-    private List routes;
+    private final List routes;
     private Optional defaultRoute;
     private Optional nextLink;
 
@@ -78,17 +78,17 @@ public Optional getDefault() {
     }
 
     private Optional findRoute(Message input) {
-        if(inputField == null) {
+        if (inputField == null) {
             throw new IllegalStateException("The routing field was not defined.");
         }
 
         Optional valueOpt = input.getField(inputField);
-        if(valueOpt.isPresent()) {
+        if (valueOpt.isPresent()) {
             FieldValue fieldValue = valueOpt.get();
 
-            for(Route route: routes) {
+            for (Route route : routes) {
                 Regex regex = route.regex;
-                if(regex.matches(fieldValue)) {
+                if (regex.matches(fieldValue)) {
                     return Optional.of(route.next);
                 }
             }
@@ -110,13 +110,13 @@ public List process(Message input) {
 
         // retrieve the last output from the route taken;
         Message output = input;
-        if(results.size() > 0) {
+        if (results.size() > 0) {
             output = results.get(results.size() - 1);
         }
 
         // if no errors, allow the next link in the chain to process the message
         boolean noError = !output.getError().isPresent();
-        if(noError && nextLink.isPresent()) {
+        if (noError && nextLink.isPresent()) {
             List nextResults = nextLink.get().process(output);
             results.addAll(nextResults);
         }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/StringFieldValue.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/StringFieldValue.java
index 31407b13..17e668ac 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/StringFieldValue.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/StringFieldValue.java
@@ -12,12 +12,10 @@
 
 package com.cloudera.parserchains.core;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import com.cloudera.cyber.parser.MessageToParse;
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-
-import static java.nio.charset.StandardCharsets.UTF_8;
 
 @EqualsAndHashCode
 public class StringFieldValue implements FieldValue {
@@ -32,7 +30,7 @@ public static FieldValue of(String fieldValue) {
      * Instead use {@link StringFieldValue#of(String)}.
      */
     private StringFieldValue(String value) {
-        if(value == null || value.length() > MAX_LENGTH) {
+        if (value == null || value.length() > MAX_LENGTH) {
             throw new IllegalArgumentException("Invalid field value.");
         }
         this.value = value;
@@ -49,6 +47,7 @@ public byte[] toBytes() {
     public String toString() {
         return (value != null) ? value : "null";
     }
+
     @Override
     public MessageToParse toMessageToParse() {
         return MessageToParse.builder().originalBytes(toBytes()).offset(0).partition(0).topic("none").build();
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Validator.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Validator.java
index c186acd6..8274ada0 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Validator.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/Validator.java
@@ -20,13 +20,14 @@ private Validator() {
 
     /**
      * Throws an exception if a value does not match a regular expression.
-     * @param value The value that must match.
-     * @param regex The regular expression that defines a valid value.
+     *
+     * @param value      The value that must match.
+     * @param regex      The regular expression that defines a valid value.
      * @param entityName The name of the entity being validated.
      * @throws IllegalArgumentException If the value is invalid.
      */
     public static void mustMatch(String value, Regex regex, String entityName) throws IllegalArgumentException {
-        if(!regex.matches(value)) {
+        if (!regex.matches(value)) {
             throw new IllegalArgumentException(String.format("Invalid %s=%s", entityName, value));
         }
     }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/AnnotationBasedParserInfoBuilder.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/AnnotationBasedParserInfoBuilder.java
index 2d22a15b..17ac077a 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/AnnotationBasedParserInfoBuilder.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/AnnotationBasedParserInfoBuilder.java
@@ -13,9 +13,8 @@
 package com.cloudera.parserchains.core.catalog;
 
 import com.cloudera.parserchains.core.Parser;
-import lombok.extern.slf4j.Slf4j;
-
 import java.util.Optional;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * Constructs a {@link ParserInfo} object using
@@ -27,23 +26,23 @@ public class AnnotationBasedParserInfoBuilder implements ParserInfoBuilder {
     public Optional build(Class clazz) {
         Optional result = Optional.empty();
         MessageParser annotation = clazz.getAnnotation(MessageParser.class);
-        if(annotation == null) {
+        if (annotation == null) {
             log.warn("Found parser class missing the '{}' annotation; class={}",
-                    MessageParser.class.getName(), clazz.getName());
+                  MessageParser.class.getName(), clazz.getName());
 
-        } else if(!Parser.class.isAssignableFrom(clazz)) {
+        } else if (!Parser.class.isAssignableFrom(clazz)) {
             log.warn("Found parser class that does not implement '{}'; class={}",
-                    Parser.class.getName(), clazz.getName());
+                  Parser.class.getName(), clazz.getName());
 
         } else {
             // found a parser.  the cast is safe because of the 'if' condition above
             @SuppressWarnings("unchecked")
             Class parserClass = (Class) clazz;
             ParserInfo parserInfo = ParserInfo.builder()
-                    .name(annotation.name())
-                    .description(annotation.description())
-                    .parserClass(parserClass)
-                    .build();
+                                              .name(annotation.name())
+                                              .description(annotation.description())
+                                              .parserClass(parserClass)
+                                              .build();
             result = Optional.of(parserInfo);
         }
         return result;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ClassIndexParserCatalog.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ClassIndexParserCatalog.java
index 5e7f591a..229c01bd 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ClassIndexParserCatalog.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ClassIndexParserCatalog.java
@@ -12,11 +12,10 @@
 
 package com.cloudera.parserchains.core.catalog;
 
-import lombok.extern.slf4j.Slf4j;
-import org.atteo.classindex.ClassIndex;
-
 import java.util.ArrayList;
 import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.atteo.classindex.ClassIndex;
 
 /**
  * A {@link ParserCatalog} that builds a catalog of parsers using a class index
@@ -25,11 +24,12 @@
  * 
      A parser must be marked using the {@link MessageParser} annotation
  * so that the parser is discoverable using this class.
  *
+ * 
      
  * https://github.com/atteo/classindex
  */
 @Slf4j
 public class ClassIndexParserCatalog implements ParserCatalog {
-    private ParserInfoBuilder parserInfoBuilder;
+    private final ParserInfoBuilder parserInfoBuilder;
 
     public ClassIndexParserCatalog(ParserInfoBuilder parserInfoBuilder) {
         this.parserInfoBuilder = parserInfoBuilder;
@@ -45,16 +45,16 @@ public List getParsers() {
 
         // search the class index for the annotation
         Iterable> knownAnnotations = ClassIndex.getAnnotated(MessageParser.class);
-        for(Class clazz: knownAnnotations) {
+        for (Class clazz : knownAnnotations) {
             parserInfoBuilder.build(clazz).ifPresent(info -> results.add(info));
         }
 
-        if(log.isDebugEnabled()) {
-            for(ParserInfo parserInfo: results) {
+        if (log.isDebugEnabled()) {
+            for (ParserInfo parserInfo : results) {
                 log.debug("Found parser: class={}, name={}, desc={}",
-                        parserInfo.getParserClass(),
-                        parserInfo.getName(),
-                        parserInfo.getDescription());
+                      parserInfo.getParserClass(),
+                      parserInfo.getName(),
+                      parserInfo.getDescription());
             }
         }
         return results;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Configurable.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Configurable.java
index eaa17a14..d4bfd245 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Configurable.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Configurable.java
@@ -64,6 +64,7 @@
 
     /**
      * The default value of this configurable parameter.
+     *
      * 
      This value is optional.
      */
     String defaultValue() default "";
@@ -71,6 +72,7 @@
     /**
      * Specifies if the parameter specifies the output field name.
      * If true, the selection will be provided with possible field names.
+     *
      * 
      This value is optional.
      */
     boolean isOutputName() default false;
@@ -78,6 +80,7 @@
     /**
      * Defines the type of widget presented to the user when
      * configuring the parameter.
+     *
      * 
      Accepts either "text" or "textarea".
      */
     WidgetType widgetType() default WidgetType.TEXT;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/MessageParser.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/MessageParser.java
index 6e3473f8..41f7685a 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/MessageParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/MessageParser.java
@@ -14,10 +14,9 @@
 
 import com.cloudera.parserchains.core.Message;
 import com.cloudera.parserchains.core.Parser;
-import org.atteo.classindex.IndexAnnotated;
-
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
+import org.atteo.classindex.IndexAnnotated;
 
 /**
  * A marker for {@link Parser} implementations that parse {@link Message}s.
@@ -38,7 +37,6 @@
 
     /**
      * Returns a description of the parser.
-     * @return
      */
     String description() default "";
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Parameter.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Parameter.java
index 742e8b6c..0b110d77 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Parameter.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Parameter.java
@@ -32,12 +32,14 @@
 
     /**
      * A unique key for this configurable parameter.
+     *
      * 
      This value is required.
      */
     String key();
 
     /**
      * A label for this configurable parameter that is displayed to the user.
+     *
      * 
      This value is optional.
      */
     String label() default "";
@@ -45,19 +47,23 @@
     /**
      * A description of this configurable parameter that is displayed to
      * the user.
+     *
      * 
      This value is optional.
      */
     String description() default "";
 
     /**
      * Whether the user is required to provide a value for this configurable parameter.
+     *
      * 
      If true, the user must provide a value. Otherwise, false.
+     *
      * 
      This value is optional.
      */
     boolean required() default false;
 
     /**
      * The default value of this configurable parameter.
+     *
      * 
      This value is optional.
      */
     String defaultValue() default "";
@@ -65,6 +71,7 @@
     /**
      * Specifies if the parameter specifies the output field name.
      * If true, the selection will be provided with possible field names.
+     *
      * 
      This value is optional.
      */
     boolean isOutputName() default false;
@@ -72,6 +79,7 @@
     /**
      * Defines the type of widget presented to the user when
      * configuring the parameter.
+     *
      * 
      This value is optional and defaults to a simple text box.
      */
     WidgetType widgetType() default WidgetType.TEXT;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ParserInfo.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ParserInfo.java
index fe98999a..fa1ff5ab 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ParserInfo.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ParserInfo.java
@@ -13,11 +13,10 @@
 package com.cloudera.parserchains.core.catalog;
 
 import com.cloudera.parserchains.core.Parser;
+import java.util.Objects;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
-import java.util.Objects;
-
 /**
  * Describes a {@link Parser} that was discovered using a {@link ParserCatalog}.
  */
@@ -70,19 +69,19 @@ public boolean equals(Object o) {
         }
         ParserInfo that = (ParserInfo) o;
         return new EqualsBuilder()
-                .append(name, that.name)
-                .append(description, that.description)
-                .append(parserClass, that.parserClass)
-                .isEquals();
+              .append(name, that.name)
+              .append(description, that.description)
+              .append(parserClass, that.parserClass)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(name)
-                .append(description)
-                .append(parserClass)
-                .toHashCode();
+              .append(name)
+              .append(description)
+              .append(parserClass)
+              .toHashCode();
     }
 
     public static class Builder {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ParserInfoBuilder.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ParserInfoBuilder.java
index 05bea867..91deef7c 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ParserInfoBuilder.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/ParserInfoBuilder.java
@@ -21,6 +21,7 @@ public interface ParserInfoBuilder {
 
     /**
      * Builds a {@link ParserInfo} object from a {@link Class}.
+     *
      * @param clazz The class definition.
      * @return A {@link ParserInfo} object, if the given class defines a valid parser. Otherwise, empty.
      */
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigDescription.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigDescription.java
index cc3d0233..245e5dff 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigDescription.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigDescription.java
@@ -12,12 +12,11 @@
 
 package com.cloudera.parserchains.core.model.config;
 
-import com.cloudera.parserchains.core.Regex;
+import static com.cloudera.parserchains.core.Validator.mustMatch;
 
+import com.cloudera.parserchains.core.Regex;
 import java.io.Serializable;
 
-import static com.cloudera.parserchains.core.Validator.mustMatch;
-
 /**
  * Describes a parser's configuration parameter for a user.
  */
@@ -30,6 +29,7 @@ public class ConfigDescription implements Serializable {
 
     /**
      * Creates a {@link ConfigDescription}.
+     *
      * @param desc The description.
      */
     public static ConfigDescription of(String desc) {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigDescriptor.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigDescriptor.java
index 45e8cdc6..1e39b80d 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigDescriptor.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigDescriptor.java
@@ -13,7 +13,6 @@
 package com.cloudera.parserchains.core.model.config;
 
 import com.cloudera.parserchains.core.Parser;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -31,6 +30,7 @@ public class ConfigDescriptor implements Serializable {
      * The unique name of the configuration parameter.
      *
      * 
      This is used to identify the parameter and is not displayed to the user.
+     *
      * 
      For example, 'fieldToRename', 'inputField', or 'outputField'.
      */
     private final ConfigName name;
@@ -116,7 +116,7 @@ public static class Builder {
         private ConfigName name;
         private ConfigDescription description;
         private boolean required;
-        private List acceptedValues;
+        private final List acceptedValues;
         private boolean cumulative;
 
         public Builder() {
@@ -151,32 +151,32 @@ public Builder acceptsValue(ConfigKey key) {
             return this;
         }
 
-        public Builder isCumulative(boolean cumulative) {
-            this.cumulative = cumulative;
-            return this;
-        }
-
         public Builder acceptsValue(String key, String label, String description) {
             ConfigKey configKey = ConfigKey.builder()
-                    .key(key)
-                    .label(label)
-                    .description(description)
-                    .build();
+                                           .key(key)
+                                           .label(label)
+                                           .description(description)
+                                           .build();
             return acceptsValue(configKey);
         }
 
+        public Builder isCumulative(boolean cumulative) {
+            this.cumulative = cumulative;
+            return this;
+        }
+
         public ConfigDescriptor build() {
-            if(acceptedValues.size() == 0) {
+            if (acceptedValues.size() == 0) {
                 throw new IllegalArgumentException("Must define at least 1 required value.");
             }
 
             // shortcut - if name not defined, use the name associated with the ConfigKey
-            if(name == null && acceptedValues.size() > 0) {
+            if (name == null && acceptedValues.size() > 0) {
                 name = ConfigName.of(acceptedValues.get(0).getKey());
             }
 
             // shortcut - if description not defined, use the name associated with the ConfigKey
-            if(description == null && acceptedValues.size() > 0) {
+            if (description == null && acceptedValues.size() > 0) {
                 description = acceptedValues.get(0).getDescription();
             }
 
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigKey.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigKey.java
index a0ead7c9..a5956c71 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigKey.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigKey.java
@@ -13,12 +13,10 @@
 package com.cloudera.parserchains.core.model.config;
 
 import com.cloudera.parserchains.core.Regex;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.io.Serializable;
-import java.util.Map;
 import java.util.Optional;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * A {@link ConfigName} is associated with one or more key/value pairs representing
@@ -31,7 +29,7 @@
  *
  * 
      Effectively, a {@link ConfigName} is associated with a Map<{@link ConfigKey}, {@link ConfigValue}>
  * of values. This relationship can be seen in the parser's primary configuration method;
- * {@link com.cloudera.parserchains.core.Parser#configure(ConfigName, Map)}.
+ * {@code com.cloudera.parserchains.core.Parser#configure(ConfigName, Map(ConfigKey, ConfigValue))}.
  */
 public class ConfigKey implements Serializable {
 
@@ -67,7 +65,7 @@ public static Builder builder() {
      * Private constructor.  See {@link #builder()}.
      */
     private ConfigKey(Builder builder) {
-        if(!isValidRegex.matches(builder.key)) {
+        if (!isValidRegex.matches(builder.key)) {
             throw new IllegalArgumentException(String.format("Invalid config key: '%s'", builder.key));
         }
         this.key = builder.key;
@@ -102,22 +100,22 @@ public boolean equals(Object o) {
         }
         ConfigKey that = (ConfigKey) o;
         return new EqualsBuilder()
-                .append(key, that.key)
-                .isEquals();
+              .append(key, that.key)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(key)
-                .toHashCode();
+              .append(key)
+              .toHashCode();
     }
 
     @Override
     public String toString() {
-        return "ConfigKey{" +
-                "key='" + key + '\'' +
-                '}';
+        return "ConfigKey{"
+               + "key='" + key + '\''
+               + '}';
     }
 
     public static class Builder {
@@ -131,6 +129,8 @@ public Builder() {
         }
 
         /**
+         * key setter.
+         *
          * @param key A unique key used to identify the configuration element.
          */
         public Builder key(String key) {
@@ -139,6 +139,8 @@ public Builder key(String key) {
         }
 
         /**
+         * label setter.
+         *
          * @param label A brief label that can be shown to the user.
          */
         public Builder label(String label) {
@@ -148,6 +150,8 @@ public Builder label(String label) {
         }
 
         /**
+         * description setter.
+         *
          * @param description A description of the configuration element.
          */
         public Builder description(String description) {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigName.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigName.java
index 81ee1be0..d47557b7 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigName.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigName.java
@@ -12,15 +12,14 @@
 
 package com.cloudera.parserchains.core.model.config;
 
+import static com.cloudera.parserchains.core.Validator.mustMatch;
+
 import com.cloudera.parserchains.core.Parser;
 import com.cloudera.parserchains.core.Regex;
+import java.io.Serializable;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
-import java.io.Serializable;
-
-import static com.cloudera.parserchains.core.Validator.mustMatch;
-
 /**
  * The name of a configuration parameter used to configure a {@link Parser}.
  */
@@ -33,6 +32,7 @@ public class ConfigName implements Serializable {
 
     /**
      * Create a {@link ConfigName}.
+     *
      * @param name The name of the configuration parameter.
      */
     public static ConfigName of(String name) {
@@ -61,21 +61,21 @@ public boolean equals(Object o) {
         }
         ConfigName that = (ConfigName) o;
         return new EqualsBuilder()
-                .append(name, that.name)
-                .isEquals();
+              .append(name, that.name)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(name)
-                .toHashCode();
+              .append(name)
+              .toHashCode();
     }
 
     @Override
     public String toString() {
-        return "ConfigName{" +
-                "name='" + name + '\'' +
-                '}';
+        return "ConfigName{"
+               + "name='" + name + '\''
+               + '}';
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigValue.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigValue.java
index ad8be31a..a762962a 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigValue.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/config/ConfigValue.java
@@ -12,14 +12,13 @@
 
 package com.cloudera.parserchains.core.model.config;
 
+import static com.cloudera.parserchains.core.Validator.mustMatch;
+
 import com.cloudera.parserchains.core.Regex;
+import java.io.Serializable;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
-import java.io.Serializable;
-
-import static com.cloudera.parserchains.core.Validator.mustMatch;
-
 /**
  * The value associated with a {@link ConfigName}.
  *
@@ -35,10 +34,11 @@ public class ConfigValue implements Serializable {
     private static final long serialVersionUID = 1L;
 
     private static final Regex validValue = Regex.of("^.{1,200}$");
-    private String value;
+    private final String value;
 
     /**
      * Creates a {@link ConfigValue} with a key and value.
+     *
      * @param value The value.
      */
     public static ConfigValue of(String value) {
@@ -59,9 +59,9 @@ public String get() {
 
     @Override
     public String toString() {
-        return "ConfigValue{" +
-                "value='" + value + '\'' +
-                '}';
+        return "ConfigValue{"
+               + "value='" + value + '\''
+               + '}';
     }
 
     @Override
@@ -74,14 +74,14 @@ public boolean equals(Object o) {
         }
         ConfigValue that = (ConfigValue) o;
         return new EqualsBuilder()
-                .append(value, that.value)
-                .isEquals();
+              .append(value, that.value)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(value)
-                .toHashCode();
+              .append(value)
+              .toHashCode();
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ConfigValueSchema.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ConfigValueSchema.java
index dbb98019..fd34219e 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ConfigValueSchema.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ConfigValueSchema.java
@@ -14,13 +14,12 @@
 
 import com.fasterxml.jackson.annotation.JsonAnyGetter;
 import com.fasterxml.jackson.annotation.JsonAnySetter;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * Describes the configuration values defined by the user when building a parser chain.
@@ -65,21 +64,21 @@ public boolean equals(Object o) {
         }
         ConfigValueSchema that = (ConfigValueSchema) o;
         return new EqualsBuilder()
-                .append(values, that.values)
-                .isEquals();
+              .append(values, that.values)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(values)
-                .toHashCode();
+              .append(values)
+              .toHashCode();
     }
 
     @Override
     public String toString() {
-        return "ConfigValue{" +
-                "values=" + values +
-                '}';
+        return "ConfigValue{"
+               + "values=" + values
+               + '}';
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserChainSchema.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserChainSchema.java
index 3f365c4d..50f0ebec 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserChainSchema.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserChainSchema.java
@@ -6,9 +6,11 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * 
+ * 
      
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,108 +21,107 @@
 package com.cloudera.parserchains.core.model.define;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * Defines the structure of a parser chain.
  */
 public class ParserChainSchema implements Serializable {
 
-  private static final long serialVersionUID = 1L;
-
-  /**
-   * The id of the parser chain.
-   *
-   * 
      This value is generated and is expected to be unique amongst
-   * all parser chains.
-   */
-  @JsonProperty("id")
-  private String id;
-
-  /**
-   * The user provided name of the parser chain.
-   */
-  @JsonProperty("name")
-  private String name;
-
-  /**
-   * The parsers in this parser chain.
-   */
-  @JsonProperty("parsers")
-  private List parsers;
-
-  public ParserChainSchema() {
-    parsers = new ArrayList<>();
-  }
-
-  public String getId() {
-    return id;
-  }
-
-  public ParserChainSchema setId(String id) {
-    this.id = id;
-    return this;
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public ParserChainSchema setName(String name) {
-    this.name = name;
-    return this;
-  }
-
-  public List getParsers() {
-    return parsers;
-  }
-
-  public ParserChainSchema setParsers(List parsers) {
-    this.parsers = parsers;
-    return this;
-  }
-
-  public ParserChainSchema addParser(ParserSchema parserSchema) {
-    this.parsers.add(parserSchema);
-    return this;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * The id of the parser chain.
+     *
+     * 
      This value is generated and is expected to be unique amongst
+     * all parser chains.
+     */
+    @JsonProperty("id")
+    private String id;
+
+    /**
+     * The user provided name of the parser chain.
+     */
+    @JsonProperty("name")
+    private String name;
+
+    /**
+     * The parsers in this parser chain.
+     */
+    @JsonProperty("parsers")
+    private List parsers;
+
+    public ParserChainSchema() {
+        parsers = new ArrayList<>();
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public ParserChainSchema setId(String id) {
+        this.id = id;
+        return this;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public ParserChainSchema setName(String name) {
+        this.name = name;
+        return this;
+    }
+
+    public List getParsers() {
+        return parsers;
+    }
+
+    public ParserChainSchema setParsers(List parsers) {
+        this.parsers = parsers;
+        return this;
+    }
+
+    public ParserChainSchema addParser(ParserSchema parserSchema) {
+        this.parsers.add(parserSchema);
+        return this;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        ParserChainSchema that = (ParserChainSchema) o;
+        return new EqualsBuilder()
+              .append(id, that.id)
+              .append(name, that.name)
+              .append(parsers, that.parsers)
+              .isEquals();
     }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder(17, 37)
+              .append(id)
+              .append(name)
+              .append(parsers)
+              .toHashCode();
+    }
+
+    @Override
+    public String toString() {
+        return "ParserChainSchema{"
+               + "id='" + id + '\''
+               + ", name='" + name + '\''
+               + ", parsers=" + parsers
+               + '}';
     }
-    ParserChainSchema that = (ParserChainSchema) o;
-    return new EqualsBuilder()
-            .append(id, that.id)
-            .append(name, that.name)
-            .append(parsers, that.parsers)
-            .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(17, 37)
-            .append(id)
-            .append(name)
-            .append(parsers)
-            .toHashCode();
-  }
-
-  @Override
-  public String toString() {
-    return "ParserChainSchema{" +
-            "id='" + id + '\'' +
-            ", name='" + name + '\'' +
-            ", parsers=" + parsers +
-            '}';
-  }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserID.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserID.java
index 56aa4de7..4d5d985c 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserID.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserID.java
@@ -13,11 +13,10 @@
 package com.cloudera.parserchains.core.model.define;
 
 import com.fasterxml.jackson.annotation.JsonValue;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.io.Serializable;
 import java.util.Objects;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * Uniquely identifies a parser.
@@ -28,11 +27,11 @@ public class ParserID implements Serializable {
 
     @JsonValue
     private final String id;
-    
+
     /**
      * Creates a new {@link ParserID} from a Parser class.
+     *
      * @param clazz The Parser class.
-     * @return
      */
     public static ParserID of(Class clazz) {
         return new ParserID(clazz.getCanonicalName());
@@ -66,21 +65,21 @@ public boolean equals(Object o) {
         }
         ParserID parserID = (ParserID) o;
         return new EqualsBuilder()
-                .append(id, parserID.id)
-                .isEquals();
+              .append(id, parserID.id)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(id)
-                .toHashCode();
+              .append(id)
+              .toHashCode();
     }
 
     @Override
     public String toString() {
-        return "ParserID{" +
-                "id='" + id + '\'' +
-                '}';
+        return "ParserID{"
+               + "id='" + id + '\''
+               + '}';
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserName.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserName.java
index 88e9b507..118f213c 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserName.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserName.java
@@ -14,11 +14,10 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.io.Serializable;
 import java.util.Objects;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * The name of a parser.
@@ -32,6 +31,7 @@ public class ParserName implements Serializable {
 
     /**
      * Create a new {@link ParserName}.
+     *
      * @param name The name of the parser.
      */
     @JsonCreator
@@ -67,21 +67,21 @@ public boolean equals(Object o) {
         }
         ParserName that = (ParserName) o;
         return new EqualsBuilder()
-                .append(name, that.name)
-                .isEquals();
+              .append(name, that.name)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(name)
-                .toHashCode();
+              .append(name)
+              .toHashCode();
     }
 
     @Override
     public String toString() {
-        return "ParserName{" +
-                "name='" + name + '\'' +
-                '}';
+        return "ParserName{"
+               + "name='" + name + '\''
+               + '}';
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserSchema.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserSchema.java
index 16789a1b..fc9a9eb0 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserSchema.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserSchema.java
@@ -6,9 +6,11 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * 
+ * 
      
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,14 +22,13 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * Describes the structure of one parser within a {@link ParserChainSchema}.
@@ -35,146 +36,146 @@
 @JsonPropertyOrder({"id", "name", "type", "config"})
 public class ParserSchema implements Serializable {
 
-  private static final long serialVersionUID = 1L;
-
-  /**
-   * A label given to this parser that is generated by the front-end.
-   *
-   * 
      This is only used by the front-end and allows errors encountered
-   * in the Live View to be mapped to the specific parser that failed.
-   */
-  @JsonProperty("id")
-  private String label;
-
-  /**
-   * The id of the parser.
-   *
-   * 
      This id should be unique amongst all of the available parsers.  This is
-   * derived from the name of the class implementing the parser.
-   *
-   * 
      The front-end refers to this as the "type" of parser, which might be a
-   * better name than id.
-   */
-  @JsonProperty("type")
-  private ParserID id;
-
-  /**
-   * A name given to this parser by the user.
-   */
-  @JsonProperty("name")
-  private ParserName name;
-
-  /**
-   * Defines how the user has configured this parser.
-   */
-  @JsonProperty("config")
-  private Map> config;
-
-  /**
-   * If the user has created a router, this describes the routing features
-   * as configured by the user.
-   */
-  @JsonProperty("routing")
-  private RoutingSchema routing;
-
-  public ParserSchema() {
-    config = new HashMap<>();
-  }
-
-  public ParserID getId() {
-    return id;
-  }
-
-  public ParserSchema setId(ParserID id) {
-    this.id = id;
-    return this;
-  }
-
-  public ParserName getName() {
-    return name;
-  }
-
-  public ParserSchema setName(ParserName name) {
-    this.name = name;
-    return this;
-  }
-
-  public String getLabel() {
-    return label;
-  }
-
-  public ParserSchema setLabel(String label) {
-    this.label = label;
-    return this;
-  }
-
-  public Map> getConfig() {
-    return config;
-  }
-
-  public ParserSchema setConfig(Map> config) {
-    this.config = config;
-    return this;
-  }
-
-  public ParserSchema addConfig(String key, ConfigValueSchema value) {
-    List values;
-    if(config.containsKey(key)) {
-      values = config.get(key);
-    } else {
-      values = new ArrayList<>();
-      config.put(key, values);
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * A label given to this parser that is generated by the front-end.
+     *
+     * 
      This is only used by the front-end and allows errors encountered
+     * in the Live View to be mapped to the specific parser that failed.
+     */
+    @JsonProperty("id")
+    private String label;
+
+    /**
+     * The id of the parser.
+     *
+     * 
      This id should be unique amongst all of the available parsers.  This is
+     * derived from the name of the class implementing the parser.
+     *
+     * 
      The front-end refers to this as the "type" of parser, which might be a
+     * better name than id.
+     */
+    @JsonProperty("type")
+    private ParserID id;
+
+    /**
+     * A name given to this parser by the user.
+     */
+    @JsonProperty("name")
+    private ParserName name;
+
+    /**
+     * Defines how the user has configured this parser.
+     */
+    @JsonProperty("config")
+    private Map> config;
+
+    /**
+     * If the user has created a router, this describes the routing features
+     * as configured by the user.
+     */
+    @JsonProperty("routing")
+    private RoutingSchema routing;
+
+    public ParserSchema() {
+        config = new HashMap<>();
+    }
+
+    public ParserID getId() {
+        return id;
+    }
+
+    public ParserSchema setId(ParserID id) {
+        this.id = id;
+        return this;
+    }
+
+    public ParserName getName() {
+        return name;
+    }
+
+    public ParserSchema setName(ParserName name) {
+        this.name = name;
+        return this;
+    }
+
+    public String getLabel() {
+        return label;
     }
-    values.add(value);
-    return this;
-  }
-
-  public RoutingSchema getRouting() {
-    return routing;
-  }
-
-  public ParserSchema setRouting(RoutingSchema routing) {
-    this.routing = routing;
-    return this;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
+
+    public ParserSchema setLabel(String label) {
+        this.label = label;
+        return this;
     }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
+
+    public Map> getConfig() {
+        return config;
+    }
+
+    public ParserSchema setConfig(Map> config) {
+        this.config = config;
+        return this;
+    }
+
+    public ParserSchema addConfig(String key, ConfigValueSchema value) {
+        List values;
+        if (config.containsKey(key)) {
+            values = config.get(key);
+        } else {
+            values = new ArrayList<>();
+            config.put(key, values);
+        }
+        values.add(value);
+        return this;
+    }
+
+    public RoutingSchema getRouting() {
+        return routing;
+    }
+
+    public ParserSchema setRouting(RoutingSchema routing) {
+        this.routing = routing;
+        return this;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        ParserSchema that = (ParserSchema) o;
+        return new EqualsBuilder()
+              .append(label, that.label)
+              .append(id, that.id)
+              .append(name, that.name)
+              .append(config, that.config)
+              .append(routing, that.routing)
+              .isEquals();
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder(17, 37)
+              .append(label)
+              .append(id)
+              .append(name)
+              .append(config)
+              .append(routing)
+              .toHashCode();
+    }
+
+    @Override
+    public String toString() {
+        return "ParserSchema{"
+               + "label='" + label + '\''
+               + ", id=" + id
+               + ", name=" + name
+               + ", config=" + config
+               + ", routing=" + routing
+               + '}';
     }
-    ParserSchema that = (ParserSchema) o;
-    return new EqualsBuilder()
-            .append(label, that.label)
-            .append(id, that.id)
-            .append(name, that.name)
-            .append(config, that.config)
-            .append(routing, that.routing)
-            .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(17, 37)
-            .append(label)
-            .append(id)
-            .append(name)
-            .append(config)
-            .append(routing)
-            .toHashCode();
-  }
-
-  @Override
-  public String toString() {
-    return "ParserSchema{" +
-            "label='" + label + '\'' +
-            ", id=" + id +
-            ", name=" + name +
-            ", config=" + config +
-            ", routing=" + routing +
-            '}';
-  }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/RouteSchema.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/RouteSchema.java
index 6f00b92f..b5f7a1ea 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/RouteSchema.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/RouteSchema.java
@@ -14,11 +14,10 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import java.io.Serializable;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
-import java.io.Serializable;
-
 /**
  * Describes the structure of one route.
  */
@@ -79,8 +78,8 @@ public boolean isDefault() {
         return isDefault;
     }
 
-    public RouteSchema setDefault(boolean aDefault) {
-        isDefault = aDefault;
+    public RouteSchema setDefault(boolean defaultValue) {
+        isDefault = defaultValue;
         return this;
     }
 
@@ -112,33 +111,33 @@ public boolean equals(Object o) {
         }
         RouteSchema that = (RouteSchema) o;
         return new EqualsBuilder()
-                .append(isDefault, that.isDefault)
-                .append(matchingValue, that.matchingValue)
-                .append(subChain, that.subChain)
-                .append(label, that.label)
-                .append(name, that.name)
-                .isEquals();
+              .append(isDefault, that.isDefault)
+              .append(matchingValue, that.matchingValue)
+              .append(subChain, that.subChain)
+              .append(label, that.label)
+              .append(name, that.name)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(matchingValue)
-                .append(isDefault)
-                .append(subChain)
-                .append(label)
-                .append(name)
-                .toHashCode();
+              .append(matchingValue)
+              .append(isDefault)
+              .append(subChain)
+              .append(label)
+              .append(name)
+              .toHashCode();
     }
 
     @Override
     public String toString() {
-        return "RouteSchema{" +
-                "matchingValue='" + matchingValue + '\'' +
-                ", isDefault=" + isDefault +
-                ", subChain=" + subChain +
-                ", label='" + label + '\'' +
-                ", name=" + name +
-                '}';
+        return "RouteSchema{"
+               + "matchingValue='" + matchingValue + '\''
+               + ", isDefault=" + isDefault
+               + ", subChain=" + subChain
+               + ", label='" + label + '\''
+               + ", name=" + name
+               + '}';
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/RoutingSchema.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/RoutingSchema.java
index 773a4616..3b27a5ef 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/RoutingSchema.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/RoutingSchema.java
@@ -14,12 +14,11 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * Defines the structure of a router contained within a parser chain.
@@ -81,24 +80,24 @@ public boolean equals(Object o) {
         }
         RoutingSchema that = (RoutingSchema) o;
         return new EqualsBuilder()
-                .append(matchingField, that.matchingField)
-                .append(routes, that.routes)
-                .isEquals();
+              .append(matchingField, that.matchingField)
+              .append(routes, that.routes)
+              .isEquals();
     }
 
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
-                .append(matchingField)
-                .append(routes)
-                .toHashCode();
+              .append(matchingField)
+              .append(routes)
+              .toHashCode();
     }
 
     @Override
     public String toString() {
-        return "RoutingSchema{" +
-                "matchingField='" + matchingField + '\'' +
-                ", routes=" + routes +
-                '}';
+        return "RoutingSchema{"
+               + "matchingField='" + matchingField + '\''
+               + ", routes=" + routes
+               + '}';
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/AnnotationUtils.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/AnnotationUtils.java
index acd73bb8..7f261939 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/AnnotationUtils.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/AnnotationUtils.java
@@ -15,7 +15,6 @@
 import com.cloudera.parserchains.core.Parser;
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.Parameter;
-
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
@@ -42,18 +41,18 @@ public static Map getAnnotatedMethods(Class getAnnotatedMethodsInOrder(Class clazz) {
         return getAnnotatedMethods(clazz).entrySet().stream()
-                .sorted(Comparator.comparing(entry -> entry.getKey().orderPriority()))
-                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
-                        (first, second) -> first,
-                        LinkedHashMap::new));
+                                         .sorted(Comparator.comparing(entry -> entry.getKey().orderPriority()))
+                                         .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
+                                               (first, second) -> first,
+                                               LinkedHashMap::new));
     }
 
     public static List getAnnotatedParameters(Method method) {
         List results = new ArrayList<>();
-        for (Annotation[] parameterAnnotations : method.getParameterAnnotations()) {
-            for (Annotation aParameterAnnotation : parameterAnnotations) {
-                if (aParameterAnnotation instanceof Parameter) {
-                    results.add((Parameter) aParameterAnnotation);
+        for (Annotation[] parameterAnnotationArray : method.getParameterAnnotations()) {
+            for (Annotation parameterAnnotation : parameterAnnotationArray) {
+                if (parameterAnnotation instanceof Parameter) {
+                    results.add((Parameter) parameterAnnotation);
                 }
             }
         }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/JSONUtils.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/JSONUtils.java
index 36f0e406..ba2ff023 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/JSONUtils.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/JSONUtils.java
@@ -26,206 +26,215 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.MapperFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
-
-import java.io.*;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
 
+@SuppressWarnings({"checkstyle:InvalidJavadocPosition", "checkstyle:RegexpSinglelineJava"})
 public enum JSONUtils {
-  INSTANCE;
-
-  public static class ReferenceSupplier implements Supplier> {
-    Type type;
-    protected ReferenceSupplier() {
-      Type superClass = this.getClass().getGenericSuperclass();
-      if(superClass instanceof Class) {
-        throw new IllegalArgumentException("Internal error: ReferenceSupplier constructed without actual type information");
-      } else {
-        this.type = ((ParameterizedType)superClass).getActualTypeArguments()[0];
-      }
-    }
+    INSTANCE;
+
+    public static class ReferenceSupplier implements Supplier> {
+        Type type;
+
+        protected ReferenceSupplier() {
+            Type superClass = this.getClass().getGenericSuperclass();
+            if (superClass instanceof Class) {
+                throw new IllegalArgumentException(
+                      "Internal error: ReferenceSupplier constructed without actual type information");
+            } else {
+                this.type = ((ParameterizedType) superClass).getActualTypeArguments()[0];
+            }
+        }
 
-    @Override
-    public TypeReference get() {
-      return new TypeReference() {
         @Override
-        public Type getType() {
-          return type;
+        public TypeReference get() {
+            return new TypeReference() {
+                @Override
+                public Type getType() {
+                    return type;
+                }
+            };
         }
-      };
     }
-  }
-
-  public final static ReferenceSupplier> MAP_SUPPLIER = new ReferenceSupplier>() {};
-  public final static ReferenceSupplier> LIST_SUPPLIER = new ReferenceSupplier>(){};
-
-  private static ThreadLocal _mapper = ThreadLocal.withInitial(() ->
-      new ObjectMapper()
-              .setSerializationInclusion(JsonInclude.Include.NON_NULL)
-              .enable(JsonParser.Feature.ALLOW_COMMENTS)
-              .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
-              .enable(MapperFeature.USE_ANNOTATIONS));
-
-  public  T convert(Object original, Class targetClass) {
-    return _mapper.get().convertValue(original, targetClass);
-  }
-
-  public ObjectMapper getMapper() {
-    return _mapper.get();
-  }
-
-
-  public  T load(InputStream is, ReferenceSupplier ref) throws IOException {
-    return _mapper.get().readValue(is, (TypeReference)ref.get());
-  }
-
-  public  T load(String is, ReferenceSupplier ref) throws IOException {
-    return _mapper.get().readValue(is, (TypeReference)ref.get());
-  }
-
-  /**
-   * Loads JSON from a file and ensures it's located in the specified class.
-   *
-   * @param f The file to read from
-   * @param ref A {@link ReferenceSupplier} for the class to be loaded into.
-   * @param  The type parameter of the class
-   * @return The JSON loaded into the provided class
-   * @throws IOException If there's an issue loading the JSON
-   */
-  public  T load(File f, ReferenceSupplier ref) throws IOException {
-    try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
-      return _mapper.get().readValue(is, (TypeReference)ref.get());
+
+    public static final ReferenceSupplier> MAP_SUPPLIER =
+          new ReferenceSupplier>() {
+          };
+    public static final ReferenceSupplier> LIST_SUPPLIER = new ReferenceSupplier>() {
+    };
+
+    private static final ThreadLocal _mapper = ThreadLocal.withInitial(() ->
+          new ObjectMapper()
+                .setSerializationInclusion(JsonInclude.Include.NON_NULL)
+                .enable(JsonParser.Feature.ALLOW_COMMENTS)
+                .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
+                .enable(MapperFeature.USE_ANNOTATIONS));
+
+    public  T convert(Object original, Class targetClass) {
+        return _mapper.get().convertValue(original, targetClass);
     }
-  }
-
-  public  T load(InputStream is, Class clazz) throws IOException {
-    return _mapper.get().readValue(is, clazz);
-  }
-
-  /**
-   * Loads JSON from a file and ensures it's located in the provided class.
-   *
-   * @param f The file to read from
-   * @param clazz The class to read into
-   * @param  The type parameter of the class
-   * @return The JSON loaded into the provided class
-   * @throws IOException If there's an issue loading the JSON
-   */
-  public  T load(File f, Class clazz) throws IOException {
-    try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
-      return _mapper.get().readValue(is, clazz);
+
+    public ObjectMapper getMapper() {
+        return _mapper.get();
     }
-  }
-
-  public  T load(String is, Class clazz) throws IOException {
-    return _mapper.get().readValue(is, clazz);
-  }
-
-  /**
-   * Converts an object to a JSON string. Can be a pretty string
-   *
-   * @param o The object to convert
-   * @param pretty Pretty formatted string if true, otherwise not pretty formatted.
-   * @return A JSON string representation of the object
-   * @throws JsonProcessingException If there's an issue converting to JSON.
-   */
-  public String toJSON(Object o, boolean pretty) throws JsonProcessingException {
-    if (pretty) {
-      return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsString(o);
-    } else {
-      return _mapper.get().writeValueAsString(o);
+
+
+    public  T load(InputStream is, ReferenceSupplier ref) throws IOException {
+        return _mapper.get().readValue(is, ref.get());
     }
-  }
-
-  public byte[] toJSONPretty(String config) throws IOException {
-    return toJSONPretty(readTree(config));
-  }
-
-  public byte[] toJSONPretty(Object config) throws JsonProcessingException {
-    return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsBytes(config);
-  }
-
-  /**
-   * Reads a JSON string into a JsonNode Object.
-   *
-   * @param json JSON value to deserialize
-   * @return deserialized JsonNode Object
-   */
-  JsonNode readTree(String json) throws IOException {
-    return _mapper.get().readTree(json);
-  }
-
-  /**
-   * Reads a JSON byte array into a JsonNode Object.
-   *
-   * @param json JSON value to deserialize
-   * @return deserialized JsonNode Object
-   */
-  JsonNode readTree(byte[] json) throws IOException {
-    return _mapper.get().readTree(json);
-  }
-
-  /**
-   * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902)
-   * Operations:
-   * 
        
-   *   
      • add
        • 
-   *   
      • remove
        • 
-   *   
      • replace
        • 
-   *   
      • move
        • 
-   *   
      • copy
        • 
-   *   
      • test
        • 
-   * 
      
-   *
-   * @param patch Array of JSON patches, e.g. [{ "op": "move", "from": "/a", "path": "/c" }]
-   * @param source Source JSON to apply patch to
-   * @return new json after applying the patch
-   */
-/*
-  public byte[] applyPatch(String patch, String source) throws IOException {
-    JsonNode patchNode = readTree(patch);
-    JsonNode sourceNode = readTree(source);
-    return toJSONPretty(JsonPatch.apply(patchNode, sourceNode));
-  }
-*/
-
-  /**
-   * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902)
-   *
-   * @param patch Array of JSON patches in raw bytes
-   * @param source Source JSON in raw bytes to apply patch to
-   * @return new json after applying the patch
-   *
-   * @see JSONUtils#applyPatch(String, String)
-   */
-/*
-  public byte[] applyPatch(byte[] patch, byte[] source) throws IOException {
-    JsonNode patchNode = readTree(patch);
-    JsonNode sourceNode = readTree(source);
-    return toJSONPretty(JsonPatch.apply(patchNode, sourceNode));
-  }
-*/
-
-
-  /**
-   * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902)
-   *
-   * @param patch List of JSON patches in map form
-   * @param source Source JSON in map form to apply patch to
-   * @return new json after applying the patch
-   *
-   * @see JSONUtils#applyPatch(String, String)
-   */
-/*
-  public Map applyPatch(List> patch, Map source) {
-    JsonNode originalNode = convert(source, JsonNode.class);
-    JsonNode patchNode = convert(patch, JsonNode.class);
-    JsonNode patched = JsonPatch.apply(patchNode, originalNode);
-    return _mapper.get().convertValue(patched, new TypeReference>() { });
-  }
-*/
+
+    public  T load(String is, ReferenceSupplier ref) throws IOException {
+        return _mapper.get().readValue(is, ref.get());
+    }
+
+    /**
+     * Loads JSON from a file and ensures it's located in the specified class.
+     *
+     * @param f   The file to read from
+     * @param ref A {@link ReferenceSupplier} for the class to be loaded into.
+     * @param  The type parameter of the class
+     * @return The JSON loaded into the provided class
+     * @throws IOException If there's an issue loading the JSON
+     */
+    public  T load(File f, ReferenceSupplier ref) throws IOException {
+        try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
+            return _mapper.get().readValue(is, ref.get());
+        }
+    }
+
+    public  T load(InputStream is, Class clazz) throws IOException {
+        return _mapper.get().readValue(is, clazz);
+    }
+
+    /**
+     * Loads JSON from a file and ensures it's located in the provided class.
+     *
+     * @param f     The file to read from
+     * @param clazz The class to read into
+     * @param    The type parameter of the class
+     * @return The JSON loaded into the provided class
+     * @throws IOException If there's an issue loading the JSON
+     */
+    public  T load(File f, Class clazz) throws IOException {
+        try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
+            return _mapper.get().readValue(is, clazz);
+        }
+    }
+
+    public  T load(String is, Class clazz) throws IOException {
+        return _mapper.get().readValue(is, clazz);
+    }
+
+    /**
+     * Converts an object to a JSON string. Can be a pretty string
+     *
+     * @param o      The object to convert
+     * @param pretty Pretty formatted string if true, otherwise not pretty formatted.
+     * @return A JSON string representation of the object
+     * @throws JsonProcessingException If there's an issue converting to JSON.
+     */
+    public String toJSON(Object o, boolean pretty) throws JsonProcessingException {
+        if (pretty) {
+            return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsString(o);
+        } else {
+            return _mapper.get().writeValueAsString(o);
+        }
+    }
+
+    public byte[] toJSONPretty(String config) throws IOException {
+        return toJSONPretty(readTree(config));
+    }
+
+    public byte[] toJSONPretty(Object config) throws JsonProcessingException {
+        return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsBytes(config);
+    }
+
+    /**
+     * Reads a JSON string into a JsonNode Object.
+     *
+     * @param json JSON value to deserialize
+     * @return deserialized JsonNode Object
+     */
+    JsonNode readTree(String json) throws IOException {
+        return _mapper.get().readTree(json);
+    }
+
+    /**
+     * Reads a JSON byte array into a JsonNode Object.
+     *
+     * @param json JSON value to deserialize
+     * @return deserialized JsonNode Object
+     */
+    JsonNode readTree(byte[] json) throws IOException {
+        return _mapper.get().readTree(json);
+    }
+
+    /**
+     * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902)
+     * Operations:
+     * 
        
+     *   
      • add
        • 
+     *   
      • remove
        • 
+     *   
      • replace
        • 
+     *   
      • move
        • 
+     *   
      • copy
        • 
+     *   
      • test
        • 
+     * 
      
+     *
+     * @param patch Array of JSON patches, e.g. [{ "op": "move", "from": "/a", "path": "/c" }]
+     * @param source Source JSON to apply patch to
+     * @return new json after applying the patch
+     */
+    /*
+      public byte[] applyPatch(String patch, String source) throws IOException {
+        JsonNode patchNode = readTree(patch);
+        JsonNode sourceNode = readTree(source);
+        return toJSONPretty(JsonPatch.apply(patchNode, sourceNode));
+      }
+    */
+
+    /**
+     * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902)
+     *
+     * @param patch Array of JSON patches in raw bytes
+     * @param source Source JSON in raw bytes to apply patch to
+     * @return new json after applying the patch
+     *
+     * @see JSONUtils#applyPatch(String, String)
+     */
+    /*
+      public byte[] applyPatch(byte[] patch, byte[] source) throws IOException {
+        JsonNode patchNode = readTree(patch);
+        JsonNode sourceNode = readTree(source);
+        return toJSONPretty(JsonPatch.apply(patchNode, sourceNode));
+      }
+    */
+
+
+    /**
+     * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902)
+     *
+     * @param patch List of JSON patches in map form
+     * @param source Source JSON in map form to apply patch to
+     * @return new json after applying the patch
+     *
+     * @see JSONUtils#applyPatch(String, String)
+     */
+    /*
+      public Map applyPatch(List> patch, Map source) {
+        JsonNode originalNode = convert(source, JsonNode.class);
+        JsonNode patchNode = convert(patch, JsonNode.class);
+        JsonNode patched = JsonPatch.apply(patchNode, originalNode);
+        return _mapper.get().convertValue(patched, new TypeReference>() { });
+      }
+    */
 
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/StringUtils.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/StringUtils.java
index b8f0bb43..d3a45499 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/StringUtils.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/utils/StringUtils.java
@@ -12,18 +12,17 @@
 
 package com.cloudera.parserchains.core.utils;
 
-import lombok.experimental.UtilityClass;
-import org.apache.commons.lang3.StringEscapeUtils;
-
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import lombok.experimental.UtilityClass;
+import org.apache.commons.lang3.StringEscapeUtils;
 
 @UtilityClass
 public final class StringUtils {
 
-    private static JSONUtils jsonUtils = JSONUtils.INSTANCE;
+    private static final JSONUtils jsonUtils = JSONUtils.INSTANCE;
 
     public static char getFirstChar(String delimiter) {
         return unescapeJava(delimiter).charAt(0);
@@ -36,25 +35,25 @@ public static String unescapeJava(String text) {
     public static Object parseProperType(String s) {
         Optional result;
         result = getLong(s);
-        if (result.isPresent()){
+        if (result.isPresent()) {
             return result.get();
         }
         result = getDouble(s);
-        if (result.isPresent()){
+        if (result.isPresent()) {
             return result.get();
         }
         result = getList(s);
-        if (result.isPresent()){
+        if (result.isPresent()) {
             return result.get();
         }
         result = getMap(s);
-        if (result.isPresent()){
+        if (result.isPresent()) {
             return result.get();
         }
         return s;
     }
 
-    public static Optional getDouble(String text){
+    public static Optional getDouble(String text) {
         if (text == null) {
             return Optional.empty();
         }
@@ -65,7 +64,7 @@ public static Optional getDouble(String text){
         }
     }
 
-    public static Optional getLong(String text){
+    public static Optional getLong(String text) {
         if (text == null) {
             return Optional.empty();
         }
@@ -76,7 +75,7 @@ public static Optional getLong(String text){
         }
     }
 
-    public static Optional> getList(String text){
+    public static Optional> getList(String text) {
         if (text == null || getMap(text).isPresent()) {
             return Optional.empty();
         }
@@ -87,7 +86,7 @@ public static Optional> getList(String text){
         }
     }
 
-    public static Optional> getMap(String text){
+    public static Optional> getMap(String text) {
         if (text == null) {
             return Optional.empty();
         }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/test/java/com/cloudera/parserchains/core/RouterLinkTest.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/test/java/com/cloudera/parserchains/core/RouterLinkTest.java
index 51ea82fc..cb3a74b3 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/test/java/com/cloudera/parserchains/core/RouterLinkTest.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/test/java/com/cloudera/parserchains/core/RouterLinkTest.java
@@ -12,7 +12,18 @@
 
 package com.cloudera.parserchains.core;
 
+import static com.cloudera.parserchains.core.ChainLinkTestUtilities.makeEchoParser;
+import static com.cloudera.parserchains.core.ChainLinkTestUtilities.makeErrorParser;
+import static com.cloudera.parserchains.core.ChainLinkTestUtilities.makeParser;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
 import com.cloudera.parserchains.core.model.define.ParserName;
+import java.util.List;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -22,16 +33,6 @@
 import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
 
-import java.util.List;
-
-import static com.cloudera.parserchains.core.ChainLinkTestUtilities.*;
-import static org.hamcrest.CoreMatchers.hasItem;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.when;
-
 @ExtendWith(MockitoExtension.class)
 @MockitoSettings(strictness = Strictness.LENIENT)
 public class RouterLinkTest {
@@ -173,8 +174,8 @@ void errorRouteWithNextLink() {
         List results = routerLink.process(input);
 
         // validate
-        assertThat("Expected 1 result showing the error caused by the route taken. " +
-                        "The next link to parser2 should not be followed.",
+        assertThat("Expected 1 result showing the error caused by the route taken. "
+                        + "The next link to parser2 should not be followed.",
                 results.size(), is(1));
 
     }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AlwaysFailParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AlwaysFailParser.java
index e563d10e..1d8343d0 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AlwaysFailParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AlwaysFailParser.java
@@ -28,8 +28,8 @@
  * to flag when unexpected conditions are encountered in the data.
  */
 @MessageParser(
-    name="Error",
-    description = "Always results in an error. Can be used with a router to flag unexpected data.")
+      name = "Error",
+      description = "Always results in an error. Can be used with a router to flag unexpected data.")
 public class AlwaysFailParser implements Parser {
     private static final String DEFAULT_ERROR_MESSAGE = "Parsing error encountered";
     private Throwable error;
@@ -41,18 +41,18 @@ public AlwaysFailParser() {
     @Override
     public Message parse(Message message) {
         return Message.builder()
-                .withFields(message)
-                .withError(error)
-                .build();
+                      .withFields(message)
+                      .withError(error)
+                      .build();
     }
 
-    @Configurable(key="errorMessage")
+    @Configurable(key = "errorMessage")
     public AlwaysFailParser withError(
-            @Parameter(key="errorMessage",
-                    label="Error Message",
-                    description="The error message explaining the error. Default value: '" + DEFAULT_ERROR_MESSAGE + "'",
-                    defaultValue=DEFAULT_ERROR_MESSAGE) String message) {
-        if(StringUtils.isNotEmpty(message)) {
+          @Parameter(key = "errorMessage",
+                label = "Error Message",
+                description = "The error message explaining the error. Default value: '" + DEFAULT_ERROR_MESSAGE + "'",
+                defaultValue = DEFAULT_ERROR_MESSAGE) String message) {
+        if (StringUtils.isNotEmpty(message)) {
             error = new IllegalStateException(message);
         }
         return this;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AvroParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AvroParser.java
index 53b77a1d..8ca2f149 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AvroParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AvroParser.java
@@ -12,12 +12,18 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
+import static java.lang.String.format;
+
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.FieldValue;
 import com.cloudera.parserchains.core.Message;
 import com.cloudera.parserchains.core.Parser;
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.MessageParser;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Optional;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.avro.AvroRuntimeException;
 import org.apache.avro.Schema;
@@ -30,16 +36,9 @@
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.util.Optional;
-
-import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
-import static java.lang.String.format;
-
 @MessageParser(
-        name = "Simple Avro parser",
-        description = "Parses Avro data by creating a field for each Avro element.")
+      name = "Simple Avro parser",
+      description = "Parses Avro data by creating a field for each Avro element.")
 @Slf4j
 public class AvroParser implements Parser {
 
@@ -54,11 +53,11 @@ public AvroParser() {
     }
 
     @Configurable(
-            key = "input",
-            label = "Input Field",
-            description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
-            defaultValue = DEFAULT_INPUT_FIELD,
-            isOutputName = true)
+          key = "input",
+          label = "Input Field",
+          description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
+          defaultValue = DEFAULT_INPUT_FIELD,
+          isOutputName = true)
     public AvroParser inputField(String fieldName) {
         if (StringUtils.isNotBlank(fieldName)) {
             this.inputField = FieldName.of(fieldName);
@@ -67,11 +66,11 @@ public AvroParser inputField(String fieldName) {
     }
 
     @Configurable(
-            key = "schemaPath",
-            label = "Schema Path",
-            description = "Path to schema of avro file. Default value: '" + DEFAULT_AVRO_SCHEMA + "'",
-            defaultValue = DEFAULT_AVRO_SCHEMA,
-            required = true)
+          key = "schemaPath",
+          label = "Schema Path",
+          description = "Path to schema of avro file. Default value: '" + DEFAULT_AVRO_SCHEMA + "'",
+          defaultValue = DEFAULT_AVRO_SCHEMA,
+          required = true)
     public AvroParser schemaPath(String pathToSchema) throws IOException {
         FileSystem fileSystem = new Path(pathToSchema).getFileSystem();
         loadSchema(pathToSchema, fileSystem);
@@ -96,8 +95,8 @@ public Message parse(Message input) {
             return doParse(field.get(), builder);
         } else {
             return builder
-                    .withError(format("Message missing expected input field '%s'", inputField.toString()))
-                    .build();
+                  .withError(format("Message missing expected input field '%s'", inputField.toString()))
+                  .build();
         }
     }
 
@@ -109,7 +108,7 @@ public Message doParse(FieldValue toParse, Message.Builder output) {
             BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(byteArrayInputStream, null);
             GenericRecord genericRecord = genericDatumReader.read(null, binaryDecoder);
             genericRecord.getSchema().getFields().forEach(
-                    field -> output.addField(field.name(), String.valueOf(genericRecord.get(field.name()))));
+                  field -> output.addField(field.name(), String.valueOf(genericRecord.get(field.name()))));
         } catch (IOException | AvroRuntimeException exception) {
             output.withError(exception).build();
         }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/CsvTextParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/CsvTextParser.java
index 53ac62bb..2b414147 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/CsvTextParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/CsvTextParser.java
@@ -12,6 +12,10 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static com.cloudera.parserchains.core.utils.StringUtils.getFirstChar;
+import static com.cloudera.parserchains.core.utils.StringUtils.unescapeJava;
+import static java.lang.String.format;
+
 import com.cloudera.parserchains.core.Constants;
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.FieldValue;
@@ -26,25 +30,20 @@
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.dataformat.csv.CsvMapper;
 import com.fasterxml.jackson.dataformat.csv.CsvSchema;
-import org.apache.commons.lang3.StringUtils;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Supplier;
-
-import static com.cloudera.parserchains.core.utils.StringUtils.getFirstChar;
-import static com.cloudera.parserchains.core.utils.StringUtils.unescapeJava;
-import static java.lang.String.format;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * Parses delimited text like CSV.
  */
 @MessageParser(
-        name = "CSV/TSV Parser",
-        description = "Parses delimited text like CSV or TSV.")
+      name = "CSV/TSV Parser",
+      description = "Parses delimited text like CSV or TSV.")
 public class CsvTextParser implements Parser {
 
     private static final String DEFAULT_DELIMITER = ",";
@@ -78,15 +77,17 @@ public CsvTextParser() {
         trimWhitespace = Boolean.parseBoolean(DEFAULT_TRIM);
         mapper = new CsvMapper();
         updateSchema(() -> mapper
-                .schemaFor(new TypeReference>() {
-                })
-                .withoutHeader()
-                .withLineSeparator("\n")
-                .withColumnSeparator(getFirstChar(DEFAULT_DELIMITER))
-                .withQuoteChar(getFirstChar(DEFAULT_QUOTE_CHAR)));
+              .schemaFor(new TypeReference>() {
+              })
+              .withoutHeader()
+              .withLineSeparator("\n")
+              .withColumnSeparator(getFirstChar(DEFAULT_DELIMITER))
+              .withQuoteChar(getFirstChar(DEFAULT_QUOTE_CHAR)));
     }
 
     /**
+     * inputField setter.
+     *
      * @param inputField The name of the field containing the text to parse.
      */
     public CsvTextParser withInputField(FieldName inputField) {
@@ -95,10 +96,10 @@ public CsvTextParser withInputField(FieldName inputField) {
     }
 
     @Configurable(key = "inputField",
-            label = "Input Field",
-            description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
-            isOutputName = true,
-            defaultValue = Constants.DEFAULT_INPUT_FIELD)
+          label = "Input Field",
+          description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
+          isOutputName = true,
+          defaultValue = Constants.DEFAULT_INPUT_FIELD)
     public CsvTextParser withInputField(String fieldName) {
         if (StringUtils.isNotEmpty(fieldName)) {
             withInputField(FieldName.of(fieldName));
@@ -111,6 +112,8 @@ public FieldName getInputField() {
     }
 
     /**
+     * quoteChar setter.
+     *
      * @param quoteChar A character replacing the quote character used for escaping when parsing CSV.
      */
     public CsvTextParser withQuoteChar(char quoteChar) {
@@ -119,9 +122,9 @@ public CsvTextParser withQuoteChar(char quoteChar) {
     }
 
     @Configurable(key = "quoteChar",
-            label = "Quote character",
-            description = "A character used escape commas in text. Default value: '" + DEFAULT_QUOTE_CHAR + "'",
-            defaultValue = DEFAULT_QUOTE_CHAR)
+          label = "Quote character",
+          description = "A character used escape commas in text. Default value: '" + DEFAULT_QUOTE_CHAR + "'",
+          defaultValue = DEFAULT_QUOTE_CHAR)
     public void withQuoteChar(String quoteChar) {
         if (StringUtils.isNotEmpty(quoteChar)) {
             withQuoteChar(getFirstChar(quoteChar));
@@ -133,6 +136,8 @@ public char getQuoteChar() {
     }
 
     /**
+     * Delimiter setter.
+     *
      * @param delimiter A character defining the delimiter used to split the text.
      */
     public CsvTextParser withDelimiter(char delimiter) {
@@ -141,9 +146,9 @@ public CsvTextParser withDelimiter(char delimiter) {
     }
 
     @Configurable(key = "delimiter",
-            label = "Delimiter",
-            description = "A character used to split the text. Default value: '" + DEFAULT_DELIMITER + "'",
-            defaultValue = DEFAULT_DELIMITER)
+          label = "Delimiter",
+          description = "A character used to split the text. Default value: '" + DEFAULT_DELIMITER + "'",
+          defaultValue = DEFAULT_DELIMITER)
     public void withDelimiter(String delimiter) {
         if (StringUtils.isNotEmpty(delimiter)) {
             withDelimiter(getFirstChar(delimiter));
@@ -155,6 +160,8 @@ public char getDelimiter() {
     }
 
     /**
+     * outputFields setter.
+     *
      * @param fieldName The name of a field to create.
      * @param index     The 0-based index defining which delimited element is added to the field.
      */
@@ -165,17 +172,17 @@ public CsvTextParser withOutputField(FieldName fieldName, int index) {
 
     @Configurable(key = "outputField", label = "Output Field", multipleValues = true)
     public void withOutputField(
-            @Parameter(key = "fieldName",
+          @Parameter(key = "fieldName",
                 label = "Field Name",
                 description = "The name of the output field.",
                 isOutputName = true,
                 required = true)
-            String fieldName,
-            @Parameter(key = "fieldIndex",
+          String fieldName,
+          @Parameter(key = "fieldIndex",
                 label = "Column Index",
                 description = "The index of the column containing the data.",
                 required = true)
-            String index) {
+          String index) {
         if (StringUtils.isNoneBlank(fieldName, index)) {
             withOutputField(FieldName.of(fieldName), Integer.parseInt(index));
         }
@@ -186,6 +193,8 @@ public List getOutputFields() {
     }
 
     /**
+     * trimWhitespace setter.
+     *
      * @param trimWhitespace True, if whitespace should be trimmed from each value. Otherwise, false.
      */
     public CsvTextParser trimWhitespace(boolean trimWhitespace) {
@@ -194,9 +203,9 @@ public CsvTextParser trimWhitespace(boolean trimWhitespace) {
     }
 
     @Configurable(key = "trim",
-            label = "Trim Whitespace",
-            description = "Trim whitespace from each value. Default value: '" + DEFAULT_TRIM + "'",
-            defaultValue = DEFAULT_TRIM)
+          label = "Trim Whitespace",
+          description = "Trim whitespace from each value. Default value: '" + DEFAULT_TRIM + "'",
+          defaultValue = DEFAULT_TRIM)
     public void trimWhitespace(String trimWhitespace) {
         if (StringUtils.isNotBlank(trimWhitespace)) {
             trimWhitespace(Boolean.parseBoolean(trimWhitespace));
@@ -222,7 +231,7 @@ public Message parse(Message input) {
     private void doParse(String valueToParse, Message.Builder output) {
         try {
             final List valueList = reader
-                    .readValue(valueToParse);
+                  .readValue(valueToParse);
 
             for (OutputField outputField : outputFields) {
                 final int index = outputField.index;
@@ -245,8 +254,8 @@ private void doParse(String valueToParse, Message.Builder output) {
     private void updateSchema(Supplier csvSchemaConsumer) {
         this.schema = csvSchemaConsumer.get();
         this.reader = mapper
-                .readerFor(new TypeReference>() {
-                })
-                .with(schema);
+              .readerFor(new TypeReference>() {
+              })
+              .with(schema);
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParser.java
index 72712a07..49ecfd5b 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParser.java
@@ -12,6 +12,8 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static java.lang.String.format;
+
 import com.cloudera.parserchains.core.Constants;
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.FieldValue;
@@ -21,22 +23,20 @@
 import com.cloudera.parserchains.core.StringFieldValue;
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.MessageParser;
+import java.util.Optional;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 
-import java.util.Optional;
-
-import static java.lang.String.format;
-
 @MessageParser(
-        name="Delimited Key Values",
-        description="Parses delimited key-value pairs."
+      name = "Delimited Key Values",
+      description = "Parses delimited key-value pairs."
 )
 @Slf4j
 public class DelimitedKeyValueParser implements Parser {
 
     /**
      * The default key-value delimiter are double pipes; ||.
+     *
      * 
      Each pipe must be escaped.
      */
     private static final String DEFAULT_DELIMITER = "\\|\\|";
@@ -56,13 +56,13 @@ public DelimitedKeyValueParser() {
     }
 
     @Configurable(
-            key="input",
-            label="Input Field",
-            description="The input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
-            defaultValue=Constants.DEFAULT_INPUT_FIELD,
-            isOutputName = true)
+          key = "input",
+          label = "Input Field",
+          description = "The input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
+          defaultValue = Constants.DEFAULT_INPUT_FIELD,
+          isOutputName = true)
     public DelimitedKeyValueParser inputField(String inputField) {
-        if(StringUtils.isNotBlank(inputField)) {
+        if (StringUtils.isNotBlank(inputField)) {
             this.inputField = FieldName.of(inputField);
         }
         return this;
@@ -73,13 +73,13 @@ public FieldName getInputField() {
     }
 
     @Configurable(
-            key="delimiter",
-            label="Key Value Delimiter",
-            description="A regex that separates different key-value pairs. Default value: '" + DEFAULT_DELIMITER + "'",
-            defaultValue=DEFAULT_DELIMITER
+          key = "delimiter",
+          label = "Key Value Delimiter",
+          description = "A regex that separates different key-value pairs. Default value: '" + DEFAULT_DELIMITER + "'",
+          defaultValue = DEFAULT_DELIMITER
     )
     public DelimitedKeyValueParser keyValueDelimiter(String keyValueDelimiter) {
-        if(StringUtils.isNotBlank(keyValueDelimiter)) {
+        if (StringUtils.isNotBlank(keyValueDelimiter)) {
             this.keyValueDelimiter = Regex.of(keyValueDelimiter);
         }
         return this;
@@ -90,13 +90,14 @@ public Regex getKeyValueDelimiter() {
     }
 
     @Configurable(
-            key="separator",
-            label="Key Value Separator",
-            description="A regex that separates a key and value within a key-value pair. Default value: '" + DEFAULT_SEPARATOR + "'",
-            defaultValue=DEFAULT_SEPARATOR
+          key = "separator",
+          label = "Key Value Separator",
+          description = "A regex that separates a key and value within a key-value pair. Default value: '"
+                        + DEFAULT_SEPARATOR + "'",
+          defaultValue = DEFAULT_SEPARATOR
     )
     public DelimitedKeyValueParser keyValueSeparator(String keyValueSeparator) {
-        if(StringUtils.isNotBlank(keyValueSeparator)) {
+        if (StringUtils.isNotBlank(keyValueSeparator)) {
             this.keyValueSeparator = Regex.of(keyValueSeparator);
         }
         return this;
@@ -107,12 +108,12 @@ public Regex getKeyValueSeparator() {
     }
 
     @Configurable(
-            key="validKey",
-            label="Valid Key Regex",
-            description="Any key not matching this regex will be ignored."
+          key = "validKey",
+          label = "Valid Key Regex",
+          description = "Any key not matching this regex will be ignored."
     )
     public DelimitedKeyValueParser validKeyRegex(String validKeyRegex) {
-        if(StringUtils.isNotBlank(validKeyRegex)) {
+        if (StringUtils.isNotBlank(validKeyRegex)) {
             this.validKeyRegex = Optional.ofNullable(Regex.of(validKeyRegex));
         } else {
             // a blank string should 'turn off' regex matching
@@ -126,12 +127,12 @@ public Optional getValidKeyRegex() {
     }
 
     @Configurable(
-            key="validValue",
-            label="Valid Value Regex",
-            description="Any value not matching this regex will be ignored."
+          key = "validValue",
+          label = "Valid Value Regex",
+          description = "Any value not matching this regex will be ignored."
     )
     public DelimitedKeyValueParser validValueRegex(String validValueRegex) {
-        if(StringUtils.isNotBlank(validValueRegex)) {
+        if (StringUtils.isNotBlank(validValueRegex)) {
             this.validValueRegex = Optional.ofNullable(Regex.of(validValueRegex));
         } else {
             // a blank string should 'turn off' regex matching
@@ -147,7 +148,7 @@ public Optional getValidValueRegex() {
     @Override
     public Message parse(Message input) {
         Message.Builder output = Message.builder().withFields(input);
-        if(!input.getField(inputField).isPresent()) {
+        if (!input.getField(inputField).isPresent()) {
             output.withError(format("Message missing expected input field '%s'", inputField.toString()));
         } else {
             input.getField(inputField).ifPresent(val -> doParse(val.toString(), output));
@@ -159,22 +160,22 @@ private void doParse(String valueToParse, Message.Builder output) {
         String[] keyValuePairs = valueToParse.split(keyValueDelimiter.toString());
         log.debug("Found {} key-value pairs.", keyValuePairs.length);
 
-        for(String keyValuePair: keyValuePairs) {
+        for (String keyValuePair : keyValuePairs) {
 
-            String [] keyValue = keyValuePair.split(keyValueSeparator.toString(), 2);
-            if(keyValue.length == 2) {
+            String[] keyValue = keyValuePair.split(keyValueSeparator.toString(), 2);
+            if (keyValue.length == 2) {
                 final String key = keyValue[0];
                 final String value = keyValue[1];
 
                 boolean validKey = validKeyRegex.map(regex -> regex.matches(key)).orElse(true);
                 boolean validValue = validValueRegex.map(regex -> regex.matches(value)).orElse(true);
-                if(validKey && validValue) {
+                if (validKey && validValue) {
                     try {
                         FieldName fieldName = FieldName.of(key);
                         FieldValue fieldValue = StringFieldValue.of(value);
                         output.addField(fieldName, fieldValue);
 
-                    } catch(IllegalArgumentException e) {
+                    } catch (IllegalArgumentException e) {
                         log.debug("Ignoring an invalid key-value pair; '{}'", keyValuePair);
                     }
                 } else {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedTextParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedTextParser.java
index f8d97185..dfa18859 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedTextParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedTextParser.java
@@ -13,6 +13,7 @@
 package com.cloudera.parserchains.parsers;
 
 import static java.lang.String.format;
+
 import com.cloudera.parserchains.core.Constants;
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.FieldValue;
@@ -34,8 +35,8 @@
  * Parses delimited text like CSV.
  */
 @MessageParser(
-        name = "Delimited Text",
-        description = "Parses delimited text like CSV or TSV.")
+      name = "Delimited Text",
+      description = "Parses delimited text like CSV or TSV.")
 public class DelimitedTextParser implements Parser {
     private static final String DEFAULT_DELIMITER = ",";
     private static final String DEFAULT_TRIM = "true";
@@ -55,7 +56,7 @@ static class OutputField {
 
     private FieldName inputField;
     private Regex delimiter;
-    private List outputFields;
+    private final List outputFields;
     private boolean trimWhitespace;
 
     public DelimitedTextParser() {
@@ -66,6 +67,8 @@ public DelimitedTextParser() {
     }
 
     /**
+     * inputField setter.
+     *
      * @param inputField The name of the field containing the text to parse.
      */
     public DelimitedTextParser withInputField(FieldName inputField) {
@@ -74,9 +77,9 @@ public DelimitedTextParser withInputField(FieldName inputField) {
     }
 
     @Configurable(key = "inputField",
-            label = "Input Field",
-            description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
-            defaultValue = Constants.DEFAULT_INPUT_FIELD)
+          label = "Input Field",
+          description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
+          defaultValue = Constants.DEFAULT_INPUT_FIELD)
     public DelimitedTextParser withInputField(String fieldName) {
         if (StringUtils.isNotEmpty(fieldName)) {
             withInputField(FieldName.of(fieldName));
@@ -89,6 +92,8 @@ public FieldName getInputField() {
     }
 
     /**
+     * delimiter setter..
+     *
      * @param delimiter A character or regular expression defining the delimiter used to split the text.
      */
     public DelimitedTextParser withDelimiter(Regex delimiter) {
@@ -97,9 +102,9 @@ public DelimitedTextParser withDelimiter(Regex delimiter) {
     }
 
     @Configurable(key = "delimiter",
-            label = "Delimiter",
-            description = "A regex used to split the text. Default value: '" + DEFAULT_DELIMITER + "'",
-            defaultValue = DEFAULT_DELIMITER)
+          label = "Delimiter",
+          description = "A regex used to split the text. Default value: '" + DEFAULT_DELIMITER + "'",
+          defaultValue = DEFAULT_DELIMITER)
     public void withDelimiter(String delimiter) {
         if (StringUtils.isNotEmpty(delimiter)) {
             withDelimiter(Regex.of(delimiter));
@@ -111,6 +116,8 @@ public Regex getDelimiter() {
     }
 
     /**
+     * outputFields setter.
+     *
      * @param fieldName The name of a field to create.
      * @param index     The 0-based index defining which delimited element is added to the field.
      */
@@ -119,20 +126,20 @@ public DelimitedTextParser withOutputField(FieldName fieldName, int index) {
         return this;
     }
 
-    @Configurable(key = "outputField", label = "Output Field",multipleValues = true)
+    @Configurable(key = "outputField", label = "Output Field", multipleValues = true)
     public void withOutputField(
-            @Parameter(
+          @Parameter(
                 key = "fieldName",
                 label = "Field Name",
                 description = "The name of the output field.",
                 isOutputName = true,
                 required = true)
-            String fieldName,
-            @Parameter(key = "fieldIndex",
+          String fieldName,
+          @Parameter(key = "fieldIndex",
                 label = "Column Index",
                 description = "The index of the column containing the data.",
                 required = true)
-            String index) {
+          String index) {
         if (StringUtils.isNoneBlank(fieldName, index)) {
             withOutputField(FieldName.of(fieldName), Integer.parseInt(index));
         }
@@ -143,6 +150,8 @@ public List getOutputFields() {
     }
 
     /**
+     * trimWhitespace setter.
+     *
      * @param trimWhitespace True, if whitespace should be trimmed from each value. Otherwise, false.
      */
     public DelimitedTextParser trimWhitespace(boolean trimWhitespace) {
@@ -151,9 +160,9 @@ public DelimitedTextParser trimWhitespace(boolean trimWhitespace) {
     }
 
     @Configurable(key = "trim",
-            label = "Trim Whitespace",
-            description = "Trim whitespace from each value. Default value: '" + DEFAULT_TRIM + "'",
-            defaultValue = DEFAULT_TRIM)
+          label = "Trim Whitespace",
+          description = "Trim whitespace from each value. Default value: '" + DEFAULT_TRIM + "'",
+          defaultValue = DEFAULT_TRIM)
     public void trimWhitespace(String trimWhitespace) {
         if (StringUtils.isNotBlank(trimWhitespace)) {
             trimWhitespace(Boolean.valueOf(trimWhitespace));
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokParser.java
index 952ee1bc..a46e576d 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokParser.java
@@ -12,6 +12,8 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static java.lang.String.format;
+
 import com.cloudera.parserchains.core.Constants;
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.Message;
@@ -22,25 +24,22 @@
 import com.cloudera.parserchains.core.catalog.WidgetType;
 import io.krakens.grok.api.Grok;
 import io.krakens.grok.api.GrokCompiler;
-import org.apache.commons.lang3.StringUtils;
-
 import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-
-import static java.lang.String.format;
+import org.apache.commons.lang3.StringUtils;
 
 @MessageParser(
-        name = "Grok",
-        description = "Parses a message using Grok expressions."
+      name = "Grok",
+      description = "Parses a message using Grok expressions."
 )
 public class GrokParser implements Parser {
     private static final String DEFAULT_ZONE_OFFSET = "+00:00";
     private FieldName inputField;
     private ZoneOffset zoneOffset;
-    private GrokCompiler grokCompiler;
-    private List grokExpressions;
+    private final GrokCompiler grokCompiler;
+    private final List grokExpressions;
 
     public GrokParser() {
         inputField = FieldName.of(Constants.DEFAULT_INPUT_FIELD);
@@ -66,21 +65,21 @@ public Message parse(Message input) {
     private void doParse(String textToParse, Message.Builder output) {
         for (Grok grokPattern : grokExpressions) {
             grokPattern.match(textToParse)
-                    .capture()
-                    .entrySet()
-                    .stream()
-                    .filter(e -> e.getKey() != null && e.getValue() != null)
-                    .forEach(e -> output.addField(e.getKey(), e.getValue().toString()));
+                       .capture()
+                       .entrySet()
+                       .stream()
+                       .filter(e -> e.getKey() != null && e.getValue() != null)
+                       .forEach(e -> output.addField(e.getKey(), e.getValue().toString()));
         }
     }
 
     @Configurable(key = "grokPattern",
-            description = "Define a Grok pattern that can be referenced from an expression.",
-            orderPriority = 1,
-            isOutputName = true)
+          description = "Define a Grok pattern that can be referenced from an expression.",
+          orderPriority = 1,
+          isOutputName = true)
     public GrokParser pattern(
-            @Parameter(key = "name", label = "Pattern Name") String patternName,
-            @Parameter(key = "regex", label = "Pattern Regex", widgetType = WidgetType.TEXTAREA) String patternRegex) {
+          @Parameter(key = "name", label = "Pattern Name") String patternName,
+          @Parameter(key = "regex", label = "Pattern Regex", widgetType = WidgetType.TEXTAREA) String patternRegex) {
         if (StringUtils.isNoneBlank(patternName, patternRegex)) {
             grokCompiler.register(patternName, patternRegex);
         }
@@ -88,9 +87,9 @@ public GrokParser pattern(
     }
 
     @Configurable(key = "grokExpression",
-            label = "Grok Expression(s)",
-            description = "The grok expression to execute.",
-            multipleValues = true)
+          label = "Grok Expression(s)",
+          description = "The grok expression to execute.",
+          multipleValues = true)
     public GrokParser expression(String grokExpression) {
         if (StringUtils.isNotBlank(grokExpression)) {
             Grok grok = grokCompiler.compile(grokExpression, zoneOffset, false);
@@ -109,9 +108,9 @@ public GrokParser inputField(FieldName inputField) {
     }
 
     @Configurable(key = "inputField",
-            label = "Input Field",
-            description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
-            defaultValue = Constants.DEFAULT_INPUT_FIELD)
+          label = "Input Field",
+          description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
+          defaultValue = Constants.DEFAULT_INPUT_FIELD)
     public GrokParser inputField(String inputField) {
         if (StringUtils.isNotBlank(inputField)) {
             this.inputField = FieldName.of(inputField);
@@ -129,9 +128,9 @@ public GrokParser zoneOffset(ZoneOffset zoneOffset) {
     }
 
     @Configurable(key = "zoneOffset",
-            label = "Zone Offset",
-            description = "Set the zone offset. For example \"+02:00\". Default value: '" + DEFAULT_ZONE_OFFSET + "'",
-            defaultValue = DEFAULT_ZONE_OFFSET)
+          label = "Zone Offset",
+          description = "Set the zone offset. For example \"+02:00\". Default value: '" + DEFAULT_ZONE_OFFSET + "'",
+          defaultValue = DEFAULT_ZONE_OFFSET)
     public void zoneOffset(String offset) {
         if (StringUtils.isNotBlank(offset)) {
             zoneOffset(ZoneOffset.of(offset));
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/JSONParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/JSONParser.java
index ff2ab416..78fff873 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/JSONParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/JSONParser.java
@@ -12,6 +12,9 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
+import static java.lang.String.format;
+
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.Message;
 import com.cloudera.parserchains.core.Parser;
@@ -21,8 +24,6 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.commons.lang3.StringUtils;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -31,18 +32,16 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
-
-import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
-import static java.lang.String.format;
+import org.apache.commons.lang3.StringUtils;
 
 @MessageParser(
-        name="Simple JSON",
-        description="Parses JSON data by creating a field for each JSON element.")
+      name = "Simple JSON",
+      description = "Parses JSON data by creating a field for each JSON element.")
 public class JSONParser implements Parser {
     private static final String DEFAULT_NORMALIZER = "UNFOLD_NESTED";
     private FieldName inputField;
-    private ObjectReader reader;
-    private List normalizers;
+    private final ObjectReader reader;
+    private final List normalizers;
 
     public JSONParser() {
         inputField = FieldName.of(DEFAULT_INPUT_FIELD);
@@ -51,44 +50,44 @@ public JSONParser() {
     }
 
     @Configurable(
-            key="input",
-            label="Input Field",
-            description= "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
-            defaultValue=DEFAULT_INPUT_FIELD,
-            isOutputName = true)
+          key = "input",
+          label = "Input Field",
+          description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
+          defaultValue = DEFAULT_INPUT_FIELD,
+          isOutputName = true)
     public JSONParser inputField(String fieldName) {
-        if(StringUtils.isNotBlank(fieldName)) {
+        if (StringUtils.isNotBlank(fieldName)) {
             this.inputField = FieldName.of(fieldName);
         }
         return this;
     }
 
     @Configurable(
-            key="norm",
-            label="Normalizer",
-            description="Defines how fields are normalized. Accepted values include: " +
-                    "'ALLOW_NESTED' Embed nested JSON string as the field value.  " +
-                    "'DISALLOW_NESTED' Stop parsing and throw an error if nested JSON exists.  " +
-                    "'DROP_NESTED' Drop and ignore any nested JSON values.  " +
-                    "'UNFOLD_NESTED' Unfold the nested JSON by creating a nested, dot-separated field name.  " +
-                    "Default value: '" + DEFAULT_NORMALIZER + "'",
-            defaultValue=DEFAULT_NORMALIZER,
-            multipleValues = true)
+          key = "norm",
+          label = "Normalizer",
+          description = "Defines how fields are normalized. Accepted values include: "
+                        + "'ALLOW_NESTED' Embed nested JSON string as the field value.  "
+                        + "'DISALLOW_NESTED' Stop parsing and throw an error if nested JSON exists.  "
+                        + "'DROP_NESTED' Drop and ignore any nested JSON values.  "
+                        + "'UNFOLD_NESTED' Unfold the nested JSON by creating a nested, dot-separated field name.  "
+                        + "Default value: '" + DEFAULT_NORMALIZER + "'",
+          defaultValue = DEFAULT_NORMALIZER,
+          multipleValues = true)
     public JSONParser normalizer(String normalizer) {
-        if(StringUtils.isNotBlank(normalizer)) {
+        if (StringUtils.isNotBlank(normalizer)) {
             addNormalizer(Normalizers.valueOf(normalizer));
         }
         return this;
     }
 
-    void addNormalizer (Normalizers normalizer) {
+    void addNormalizer(Normalizers normalizer) {
         normalizers.add(Objects.requireNonNull(normalizer, "A normalizer is required."));
     }
 
     @Override
     public Message parse(Message input) {
         Message.Builder output = Message.builder().withFields(input);
-        if(!input.getField(inputField).isPresent()) {
+        if (!input.getField(inputField).isPresent()) {
             output.withError(format("Message missing expected input field '%s'", inputField.toString()));
         } else {
             input.getField(inputField).ifPresent(val -> doParse(val.toString(), output));
@@ -109,14 +108,14 @@ public void doParse(String toParse, Message.Builder output) {
 
     private Map normalize(Map valueToNormalize, Message.Builder output) {
         // use the default normalizer, if none other specified
-        if(normalizers.size() == 0) {
+        if (normalizers.size() == 0) {
             normalizer(DEFAULT_NORMALIZER);
         }
         try {
             for (Normalizer normalizer : normalizers) {
                 valueToNormalize = normalizer.normalize(valueToNormalize);
             }
-        } catch(IOException e) {
+        } catch (IOException e) {
             output.withError("Failed to normalize.", e);
         }
         return valueToNormalize;
@@ -131,7 +130,7 @@ private enum Normalizers implements Normalizer {
         DROP_NESTED(new DropNestedObjects()),
         UNFOLD_NESTED(new UnfoldNestedObjects());
 
-        private Normalizer normalizer;
+        private final Normalizer normalizer;
 
         Normalizers(Normalizer normalizer) {
             this.normalizer = normalizer;
@@ -157,7 +156,7 @@ private static class AllowNestedObjects implements Normalizer {
         @Override
         public Map normalize(Map input) throws JsonProcessingException {
             Map output = new HashMap<>();
-            for(Map.Entry entry: input.entrySet()) {
+            for (Map.Entry entry : input.entrySet()) {
                 String key = entry.getKey();
                 Object value = entry.getValue();
                 serializeNested(key, value, output);
@@ -165,7 +164,8 @@ public Map normalize(Map input) throws JsonProce
             return output;
         }
 
-        private void serializeNested(String rootKey, Object valueToUnfold, Map output) throws JsonProcessingException {
+        private void serializeNested(String rootKey, Object valueToUnfold, Map output)
+              throws JsonProcessingException {
             if (valueToUnfold instanceof Map) {
                 // handle nested JSON objects
                 String serialized = JSONUtils.INSTANCE.toJSON(valueToUnfold, false);
@@ -190,10 +190,10 @@ private static class DisallowNestedObjects implements Normalizer {
         public Map normalize(Map input) throws IOException {
             // throw an exception if any nested objects exist
             Optional nestedObject = input.values()
-                    .stream()
-                    .filter(v -> v instanceof Map || v instanceof List)
-                    .findFirst();
-            if(nestedObject.isPresent()) {
+                                                 .stream()
+                                                 .filter(v -> v instanceof Map || v instanceof List)
+                                                 .findFirst();
+            if (nestedObject.isPresent()) {
                 throw new IOException("Nested objects are not allowed.");
             }
             return input;
@@ -208,9 +208,9 @@ private static class DropNestedObjects implements Normalizer {
         public Map normalize(Map input) {
             // drop any that is a JSON object (aka Map)
             return input.entrySet()
-                    .stream()
-                    .filter(e -> !(e.getValue() instanceof Map) && !(e.getValue() instanceof List))
-                    .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
+                        .stream()
+                        .filter(e -> !(e.getValue() instanceof Map) && !(e.getValue() instanceof List))
+                        .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
         }
     }
 
@@ -237,7 +237,7 @@ private void unfold(String rootKey, Object valueToUnfold, Map ou
             } else if (valueToUnfold instanceof List) {
                 // handle JSON arrays
                 List listValue = (List) valueToUnfold;
-                for(int i=0; iSee https://github.com/json-path/JsonPath.
  */
 @MessageParser(
-        name = "JSON Path",
-        description = "Parse JSON using JSONPath expressions.")
+      name = "JSON Path",
+      description = "Parse JSON using JSONPath expressions.")
 @Slf4j
 public class JSONPathParser implements Parser {
-    private FieldName inputField;
-    private LinkedHashMap expressions;
+    private final FieldName inputField;
+    private final LinkedHashMap expressions;
 
     public JSONPathParser() {
         inputField = FieldName.of(DEFAULT_INPUT_FIELD);
@@ -57,6 +58,7 @@ public JSONPathParser() {
     /**
      * Add a JSONPath expression that will be executed. The result of the JSONPath expression
      * is used to add or modify a field.
+     *
      * 
      Multiple expressions can be provided to create or modify multiple fields.
      *
      * @param fieldName The name of the field to create or modify.
@@ -64,17 +66,17 @@ public JSONPathParser() {
      */
     @Configurable(key = "expr", multipleValues = true)
     public JSONPathParser expression(
-            @Parameter(key = "field",
+          @Parameter(key = "field",
                 label = "Field Name",
                 description = "The field to create or modify. Default value: '" + DEFAULT_INPUT_FIELD + "'",
                 defaultValue = DEFAULT_INPUT_FIELD,
                 isOutputName = true)
-            String fieldName,
-            @Parameter(key = "expr",
+          String fieldName,
+          @Parameter(key = "expr",
                 label = "Path Expression",
                 description = "The path expression.",
                 required = true)
-            String expr) {
+          String expr) {
         if (StringUtils.isNoneBlank(fieldName, expr)) {
             expressions.put(FieldName.of(fieldName), JsonPath.compile(expr));
         }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/RemoveFieldParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/RemoveFieldParser.java
index 39f70bcd..d8d70524 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/RemoveFieldParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/RemoveFieldParser.java
@@ -17,19 +17,18 @@
 import com.cloudera.parserchains.core.Parser;
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.MessageParser;
-import org.apache.commons.lang3.StringUtils;
-
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * A parser which can remove fields from a message.
  */
 @MessageParser(
-    name="Remove Field(s)",
-    description="Removes unwanted message field(s).")
+      name = "Remove Field(s)",
+      description = "Removes unwanted message field(s).")
 public class RemoveFieldParser implements Parser {
-    private List fieldsToRemove;
+    private final List fieldsToRemove;
 
     public RemoveFieldParser() {
         fieldsToRemove = new ArrayList<>();
@@ -40,27 +39,27 @@ public RemoveFieldParser removeField(FieldName fieldName) {
         return this;
     }
 
+    @Configurable(
+          key = "fieldToRemove",
+          label = "Field to Remove",
+          description = "The name of a field to remove.",
+          multipleValues = true,
+          required = true)
+    public void removeField(String fieldName) {
+        if (StringUtils.isNotBlank(fieldName)) {
+            removeField(FieldName.of(fieldName));
+        }
+    }
+
     @Override
     public Message parse(Message message) {
         return Message.builder()
-                .withFields(message)
-                .removeFields(fieldsToRemove)
-                .build();
+                      .withFields(message)
+                      .removeFields(fieldsToRemove)
+                      .build();
     }
 
     List getFieldsToRemove() {
         return fieldsToRemove;
     }
-
-    @Configurable(
-            key="fieldToRemove",
-            label="Field to Remove",
-            description="The name of a field to remove.",
-            multipleValues=true,
-            required=true)
-    public void removeField(String fieldName) {
-        if(StringUtils.isNotBlank(fieldName)) {
-            removeField(FieldName.of(fieldName));
-        }
-    }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/RenameFieldParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/RenameFieldParser.java
index def64208..a26ca6aa 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/RenameFieldParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/RenameFieldParser.java
@@ -27,10 +27,10 @@
  * A parser that can rename message fields.
  */
 @MessageParser(
-    name="Rename Field(s)", 
-    description="Renames message field(s).")
+      name = "Rename Field(s)",
+      description = "Renames message field(s).")
 public class RenameFieldParser implements Parser {
-    private Map fieldsToRename;
+    private final Map fieldsToRename;
 
     public RenameFieldParser() {
         this.fieldsToRename = new HashMap<>();
@@ -38,14 +38,35 @@ public RenameFieldParser() {
 
     /**
      * Configure the parser to rename a field.
+     *
      * @param from The original field name.
-     * @param to The new field name.
+     * @param to   The new field name.
      */
     public RenameFieldParser renameField(FieldName from, FieldName to) {
         fieldsToRename.put(from, to);
         return this;
     }
 
+    @Configurable(
+          key = "fieldToRename",
+          multipleValues = true)
+    public void renameField(
+          @Parameter(key = "from",
+                label = "Rename From",
+                description = "The original name of the field.",
+                required = true)
+          String from,
+          @Parameter(key = "to",
+                label = "Rename To",
+                description = "The new name of the field.",
+                isOutputName = true,
+                required = true)
+          String to) {
+        if (StringUtils.isNoneBlank(from, to)) {
+            renameField(FieldName.of(from), FieldName.of(to));
+        }
+    }
+
     Map getFieldsToRename() {
         return Collections.unmodifiableMap(fieldsToRename);
     }
@@ -53,28 +74,8 @@ Map getFieldsToRename() {
     @Override
     public Message parse(Message input) {
         Message.Builder output = Message.builder()
-                .withFields(input);
+                                        .withFields(input);
         fieldsToRename.forEach((from, to) -> output.renameField(from, to));
         return output.build();
     }
-
-    @Configurable(
-            key="fieldToRename",
-            multipleValues=true)
-    public void renameField(
-            @Parameter(key="from",
-                label="Rename From",
-                description="The original name of the field.",
-                required = true)
-            String from,
-            @Parameter(key="to",
-                label="Rename To",
-                description="The new name of the field.",
-                isOutputName = true,
-                required = true)
-            String to) {
-        if(StringUtils.isNoneBlank(from, to)) {
-            renameField(FieldName.of(from), FieldName.of(to));
-        }
-    }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SimpleStellarParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SimpleStellarParser.java
index 911107c3..187875c8 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SimpleStellarParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SimpleStellarParser.java
@@ -26,14 +26,14 @@
 import org.json.simple.JSONObject;
 
 @MessageParser(
-        name = "Simple Stellar parser",
-        description = "Metron compatibility parser.")
+      name = "Simple Stellar parser",
+      description = "Metron compatibility parser.")
 @Slf4j
 public class SimpleStellarParser implements Parser {
 
     private ConfigHandler configHandler;
-    private StellarProcessor processor;
-    private Context stellarContext;
+    private final StellarProcessor processor;
+    private final Context stellarContext;
 
 
     public SimpleStellarParser() {
@@ -43,10 +43,10 @@ public SimpleStellarParser() {
     }
 
     @Configurable(
-            key = "stellarPath",
-            label = "Stellar File Path",
-            description = "Path to stellar file",
-            required = true)
+          key = "stellarPath",
+          label = "Stellar File Path",
+          description = "Path to stellar file",
+          required = true)
     public SimpleStellarParser stellarPath(String pathToStellar) throws IOException {
         FileSystem fileSystem = new Path(pathToStellar).getFileSystem();
         loadExpressions(pathToStellar, fileSystem);
@@ -77,14 +77,16 @@ private void updateExpressionList(List expressionList) {
     public Message parse(Message input) {
         Message.Builder builder = Message.builder().withFields(input);
         final Map fieldMap = input.getFields().entrySet().stream()
-                .collect(Collectors.toMap(e -> e.getKey().get(), e -> StringUtils.parseProperType(e.getValue().get())));
+                                                  .collect(Collectors.toMap(e -> e.getKey().get(),
+                                                        e -> StringUtils.parseProperType(e.getValue().get())));
         return doParse(fieldMap, builder);
     }
 
     private Message doParse(Map toParse, Message.Builder output) {
         final MapVariableResolver resolver = new MapVariableResolver(toParse);
         try {
-            final JSONObject result = StellarAdapter.process(toParse, configHandler, "", 1000L, processor, resolver, stellarContext);
+            final JSONObject result =
+                  StellarAdapter.process(toParse, configHandler, "", 1000L, processor, resolver, stellarContext);
             result.forEach((key, value) -> output.addField(String.valueOf(key), String.valueOf(value)));
         } catch (Exception e) {
             output.withError("Parser did not return a message result").build();
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java
index 646d50cb..804af2e6 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java
@@ -12,6 +12,8 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
+
 import com.cloudera.cyber.parser.MessageToParse;
 import com.cloudera.cyber.stellar.MetronCompatibilityParser;
 import com.cloudera.parserchains.core.FieldName;
@@ -20,6 +22,11 @@
 import com.cloudera.parserchains.core.Parser;
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.MessageParser;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -29,17 +36,9 @@
 import org.apache.metron.parsers.interfaces.MessageParserResult;
 import org.json.simple.JSONObject;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
-
 @MessageParser(
-        name = "Metron Stellar parser",
-        description = "Metron compatibility parser.")
+      name = "Metron Stellar parser",
+      description = "Metron compatibility parser.")
 @Slf4j
 public class StellarParser implements Parser {
 
@@ -52,11 +51,11 @@ public StellarParser() {
     }
 
     @Configurable(
-            key = "input",
-            label = "Input Field",
-            description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
-            defaultValue = DEFAULT_INPUT_FIELD,
-            isOutputName = true)
+          key = "input",
+          label = "Input Field",
+          description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
+          defaultValue = DEFAULT_INPUT_FIELD,
+          isOutputName = true)
     public StellarParser inputField(String fieldName) {
         if (StringUtils.isNotBlank(fieldName)) {
             this.inputField = FieldName.of(fieldName);
@@ -65,10 +64,10 @@ public StellarParser inputField(String fieldName) {
     }
 
     @Configurable(
-            key = "configurationPath",
-            label = "Configuration File Path",
-            description = "Path to parser config file",
-            required = true)
+          key = "configurationPath",
+          label = "Configuration File Path",
+          description = "Path to parser config file",
+          required = true)
     public StellarParser configurationPath(String pathToSchema) throws IOException {
         FileSystem fileSystem = new Path(pathToSchema).getFileSystem();
         loadParser(pathToSchema, fileSystem);
@@ -95,8 +94,8 @@ public Message parse(Message input) {
             return doParse(field.get(), builder);
         } else {
             return builder
-                    .withError(String.format("Message missing expected input field '%s'", inputField.toString()))
-                    .build();
+                  .withError(String.format("Message missing expected input field '%s'", inputField.toString()))
+                  .build();
         }
     }
 
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SyslogParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SyslogParser.java
index c40d7318..326431d6 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SyslogParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SyslogParser.java
@@ -12,6 +12,8 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static java.lang.String.format;
+
 import com.cloudera.parserchains.core.Constants;
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.Message;
@@ -21,15 +23,12 @@
 import com.cloudera.parserchains.core.catalog.MessageParser;
 import com.github.palindromicity.syslog.SyslogParserBuilder;
 import com.github.palindromicity.syslog.SyslogSpecification;
-import org.apache.commons.lang3.StringUtils;
-
 import java.util.Objects;
-
-import static java.lang.String.format;
+import org.apache.commons.lang3.StringUtils;
 
 @MessageParser(
-    name="Syslog",
-    description="Parses Syslog according to RFC 3164 and 5424.")
+      name = "Syslog",
+      description = "Parses Syslog according to RFC 3164 and 5424.")
 public class SyslogParser implements Parser {
     private static final String DEFAULT_SYSLOG_SPEC = "RFC_5424";
     private FieldName inputField;
@@ -41,12 +40,13 @@ public SyslogParser() {
     }
 
     @Configurable(
-            key="specification",
-            label="Specification",
-            description="The Syslog specification; 'RFC_5424' or 'RFC_3164'. Default value: '" + DEFAULT_SYSLOG_SPEC + "'",
-            defaultValue=DEFAULT_SYSLOG_SPEC)
+          key = "specification",
+          label = "Specification",
+          description = "The Syslog specification; 'RFC_5424' or 'RFC_3164'. Default value: '" + DEFAULT_SYSLOG_SPEC
+                        + "'",
+          defaultValue = DEFAULT_SYSLOG_SPEC)
     public void withSpecification(String specification) {
-        if(StringUtils.isNotBlank(specification)) {
+        if (StringUtils.isNotBlank(specification)) {
             SyslogSpecification spec = SyslogSpecification.valueOf(specification);
             withSpecification(spec);
         }
@@ -61,13 +61,13 @@ public SyslogSpecification getSpecification() {
         return specification;
     }
 
-    @Configurable(key="inputField",
-            label="Input Field",
-            description="The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
-            defaultValue = Constants.DEFAULT_INPUT_FIELD,
-            isOutputName = true)
+    @Configurable(key = "inputField",
+          label = "Input Field",
+          description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
+          defaultValue = Constants.DEFAULT_INPUT_FIELD,
+          isOutputName = true)
     public SyslogParser withInputField(String inputField) {
-        if(StringUtils.isNotBlank(inputField)) {
+        if (StringUtils.isNotBlank(inputField)) {
             this.inputField = FieldName.of(inputField);
         }
         return this;
@@ -80,10 +80,10 @@ public FieldName getInputField() {
     @Override
     public Message parse(Message input) {
         Message.Builder output = Message.builder().withFields(input);
-        if(inputField == null) {
+        if (inputField == null) {
             output.withError("Input Field has not been defined.");
 
-        } else if(!input.getField(inputField).isPresent()) {
+        } else if (!input.getField(inputField).isPresent()) {
             output.withError(format("Message missing expected input field '%s'", inputField.toString()));
 
         } else {
@@ -95,12 +95,12 @@ public Message parse(Message input) {
     private void doParse(String valueToParse, Message.Builder output) {
         try {
             new SyslogParserBuilder()
-                    .forSpecification(specification)
-                    .build()
-                    .parseLine(valueToParse)
-                    .forEach((k, v) -> output.addField(FieldName.of(k), StringFieldValue.of(v.toString())));
+                  .forSpecification(specification)
+                  .build()
+                  .parseLine(valueToParse)
+                  .forEach((k, v) -> output.addField(FieldName.of(k), StringFieldValue.of(v.toString())));
 
-        } catch(Exception e) {
+        } catch (Exception e) {
             output.withError(e);
         }
     }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampFormatParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampFormatParser.java
index 625ce829..283e98b8 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampFormatParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampFormatParser.java
@@ -12,6 +12,8 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static java.util.stream.Collectors.toList;
+
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.FieldValue;
 import com.cloudera.parserchains.core.Message;
@@ -20,8 +22,6 @@
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.MessageParser;
 import com.cloudera.parserchains.core.catalog.Parameter;
-import org.apache.commons.lang3.StringUtils;
-
 import java.io.Serializable;
 import java.time.DateTimeException;
 import java.time.Instant;
@@ -30,28 +30,28 @@
 import java.time.temporal.TemporalAccessor;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.TimeZone;
-
-import static java.util.stream.Collectors.toList;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * A parser to extract formatted timestamps and express them as epoch time, preserving the
- * timezone of the original in a complementary field
+ * timezone of the original in a complementary field.
  */
 @MessageParser(
-        name = "TimestampFormat",
-        description = "Parse a formatted timestamp into usable unix epoch time")
+      name = "TimestampFormat",
+      description = "Parse a formatted timestamp into usable unix epoch time")
 public class TimestampFormatParser implements Parser {
-    private final String DEFAULT_TIMEFORMAT = "yyyyMMdd'T'hh:mm:ss.SSS'Z'";
+    private static final String DEFAULT_TIMEFORMAT = "yyyyMMdd'T'hh:mm:ss.SSS'Z'";
     private static final String DEFAULT_TIMEZONE = TimeZone.getDefault().getID();
-    private List fields = new ArrayList<>();
+    private final List fields = new ArrayList<>();
 
     @Override
     public Message parse(Message input) {
         Message.Builder builder = Message.builder()
-                .withFields(input);
+                                         .withFields(input);
         for (Config c : fields) {
             FieldName fieldName = FieldName.of(c.fieldName);
             String inputValue = input.getField(fieldName).get().get();
@@ -80,32 +80,36 @@ private Long parseDate(String inputValue, DateTimeFormatter format, String tz) t
     }
 
     @Configurable(key = "fields",
-            label = "Time Fields",
-            description = "The field that will contain the timestamp.",
-            multipleValues = true)
+          label = "Time Fields",
+          description = "The field that will contain the timestamp.",
+          multipleValues = true)
     public TimestampFormatParser withOutputField(
-            @Parameter(key = "field", label = "Input Field", description = "Field to be parsed", required = true, isOutputName = true) String fieldName,
-            @Parameter(key = "format", label = "Time format", description = "A compatible time format", required = true) String format,
-            @Parameter(key = "tz", label = "Timezone", description = "Optionally set the expected timezone", required = true) String tz
+          @Parameter(key = "field", label = "Input Field", description = "Field to be parsed", required = true, isOutputName = true)
+          String fieldName,
+          @Parameter(key = "format", label = "Time format", description = "A compatible time format", required = true)
+          String format,
+          @Parameter(key = "tz", label = "Timezone", description = "Optionally set the expected timezone", required = true)
+          String tz
     ) {
 
         List formats = Arrays.asList(format.split(","));
 
-        if (formats == null)
-            formats = Arrays.asList(DEFAULT_TIMEFORMAT);
+        if (formats == null) {
+            formats = Collections.singletonList(DEFAULT_TIMEFORMAT);
+        }
 
         if (StringUtils.isNotBlank(fieldName)) {
             this.fields.add(new TimestampFormatParser.Config(fieldName,
-                    formats.stream().map(f -> DateTimeFormatter.ofPattern(f)).collect(toList()),
-                    StringUtils.isNotBlank(tz) ? tz : DEFAULT_TIMEZONE));
+                  formats.stream().map(f -> DateTimeFormatter.ofPattern(f)).collect(toList()),
+                  StringUtils.isNotBlank(tz) ? tz : DEFAULT_TIMEZONE));
         }
         return this;
     }
 
     public class Config implements Serializable {
-        private String fieldName;
+        private final String fieldName;
         protected List dateTimeFormatter;
-        private String tz;
+        private final String tz;
 
         public Config(String fieldName, List dateTimeFormatter, String tz) {
             this.fieldName = fieldName;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampParser.java
index 09217cb9..56fdaba9 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampParser.java
@@ -27,8 +27,8 @@
  * tracking the time when a message was parsed.
  */
 @MessageParser(
-    name="Timestamp",
-    description="Adds a timestamp to a message. Can be used to mark processing time.")
+      name = "Timestamp",
+      description = "Adds a timestamp to a message. Can be used to mark processing time.")
 public class TimestampParser implements Parser {
     private static final String DEFAULT_OUTPUT_FIELD = "timestamp";
     private FieldName outputField;
@@ -44,24 +44,26 @@ public Message parse(Message input) {
         long now = clock.currentTimeMillis();
         FieldValue timestamp = StringFieldValue.of(Long.toString(now));
         return Message.builder()
-                .withFields(input)
-                .addField(outputField, timestamp)
-                .build();
+                      .withFields(input)
+                      .addField(outputField, timestamp)
+                      .build();
     }
 
-    @Configurable(key="outputField",
-            label="Output Field",
-            description="The field that will contain the timestamp. Default value: '" + DEFAULT_OUTPUT_FIELD + "'",
-            isOutputName=true,
-            defaultValue=DEFAULT_OUTPUT_FIELD)
+    @Configurable(key = "outputField",
+          label = "Output Field",
+          description = "The field that will contain the timestamp. Default value: '" + DEFAULT_OUTPUT_FIELD + "'",
+          isOutputName = true,
+          defaultValue = DEFAULT_OUTPUT_FIELD)
     public TimestampParser withOutputField(String fieldName) {
-        if(StringUtils.isNotBlank(fieldName)) {
+        if (StringUtils.isNotBlank(fieldName)) {
             this.outputField = FieldName.of(fieldName);
         }
         return this;
     }
 
     /**
+     * Clock setter.
+     *
      * @param clock A {@link Clock} to use during testing.
      */
     public TimestampParser withClock(Clock clock) {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XMLFlattener.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XMLFlattener.java
index 3e383b4d..62ed3690 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XMLFlattener.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XMLFlattener.java
@@ -12,6 +12,9 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static java.lang.String.format;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
 import com.cloudera.parserchains.core.Constants;
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.FieldValue;
@@ -21,6 +24,9 @@
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.MessageParser;
 import com.github.wnameless.json.flattener.JsonFlattener;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.json.JSONException;
@@ -28,16 +34,9 @@
 import org.json.XML;
 import org.json.XMLParserConfiguration;
 
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-
-import static java.lang.String.format;
-import static org.apache.commons.lang3.StringUtils.isNotBlank;
-
 @MessageParser(
-        name = "XML Flattener",
-        description = "Flattens XML data."
+      name = "XML Flattener",
+      description = "Flattens XML data."
 )
 @Slf4j
 public class XMLFlattener implements Parser {
@@ -53,10 +52,10 @@ public XMLFlattener() {
     }
 
     @Configurable(key = "inputField",
-            label = "Input Field",
-            description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
-            defaultValue = Constants.DEFAULT_INPUT_FIELD,
-            isOutputName = true)
+          label = "Input Field",
+          description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
+          defaultValue = Constants.DEFAULT_INPUT_FIELD,
+          isOutputName = true)
     public XMLFlattener inputField(String fieldName) {
         if (StringUtils.isNotEmpty(fieldName)) {
             this.inputField = FieldName.of(fieldName);
@@ -65,9 +64,10 @@ public XMLFlattener inputField(String fieldName) {
     }
 
     @Configurable(key = "separator",
-            label = "Separator",
-            description = "The character used to separate each nested XML element. Default value: '" + DEFAULT_SEPARATOR + "'",
-            defaultValue = DEFAULT_SEPARATOR
+          label = "Separator",
+          description = "The character used to separate each nested XML element. Default value: '" + DEFAULT_SEPARATOR
+                        + "'",
+          defaultValue = DEFAULT_SEPARATOR
     )
     public XMLFlattener separator(String separator) {
         if (StringUtils.isNotEmpty(separator)) {
@@ -96,14 +96,14 @@ private void doParse(String valueToParse, Message.Builder output) {
             // flatten the JSON
             final String json = result.toString();
             Map values = new JsonFlattener(json)
-                    .withSeparator(separator)
-                    .flattenAsMap();
+                  .withSeparator(separator)
+                  .flattenAsMap();
 
             // add each value to the message
             values.entrySet()
-                    .stream()
-                    .filter(e -> e.getValue() != null && isNotBlank(e.getKey()))
-                    .forEach(e -> output.addField(fieldName(e.getKey()), fieldValue(e.getValue())));
+                  .stream()
+                  .filter(e -> e.getValue() != null && isNotBlank(e.getKey()))
+                  .forEach(e -> output.addField(fieldName(e.getKey()), fieldValue(e.getValue())));
 
         } catch (JSONException e) {
             output.withError("Unable to convert XML to JSON.", e);
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XPathParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XPathParser.java
index e43fb178..24c9f303 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XPathParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XPathParser.java
@@ -14,6 +14,7 @@
 
 import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
 import static java.lang.String.format;
+
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.Message;
 import com.cloudera.parserchains.core.Parser;
@@ -38,16 +39,16 @@
 import org.xml.sax.SAXException;
 
 @MessageParser(
-        name="XPath",
-        description="Parse XML using XPath expressions."
+      name = "XPath",
+      description = "Parse XML using XPath expressions."
 )
 @Slf4j
 public class XPathParser implements Parser {
     private static final String DEFAULT_NAMESPACE_AWARE = "false";
-    private LinkedHashMap compiledExpressions;
-    private LinkedHashMap expressions;
+    private final LinkedHashMap compiledExpressions;
+    private final LinkedHashMap expressions;
     private FieldName inputField;
-    private XPath xpath;
+    private final XPath xpath;
     private boolean namespaceAware;
 
     public XPathParser() {
@@ -59,32 +60,32 @@ public XPathParser() {
     }
 
     @Configurable(
-            key="input",
-            label="Input Field",
-            description= "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
-            defaultValue=DEFAULT_INPUT_FIELD)
+          key = "input",
+          label = "Input Field",
+          description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
+          defaultValue = DEFAULT_INPUT_FIELD)
     public XPathParser inputField(String inputField) {
-        if(StringUtils.isNotBlank(inputField)) {
+        if (StringUtils.isNotBlank(inputField)) {
             this.inputField = FieldName.of(inputField);
         }
         return this;
     }
 
-    @Configurable(key="xpath",
-            multipleValues = true)
+    @Configurable(key = "xpath",
+          multipleValues = true)
     public XPathParser expression(
-            @Parameter(key="field",
-                label="Field Name",
-                description="The field to create or modify.",
+          @Parameter(key = "field",
+                label = "Field Name",
+                description = "The field to create or modify.",
                 isOutputName = true,
                 required = true)
-            String fieldName,
-            @Parameter(key="expr",
-                label="XPath",
-                description="The XPath expression.",
+          String fieldName,
+          @Parameter(key = "expr",
+                label = "XPath",
+                description = "The XPath expression.",
                 required = true)
-            String expression) {
-        if(StringUtils.isNoneBlank(fieldName, expression)) {
+          String expression) {
+        if (StringUtils.isNoneBlank(fieldName, expression)) {
             FieldName field = FieldName.of(fieldName);
             this.compiledExpressions.put(field, compile(expression));
             // save the text of the expression so that it can be reported when an error occurs
@@ -104,12 +105,12 @@ private XPathExpression compile(String expression) {
     }
 
     @Configurable(
-            key="nsAware",
-            label="Namespace Aware",
-            description="Should the parser support XML namespaces. Default value: '" + DEFAULT_NAMESPACE_AWARE + "'",
-            defaultValue=DEFAULT_NAMESPACE_AWARE)
+          key = "nsAware",
+          label = "Namespace Aware",
+          description = "Should the parser support XML namespaces. Default value: '" + DEFAULT_NAMESPACE_AWARE + "'",
+          defaultValue = DEFAULT_NAMESPACE_AWARE)
     public XPathParser namespaceAware(String namespaceAware) {
-        if(StringUtils.isNotBlank(namespaceAware)) {
+        if (StringUtils.isNotBlank(namespaceAware)) {
             this.namespaceAware = Boolean.valueOf(namespaceAware);
         }
         return this;
@@ -118,7 +119,7 @@ public XPathParser namespaceAware(String namespaceAware) {
     @Override
     public Message parse(Message input) {
         Message.Builder output = Message.builder().withFields(input);
-        if(!input.getField(inputField).isPresent()) {
+        if (!input.getField(inputField).isPresent()) {
             output.withError(format("Message missing expected input field '%s'", inputField.toString()));
         } else {
             input.getField(inputField).ifPresent(val -> doParse(val.toString(), output));
@@ -128,16 +129,16 @@ public Message parse(Message input) {
 
     private void doParse(String valueToParse, Message.Builder output) {
         Document document = buildDocument(valueToParse, output);
-        if(document != null) {
+        if (document != null) {
             compiledExpressions.forEach(((field, expr) -> execute(document, field, expr, output)));
         }
     }
 
     /**
      * Builds the XML document.
+     *
      * @param valueToParse The raw XML to parse.
-     * @param output The output message.
-     * @return
+     * @param output       The output message.
      */
     private Document buildDocument(String valueToParse, Message.Builder output) {
         Document document = null;
@@ -145,9 +146,9 @@ private Document buildDocument(String valueToParse, Message.Builder output) {
             DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
             factory.setNamespaceAware(namespaceAware);
             document = factory.newDocumentBuilder()
-                    .parse(IOUtils.toInputStream(valueToParse, Charset.defaultCharset()));
+                              .parse(IOUtils.toInputStream(valueToParse, Charset.defaultCharset()));
 
-        } catch(ParserConfigurationException | SAXException | IOException e) {
+        } catch (ParserConfigurationException | SAXException | IOException e) {
             log.debug("Unable to parse XML document.", e);
             output.withError("Unable to parse XML document.", e);
         }
@@ -156,10 +157,11 @@ private Document buildDocument(String valueToParse, Message.Builder output) {
 
     /**
      * Execute the XPath expression and add the value to the message.
-     * @param document The XML document.
-     * @param fieldName The name of the field to create or modify.
+     *
+     * @param document   The XML document.
+     * @param fieldName  The name of the field to create or modify.
      * @param expression The XPath expression.
-     * @param output The output message.
+     * @param output     The output message.
      */
     private void execute(Document document, FieldName fieldName, XPathExpression expression, Message.Builder output) {
         try {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParserTest.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParserTest.java
index 9e94af19..af38d533 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParserTest.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParserTest.java
@@ -12,14 +12,14 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
 import com.cloudera.parserchains.core.Constants;
 import com.cloudera.parserchains.core.Message;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-
 public class DelimitedKeyValueParserTest {
     private DelimitedKeyValueParser parser;
 
@@ -261,17 +261,17 @@ void unsetValidValueRegex() {
 
     @Test
     void complex() {
-        final String textToParse = "2020-03-04 13:05:21.322- AuditOperation - HostName=PRDPWSLNX0067" +
-                "||Operation=getUserInfo" +
-                "||Is_Operation_Success=SUCCESS" +
-                "||AuditMessage; Interaction_Id=" +
-                "||PayType=DEFAULT" +
-                "||Operation_Type=Tibco_Service_Call" +
-                "||Service_Name=getUserInfo" +
-                "||Message=Invoking:#UserInfoService.getUserInfo tibco call with request:#Invoke of Streamline.Tibco.Services.GetUserInfoService.UserInfoService.getUserInfo started at 03/04/2020 13:05:21.322 with parameters: parameter 0 (serialized .NET Object):   
       STL 87c9-d8c7-4e3f-aafd-8f4b8c2fafe1 2020-03-04T13:05:21.3222285-08:00 STL STL 0000002 
       CAcuna6 Streamline true  URL = https://rspservices.t-mobile.com/rsp/UserInfoService Timeout = 10000 " +
-                "||LogType=Call" +
-                "||Method_Name=Invoke" +
-                "||Reason=";
+        final String textToParse = "2020-03-04 13:05:21.322- AuditOperation - HostName=PRDPWSLNX0067"
+                + "||Operation=getUserInfo"
+                + "||Is_Operation_Success=SUCCESS"
+                + "||AuditMessage; Interaction_Id="
+                + "||PayType=DEFAULT"
+                + "||Operation_Type=Tibco_Service_Call"
+                + "||Service_Name=getUserInfo"
+                + "||Message=Invoking:#UserInfoService.getUserInfo tibco call with request:#Invoke of Streamline.Tibco.Services.GetUserInfoService.UserInfoService.getUserInfo started at 03/04/2020 13:05:21.322 with parameters: parameter 0 (serialized .NET Object):   
       STL 87c9-d8c7-4e3f-aafd-8f4b8c2fafe1 2020-03-04T13:05:21.3222285-08:00 STL STL 0000002 
       CAcuna6 Streamline true        URL = https://rspservices.t-mobile.com/rsp/UserInfoService Timeout = 10000 "
+                + "||LogType=Call"
+                + "||Method_Name=Invoke"
+                + "||Reason=";
         Message input = Message.builder()
                 .addField(Constants.DEFAULT_INPUT_FIELD, textToParse)
                 .build();
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/GrokParserTest.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/GrokParserTest.java
index 67ea4db4..f698749c 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/GrokParserTest.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/GrokParserTest.java
@@ -12,17 +12,16 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
 import com.cloudera.parserchains.core.Constants;
 import com.cloudera.parserchains.core.FieldName;
 import com.cloudera.parserchains.core.Message;
+import java.time.ZoneOffset;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-import java.time.ZoneOffset;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-
 public class GrokParserTest {
     private GrokParser grokParser;
 
@@ -144,10 +143,10 @@ void defaultInputField() {
 
     @Test
     void defaultPatterns() {
-        final String apacheLog = "112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] " +
-                "\"GET / HTTP/1.1\" 200 44346 \"-\" " +
-                "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) " +
-                "AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152 Safari/537.22\"";
+        final String apacheLog = "112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] "
+                + "\"GET / HTTP/1.1\" 200 44346 \"-\" "
+                + "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) "
+                + "AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152 Safari/537.22\"";
         Message input = Message.builder()
                 .addField(Constants.DEFAULT_INPUT_FIELD, apacheLog)
                 .build();
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/StellarParserTest.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/StellarParserTest.java
index b5d222c4..3028da3d 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/StellarParserTest.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/StellarParserTest.java
@@ -12,20 +12,19 @@
 
 package com.cloudera.parserchains.parsers;
 
-import com.cloudera.parserchains.core.FieldName;
-import com.cloudera.parserchains.core.Message;
-import com.cloudera.parserchains.core.StringFieldValue;
-import org.junit.jupiter.api.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
 import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
 import static com.cloudera.parserchains.parsers.ResourceTestUtil.getFileFromResource;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.assertj.core.api.Assertions.entry;
 
+import com.cloudera.parserchains.core.FieldName;
+import com.cloudera.parserchains.core.Message;
+import com.cloudera.parserchains.core.StringFieldValue;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import org.junit.jupiter.api.Test;
+
 public class StellarParserTest {
 
     private static final String CONFIG_PATH = "/stellar/test_stellar_parser_config.json";
@@ -61,8 +60,8 @@ public void testParserWithNonDefaultInputField() throws IOException {
 
     @Test
     public void testParserErrorConfigFileDoesNotExist() {
-        assertThatThrownBy(() -> new StellarParser().configurationPath(DOESNT_EXIST_CONFIG_FILE)).
-                isInstanceOf(FileNotFoundException.class).hasMessageContaining(String.format("%s (No such file or directory)", DOESNT_EXIST_CONFIG_FILE));
+        assertThatThrownBy(() -> new StellarParser().configurationPath(DOESNT_EXIST_CONFIG_FILE))
+                .isInstanceOf(FileNotFoundException.class).hasMessageContaining(String.format("%s (No such file or directory)", DOESNT_EXIST_CONFIG_FILE));
     }
 
     @Test
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/SyslogParserTest.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/SyslogParserTest.java
index 25f369b6..fdb2af9a 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/SyslogParserTest.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/SyslogParserTest.java
@@ -12,14 +12,27 @@
 
 package com.cloudera.parserchains.parsers;
 
-import com.cloudera.parserchains.core.*;
-import com.github.palindromicity.syslog.SyslogSpecification;
-import org.junit.jupiter.api.Test;
-
-import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.*;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_APPNAME;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_HOSTNAME;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_MSGID;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_PRI;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_PRI_FACILITY;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_PRI_SEVERITY;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_PROCID;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_TIMESTAMP;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.HEADER_VERSION;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.MESSAGE;
+import static com.github.palindromicity.syslog.dsl.SyslogFieldKeys.STRUCTURED_BASE;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import com.cloudera.parserchains.core.Constants;
+import com.cloudera.parserchains.core.FieldName;
+import com.cloudera.parserchains.core.Message;
+import com.cloudera.parserchains.core.StringFieldValue;
+import com.github.palindromicity.syslog.SyslogSpecification;
+import org.junit.jupiter.api.Test;
+
 public class SyslogParserTest {
 
     private static final String SYSLOG_5424 = "<14>1 2014-06-20T09:14:07+00:00 loggregator"
@@ -57,8 +70,8 @@ void parse5424() {
         assertEquals(expected, output);
     }
 
-    private static final String SYSLOG_3164 = "<181>2018-09-14T00:54:09+00:00 lzpqrst-admin.in.mycompany.com.lg " +
-            "CISE_RADIUS_Accounting 0018032501 1 0 2018-09-14 10:54:09.095 +10:00";
+    private static final String SYSLOG_3164 = "<181>2018-09-14T00:54:09+00:00 lzpqrst-admin.in.mycompany.com.lg "
+            + "CISE_RADIUS_Accounting 0018032501 1 0 2018-09-14 10:54:09.095 +10:00";
 
     @Test
     void parse3164() {
diff --git a/flink-cyber/nifi-parser-bundle/nifi-parser-services/src/main/java/com/cloudera/cyber/nifi/CDPDCCyberConnectionService.java b/flink-cyber/nifi-parser-bundle/nifi-parser-services/src/main/java/com/cloudera/cyber/nifi/CDPDCCyberConnectionService.java
index 5b363533..9467b206 100644
--- a/flink-cyber/nifi-parser-bundle/nifi-parser-services/src/main/java/com/cloudera/cyber/nifi/CDPDCCyberConnectionService.java
+++ b/flink-cyber/nifi-parser-bundle/nifi-parser-services/src/main/java/com/cloudera/cyber/nifi/CDPDCCyberConnectionService.java
@@ -9,7 +9,8 @@
 import java.util.List;
 
 /**
- * Connection service for the very opinionated way we setup and connect to Cyber Clusters
+ * Connection service for the very opinionated way we setup and connect to Cyber Clusters.
+ *
  * 
      
  * This will also build the Kafka Producers and predetermine the schema registry requirements, kerberos etc.
  */
diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ChainParserMapFunction.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ChainParserMapFunction.java
index 4ab197d6..851451e2 100644
--- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ChainParserMapFunction.java
+++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ChainParserMapFunction.java
@@ -12,12 +12,34 @@
 
 package com.cloudera.cyber.parser;
 
+import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
+
 import com.cloudera.cyber.DataQualityMessage;
 import com.cloudera.cyber.DataQualityMessageLevel;
 import com.cloudera.cyber.Message;
 import com.cloudera.cyber.SignedSourceKey;
-import com.cloudera.parserchains.core.*;
+import com.cloudera.parserchains.core.ChainBuilder;
+import com.cloudera.parserchains.core.ChainLink;
+import com.cloudera.parserchains.core.ChainRunner;
+import com.cloudera.parserchains.core.DefaultChainBuilder;
+import com.cloudera.parserchains.core.DefaultChainRunner;
+import com.cloudera.parserchains.core.FieldName;
+import com.cloudera.parserchains.core.FieldValue;
+import com.cloudera.parserchains.core.InvalidParserException;
+import com.cloudera.parserchains.core.ReflectiveParserBuilder;
 import com.cloudera.parserchains.core.catalog.ClassIndexParserCatalog;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -31,16 +53,6 @@
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.OutputTag;
 
-import java.security.PrivateKey;
-import java.security.Signature;
-import java.security.SignatureException;
-import java.time.Instant;
-import java.util.*;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
-
 @RequiredArgsConstructor
 @Slf4j
 public class ChainParserMapFunction extends ProcessFunction {
@@ -62,35 +74,35 @@ public class ChainParserMapFunction extends ProcessFunction chains;
     private transient Signature signature;
     private transient Meter messageMeter;
-    private final OutputTag errorOutputTag = new OutputTag(ParserJob.ERROR_MESSAGE_SIDE_OUTPUT){};
+    private final OutputTag errorOutputTag = new OutputTag(ParserJob.ERROR_MESSAGE_SIDE_OUTPUT) {
+    };
     private transient Map topicNameToChain;
     private transient List> topicPatternToChain;
 
     @Override
     public void open(Configuration parameters) throws Exception {
         super.open(parameters);
-        log.info( "Chain config {}", chainConfig);
-        log.info( "Topic map {}", topicMap);
+        log.info("Chain config {}", chainConfig);
+        log.info("Topic map {}", topicMap);
         topicNameToChain = new HashMap<>();
-        topicPatternToChain = topicMap.entrySet().stream().
-                map( e -> new Tuple2<>(Pattern.compile(e.getKey()), e.getValue())).
-                collect(Collectors.toList());
+        topicPatternToChain = topicMap.entrySet().stream()
+                                      .map(e -> new Tuple2<>(Pattern.compile(e.getKey()), e.getValue()))
+                                      .collect(Collectors.toList());
         ChainBuilder chainBuilder = new DefaultChainBuilder(new ReflectiveParserBuilder(),
-                new ClassIndexParserCatalog());
+              new ClassIndexParserCatalog());
         ArrayList errors = new ArrayList<>();
         try {
             chains = chainConfig.entrySet().stream().collect(Collectors.toMap(
-                    Map.Entry::getKey,
-                    v ->
-                    {
-                        try {
-                            return chainBuilder.build(v.getValue());
-                        } catch (InvalidParserException e) {
-                            log.error("Cannot build parser chain", e);
-                            errors.add(e);
-                            return null;
-                        }
-                    }));
+                  Map.Entry::getKey,
+                  v -> {
+                      try {
+                          return chainBuilder.build(v.getValue());
+                      } catch (InvalidParserException e) {
+                          log.error("Cannot build parser chain", e);
+                          errors.add(e);
+                          return null;
+                      }
+                  }));
         } catch (NullPointerException e) {
             if (CollectionUtils.isNotEmpty(errors)) {
                 throw errors.get(0);
@@ -142,27 +154,28 @@ public void processElement(MessageToParse message, Context context, Collector dataQualityMessages = errorMessage.
-                    map(messageText -> Collections.singletonList(
-                            DataQualityMessage.builder().
-                                    field(DEFAULT_INPUT_FIELD).
-                                    feature(CHAIN_PARSER_FEATURE).
-                                    level(DataQualityMessageLevel.ERROR.name()).
-                                    message(messageText).
-                                    build())).
-                    orElse(null);
-
-            Message parsedMessage = Message.builder().extensions(fieldsFromChain(errorMessage.isPresent(), m.getFields()))
-                    .source(topicParserConfig.getSource())
-                    .originalSource(SignedSourceKey.builder()
-                            .topic(topic)
-                            .partition(message.getPartition())
-                            .offset(message.getOffset())
-                            .signature(signOriginalText(m))
-                            .build())
-                    .ts(messageTimestamp)
-                    .dataQualityMessages(dataQualityMessages)
-                    .build();
+            List dataQualityMessages = errorMessage
+                  .map(messageText -> Collections.singletonList(
+                        DataQualityMessage.builder()
+                                          .field(DEFAULT_INPUT_FIELD)
+                                          .feature(CHAIN_PARSER_FEATURE)
+                                          .level(DataQualityMessageLevel.ERROR.name())
+                                          .message(messageText)
+                                          .build()))
+                  .orElse(null);
+
+            Message parsedMessage =
+                  Message.builder().extensions(fieldsFromChain(errorMessage.isPresent(), m.getFields()))
+                         .source(topicParserConfig.getSource())
+                         .originalSource(SignedSourceKey.builder()
+                                                        .topic(topic)
+                                                        .partition(message.getPartition())
+                                                        .offset(message.getOffset())
+                                                        .signature(signOriginalText(m))
+                                                        .build())
+                         .ts(messageTimestamp)
+                         .dataQualityMessages(dataQualityMessages)
+                         .build();
 
             if (dataQualityMessages != null) {
                 context.output(errorOutputTag, parsedMessage);
@@ -173,7 +186,7 @@ public void processElement(MessageToParse message, Context context, Collector originalMessage = m.getField(FieldName.of(DEFAULT_INPUT_FIELD));
@@ -195,15 +208,16 @@ private byte[] signOriginalText(com.cloudera.parserchains.core.Message m)  {
     private static Map fieldsFromChain(boolean hasError, Map fields) {
         return fields.entrySet().stream().filter(mapEntry -> {
             String fieldName = mapEntry.getKey().get();
-            return (!StringUtils.equals(fieldName,DEFAULT_INPUT_FIELD) || hasError) && !fieldName.equals("timestamp");
+            return (!StringUtils.equals(fieldName, DEFAULT_INPUT_FIELD) || hasError) && !fieldName.equals("timestamp");
         }).collect(Collectors.toMap(entryMap -> entryMap.getKey().get(), entryMap -> entryMap.getValue().get()));
 
     }
 
     private TopicParserConfig getChainForTopic(String topicName) {
         return topicNameToChain.computeIfAbsent(topicName, top ->
-                topicPatternToChain.stream().filter(t -> t.f0.matcher(top).
-                        matches()).findFirst().map(t -> t.f1).orElse(new TopicParserConfig(top, top, defaultKafkaBootstrap)));
+              topicPatternToChain.stream().filter(t -> t.f0.matcher(top)
+                                                           .matches()).findFirst().map(t -> t.f1)
+                                 .orElse(new TopicParserConfig(top, top, defaultKafkaBootstrap)));
     }
 
 }
diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/MessageToEnrichmentCommandFunction.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/MessageToEnrichmentCommandFunction.java
index 43e5fde3..3afa84ea 100644
--- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/MessageToEnrichmentCommandFunction.java
+++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/MessageToEnrichmentCommandFunction.java
@@ -12,11 +12,23 @@
 
 package com.cloudera.cyber.parser;
 
-import com.cloudera.cyber.*;
+import com.cloudera.cyber.DataQualityMessage;
+import com.cloudera.cyber.DataQualityMessageLevel;
+import com.cloudera.cyber.EnrichmentEntry;
+import com.cloudera.cyber.Message;
+import com.cloudera.cyber.MessageUtils;
 import com.cloudera.cyber.commands.CommandType;
 import com.cloudera.cyber.commands.EnrichmentCommand;
 import com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfig;
 import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig;
+import java.io.Serializable;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
@@ -24,18 +36,17 @@
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.OutputTag;
 
-import java.io.Serializable;
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 public class MessageToEnrichmentCommandFunction extends ProcessFunction {
 
     public static final String STREAMING_ENRICHMENT_FEATURE = "stream_enrich";
-    public static final String STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET = "Message does not define values for key field for enrichment type '%s'";
-    public static final String STREAMING_ENRICHMENT_VALUE_FIELD_NOT_SET = "Message does not contain any values for enrichment type '%s'";
+    public static final String STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET =
+          "Message does not define values for key field for enrichment type '%s'";
+    public static final String STREAMING_ENRICHMENT_VALUE_FIELD_NOT_SET =
+          "Message does not contain any values for enrichment type '%s'";
     public static final String VALUE_FIELDS = "valueFields";
-    private final OutputTag errorOutputTag = new OutputTag(ParserJob.ERROR_MESSAGE_SIDE_OUTPUT){};
+    private final OutputTag errorOutputTag = new OutputTag(ParserJob.ERROR_MESSAGE_SIDE_OUTPUT) {
+    };
 
     @Data
     @AllArgsConstructor
@@ -47,7 +58,8 @@ private static class EnrichmentTypeConfig implements Serializable {
     private final HashMap> sourceToEnrichmentsConfig = new HashMap<>();
 
 
-    public MessageToEnrichmentCommandFunction(List streamingEnrichmentSources, EnrichmentsConfig streamingEnrichmentConfig) {
+    public MessageToEnrichmentCommandFunction(List streamingEnrichmentSources,
+                                              EnrichmentsConfig streamingEnrichmentConfig) {
         streamingEnrichmentSources.forEach(source -> sourceToEnrichmentsConfig.put(source, new ArrayList<>()));
         streamingEnrichmentConfig.getEnrichmentConfigs().forEach((key, value) -> {
             List configSources = value.getFieldMapping().getStreamingSources();
@@ -74,64 +86,73 @@ public void processElement(Message message, Context context, Collector dataQualityMessages = new ArrayList<>();
             Map valueFieldValues = null;
 
-            extractKeyFields(typeConfig.getEnrichmentType(), fieldsConfig, message.getExtensions(), keyFieldValues, dataQualityMessages);
+            extractKeyFields(typeConfig.getEnrichmentType(), fieldsConfig, message.getExtensions(), keyFieldValues,
+                  dataQualityMessages);
             if (dataQualityMessages.isEmpty() && extensions != null) {
                 valueFieldValues = extractValueFields(enrichmentType, fieldsConfig, extensions, dataQualityMessages);
             }
             if (dataQualityMessages.isEmpty()) {
-                String enrichmentKey = keyFieldValues.stream().collect(Collectors.joining(fieldsConfig.getKeyDelimiter()));
-                EnrichmentEntry enrichmentEntry = EnrichmentEntry.builder().ts(message.getTs()).
-                        type(typeConfig.getEnrichmentType()).
-                        key(enrichmentKey).entries(valueFieldValues).build();
+                String enrichmentKey =
+                      keyFieldValues.stream().collect(Collectors.joining(fieldsConfig.getKeyDelimiter()));
+                EnrichmentEntry enrichmentEntry = EnrichmentEntry.builder().ts(message.getTs())
+                                                                 .type(typeConfig.getEnrichmentType())
+                                                                 .key(enrichmentKey).entries(valueFieldValues).build();
                 log.info("Writing enrichment key {}", enrichmentKey);
-                collector.collect(EnrichmentCommand.builder().
-                        headers(Collections.emptyMap()).
-                        type(commandType).payload(enrichmentEntry).build());
-            }
-            else {
-                context.output(errorOutputTag, MessageUtils.enrich(message, Collections.emptyMap(), dataQualityMessages));
+                collector.collect(EnrichmentCommand.builder()
+                                                   .headers(Collections.emptyMap())
+                                                   .type(commandType).payload(enrichmentEntry).build());
+            } else {
+                context.output(errorOutputTag,
+                      MessageUtils.enrich(message, Collections.emptyMap(), dataQualityMessages));
             }
         }
     }
 
-    private void extractKeyFields(String enrichmentType, EnrichmentFieldsConfig fieldsConfig, Map extensions,
-                                          List keyFieldValues, List dataQualityMessages) {
+    private void extractKeyFields(String enrichmentType, EnrichmentFieldsConfig fieldsConfig,
+                                  Map extensions,
+                                  List keyFieldValues, List dataQualityMessages) {
         for (String keyFieldName : fieldsConfig.getKeyFields()) {
             String keyFieldValue = extensions != null ? extensions.get(keyFieldName) : null;
             if (keyFieldValue != null) {
                 keyFieldValues.add(keyFieldValue);
             } else {
-                dataQualityMessages.add(DataQualityMessage.builder().
-                        level(DataQualityMessageLevel.ERROR.name()).
-                        feature(STREAMING_ENRICHMENT_FEATURE).
-                        field(keyFieldName).
-                        message(String.format(STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET, enrichmentType)).
-                        build());
+                dataQualityMessages.add(DataQualityMessage.builder()
+                                                          .level(DataQualityMessageLevel.ERROR.name())
+                                                          .feature(STREAMING_ENRICHMENT_FEATURE)
+                                                          .field(keyFieldName)
+                                                          .message(String.format(STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET,
+                                                                enrichmentType))
+                                                          .build());
             }
         }
     }
 
-    private Map extractValueFields(String enrichmentType, EnrichmentFieldsConfig fieldsConfig, Map extensions,
-                                    List dataQualityMessages) {
-        List  keyFieldNames = fieldsConfig.getKeyFields();
+    private Map extractValueFields(String enrichmentType, EnrichmentFieldsConfig fieldsConfig,
+                                                   Map extensions,
+                                                   List dataQualityMessages) {
+        List keyFieldNames = fieldsConfig.getKeyFields();
         Map valueFieldValues;
         List valueFieldNames = fieldsConfig.getValueFields();
 
         if (valueFieldNames == null) {
             // no fields specified, write all the extensions minus the key fields
-            valueFieldValues = extensions.entrySet().stream().filter(e -> !keyFieldNames.contains(e.getKey())).
-                    collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+            valueFieldValues = extensions.entrySet().stream().filter(e -> !keyFieldNames.contains(e.getKey()))
+                                         .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         } else {
-            valueFieldValues = valueFieldNames.stream().map(fieldValueName -> new AbstractMap.SimpleEntry<>(fieldValueName, extensions.get(fieldValueName))).
-                    filter(e -> (e.getValue() != null)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+            valueFieldValues = valueFieldNames.stream()
+                                              .map(fieldValueName -> new AbstractMap.SimpleEntry<>(fieldValueName,
+                                                    extensions.get(fieldValueName)))
+                                              .filter(e -> (e.getValue() != null))
+                                              .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         }
         if (valueFieldValues.isEmpty()) {
-            dataQualityMessages.add(DataQualityMessage.builder().
-                    level(DataQualityMessageLevel.ERROR.name()).
-                    feature(STREAMING_ENRICHMENT_FEATURE).
-                    field(VALUE_FIELDS).
-                    message(String.format(STREAMING_ENRICHMENT_VALUE_FIELD_NOT_SET, enrichmentType)).
-                    build());
+            dataQualityMessages.add(DataQualityMessage.builder()
+                                                      .level(DataQualityMessageLevel.ERROR.name())
+                                                      .feature(STREAMING_ENRICHMENT_FEATURE)
+                                                      .field(VALUE_FIELDS)
+                                                      .message(String.format(STREAMING_ENRICHMENT_VALUE_FIELD_NOT_SET,
+                                                            enrichmentType))
+                                                      .build());
         }
         return valueFieldValues;
     }
diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserChainMap.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserChainMap.java
index 46b4b47d..1d7bbcf2 100644
--- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserChainMap.java
+++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserChainMap.java
@@ -13,7 +13,6 @@
 package com.cloudera.cyber.parser;
 
 import com.cloudera.parserchains.core.model.define.ParserChainSchema;
-
 import java.util.HashMap;
 
 public class ParserChainMap extends HashMap {
diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserJob.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserJob.java
index b409c65b..26d48eb6 100644
--- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserJob.java
+++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserJob.java
@@ -19,20 +19,6 @@
 import com.cloudera.cyber.flink.Utils;
 import com.cloudera.parserchains.core.model.define.ParserChainSchema;
 import com.cloudera.parserchains.core.utils.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.core.fs.FSDataInputStream;
-import org.apache.flink.core.fs.FileStatus;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.OutputTag;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -48,9 +34,22 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.OutputTag;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
 
 /**
- * Host for the chain parser jobs
+ * Host for the chain parser jobs.
  */
 @Slf4j
 public abstract class ParserJob {
@@ -59,7 +58,7 @@ public abstract class ParserJob {
     protected static final String PARAM_CHAIN_CONFIG_FILE = "chain.file";
     protected static final String PARAM_CHAIN_CONFIG_DIRECTORY = "chain.dir";
     protected static final List PARAM_CHAIN_CONFIG_EXCLUSIVE_LIST = Arrays.asList(
-            PARAM_CHAIN_CONFIG, PARAM_CHAIN_CONFIG_FILE, PARAM_CHAIN_CONFIG_DIRECTORY);
+          PARAM_CHAIN_CONFIG, PARAM_CHAIN_CONFIG_FILE, PARAM_CHAIN_CONFIG_DIRECTORY);
 
     protected static final String PARAM_TOPIC_MAP_CONFIG = "chain.topic.map";
     protected static final String PARAM_TOPIC_MAP_CONFIG_FILE = "chain.topic.map.file";
@@ -70,12 +69,14 @@ public abstract class ParserJob {
     public static final String PARAM_STREAMING_ENRICHMENTS_CONFIG = "chain.enrichments.file";
 
     protected StreamExecutionEnvironment createPipeline(ParameterTool params)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+          throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
         FlinkUtils.setupEnv(env, params);
         validateParams(params);
 
-        String chainConfig = readConfigChainMap(PARAM_CHAIN_CONFIG_FILE, PARAM_CHAIN_CONFIG, PARAM_CHAIN_CONFIG_DIRECTORY, params, null);
+        String chainConfig =
+              readConfigChainMap(PARAM_CHAIN_CONFIG_FILE, PARAM_CHAIN_CONFIG, PARAM_CHAIN_CONFIG_DIRECTORY, params,
+                    null);
         String topicConfig = readConfigMap(PARAM_TOPIC_MAP_CONFIG_FILE, PARAM_TOPIC_MAP_CONFIG, params, "{}");
 
 
@@ -91,8 +92,9 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params)
             streamingEnrichmentsConfig = EnrichmentsConfig.load(enrichmentsConfigFile);
 
             List sourcesProduced = topicMap.getSourcesProduced();
-            streamingSourcesProduced = streamingEnrichmentsConfig.getStreamingEnrichmentSources().stream().
-                    filter(sourcesProduced::contains).collect(Collectors.toList());
+            streamingSourcesProduced = streamingEnrichmentsConfig.getStreamingEnrichmentSources().stream()
+                                                                 .filter(sourcesProduced::contains)
+                                                                 .collect(Collectors.toList());
             if (!streamingSourcesProduced.isEmpty()) {
                 streamingEnrichFilter = new StreamingEnrichmentsSourceFilter(streamingSourcesProduced);
             }
@@ -102,9 +104,9 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params)
 
         PrivateKey privateKey = null;
         if (params.getBoolean(SIGNATURE_ENABLED, true)) {
-            byte[] privKeyBytes = params.has(PARAM_PRIVATE_KEY_FILE) ?
-                    Files.readAllBytes(Paths.get(params.get(PARAM_PRIVATE_KEY_FILE))) :
-                    Base64.getDecoder().decode(params.getRequired(PARAM_PRIVATE_KEY));
+            byte[] privKeyBytes = params.has(PARAM_PRIVATE_KEY_FILE)
+                  ? Files.readAllBytes(Paths.get(params.get(PARAM_PRIVATE_KEY_FILE)))
+                  : Base64.getDecoder().decode(params.getRequired(PARAM_PRIVATE_KEY));
 
             KeyFactory keyFactory = KeyFactory.getInstance("RSA");
             PKCS8EncodedKeySpec privSpec = new PKCS8EncodedKeySpec(privKeyBytes);
@@ -112,8 +114,9 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params)
         }
 
         SingleOutputStreamOperator results =
-                source.process(new ChainParserMapFunction(chainSchema, topicMap, privateKey, defaultKafkaBootstrap))
-                        .name("Parser " + source.getTransformation().getName()).uid("parser" + source.getTransformation().getUid());
+              source.process(new ChainParserMapFunction(chainSchema, topicMap, privateKey, defaultKafkaBootstrap))
+                    .name("Parser " + source.getTransformation().getName())
+                    .uid("parser" + source.getTransformation().getUid());
         final OutputTag errorMessageSideOutput = new OutputTag(ERROR_MESSAGE_SIDE_OUTPUT) {
         };
         DataStream errorMessages = results.getSideOutput(errorMessageSideOutput);
@@ -122,8 +125,13 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params)
         writeOriginalsResults(params, source);
 
         if (streamingEnrichFilter != null) {
-            DataStream streamingEnrichmentMessages = results.filter(streamingEnrichFilter).name("Streaming Enrichment Sources Filter");
-            SingleOutputStreamOperator enrichmentCommands = streamingEnrichmentMessages.process(new MessageToEnrichmentCommandFunction(streamingSourcesProduced, streamingEnrichmentsConfig)).name("Message to EnrichmentCommand");
+            DataStream streamingEnrichmentMessages =
+                  results.filter(streamingEnrichFilter).name("Streaming Enrichment Sources Filter");
+            SingleOutputStreamOperator enrichmentCommands =
+                  streamingEnrichmentMessages.process(
+                                                   new MessageToEnrichmentCommandFunction(
+                                                         streamingSourcesProduced, streamingEnrichmentsConfig))
+                                             .name("Message to EnrichmentCommand");
             errorMessages = errorMessages.union(enrichmentCommands.getSideOutput(errorMessageSideOutput));
             writeEnrichments(params, enrichmentCommands, streamingSourcesProduced, streamingEnrichmentsConfig);
         }
@@ -136,11 +144,11 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params)
     private void validateParams(ParameterTool params) {
         //Check for mutually exclusive chain params
         final long chainParamAmount = PARAM_CHAIN_CONFIG_EXCLUSIVE_LIST.stream()
-                .filter(params::has)
-                .count();
+                                                                       .filter(params::has)
+                                                                       .count();
         if (chainParamAmount > 1) {
-            throw new RuntimeException("It's not allowed to provide more than one chain param! " +
-                    "Select one of the following: " + PARAM_CHAIN_CONFIG_EXCLUSIVE_LIST);
+            throw new RuntimeException("It's not allowed to provide more than one chain param! "
+                                       + "Select one of the following: " + PARAM_CHAIN_CONFIG_EXCLUSIVE_LIST);
         }
     }
 
@@ -153,12 +161,13 @@ private String readConfigChainMap(String fileParamKey, String inlineConfigKey, S
             final FileSystem fileSystem = path.getFileSystem();
 
             final FileStatus[] fileStatusList = fileSystem.listStatus(path);
-            if (ArrayUtils.isEmpty(fileStatusList)){
-                throw new RuntimeException(String.format("Provided config directory doesn't exist or empty [%s]!", path));
+            if (ArrayUtils.isEmpty(fileStatusList)) {
+                throw new RuntimeException(
+                      String.format("Provided config directory doesn't exist or empty [%s]!", path));
             }
             final FileStatus[] sortedFileList = Arrays.stream(fileStatusList)
-                    .sorted(Comparator.comparing(fs -> fs.getPath().getName()))
-                    .toArray(FileStatus[]::new);
+                                                      .sorted(Comparator.comparing(fs -> fs.getPath().getName()))
+                                                      .toArray(FileStatus[]::new);
             for (FileStatus fileStatus : sortedFileList) {
                 final Path filePath = fileStatus.getPath();
                 if (filePath.getName().endsWith(".json")) {
@@ -173,7 +182,9 @@ private String readConfigChainMap(String fileParamKey, String inlineConfigKey, S
 
                     final String schemaName = chainSchema.getName();
                     if (result.containsKey(schemaName)) {
-                        throw new RuntimeException(String.format("Found a duplicate schema named [%s] in the [%s] file, which isn't allowed!", schemaName, filePath));
+                        throw new RuntimeException(String.format(
+                              "Found a duplicate schema named [%s] in the [%s] file, which isn't allowed!", schemaName,
+                              filePath));
                     }
                     result.put(schemaName, chainSchema);
                 }
@@ -189,7 +200,7 @@ private String readConfigMap(String fileParamKey, String inlineConfigKey, Parame
             return new String(Files.readAllBytes(Paths.get(params.getRequired(fileParamKey))), StandardCharsets.UTF_8);
         }
         return defaultConfig == null ? params.getRequired(inlineConfigKey)
-                : params.get(inlineConfigKey, defaultConfig);
+              : params.get(inlineConfigKey, defaultConfig);
 
     }
 
@@ -197,7 +208,10 @@ private String readConfigMap(String fileParamKey, String inlineConfigKey, Parame
 
     protected abstract void writeOriginalsResults(ParameterTool params, DataStream results);
 
-    protected abstract void writeEnrichments(ParameterTool params, DataStream streamingEnrichmentResults, List streamingEnrichmentSources, EnrichmentsConfig streamingEnrichmentConfig);
+    protected abstract void writeEnrichments(ParameterTool params,
+                                             DataStream streamingEnrichmentResults,
+                                             List streamingEnrichmentSources,
+                                             EnrichmentsConfig streamingEnrichmentConfig);
 
     protected abstract void writeErrors(ParameterTool params, DataStream errors);
 
diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserJobKafka.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserJobKafka.java
index 865725eb..9a3e67f9 100644
--- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserJobKafka.java
+++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/ParserJobKafka.java
@@ -19,6 +19,14 @@
 import com.cloudera.cyber.flink.ConfigConstants;
 import com.cloudera.cyber.flink.FlinkUtils;
 import com.cloudera.cyber.flink.Utils;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
@@ -38,15 +46,6 @@
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.springframework.util.DigestUtils;
 
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
 @Slf4j
 public class ParserJobKafka extends ParserJob {
 
@@ -60,17 +59,17 @@ public static void main(String[] args) throws Exception {
         Preconditions.checkArgument(args.length >= 1, "Arguments must consist of a properties files");
         ParameterTool params = Utils.getParamToolsFromProperties(args);
         FlinkUtils.executeEnv(new ParserJobKafka()
-                .createPipeline(params), "Flink Parser", params);
+              .createPipeline(params), "Flink Parser", params);
 
     }
 
     @Override
     protected void writeResults(ParameterTool params, DataStream results) {
         KafkaSink sink = new FlinkUtils<>(Message.class).createKafkaSink(
-                params.getRequired(ConfigConstants.PARAMS_TOPIC_OUTPUT), "cyber-parser",
-                params);
+              params.getRequired(ConfigConstants.PARAMS_TOPIC_OUTPUT), "cyber-parser",
+              params);
         results.sinkTo(sink).name("Kafka Parser Results ")
-                .uid("kafka.results." + results.getTransformation().getUid());
+               .uid("kafka.results." + results.getTransformation().getUid());
     }
 
     @Override
@@ -89,66 +88,77 @@ protected void writeOriginalsResults(ParameterTool params, DataStream sink = StreamingFileSink
-                .forBulkFormat(path, ParquetAvroWriters.forReflectRecord(MessageToParse.class))
-                .withRollingPolicy(OnCheckpointRollingPolicy.build())
-                .withOutputFileConfig(OutputFileConfig
-                        .builder()
-                        .withPartPrefix("logs")
-                        .withPartSuffix(".parquet")
-                        .build())
-                .build();
+              .forBulkFormat(path, ParquetAvroWriters.forReflectRecord(MessageToParse.class))
+              .withRollingPolicy(OnCheckpointRollingPolicy.build())
+              .withOutputFileConfig(OutputFileConfig
+                    .builder()
+                    .withPartPrefix("logs")
+                    .withPartSuffix(".parquet")
+                    .build())
+              .build();
 
         results.addSink(sink).name("Original Archiver ")
-                .uid("original.archiver." + results.getTransformation().getUid());
+               .uid("original.archiver." + results.getTransformation().getUid());
     }
 
     @Override
-    protected void writeEnrichments(ParameterTool params, DataStream streamingEnrichmentResults, List streamingEnrichmentSources, EnrichmentsConfig streamingEnrichmentConfig) {
-        List tables = streamingEnrichmentSources.stream().map(streamingEnrichmentConfig::getReferencedTablesForSource).flatMap(Collection::stream).
-                distinct().collect(Collectors.toList());
-        HbaseJobRawKafka.enrichmentCommandsToHbase(params, streamingEnrichmentResults, streamingEnrichmentConfig, tables);
+    protected void writeEnrichments(ParameterTool params, DataStream streamingEnrichmentResults,
+                                    List streamingEnrichmentSources,
+                                    EnrichmentsConfig streamingEnrichmentConfig) {
+        List tables =
+              streamingEnrichmentSources.stream().map(streamingEnrichmentConfig::getReferencedTablesForSource)
+                                        .flatMap(Collection::stream)
+                                        .distinct().collect(Collectors.toList());
+        HbaseJobRawKafka.enrichmentCommandsToHbase(params, streamingEnrichmentResults, streamingEnrichmentConfig,
+              tables);
     }
 
     @Override
     protected void writeErrors(ParameterTool params, DataStream errors) {
         KafkaSink sink = new FlinkUtils<>(Message.class).createKafkaSink(
-                params.getRequired(ConfigConstants.PARAMS_TOPIC_ERROR), "cyber-parser",
-                params);
+              params.getRequired(ConfigConstants.PARAMS_TOPIC_ERROR), "cyber-parser",
+              params);
         errors.sinkTo(sink).name("Kafka Error Results " + atomicInteger.get())
-                .uid("kafka.error.results." + atomicInteger.getAndIncrement());
+              .uid("kafka.error.results." + atomicInteger.getAndIncrement());
     }
 
     @Override
     protected DataStream createSource(StreamExecutionEnvironment env, ParameterTool params,
-            TopicPatternToChainMap topicPatternToChainMap) {
+                                                      TopicPatternToChainMap topicPatternToChainMap) {
         return createDataStreamFromMultipleKafkaBrokers(env, params, createGroupId(
-                params.get(ConfigConstants.PARAMS_TOPIC_INPUT, "") + params
-                        .get(ConfigConstants.PARAMS_TOPIC_PATTERN, "")), topicPatternToChainMap);
+              params.get(ConfigConstants.PARAMS_TOPIC_INPUT, "") + params
+                    .get(ConfigConstants.PARAMS_TOPIC_PATTERN, "")), topicPatternToChainMap);
     }
 
     public DataStream createDataStreamFromMultipleKafkaBrokers(StreamExecutionEnvironment env,
-            ParameterTool params, String groupId, TopicPatternToChainMap topicPatternToChainMap) {
+                                                                               ParameterTool params, String groupId,
+                                                                               TopicPatternToChainMap topicPatternToChainMap) {
         DataStream firstSource = null;
         Map brokerTopicPatternMap = topicPatternToChainMap.getBrokerPrefixTopicPatternMap();
-        for(Map.Entry kafkaPrefixToTopicPattern : brokerTopicPatternMap.entrySet()) {
+        for (Map.Entry kafkaPrefixToTopicPattern : brokerTopicPatternMap.entrySet()) {
             String kafkaPrefixConf = kafkaPrefixToTopicPattern.getKey();
             Pattern topicNamePattern = kafkaPrefixToTopicPattern.getValue();
             log.info(String.format("createRawKafkaSource  pattern: '%s', good: %b", topicNamePattern,
-                    StringUtils.isNotEmpty(kafkaPrefixConf)));
+                  StringUtils.isNotEmpty(kafkaPrefixConf)));
             Preconditions.checkArgument(StringUtils.isNotEmpty(topicNamePattern.toString()),
-                    "Topic name must be specified in chain.topic.map property variable");
+                  "Topic name must be specified in chain.topic.map property variable");
             Properties kafkaProperties = Utils.readKafkaProperties(params, groupId, true);
             if (!StringUtils.equals(kafkaPrefixConf, TopicPatternToChainMap.DEFAULT_PREFIX)) {
                 Properties brokerSpecificProperties = Utils
-                        .readProperties(params.getProperties(), kafkaPrefixConf + '.' + Utils.KAFKA_PREFIX);
+                      .readProperties(params.getProperties(), kafkaPrefixConf + '.' + Utils.KAFKA_PREFIX);
                 kafkaProperties.putAll(brokerSpecificProperties);
             }
             KafkaSource rawMessages = KafkaSource.builder()
-                    .setTopicPattern(topicNamePattern).setBootstrapServers(kafkaProperties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)).setDeserializer(new MessageToParseDeserializer()).setProperties(kafkaProperties).build();
-            DataStreamSource streamSource = env.fromSource(rawMessages, WatermarkStrategy.noWatermarks(), "Kafka Raw Messages");
+                                                                 .setTopicPattern(topicNamePattern).setBootstrapServers(
+                        kafkaProperties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG))
+                                                                 .setDeserializer(new MessageToParseDeserializer())
+                                                                 .setProperties(kafkaProperties).build();
+            DataStreamSource streamSource =
+                  env.fromSource(rawMessages, WatermarkStrategy.noWatermarks(), "Kafka Raw Messages");
             SingleOutputStreamOperator newSource = streamSource
-                    .name(String.format("Kafka Source topic='%s' kafka prefix configuration='%s'", topicNamePattern.toString(), kafkaPrefixConf))
-                    .uid("kafka.input." + kafkaPrefixConf);
+                  .name(String.format("Kafka Source topic='%s' kafka prefix configuration='%s'",
+                        topicNamePattern, kafkaPrefixConf))
+                  .uid("kafka.input." + kafkaPrefixConf);
 
             if (firstSource == null) {
                 firstSource = newSource;
diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/StreamingEnrichmentsSourceFilter.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/StreamingEnrichmentsSourceFilter.java
index fc0ee8c9..36101eef 100644
--- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/StreamingEnrichmentsSourceFilter.java
+++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/StreamingEnrichmentsSourceFilter.java
@@ -13,10 +13,9 @@
 package com.cloudera.cyber.parser;
 
 import com.cloudera.cyber.Message;
-import org.apache.flink.api.common.functions.FilterFunction;
-
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.flink.api.common.functions.FilterFunction;
 
 public class StreamingEnrichmentsSourceFilter implements FilterFunction {
     private final ArrayList streamingEnrichmentSources;
diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/TopicParserConfig.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/TopicParserConfig.java
index 957864ac..53bc12a6 100644
--- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/TopicParserConfig.java
+++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/TopicParserConfig.java
@@ -12,12 +12,11 @@
 
 package com.cloudera.cyber.parser;
 
+import java.io.Serializable;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.io.Serializable;
-
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
diff --git a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/TopicPatternToChainMap.java b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/TopicPatternToChainMap.java
index 4542d016..12786a57 100644
--- a/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/TopicPatternToChainMap.java
+++ b/flink-cyber/parser-chains-flink/src/main/java/com/cloudera/cyber/parser/TopicPatternToChainMap.java
@@ -26,15 +26,16 @@ public class TopicPatternToChainMap extends HashMap {
 
     public Map getBrokerPrefixTopicNameMap() {
         return this.entrySet().stream()
-                .collect(Collectors.groupingBy(
-                        mapEntry -> StringUtils.defaultIfEmpty(mapEntry.getValue().getBroker(), DEFAULT_PREFIX),
-                        Collectors.mapping(
-                                Entry::getKey, Collectors.joining("|"))));
+                   .collect(Collectors.groupingBy(
+                         mapEntry -> StringUtils.defaultIfEmpty(mapEntry.getValue().getBroker(), DEFAULT_PREFIX),
+                         Collectors.mapping(
+                               Entry::getKey, Collectors.joining("|"))));
     }
 
     public Map getBrokerPrefixTopicPatternMap() {
         return getBrokerPrefixTopicNameMap().entrySet().stream()
-                .collect(Collectors.toMap(Entry::getKey, entry -> Pattern.compile(entry.getValue())));
+                                            .collect(Collectors.toMap(Entry::getKey,
+                                                  entry -> Pattern.compile(entry.getValue())));
     }
 
     public List getSourcesProduced() {
diff --git a/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/ChainParserMapFunctionTest.java b/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/ChainParserMapFunctionTest.java
index 20f84bec..629da02e 100644
--- a/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/ChainParserMapFunctionTest.java
+++ b/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/ChainParserMapFunctionTest.java
@@ -12,19 +12,20 @@
 
 package com.cloudera.cyber.parser;
 
+import static com.cloudera.cyber.parser.ChainParserMapFunction.CHAIN_PARSER_FEATURE;
+import static com.cloudera.cyber.parser.ChainParserMapFunction.EMPTY_SIGNATURE;
+import static com.cloudera.cyber.parser.ChainParserMapFunction.NO_TIMESTAMP_FIELD_MESSAGE;
+import static com.cloudera.cyber.parser.ChainParserMapFunction.TIMESTAMP_NOT_EPOCH;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
 import com.cloudera.cyber.DataQualityMessage;
 import com.cloudera.cyber.DataQualityMessageLevel;
 import com.cloudera.cyber.Message;
 import com.cloudera.parserchains.core.InvalidParserException;
 import com.cloudera.parserchains.core.utils.JSONUtils;
 import com.google.common.io.Resources;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
-import org.apache.flink.streaming.util.ProcessFunctionTestHarnesses;
-import org.apache.flink.util.OutputTag;
-import org.junit.Test;
-
 import java.io.IOException;
 import java.net.URL;
 import java.security.InvalidKeyException;
@@ -38,14 +39,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-
-import static com.cloudera.cyber.parser.ChainParserMapFunction.CHAIN_PARSER_FEATURE;
-import static com.cloudera.cyber.parser.ChainParserMapFunction.EMPTY_SIGNATURE;
-import static com.cloudera.cyber.parser.ChainParserMapFunction.NO_TIMESTAMP_FIELD_MESSAGE;
-import static com.cloudera.cyber.parser.ChainParserMapFunction.TIMESTAMP_NOT_EPOCH;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.ProcessFunctionTestHarnesses;
+import org.apache.flink.util.OutputTag;
+import org.junit.Test;
 
 @SuppressWarnings("UnstableApiUsage")
 public class ChainParserMapFunctionTest {
@@ -107,8 +106,8 @@ private void testMessageWithError(String messageText, String timestampNotEpoch)
 
     @Test
     public void testInvalidParser() {
-        assertThatThrownBy(() ->createTestHarness("ErrorParserChain.json", null)).
-                isInstanceOf(InvalidParserException.class).hasMessageContaining("Unable to find parser in catalog");
+        assertThatThrownBy(() ->createTestHarness("ErrorParserChain.json", null))
+                .isInstanceOf(InvalidParserException.class).hasMessageContaining("Unable to find parser in catalog");
     }
 
     @Test
diff --git a/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/MessageToEnrichmentCommandFunctionTest.java b/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/MessageToEnrichmentCommandFunctionTest.java
index c16aa247..7c4af8ca 100644
--- a/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/MessageToEnrichmentCommandFunctionTest.java
+++ b/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/MessageToEnrichmentCommandFunctionTest.java
@@ -12,7 +12,17 @@
 
 package com.cloudera.cyber.parser;
 
-import com.cloudera.cyber.*;
+import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfig.DEFAULT_KEY_DELIMITER;
+import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_METRON;
+import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.cloudera.cyber.DataQualityMessage;
+import com.cloudera.cyber.DataQualityMessageLevel;
+import com.cloudera.cyber.EnrichmentEntry;
+import com.cloudera.cyber.Message;
+import com.cloudera.cyber.MessageUtils;
+import com.cloudera.cyber.TestUtils;
 import com.cloudera.cyber.commands.CommandType;
 import com.cloudera.cyber.commands.EnrichmentCommand;
 import com.cloudera.cyber.enrichment.hbase.config.EnrichmentConfig;
@@ -21,20 +31,19 @@
 import com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
 import org.apache.flink.streaming.util.ProcessFunctionTestHarnesses;
 import org.apache.flink.util.OutputTag;
 import org.junit.Test;
 
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentStorageFormat.HBASE_METRON;
-import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentsConfig.DEFAULT_ENRICHMENT_STORAGE_NAME;
-import static com.cloudera.cyber.enrichment.hbase.config.EnrichmentFieldsConfig.DEFAULT_KEY_DELIMITER;
-import static org.assertj.core.api.Assertions.assertThat;
-
 public class MessageToEnrichmentCommandFunctionTest {
 
     private static final String TEST_SOURCE = "test_source";
@@ -84,8 +93,8 @@ public void testExtractAllValueEnrichment() throws Exception {
         Map messageExtensions = ImmutableMap.of(KEY_1, KEY_VALUE_1,
                 VALUE_1, VALUE_1_VALUE,
                 "extra", "include");
-        Map enrichmentValues = messageExtensions.entrySet().stream().filter(e -> e.getKey().equals(KEY_1)).
-                collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        Map enrichmentValues = messageExtensions.entrySet().stream().filter(e -> e.getKey().equals(KEY_1))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         testSuccessful(messageExtensions, KEY_FIELDS, null, null,
                 KEY_FIELD_VALUES, VALUE_FIELD_VALUES);
         testSuccessful(messageExtensions, KEY_FIELDS, ".", null,
@@ -107,18 +116,18 @@ public void testExtractMultiFieldKey() throws Exception {
 
     @Test
     public void testNullExtensions() throws Exception {
-        DataQualityMessage expectedMessageKey1 = DataQualityMessage.builder().
-                level(DataQualityMessageLevel.ERROR.name()).
-                feature(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_FEATURE).
-                field(KEY_1).
-                message(String.format(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET, TEST_ENRICHMENT_TYPE)).
-                build();
-        DataQualityMessage expectedMessageKey2 = DataQualityMessage.builder().
-                level(DataQualityMessageLevel.ERROR.name()).
-                feature(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_FEATURE).
-                field(KEY_2).
-                message(String.format(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET, TEST_ENRICHMENT_TYPE)).
-                build();
+        DataQualityMessage expectedMessageKey1 = DataQualityMessage.builder()
+                .level(DataQualityMessageLevel.ERROR.name())
+                .feature(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_FEATURE)
+                .field(KEY_1)
+                .message(String.format(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET, TEST_ENRICHMENT_TYPE))
+                .build();
+        DataQualityMessage expectedMessageKey2 = DataQualityMessage.builder()
+                .level(DataQualityMessageLevel.ERROR.name())
+                .feature(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_FEATURE)
+                .field(KEY_2)
+                .message(String.format(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET, TEST_ENRICHMENT_TYPE))
+                .build();
         testError(null,
                 Lists.newArrayList(expectedMessageKey1, expectedMessageKey2));
     }
@@ -131,12 +140,12 @@ public void testMissingKeyField() throws Exception {
                 VALUE_2, VALUE_2_VALUE,
                 "extra", "ignore");
 
-        DataQualityMessage expectedMessage = DataQualityMessage.builder().
-                level(DataQualityMessageLevel.ERROR.name()).
-                feature(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_FEATURE).
-                field(KEY_2).
-                message(String.format(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET, TEST_ENRICHMENT_TYPE)).
-                build();
+        DataQualityMessage expectedMessage = DataQualityMessage.builder()
+                .level(DataQualityMessageLevel.ERROR.name())
+                .feature(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_FEATURE)
+                .field(KEY_2)
+                .message(String.format(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_KEY_FIELD_NOT_SET, TEST_ENRICHMENT_TYPE))
+                .build();
         testError(messageExtensions,
                 Collections.singletonList(expectedMessage));
     }
@@ -148,12 +157,12 @@ public void testNoValues() throws Exception {
                 KEY_2, KEY_VALUE_2,
                 "extra", "ignore");
 
-        DataQualityMessage expectedMessage = DataQualityMessage.builder().
-                level(DataQualityMessageLevel.ERROR.name()).
-                feature(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_FEATURE).
-                field(MessageToEnrichmentCommandFunction.VALUE_FIELDS).
-                message(String.format(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_VALUE_FIELD_NOT_SET, TEST_ENRICHMENT_TYPE)).
-                build();
+        DataQualityMessage expectedMessage = DataQualityMessage.builder()
+                .level(DataQualityMessageLevel.ERROR.name())
+                .feature(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_FEATURE)
+                .field(MessageToEnrichmentCommandFunction.VALUE_FIELDS)
+                .message(String.format(MessageToEnrichmentCommandFunction.STREAMING_ENRICHMENT_VALUE_FIELD_NOT_SET, TEST_ENRICHMENT_TYPE))
+                .build();
         testError(messageExtensions,
                 Collections.singletonList(expectedMessage));
     }
@@ -204,12 +213,12 @@ private EnrichmentsConfig createEnrichmentsConfig(ArrayList keyFieldsNam
     private EnrichmentCommand createEnrichmentCommand(List keyValues, String delimiter, Map values) {
         String joiningDelimiter = delimiter != null ? delimiter : DEFAULT_KEY_DELIMITER;
         String enrichmentKey = String.join(joiningDelimiter, keyValues);
-        EnrichmentEntry enrichmentEntry = EnrichmentEntry.builder().ts(MessageUtils.getCurrentTimestamp()).
-                type(MessageToEnrichmentCommandFunctionTest.TEST_ENRICHMENT_TYPE).
-                key(enrichmentKey).entries(values).build();
-        return EnrichmentCommand.builder().
-                headers(Collections.emptyMap()).
-                type(CommandType.ADD).payload(enrichmentEntry).build();
+        EnrichmentEntry enrichmentEntry = EnrichmentEntry.builder().ts(MessageUtils.getCurrentTimestamp())
+                .type(MessageToEnrichmentCommandFunctionTest.TEST_ENRICHMENT_TYPE)
+                .key(enrichmentKey).entries(values).build();
+        return EnrichmentCommand.builder()
+                .headers(Collections.emptyMap())
+                .type(CommandType.ADD).payload(enrichmentEntry).build();
     }
 
     private OneInputStreamOperatorTestHarness createTestHarness(List streamingEnrichmentSources, EnrichmentsConfig streamingEnrichmentConfig) throws Exception {
diff --git a/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/TestNetflowBParser.java b/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/TestNetflowBParser.java
index 51010c28..9d77c43c 100644
--- a/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/TestNetflowBParser.java
+++ b/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/TestNetflowBParser.java
@@ -12,72 +12,74 @@
 
 package com.cloudera.cyber.parser;
 
+import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+
 import com.cloudera.cyber.Message;
 import com.cloudera.cyber.TestUtils;
 import com.google.common.io.Resources;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.test.util.JobTester;
-import org.junit.Test;
-
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.HashMap;
-
-import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.*;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.util.JobTester;
+import org.junit.Test;
 
 public class TestNetflowBParser extends AbstractParserJobTest {
-    private static final String config = "{\n" +
-            "    \"netflow\": {\n" +
-            "        \"id\": \"220ee8c5-07d7-48d9-8df5-7d23376cb664\",\n" +
-            "        \"name\": \"Netflow Parser\",\n" +
-            "        \"parsers\": [\n" +
-            "            {\n" +
-            "                \"id\": \"f812c6dc-40cc-4c77-abf8-e15fccdfea32\",\n" +
-            "                \"name\": \"Netflow as JSON\",\n" +
-            "                \"type\": \"com.cloudera.parserchains.parsers.JSONParser\",\n" +
-            "                \"config\": {\n" +
-            "                    \"input\": {\n" +
-            "                        \"input\": \"original_string\"\n" +
-            "                    },\n" +
-            "                    \"norm\": {\n" +
-            "                        \"norm\": \"UNFOLD_NESTED\"\n" +
-            "                    }\n" +
-            "                }\n" +
-            "            },\n" +
-            "            {\n" +
-            "                \"id\": \"6b8797a2-95df-4021-83c2-60ac4c786e67\",\n" +
-            "                \"name\": \"Field Renamer\",\n" +
-            "                \"type\": \"com.cloudera.parserchains.parsers.RenameFieldParser\",\n" +
-            "                \"config\": {\n" +
-            "                    \"fieldToRename\": [\n" +
-            "                        {\n" +
-            "                            \"from\": \"timestamp_end\",\n" +
-            "                            \"to\": \"timestamp\"\n" +
-            "                        }\n" +
-            "                    ]\n" +
-            "                }\n" +
-            "            },\n" +
-            "            {\n" +
-            "                \"id\": \"9549004f-83e4-4d24-8baa-abdbdad06e61\",\n" +
-            "                \"name\": \"Timestamp Parser\",\n" +
-            "                \"type\": \"com.cloudera.parserchains.parsers.TimestampFormatParser\",\n" +
-            "                \"config\": {\n" +
-            "                    \"fields\": [\n" +
-            "                        {\n" +
-            "                            \"field\": \"timestamp\",\n" +
-            "                            \"format\": \"yyyy-MM-dd HH:mm:ss.SSSSSS\",\n" +
-            "                            \"tz\": \"UTC\"\n" +
-            "                        }\n" +
-            "                    ]\n" +
-            "                }\n" +
-            "            }\n" +
-            "        ]\n" +
-            "    }\n" +
-            "}\n";
+    private static final String config = "{\n"
+            + "    \"netflow\": {\n"
+            + "        \"id\": \"220ee8c5-07d7-48d9-8df5-7d23376cb664\",\n"
+            + "        \"name\": \"Netflow Parser\",\n"
+            + "        \"parsers\": [\n"
+            + "            {\n"
+            + "                \"id\": \"f812c6dc-40cc-4c77-abf8-e15fccdfea32\",\n"
+            + "                \"name\": \"Netflow as JSON\",\n"
+            + "                \"type\": \"com.cloudera.parserchains.parsers.JSONParser\",\n"
+            + "                \"config\": {\n"
+            + "                    \"input\": {\n"
+            + "                        \"input\": \"original_string\"\n"
+            + "                    },\n"
+            + "                    \"norm\": {\n"
+            + "                        \"norm\": \"UNFOLD_NESTED\"\n"
+            + "                    }\n"
+            + "                }\n"
+            + "            },\n"
+            + "            {\n"
+            + "                \"id\": \"6b8797a2-95df-4021-83c2-60ac4c786e67\",\n"
+            + "                \"name\": \"Field Renamer\",\n"
+            + "                \"type\": \"com.cloudera.parserchains.parsers.RenameFieldParser\",\n"
+            + "                \"config\": {\n"
+            + "                    \"fieldToRename\": [\n"
+            + "                        {\n"
+            + "                            \"from\": \"timestamp_end\",\n"
+            + "                            \"to\": \"timestamp\"\n"
+            + "                        }\n"
+            + "                    ]\n"
+            + "                }\n"
+            + "            },\n"
+            + "            {\n"
+            + "                \"id\": \"9549004f-83e4-4d24-8baa-abdbdad06e61\",\n"
+            + "                \"name\": \"Timestamp Parser\",\n"
+            + "                \"type\": \"com.cloudera.parserchains.parsers.TimestampFormatParser\",\n"
+            + "                \"config\": {\n"
+            + "                    \"fields\": [\n"
+            + "                        {\n"
+            + "                            \"field\": \"timestamp\",\n"
+            + "                            \"format\": \"yyyy-MM-dd HH:mm:ss.SSSSSS\",\n"
+            + "                            \"tz\": \"UTC\"\n"
+            + "                        }\n"
+            + "                    ]\n"
+            + "                }\n"
+            + "            }\n"
+            + "        ]\n"
+            + "    }\n"
+            + "}\n";
 
 
     @Test
diff --git a/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/TestParserJobChainDirectory.java b/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/TestParserJobChainDirectory.java
index 1d7effad..48216b83 100644
--- a/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/TestParserJobChainDirectory.java
+++ b/flink-cyber/parser-chains-flink/src/test/java/com/cloudera/cyber/parser/TestParserJobChainDirectory.java
@@ -12,9 +12,20 @@
 
 package com.cloudera.cyber.parser;
 
+import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+
 import com.cloudera.cyber.Message;
 import com.cloudera.cyber.TestUtils;
 import com.google.common.io.Resources;
+import java.util.HashMap;
+import java.util.stream.Stream;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.utils.ParameterTool;
@@ -26,18 +37,6 @@
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
-import java.util.HashMap;
-import java.util.stream.Stream;
-
-import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.hasKey;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.notNullValue;
-
 public class TestParserJobChainDirectory extends AbstractParserJobTest {
 
     public static final String CHAIN_DIR_ERROR = "chain-dir/error";
@@ -134,8 +133,8 @@ public void testParserFail() throws Exception {
     @ParameterizedTest
     @MethodSource("mutuallyExclusiveParams")
     public void testParserMutuallyExclusiveConfig(ParameterTool params) throws Exception {
-        final String message = "It's not allowed to provide more than one chain param! " +
-                "Select one of the following: " + PARAM_CHAIN_CONFIG_EXCLUSIVE_LIST;
+        final String message = "It's not allowed to provide more than one chain param! "
+                + "Select one of the following: " + PARAM_CHAIN_CONFIG_EXCLUSIVE_LIST;
 
         try {
             createPipeline(params);
diff --git a/flink-cyber/pom.xml b/flink-cyber/pom.xml
index 023fa2c2..a9cfc0a7 100644
--- a/flink-cyber/pom.xml
+++ b/flink-cyber/pom.xml
@@ -109,6 +109,7 @@
         5.7.0
         1.5.1.${cdh.version}
         1.6.0
+        0.1.3
 
         
         3.6.1
@@ -128,7 +129,7 @@
         3.2.5
         3.3.1
         3.1.2
-        3.0.0
+        3.3.1
     
 
     
@@ -658,7 +659,7 @@
                 maven-checkstyle-plugin
                 ${maven-checkstyle-plugin.version}
                 
-                    google_checks.xml
+                    checkstyle.xml
                     true
                     warning
                     true
@@ -674,6 +675,33 @@
                     
                 
             
+            
+                org.ec4j.maven
+                editorconfig-maven-plugin
+                ${editorconfig-maven-plugin.version}
+                
+                    
+                        check
+                        validate
+                        
+                            check
+                        
+                    
+                
+                
+                    
+                        src/main/resources/**
+                        src/test/resources/**
+                        src/test/classpath-resources/**
+                        **/*.jj
+                        **/node_modules/**
+                        **/.angular/**
+                        **/node/**
+                        **/dist/**
+                        **/target/**
+                    
+                
+            
         
         
             

From acb5448fd6f5eac5aa67960bb535276092152220 Mon Sep 17 00:00:00 2001
From: Stas Panasiuk 
Date: Thu, 24 Oct 2024 01:48:05 -0400
Subject: [PATCH 09/15] compilation errors fixed

---
 .github/workflows/build_and_test.yml                            | 2 +-
 .../src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java | 2 +-
 .../cyber/profiler/FieldValueProfileAggregateFunction.java      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 4d77151f..729e50f7 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -28,7 +28,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Check codestyle
-        run: mvn -B validate --file flink-cyber/pom.xml
+        run: mvn -B compile -T 1C --file flink-cyber/pom.xml
       - name: Build and Test with Maven
         run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B verify --file flink-cyber/pom.xml
 
diff --git a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java
index 8d3346fd..62804682 100644
--- a/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java
+++ b/flink-cyber/flink-dedupe/src/main/java/com/cloudera/cyber/dedupe/DedupeJobKafka.java
@@ -69,7 +69,7 @@ protected DataStream createSource(StreamExecutionEnvironment env, Param
 
         WatermarkStrategy watermarkStrategy = WatermarkStrategy.forBoundedOutOfOrderness(
                                                                               Duration.ofMillis(params.getLong(PARAM_DEDUPE_LATENESS, 0L)))
-                                                                        .thTimestampAssigner(
+                                                                        .withTimestampAssigner(
                                                                               (event, timestamp) -> event.getTs());
         return env.fromSource(createKafkaSource(inputTopic,
                         params,
diff --git a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FieldValueProfileAggregateFunction.java b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FieldValueProfileAggregateFunction.java
index c98aefe8..40137d08 100644
--- a/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FieldValueProfileAggregateFunction.java
+++ b/flink-cyber/flink-profiler-java/src/main/java/com/cloudera/cyber/profiler/FieldValueProfileAggregateFunction.java
@@ -32,7 +32,7 @@ public ProfileGroupAcc createAccumulator() {
     @Override
     protected Map getMeasurementFormats() {
         return profileGroupConfig.getMeasurements().stream()
-                                 .llect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName,
+                                 .collect(Collectors.toMap(ProfileMeasurementConfig::getResultExtensionName,
                                        ProfileMeasurementConfig::getDecimalFormat));
     }
 }

From 929768da87ba8e73fe0862c7985d3e487512e71e Mon Sep 17 00:00:00 2001
From: Stas Panasiuk 
Date: Thu, 24 Oct 2024 01:54:09 -0400
Subject: [PATCH 10/15] disabled compilation multithreading

---
 .github/workflows/build_and_test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 729e50f7..d16c1174 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -28,7 +28,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Check codestyle
-        run: mvn -B compile -T 1C --file flink-cyber/pom.xml
+        run: mvn -B compile --file flink-cyber/pom.xml
       - name: Build and Test with Maven
         run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B verify --file flink-cyber/pom.xml
 

From 09af6e4c301c8548581ac217830086a87b6fe919 Mon Sep 17 00:00:00 2001
From: Stas Panasiuk 
Date: Thu, 24 Oct 2024 01:57:53 -0400
Subject: [PATCH 11/15] switched from compilation to checkstyle only

---
 .github/workflows/build_and_test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index d16c1174..4d77151f 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -28,7 +28,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Check codestyle
-        run: mvn -B compile --file flink-cyber/pom.xml
+        run: mvn -B validate --file flink-cyber/pom.xml
       - name: Build and Test with Maven
         run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B verify --file flink-cyber/pom.xml
 

From b51ac4719a19b899f2e52e706d8a1f85a84c7e15 Mon Sep 17 00:00:00 2001
From: Stas Panasiuk 
Date: Sun, 27 Oct 2024 23:37:27 -0400
Subject: [PATCH 12/15] tests fix

---
 .../com/cloudera/cyber/caracal/SplitJob.java  | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java
index 13cca1eb..98f00a92 100644
--- a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java
+++ b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java
@@ -79,14 +79,18 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws
         SingleOutputStreamOperator parsed = results.map(new ParserChainMapFunction(configMap));
 
 
-        DataStream> counts = parsed
-              .map((MapFunction>) message -> Tuple2.of(message.getSource(), 1L))
-              .keyBy(0)
-              .timeWindow(Time.milliseconds(
-                    params.getLong(PARAM_COUNT_INTERVAL,
-                          DEFAULT_COUNT_INTERVAL)))
-              .allowedLateness(Time.milliseconds(0))
-              .sum(1);
+        DataStream> counts = parsed.map(new MapFunction>() {
+                                                            @Override
+                                                            public Tuple2 map(Message message) throws Exception {
+                                                                return Tuple2.of(message.getSource(), 1L);
+                                                            }
+                                                        })
+                                                        .keyBy(0)
+                                                        .timeWindow(Time.milliseconds(
+                                                              params.getLong(PARAM_COUNT_INTERVAL,
+                                                                    DEFAULT_COUNT_INTERVAL)))
+                                                        .allowedLateness(Time.milliseconds(0))
+                                                        .sum(1);
         writeCounts(params, counts);
 
 

From 52c17ce658e9470a2038ce5e703004b1f092bc48 Mon Sep 17 00:00:00 2001
From: Stas Panasiuk 
Date: Mon, 28 Oct 2024 00:03:26 -0400
Subject: [PATCH 13/15] checkstyle fix

---
 .../com/cloudera/cyber/caracal/SplitJob.java  | 25 ++++++++++---------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java
index 98f00a92..c5467a46 100644
--- a/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java
+++ b/flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitJob.java
@@ -79,18 +79,19 @@ protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws
         SingleOutputStreamOperator parsed = results.map(new ParserChainMapFunction(configMap));
 
 
-        DataStream> counts = parsed.map(new MapFunction>() {
-                                                            @Override
-                                                            public Tuple2 map(Message message) throws Exception {
-                                                                return Tuple2.of(message.getSource(), 1L);
-                                                            }
-                                                        })
-                                                        .keyBy(0)
-                                                        .timeWindow(Time.milliseconds(
-                                                              params.getLong(PARAM_COUNT_INTERVAL,
-                                                                    DEFAULT_COUNT_INTERVAL)))
-                                                        .allowedLateness(Time.milliseconds(0))
-                                                        .sum(1);
+        DataStream> counts =
+              parsed.map(new MapFunction>() {
+                        @Override
+                        public Tuple2 map(Message message) throws Exception {
+                            return Tuple2.of(message.getSource(), 1L);
+                        }
+                    })
+                    .keyBy(0)
+                    .timeWindow(Time.milliseconds(
+                          params.getLong(PARAM_COUNT_INTERVAL,
+                                DEFAULT_COUNT_INTERVAL)))
+                    .allowedLateness(Time.milliseconds(0))
+                    .sum(1);
         writeCounts(params, counts);
 
 

From 5d5fee469722a6578745d238723b104ba90e2f7e Mon Sep 17 00:00:00 2001
From: Stas Panasiuk 
Date: Tue, 29 Oct 2024 09:07:49 -0400
Subject: [PATCH 14/15] Hidden objects with unknown types

---
 .../indexing/hive/util/FlinkSchemaUtil.java   |  18 +-
 .../config/kafka/KafkaConfig.java             |   9 +-
 .../describe/OcsfIndexMappingDescriptor.java  |   3 +-
 .../queryservice/service/OcsfService.java     |   7 +-
 .../core/ReflectiveParserBuilder.java         |   4 +-
 .../core/catalog/Configurable.java            |   1 +
 .../parserchains/core/catalog/Parameter.java  |   1 +
 .../core/model/define/ParserSchema.java       | 293 +++++++++---------
 .../parsers/SimpleStellarParser.java          |  38 +--
 .../parserchains/parsers/StellarParser.java   |  44 +--
 10 files changed, 210 insertions(+), 208 deletions(-)

diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java
index 8eb8c331..297ee35a 100644
--- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java
+++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java
@@ -1,29 +1,29 @@
 package com.cloudera.cyber.indexing.hive.util;
 
 import com.cloudera.cyber.indexing.TableColumnDto;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
 import org.apache.flink.table.api.DataTypes;
 import org.apache.flink.table.api.Schema;
 import org.apache.flink.table.catalog.Column;
 import org.apache.flink.table.catalog.ResolvedSchema;
 import org.apache.flink.table.types.DataType;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
 public final class FlinkSchemaUtil {
 
     public static Schema buildSchema(ResolvedSchema resolvedSchema) {
         return Schema.newBuilder()
-              .fromResolvedSchema(resolvedSchema)
-              .build();
+                     .fromResolvedSchema(resolvedSchema)
+                     .build();
     }
 
     public static ResolvedSchema getResolvedSchema(List columnList) {
         final List flinkColumnList = columnList.stream()
-              .map(col -> Column.physical(col.getName(), getFlinkType(col.getType(), col.getNullable())))
-              .collect(Collectors.toList());
+                                                       .map(col -> Column.physical(col.getName(),
+                                                             getFlinkType(col.getType(), col.getNullable())))
+                                                       .collect(Collectors.toList());
         return ResolvedSchema.of(flinkColumnList);
     }
 
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java
index c7075965..15a027c5 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java
@@ -40,7 +40,7 @@
 public class KafkaConfig {
 
     /**
-     * Provides the default kafka properties for the consumers
+     * Provides the default kafka properties for the consumers.
      *
      * @return Default kafka properties for the consumers
      */
@@ -50,10 +50,11 @@ public ClouderaKafkaProperties kafkaProperties() {
         return new ClouderaKafkaProperties();
     }
 
-    /***
-     * Provides a map with key=clusterId and value=ClouderaKafkaProperties
+    /**
+     * Provides a map with key=clusterId and value=ClouderaKafkaProperties.
      *
-     * @return Map with key of clusterId and value - kafkaProperties. This map is a mapping between clusterId and connection details for that cluster
+     * @return Map with key of clusterId and value - kafkaProperties.
+     *       This map is a mapping between clusterId and connection details for that cluster
      */
     @Bean(name = "kafka-external-cluster-map")
     @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/OcsfIndexMappingDescriptor.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/OcsfIndexMappingDescriptor.java
index acd2ddad..863df66d 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/OcsfIndexMappingDescriptor.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/OcsfIndexMappingDescriptor.java
@@ -1,11 +1,10 @@
 package com.cloudera.parserchains.queryservice.model.describe;
 
 import com.cloudera.cyber.indexing.MappingDto;
+import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
 
-import java.util.Map;
-
 @Getter
 @Setter
 public class OcsfIndexMappingDescriptor extends IndexMappingDescriptor {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/OcsfService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/OcsfService.java
index 1cf1cd0f..eac07af1 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/OcsfService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/OcsfService.java
@@ -1,10 +1,6 @@
 package com.cloudera.parserchains.queryservice.service;
 
 
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -14,6 +10,9 @@
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Collections;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
 
 @Slf4j
 @Service
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ReflectiveParserBuilder.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ReflectiveParserBuilder.java
index a556b386..1629c11b 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ReflectiveParserBuilder.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/ReflectiveParserBuilder.java
@@ -110,8 +110,8 @@ private void configureParams(ParserSchema parserSchema, List
                     throw new InvalidParserException(parserSchema,
                           String.format("Required parameter isn't provided: %s", annotationKey));
                 }
-                if (paramAnnotation.isPath() && parserSchema.getBasePath() != null &&
-                    !parserSchema.getBasePath().equals("null")) {
+                if (paramAnnotation.isPath() && parserSchema.getBasePath() != null
+                        && !parserSchema.getBasePath().equals("null")) {
                     finalValue = Paths.get(parserSchema.getBasePath(), finalValue).toString();
                 }
                 valueMap.put(annotationKey, finalValue);
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Configurable.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Configurable.java
index b2945928..1849e114 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Configurable.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Configurable.java
@@ -79,6 +79,7 @@
 
     /**
      * If true, the value will be treated as path and can be appended with base directory.
+     *
      * 
      This value is optional.
      */
     boolean isPath() default false;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Parameter.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Parameter.java
index d40ddf0f..96273056 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Parameter.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/catalog/Parameter.java
@@ -78,6 +78,7 @@
 
     /**
      * If true, the value will be treated as path and can be appended with base directory.
+     *
      * 
      This value is optional.
      */
     boolean isPath() default false;
diff --git a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserSchema.java b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserSchema.java
index 9b21d29b..4dceeaed 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserSchema.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-core/src/main/java/com/cloudera/parserchains/core/model/define/ParserSchema.java
@@ -21,14 +21,13 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 /**
  * Describes the structure of one parser within a {@link ParserChainSchema}.
@@ -36,150 +35,150 @@
 @JsonPropertyOrder({"id", "name", "type", "config"})
 public class ParserSchema implements Serializable {
 
-  private static final long serialVersionUID = 1L;
-
-  /**
-   * A label given to this parser that is generated by the front-end.
-   *
-   * 
      This is only used by the front-end and allows errors encountered
-   * in the Live View to be mapped to the specific parser that failed.
-   */
-  @JsonProperty("id")
-  private String label;
-
-  /**
-   * The id of the parser.
-   *
-   * 
      This id should be unique amongst all of the available parsers.  This is
-   * derived from the name of the class implementing the parser.
-   *
-   * 
      The front-end refers to this as the "type" of parser, which might be a
-   * better name than id.
-   */
-  @JsonProperty("type")
-  private ParserID id;
-
-  /**
-   * A name given to this parser by the user.
-   */
-  @JsonProperty("name")
-  private ParserName name;
-
-  /**
-   * Defines how the user has configured this parser.
-   */
-  @JsonProperty("config")
-  private Map> config;
-
-  /**
-   * If the user has created a router, this describes the routing features
-   * as configured by the user.
-   */
-  @JsonProperty("routing")
-  private RoutingSchema routing;
-  /**
-   * The base path for any path-related parser configs.
-   */
-  @JsonIgnore
-  private String basePath;
-
-  public ParserSchema() {
-    config = new HashMap<>();
-  }
-
-  public ParserID getId() {
-    return id;
-  }
-
-  public ParserSchema setId(ParserID id) {
-    this.id = id;
-    return this;
-  }
-
-  public ParserName getName() {
-    return name;
-  }
-
-  public ParserSchema setName(ParserName name) {
-    this.name = name;
-    return this;
-  }
-
-  public String getLabel() {
-    return label;
-  }
-
-  public ParserSchema setLabel(String label) {
-    this.label = label;
-    return this;
-  }
-
-  public Map> getConfig() {
-    return config;
-  }
-
-  public ParserSchema setConfig(Map> config) {
-    this.config = config;
-    return this;
-  }
-
-  public ParserSchema addConfig(String key, ConfigValueSchema value) {
-    List values;
-    if(config.containsKey(key)) {
-      values = config.get(key);
-    } else {
-      values = new ArrayList<>();
-      config.put(key, values);
-    }
-    values.add(value);
-    return this;
-  }
-
-  public RoutingSchema getRouting() {
-    return routing;
-  }
-
-  public ParserSchema setRouting(RoutingSchema routing) {
-    this.routing = routing;
-    return this;
-  }
-
-  public String getBasePath() {
-    return basePath;
-  }
-
-  public void setBasePath(String basePath) {
-    this.basePath = basePath;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-    ParserSchema that = (ParserSchema) o;
-    return new EqualsBuilder()
-            .append(label, that.label)
-            .append(id, that.id)
-            .append(name, that.name)
-            .append(config, that.config)
-            .append(routing, that.routing)
-            .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(17, 37)
-            .append(label)
-            .append(id)
-            .append(name)
-            .append(config)
-            .append(routing)
-            .toHashCode();
-  }
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * A label given to this parser that is generated by the front-end.
+     *
+     * 
      This is only used by the front-end and allows errors encountered
+     * in the Live View to be mapped to the specific parser that failed.
+     */
+    @JsonProperty("id")
+    private String label;
+
+    /**
+     * The id of the parser.
+     *
+     * 
      This id should be unique amongst all of the available parsers.  This is
+     * derived from the name of the class implementing the parser.
+     *
+     * 
      The front-end refers to this as the "type" of parser, which might be a
+     * better name than id.
+     */
+    @JsonProperty("type")
+    private ParserID id;
+
+    /**
+     * A name given to this parser by the user.
+     */
+    @JsonProperty("name")
+    private ParserName name;
+
+    /**
+     * Defines how the user has configured this parser.
+     */
+    @JsonProperty("config")
+    private Map> config;
+
+    /**
+     * If the user has created a router, this describes the routing features
+     * as configured by the user.
+     */
+    @JsonProperty("routing")
+    private RoutingSchema routing;
+    /**
+     * The base path for any path-related parser configs.
+     */
+    @JsonIgnore
+    private String basePath;
+
+    public ParserSchema() {
+        config = new HashMap<>();
+    }
+
+    public ParserID getId() {
+        return id;
+    }
+
+    public ParserSchema setId(ParserID id) {
+        this.id = id;
+        return this;
+    }
+
+    public ParserName getName() {
+        return name;
+    }
+
+    public ParserSchema setName(ParserName name) {
+        this.name = name;
+        return this;
+    }
+
+    public String getLabel() {
+        return label;
+    }
+
+    public ParserSchema setLabel(String label) {
+        this.label = label;
+        return this;
+    }
+
+    public Map> getConfig() {
+        return config;
+    }
+
+    public ParserSchema setConfig(Map> config) {
+        this.config = config;
+        return this;
+    }
+
+    public ParserSchema addConfig(String key, ConfigValueSchema value) {
+        List values;
+        if (config.containsKey(key)) {
+            values = config.get(key);
+        } else {
+            values = new ArrayList<>();
+            config.put(key, values);
+        }
+        values.add(value);
+        return this;
+    }
+
+    public RoutingSchema getRouting() {
+        return routing;
+    }
+
+    public ParserSchema setRouting(RoutingSchema routing) {
+        this.routing = routing;
+        return this;
+    }
+
+    public String getBasePath() {
+        return basePath;
+    }
+
+    public void setBasePath(String basePath) {
+        this.basePath = basePath;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        ParserSchema that = (ParserSchema) o;
+        return new EqualsBuilder()
+              .append(label, that.label)
+              .append(id, that.id)
+              .append(name, that.name)
+              .append(config, that.config)
+              .append(routing, that.routing)
+              .isEquals();
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder(17, 37)
+              .append(label)
+              .append(id)
+              .append(name)
+              .append(config)
+              .append(routing)
+              .toHashCode();
+    }
 
     @Override
     public String toString() {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SimpleStellarParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SimpleStellarParser.java
index dd70fe73..f29bb2a8 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SimpleStellarParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SimpleStellarParser.java
@@ -6,6 +6,14 @@
 import com.cloudera.parserchains.core.catalog.MessageParser;
 import com.cloudera.parserchains.core.catalog.Parameter;
 import com.cloudera.parserchains.core.utils.StringUtils;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
 import org.apache.flink.core.fs.FSDataInputStream;
@@ -18,18 +26,9 @@
 import org.apache.metron.stellar.dsl.MapVariableResolver;
 import org.json.simple.JSONObject;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
 @MessageParser(
-        name = "Simple Stellar parser",
-        description = "Metron compatibility parser.")
+      name = "Simple Stellar parser",
+      description = "Metron compatibility parser.")
 @Slf4j
 public class SimpleStellarParser implements Parser {
 
@@ -45,11 +44,12 @@ public SimpleStellarParser() {
     }
 
     @Configurable(
-            key = "stellarPath",
-            label = "Stellar File Path",
-            description = "Path to stellar file",
-            required = true)
-    public SimpleStellarParser stellarPath(@Parameter(key = "stellarPath", isPath = true) String pathToStellar) throws IOException {
+          key = "stellarPath",
+          label = "Stellar File Path",
+          description = "Path to stellar file",
+          required = true)
+    public SimpleStellarParser stellarPath(@Parameter(key = "stellarPath", isPath = true) String pathToStellar)
+          throws IOException {
         FileSystem fileSystem = new Path(pathToStellar).getFileSystem();
         loadExpressions(pathToStellar, fileSystem);
         return this;
@@ -79,14 +79,16 @@ private void updateExpressionList(List expressionList) {
     public Message parse(Message input) {
         Message.Builder builder = Message.builder().withFields(input);
         final Map fieldMap = input.getFields().entrySet().stream()
-                .collect(Collectors.toMap(e -> e.getKey().get(), e -> StringUtils.parseProperType(e.getValue().get())));
+                                                  .collect(Collectors.toMap(e -> e.getKey().get(),
+                                                        e -> StringUtils.parseProperType(e.getValue().get())));
         return doParse(fieldMap, builder);
     }
 
     private Message doParse(Map toParse, Message.Builder output) {
         final MapVariableResolver resolver = new MapVariableResolver(toParse);
         try {
-            final JSONObject result = StellarAdapter.process(toParse, configHandler, "", 1000L, processor, resolver, stellarContext);
+            final JSONObject result =
+                  StellarAdapter.process(toParse, configHandler, "", 1000L, processor, resolver, stellarContext);
             result.forEach((key, value) -> output.addField(String.valueOf(key), String.valueOf(value)));
         } catch (Exception e) {
             output.withError("Parser did not return a message result").build();
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java
index bf7b996c..031efefe 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java
@@ -12,6 +12,8 @@
 
 package com.cloudera.parserchains.parsers;
 
+import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
+
 import com.cloudera.cyber.parser.MessageToParse;
 import com.cloudera.cyber.stellar.MetronCompatibilityParser;
 import com.cloudera.parserchains.core.FieldName;
@@ -21,6 +23,11 @@
 import com.cloudera.parserchains.core.catalog.Configurable;
 import com.cloudera.parserchains.core.catalog.MessageParser;
 import com.cloudera.parserchains.core.catalog.Parameter;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -30,17 +37,9 @@
 import org.apache.metron.parsers.interfaces.MessageParserResult;
 import org.json.simple.JSONObject;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
-
 @MessageParser(
-        name = "Metron Stellar parser",
-        description = "Metron compatibility parser.")
+      name = "Metron Stellar parser",
+      description = "Metron compatibility parser.")
 @Slf4j
 public class StellarParser implements Parser {
 
@@ -53,11 +52,11 @@ public StellarParser() {
     }
 
     @Configurable(
-            key = "input",
-            label = "Input Field",
-            description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
-            defaultValue = DEFAULT_INPUT_FIELD,
-            isOutputName = true)
+          key = "input",
+          label = "Input Field",
+          description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
+          defaultValue = DEFAULT_INPUT_FIELD,
+          isOutputName = true)
     public StellarParser inputField(String fieldName) {
         if (StringUtils.isNotBlank(fieldName)) {
             this.inputField = FieldName.of(fieldName);
@@ -66,11 +65,12 @@ public StellarParser inputField(String fieldName) {
     }
 
     @Configurable(
-            key = "configurationPath",
-            label = "Configuration File Path",
-            description = "Path to parser config file",
-            required = true)
-    public StellarParser configurationPath(@Parameter(key = "configurationPath", isPath = true) String pathToSchema) throws IOException {
+          key = "configurationPath",
+          label = "Configuration File Path",
+          description = "Path to parser config file",
+          required = true)
+    public StellarParser configurationPath(@Parameter(key = "configurationPath", isPath = true) String pathToSchema)
+          throws IOException {
         FileSystem fileSystem = new Path(pathToSchema).getFileSystem();
         loadParser(pathToSchema, fileSystem);
         return this;
@@ -96,8 +96,8 @@ public Message parse(Message input) {
             return doParse(field.get(), builder);
         } else {
             return builder
-                    .withError(String.format("Message missing expected input field '%s'", inputField.toString()))
-                    .build();
+                  .withError(String.format("Message missing expected input field '%s'", inputField.toString()))
+                  .build();
         }
     }
 

From 0b140f445dabe8f32e611fa0cd313dde0979d7b1 Mon Sep 17 00:00:00 2001
From: Stas Panasiuk 
Date: Mon, 4 Nov 2024 23:18:59 -0500
Subject: [PATCH 15/15] Codestyle updates after main merge

---
 .../src/main/assemblies/cloudera.xml          | 332 +++++++++---------
 .../service/common/request/RequestBody.java   |   3 +-
 .../service/common/request/RequestType.java   |  10 +-
 .../service/common/response/ResponseType.java |  11 +-
 .../controller/KafkaListenerController.java   |  73 ++--
 .../restcli/service/FilePipelineService.java  |  41 ++-
 .../cyber/restcli/service/JobService.java     |  83 +++--
 .../queryservice/Application.java             |   4 +-
 .../queryservice/StartupComponent.java        |  37 +-
 .../common/ApplicationConstants.java          |   4 +-
 .../common/exception/ValidationException.java |  17 +-
 .../common/utils/CollectionsUtils.java        |   4 +-
 .../queryservice/common/utils/Utils.java      |  20 +-
 .../queryservice/config/AppProperties.java    |   4 +-
 .../config/kafka/KafkaConfig.java             | 207 +++++------
 .../controller/ChainController.java           | 134 +++----
 .../controller/ClusterController.java         |  50 +--
 .../controller/ParserController.java          |  31 +-
 .../controller/ParserSampleController.java    |  62 ++--
 .../controller/PipelineController.java        |  10 +-
 ...ocumentationPluginsManagerBootAdapter.java |   4 +-
 .../hack/TypeNameExtractorBootAdapter.java    |   4 +-
 .../model/describe/ParserDescriptor.java      |   4 +-
 .../queryservice/service/ClusterService.java  |  93 ++---
 .../queryservice/service/KafkaService.java    |  53 +--
 .../service/KafkaServiceInterface.java        |   3 +-
 .../service/MockKafkaService.java             | 164 ++++-----
 .../impl/DefaultParserDiscoveryService.java   |   4 +-
 28 files changed, 772 insertions(+), 694 deletions(-)

diff --git a/flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml b/flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml
index 6ea1bad8..99419fa7 100644
--- a/flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml
+++ b/flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml
@@ -20,170 +20,170 @@
 	
-	cloudera
-	
-		tar.gz
-	
-
-	true
-	${cloudera.tar.name}
-
-	
-		
-			lib
-			false
-			false
-			false
-			true
-			true
-			
-				org.slf4j:slf4j-api
-				org.apache.logging.log4j:log4j-api
-				org.apache.logging.log4j:log4j-core
-				org.apache.logging.log4j:log4j-slf4j-impl
-			
-		
-	
-
-	
-		
-			../caracal-generator/target/caracal-generator-${project.version}.jar
-			jobs/
-			caracal-generator-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../flink-enrichment/flink-enrichment-combined/target/flink-enrichment-combined-${project.version}.jar
-			jobs/
-			flink-enrichment-combined-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../flink-enrichment/flink-enrichment-load/target/flink-enrichment-loading-${project.version}.jar
-			jobs/
-			flink-enrichment-loading-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../flink-profiler-java/target/flink-profiler-java-${project.version}.jar
-			jobs/
-			flink-profiler-java-${cybersec.full.version}.jar
-			0644
-		
-		
-			../parser-chains-flink/target/parser-chains-flink-${project.version}.jar
-			jobs/
-			parser-chains-flink-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../flink-indexing/flink-indexing-hive/target/flink-indexing-hive-${project.version}.jar
-			jobs/
-			flink-indexing-hive-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../cyber-services/cyber-worker-service/target/cyber-worker-service-${project.version}.jar
-			tools/
-			cyber-worker-service-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../flink-commands/scoring-commands/target/scoring-commands-${project.version}.jar
-			tools/
-			scoring-commands-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../flink-commands/scoring-commands/README.md
-			tools/
-			scoring-commands-README.md
-			0644
-		
-
-		
-			../flink-commands/kafka-commands/target/kafka-commands-${project.version}.jar
-			tools/
-			kafka-commands-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../flink-commands/json-commands/target/json-commands-${project.version}.jar
-			tools/
-			json-commands-${cybersec.full.version}.jar
-			0644
-		
-
-		
-			../metron-parser-chain/parser-chains-config-service/target/parser-chains-config-service-${project.version}.jar
-			tools/
-			parser-chains-config-service-${project.version}.jar
-			0644
-		
-
-	
-
-	
-		
-			src/main/resources/conf
-			conf
-			0644
-			
-				*.json
-				log4j.properties
-			
-		
-		
-			src/main/resources/conf/templates
-			conf/templates
-			0644
-		
-		
-			src/main/resources/conf/templates/generate
-			conf/templates/generate
-			0644
-		
-		
-			src/main/resources/conf/templates/index
-			conf/templates/index
-			0644
-		
-		
-			src/main/resources/conf/templates/parse
-			conf/templates/parse
-			0644
-		
-		
-			src/main/resources/conf/templates/profile
-			conf/templates/profile
-			0644
-		
-		
-			src/main/resources/conf/templates/triage
-			conf/templates/triage
-			0644
-		
-		
-			target/classes/scripts
-			bin
-			0755
-		
-		
-			src/main/resources/conf
-			tools
-			0644
-			
-				sample-rule.json
-				log4j.properties
-			
-		
-	
+    cloudera
+    
+        tar.gz
+    
+
+    true
+    ${cloudera.tar.name}
+
+    
+        
+            lib
+            false
+            false
+            false
+            true
+            true
+            
+                org.slf4j:slf4j-api
+                org.apache.logging.log4j:log4j-api
+                org.apache.logging.log4j:log4j-core
+                org.apache.logging.log4j:log4j-slf4j-impl
+            
+        
+    
+
+    
+        
+            ../caracal-generator/target/caracal-generator-${project.version}.jar
+            jobs/
+            caracal-generator-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../flink-enrichment/flink-enrichment-combined/target/flink-enrichment-combined-${project.version}.jar
+            jobs/
+            flink-enrichment-combined-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../flink-enrichment/flink-enrichment-load/target/flink-enrichment-loading-${project.version}.jar
+            jobs/
+            flink-enrichment-loading-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../flink-profiler-java/target/flink-profiler-java-${project.version}.jar
+            jobs/
+            flink-profiler-java-${cybersec.full.version}.jar
+            0644
+        
+        
+            ../parser-chains-flink/target/parser-chains-flink-${project.version}.jar
+            jobs/
+            parser-chains-flink-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../flink-indexing/flink-indexing-hive/target/flink-indexing-hive-${project.version}.jar
+            jobs/
+            flink-indexing-hive-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../cyber-services/cyber-worker-service/target/cyber-worker-service-${project.version}.jar
+            tools/
+            cyber-worker-service-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../flink-commands/scoring-commands/target/scoring-commands-${project.version}.jar
+            tools/
+            scoring-commands-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../flink-commands/scoring-commands/README.md
+            tools/
+            scoring-commands-README.md
+            0644
+        
+
+        
+            ../flink-commands/kafka-commands/target/kafka-commands-${project.version}.jar
+            tools/
+            kafka-commands-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../flink-commands/json-commands/target/json-commands-${project.version}.jar
+            tools/
+            json-commands-${cybersec.full.version}.jar
+            0644
+        
+
+        
+            ../metron-parser-chain/parser-chains-config-service/target/parser-chains-config-service-${project.version}.jar
+            tools/
+            parser-chains-config-service-${project.version}.jar
+            0644
+        
+
+    
+
+    
+        
+            src/main/resources/conf
+            conf
+            0644
+            
+                *.json
+                log4j.properties
+            
+        
+        
+            src/main/resources/conf/templates
+            conf/templates
+            0644
+        
+        
+            src/main/resources/conf/templates/generate
+            conf/templates/generate
+            0644
+        
+        
+            src/main/resources/conf/templates/index
+            conf/templates/index
+            0644
+        
+        
+            src/main/resources/conf/templates/parse
+            conf/templates/parse
+            0644
+        
+        
+            src/main/resources/conf/templates/profile
+            conf/templates/profile
+            0644
+        
+        
+            src/main/resources/conf/templates/triage
+            conf/templates/triage
+            0644
+        
+        
+            target/classes/scripts
+            bin
+            0755
+        
+        
+            src/main/resources/conf
+            tools
+            0644
+            
+                sample-rule.json
+                log4j.properties
+            
+        
+    
 
diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestBody.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestBody.java
index 1a954807..c3632642 100644
--- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestBody.java
+++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestBody.java
@@ -1,12 +1,11 @@
 package com.cloudera.service.common.request;
 
+import java.util.List;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.List;
-
 @Data
 @Builder
 @NoArgsConstructor
diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java
index 424066ac..6154252e 100644
--- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java
+++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java
@@ -1,5 +1,13 @@
 package com.cloudera.service.common.request;
 
 public enum RequestType {
-    GET_ALL_CLUSTERS_SERVICE_REQUEST, GET_CLUSTER_SERVICE_REQUEST, START_JOB_REQUEST, RESTART_JOB_REQUEST, STOP_JOB_REQUEST, GET_JOB_CONFIG_REQUEST, CREATE_EMPTY_PIPELINE, START_ARCHIVE_PIPELINE, UPDATE_JOB_CONFIG_REQUEST
+    GET_ALL_CLUSTERS_SERVICE_REQUEST,
+    GET_CLUSTER_SERVICE_REQUEST,
+    START_JOB_REQUEST,
+    RESTART_JOB_REQUEST,
+    STOP_JOB_REQUEST,
+    GET_JOB_CONFIG_REQUEST,
+    CREATE_EMPTY_PIPELINE,
+    START_ARCHIVE_PIPELINE,
+    UPDATE_JOB_CONFIG_REQUEST
 }
diff --git a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java
index c159f4f3..f4ac3c6f 100644
--- a/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java
+++ b/flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java
@@ -1,5 +1,14 @@
 package com.cloudera.service.common.response;
 
 public enum ResponseType {
-    GET_ALL_CLUSTERS_SERVICE_RESPONSE, GET_CLUSTER_SERVICE_RESPONSE, START_JOB_RESPONSE, RESTART_JOB_RESPONSE, STOP_JOB_RESPONSE, GET_JOB_CONFIG_RESPONSE, UPDATE_JOB_CONFIG_RESPONSE, CREATE_EMPTY_PIPELINE_RESPONSE, START_ARCHIVE_PIPELINE_RESPONSE, ERROR_RESPONSE
+    GET_ALL_CLUSTERS_SERVICE_RESPONSE,
+    GET_CLUSTER_SERVICE_RESPONSE,
+    START_JOB_RESPONSE,
+    RESTART_JOB_RESPONSE,
+    STOP_JOB_RESPONSE,
+    GET_JOB_CONFIG_RESPONSE,
+    UPDATE_JOB_CONFIG_RESPONSE,
+    CREATE_EMPTY_PIPELINE_RESPONSE,
+    START_ARCHIVE_PIPELINE_RESPONSE,
+    ERROR_RESPONSE
 }
diff --git a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java
index b85a6135..57399435 100644
--- a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java
+++ b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java
@@ -1,8 +1,8 @@
 package com.cloudera.cyber.restcli.controller;
 
 import com.cloudera.cyber.restcli.configuration.AppWorkerConfig;
-import com.cloudera.cyber.restcli.service.JobService;
 import com.cloudera.cyber.restcli.service.FilePipelineService;
+import com.cloudera.cyber.restcli.service.JobService;
 import com.cloudera.service.common.Utils;
 import com.cloudera.service.common.request.RequestBody;
 import com.cloudera.service.common.request.RequestType;
@@ -10,9 +10,11 @@
 import com.cloudera.service.common.response.Job;
 import com.cloudera.service.common.response.ResponseBody;
 import com.cloudera.service.common.response.ResponseType;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.kafka.annotation.KafkaListener;
 import org.springframework.kafka.support.KafkaHeaders;
 import org.springframework.messaging.Message;
@@ -23,10 +25,6 @@
 import org.springframework.messaging.support.MessageHeaderAccessor;
 import org.springframework.stereotype.Component;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
 @Component
 @RequiredArgsConstructor
 @Slf4j
@@ -38,7 +36,9 @@ public class KafkaListenerController {
     //TODO:  Rewrite to Spring events. Probably split the events into separate types, such as cluster event, job event, pipeline event, etc.
     @KafkaListener(topics = "#{kafkaProperties.getRequestTopic()}", containerFactory = "kafkaListenerContainerFactory")
     @SendTo({"#{kafkaProperties.getReplyTopic()}"})
-    public Message handleMessage(RequestBody requestBody, @Header(KafkaHeaders.RECEIVED_MESSAGE_KEY) String key, @Header(KafkaHeaders.REPLY_TOPIC) byte[] replyTo,
+    public Message handleMessage(RequestBody requestBody,
+                                               @Header(KafkaHeaders.RECEIVED_MESSAGE_KEY) String key,
+                                               @Header(KafkaHeaders.REPLY_TOPIC) byte[] replyTo,
                                                @Header(KafkaHeaders.CORRELATION_ID) byte[] correlationId) {
         log.info("Start processing message\n Message key: '{}' \n value: '{}'", key, requestBody);
 
@@ -54,9 +54,10 @@ public Message handleMessage(RequestBody requestBody, @Header(Kafk
                 try {
                     Job job = jobService.restartJob(requestBody.getJobIdHex());
                     ResponseBody responseBody = ResponseBody.builder()
-                            .jobs(Collections.singletonList(job))
-                            .build();
-                    return buildResponseMessage(responseBody, ResponseType.RESTART_JOB_RESPONSE, replyTo, correlationId);
+                                                            .jobs(Collections.singletonList(job))
+                                                            .build();
+                    return buildResponseMessage(responseBody, ResponseType.RESTART_JOB_RESPONSE, replyTo,
+                          correlationId);
                 } catch (IOException e) {
                     return handleErrorResponse(e, replyTo, correlationId);
                 }
@@ -64,8 +65,8 @@ public Message handleMessage(RequestBody requestBody, @Header(Kafk
                 try {
                     Job job = jobService.stopJob(requestBody.getJobIdHex());
                     ResponseBody responseBody = ResponseBody.builder()
-                            .jobs(Collections.singletonList(job))
-                            .build();
+                                                            .jobs(Collections.singletonList(job))
+                                                            .build();
                     return buildResponseMessage(responseBody, ResponseType.STOP_JOB_RESPONSE, replyTo, correlationId);
                 } catch (IOException e) {
                     return handleErrorResponse(e, replyTo, correlationId);
@@ -76,7 +77,8 @@ public Message handleMessage(RequestBody requestBody, @Header(Kafk
                 try {
                     jobService.updateConfig(requestBody.getPayload());
                     final ResponseBody responseBody = ResponseBody.builder().build();
-                    return buildResponseMessage(responseBody, ResponseType.UPDATE_JOB_CONFIG_RESPONSE, replyTo, correlationId);
+                    return buildResponseMessage(responseBody, ResponseType.UPDATE_JOB_CONFIG_RESPONSE, replyTo,
+                          correlationId);
                 } catch (IOException e) {
                     return handleErrorResponse(e, replyTo, correlationId);
                 }
@@ -84,38 +86,45 @@ public Message handleMessage(RequestBody requestBody, @Header(Kafk
                 try {
                     pipelineService.createEmptyPipeline(requestBody.getPipelineName(), requestBody.getBranch());
                     final ResponseBody responseBody = ResponseBody.builder().build();
-                    return buildResponseMessage(responseBody, ResponseType.CREATE_EMPTY_PIPELINE_RESPONSE, replyTo, correlationId);
+                    return buildResponseMessage(responseBody, ResponseType.CREATE_EMPTY_PIPELINE_RESPONSE, replyTo,
+                          correlationId);
                 } catch (Exception e) {
                     return handleErrorResponse(e, replyTo, correlationId);
                 }
             case START_ARCHIVE_PIPELINE:
                 try {
-                    pipelineService.extractPipeline(requestBody.getPayload(), requestBody.getPipelineName(), requestBody.getBranch());
-                    pipelineService.startPipelineJob(requestBody.getPipelineName(), requestBody.getBranch(), requestBody.getProfileName(), requestBody.getJobs());
+                    pipelineService.extractPipeline(requestBody.getPayload(), requestBody.getPipelineName(),
+                          requestBody.getBranch());
+                    pipelineService.startPipelineJob(requestBody.getPipelineName(), requestBody.getBranch(),
+                          requestBody.getProfileName(), requestBody.getJobs());
                     final ResponseBody responseBody = ResponseBody.builder().build();
-                    return buildResponseMessage(responseBody, ResponseType.START_ARCHIVE_PIPELINE_RESPONSE, replyTo, correlationId);
+                    return buildResponseMessage(responseBody, ResponseType.START_ARCHIVE_PIPELINE_RESPONSE, replyTo,
+                          correlationId);
                 } catch (Exception e) {
                     log.error("Exception while processing the Start All request {}", e.getMessage());
                     return handleErrorResponse(e, replyTo, correlationId);
-
                 }
-
+            default:
+                String errorMessage = String.format("Unknown request type %s", requestType);
+                log.error(errorMessage);
+                return handleErrorResponse(new RuntimeException(errorMessage), replyTo, correlationId);
         }
         return null;
     }
 
-    private Message getResponseBodyMessage(byte[] replyTo, byte[] correlationId, ResponseType responseType) {
+    private Message getResponseBodyMessage(byte[] replyTo, byte[] correlationId,
+                                                         ResponseType responseType) {
         try {
             List jobs = jobService.getJobs();
             ResponseBody responseBody = ResponseBody.builder()
-                    .jobs(jobs)
-                    .clusterMeta(ClusterMeta.builder()
-                            .name(config.getName())
-                            .clusterId(config.getId())
-                            .clusterStatus(config.getStatus())
-                            .version(config.getVersion())
-                            .build())
-                    .build();
+                                                    .jobs(jobs)
+                                                    .clusterMeta(ClusterMeta.builder()
+                                                                            .name(config.getName())
+                                                                            .clusterId(config.getId())
+                                                                            .clusterStatus(config.getStatus())
+                                                                            .version(config.getVersion())
+                                                                            .build())
+                                                    .build();
             return buildResponseMessage(responseBody, responseType, replyTo, correlationId);
         } catch (IOException e) {
             return handleErrorResponse(e, replyTo, correlationId);
@@ -124,12 +133,14 @@ private Message getResponseBodyMessage(byte[] replyTo, byte[] corr
 
     private Message handleErrorResponse(Exception e, byte[] replyTo, byte[] correlationId) {
         ResponseBody responseBody = ResponseBody.builder()
-                .errorMessage(Collections.singletonMap(e.getClass().toString(), e.getMessage()))
-                .build();
+                                                .errorMessage(
+                                                      Collections.singletonMap(e.getClass().toString(), e.getMessage()))
+                                                .build();
         return buildResponseMessage(responseBody, ResponseType.ERROR_RESPONSE, replyTo, correlationId);
     }
 
-    private Message buildResponseMessage(ResponseBody body, ResponseType responseType, byte[] replyTo, byte[] correlationId) {
+    private Message buildResponseMessage(ResponseBody body, ResponseType responseType, byte[] replyTo,
+                                                       byte[] correlationId) {
         MessageHeaderAccessor accessor = new MessageHeaderAccessor();
         accessor.setHeader(KafkaHeaders.MESSAGE_KEY, responseType.name());
         accessor.setHeader(KafkaHeaders.CORRELATION_ID, correlationId);
diff --git a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/FilePipelineService.java b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/FilePipelineService.java
index 53e8de26..6b355cd0 100644
--- a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/FilePipelineService.java
+++ b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/FilePipelineService.java
@@ -4,17 +4,16 @@
 import com.cloudera.service.common.Utils;
 import com.cloudera.service.common.response.Job;
 import com.cloudera.service.common.utils.ArchiveUtil;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.stereotype.Service;
-
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.stream.Collectors;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
 
 @Slf4j
 @Service
@@ -24,8 +23,10 @@ public class FilePipelineService {
 
 
     public void createEmptyPipeline(String pipelineName, String branchName) {
-        String fullPath = this.config.getPipelineDir().endsWith("/") ? this.config.getPipelineDir() + pipelineName + "/" + branchName
-                : this.config.getPipelineDir() + "/" + pipelineName + "/" + branchName;
+        String fullPath =
+              this.config.getPipelineDir().endsWith("/")
+                    ? this.config.getPipelineDir() + pipelineName + "/" + branchName
+                    : this.config.getPipelineDir() + "/" + pipelineName + "/" + branchName;
         File directory = new File(fullPath);
         if (directory.mkdirs()) {
             log.info("Create full path {}", fullPath);
@@ -43,20 +44,26 @@ public void createEmptyPipeline(String pipelineName, String branchName) {
     }
 
     public void extractPipeline(byte[] payload, String pipelineName, String branch) throws IOException {
-        String fullPipelinePath = pipelineName.endsWith("/") ? this.config.getPipelineDir() + pipelineName + "/" + branch : this.config.getPipelineDir() + "/" + pipelineName + "/" + branch;
+        String fullPipelinePath =
+              pipelineName.endsWith("/") ? this.config.getPipelineDir() + pipelineName + "/" + branch
+                    : this.config.getPipelineDir() + "/" + pipelineName + "/" + branch;
         ArchiveUtil.decompressFromTarGzInMemory(payload, fullPipelinePath, true);
     }
 
-    public void startPipelineJob(String pipelineName, String branch, String profileName, List jobsNames) throws IOException {
-        String fullPipelinePath = pipelineName.endsWith("/") ?this.config.getPipelineDir() + pipelineName + "/" + branch
-                : this.config.getPipelineDir() + "/" + pipelineName + "/" + branch;
+    public void startPipelineJob(String pipelineName, String branch, String profileName, List jobsNames)
+          throws IOException {
+        String fullPipelinePath =
+              pipelineName.endsWith("/") ? this.config.getPipelineDir() + pipelineName + "/" + branch
+                    : this.config.getPipelineDir() + "/" + pipelineName + "/" + branch;
 
         List jobs = jobsNames.stream().map(jobName -> Job.builder()
-                .jobPipeline(pipelineName)
-                .jobType(Utils.getEnumFromString(jobName, Job.JobType.class, Job.JobType::getName))
-                .jobBranch(branch)
-                .jobName(StringUtils.defaultString(profileName, "main"))
-                .build()).collect(Collectors.toList());
+                                                              .jobPipeline(pipelineName)
+                                                              .jobType(
+                                                                    Utils.getEnumFromString(jobName, Job.JobType.class,
+                                                                          Job.JobType::getName))
+                                                              .jobBranch(branch)
+                                                              .jobName(StringUtils.defaultString(profileName, "main"))
+                                                              .build()).collect(Collectors.toList());
         for (Job job : jobs) {
             job.getJobType().getScript(job);
             ProcessBuilder processBuilder = new ProcessBuilder(job.getJobType().getScript(job));
diff --git a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java
index d42f4588..eaf143b2 100644
--- a/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java
+++ b/flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java
@@ -4,14 +4,6 @@
 import com.cloudera.service.common.Utils;
 import com.cloudera.service.common.response.Job;
 import com.cloudera.service.common.utils.ArchiveUtil;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobStatus;
-import org.springframework.stereotype.Service;
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
@@ -25,13 +17,22 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.springframework.stereotype.Service;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class JobService {
-    public static final String LOG_CLI_JOB_INFO = "Successfully read jobs from cli with exit code {}. job count '{}' jobs data '[{}]'";
-    private final Pattern pattern = Pattern.compile("^(?[\\d.:\\s]+)\\s:\\s(?[a-fA-F0-9]+)\\s:\\s(?[\\w.-]+)\\s\\((?\\w+)\\)$");
+    public static final String LOG_CLI_JOB_INFO =
+          "Successfully read jobs from cli with exit code {}. job count '{}' jobs data '[{}]'";
+    private final Pattern pattern = Pattern.compile(
+          "^(?[\\d.:\\s]+)\\s:\\s(?[a-fA-F0-9]+)\\s:\\s(?[\\w.-]+)\\s\\((?\\w+)\\)$");
 
     private final AppWorkerConfig config;
 
@@ -39,16 +40,17 @@ public class JobService {
     public List getJobs() throws IOException {
         List jobs = new ArrayList<>();
         int exitValue = fillJobList(jobs);
-        log.info(LOG_CLI_JOB_INFO, exitValue, jobs.size(), jobs.stream().map(Objects::toString).collect(Collectors.joining(",")));
+        log.info(LOG_CLI_JOB_INFO, exitValue, jobs.size(),
+              jobs.stream().map(Objects::toString).collect(Collectors.joining(",")));
         return jobs;
     }
 
     public Job getJob(String id) throws IOException {
         List jobs = getJobs();
         return jobs.stream()
-                .filter(job -> StringUtils.equalsIgnoreCase(job.getJobId().toHexString(), id))
-                .findFirst()
-                .orElse(null);
+                   .filter(job -> StringUtils.equalsIgnoreCase(job.getJobId().toHexString(), id))
+                   .findFirst()
+                   .orElse(null);
     }
 
     public Job restartJob(String id) throws IOException {
@@ -62,18 +64,26 @@ public Job restartJob(String id) throws IOException {
                     processBuilder.directory(new File(config.getPipelineDir()));
                 }
                 Process process = processBuilder.start();
-                log.debug("Command input stream '{}' \n Command error stream '{}'", IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
+                log.debug("Command input stream '{}' \n Command error stream '{}'",
+                      IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8),
+                      IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
                 int waitForCode = process.waitFor();
                 if (process.exitValue() != 0) {
-                    log.error("Failed to run job with exit code '{}' wait fore code '{}'. Command input stream '{}' \n Command error stream '{}'", process.exitValue(), waitForCode,
-                            IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
+                    log.error(
+                          "Failed to run job with exit code '{}' wait fore code '{}'. "
+                          + "Command input stream '{}' \n Command error stream '{}'",
+                          process.exitValue(), waitForCode,
+                          IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8),
+                          IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
                     return job;
                 }
             } catch (IOException ioe) {
-                log.error("There was an error when starting the restart operation for the job {}. {}", job, ioe.getMessage());
+                log.error("There was an error when starting the restart operation for the job {}. {}", job,
+                      ioe.getMessage());
                 throw ioe;
             } catch (InterruptedException ie) {
-                log.error("An unexpected event occurred while waiting for the restart to complete for the job {}. {}", job, ie.getMessage());
+                log.error("An unexpected event occurred while waiting for the restart to complete for the job {}. {}",
+                      job, ie.getMessage());
                 Thread.currentThread().interrupt();
             }
         }
@@ -86,18 +96,25 @@ public Job stopJob(String id) throws IOException {
             try {
                 ProcessBuilder processBuilder = new ProcessBuilder("flink", "cancel", id);
                 Process process = processBuilder.start();
-                log.info("Command input stream '{}' \n Command error stream '{}'", IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
+                log.info("Command input stream '{}' \n Command error stream '{}'",
+                      IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8),
+                      IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
                 process.waitFor();
                 if (process.exitValue() != 0) {
-                    log.error("Failed to run job with exit code {}. Command output stream '{}' \n Command error stream '{}'", process.exitValue(),
-                            IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
+                    log.error(
+                          "Failed to run job with exit code {}. Command output stream '{}' \n Command error stream '{}'",
+                          process.exitValue(),
+                          IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8),
+                          IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
                     return job;
                 }
             } catch (IOException ioe) {
-                log.error("There was an error when starting the restart operation for the job {}. {}", job, ioe.getMessage());
+                log.error("There was an error when starting the restart operation for the job {}. {}", job,
+                      ioe.getMessage());
                 throw ioe;
             } catch (InterruptedException ie) {
-                log.error("An unexpected event occurred while waiting for the restart to complete for the job {}. {}", job, ie.getMessage());
+                log.error("An unexpected event occurred while waiting for the restart to complete for the job {}. {}",
+                      job, ie.getMessage());
                 Thread.currentThread().interrupt();
             }
         }
@@ -131,16 +148,18 @@ private List readJobFromInputStream(InputStream inputStream) throws IOExcep
                 Matcher stringMatcher = pattern.matcher(stringLine);
                 if (stringMatcher.matches()) {
                     String jobFullName = stringMatcher.group("jobFullName");
-                    Job.JobType jobType = Utils.getEnumFromStringContains(jobFullName, Job.JobType.class, Job.JobType::getName);
+                    Job.JobType jobType =
+                          Utils.getEnumFromStringContains(jobFullName, Job.JobType.class, Job.JobType::getName);
                     if (jobType != null) {
                         Job job = Job.builder()
-                                .jobId(JobID.fromHexString(stringMatcher.group("jobId")))
-                                .jobIdString(stringMatcher.group("jobId"))
-                                .jobFullName(jobFullName)
-                                .jobType(jobType)
-                                .jobState(JobStatus.valueOf(StringUtils.toRootUpperCase(stringMatcher.group("jobStatus"))))
-                                .startTime(stringMatcher.group("date"))
-                                .build();
+                                     .jobId(JobID.fromHexString(stringMatcher.group("jobId")))
+                                     .jobIdString(stringMatcher.group("jobId"))
+                                     .jobFullName(jobFullName)
+                                     .jobType(jobType)
+                                     .jobState(JobStatus.valueOf(
+                                           StringUtils.toRootUpperCase(stringMatcher.group("jobStatus"))))
+                                     .startTime(stringMatcher.group("date"))
+                                     .build();
                         setJobParameters(job, jobFullName);
                         jobs.add(job);
                     }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/Application.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/Application.java
index 48bf5f65..47ec0ab3 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/Application.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/Application.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java
index 2901675b..dfd31ce7 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java
@@ -7,9 +7,9 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *
      http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
+ *
      Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
@@ -19,13 +19,12 @@
 package com.cloudera.parserchains.queryservice;
 
 import com.cloudera.parserchains.queryservice.config.AppProperties;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.stereotype.Component;
 
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
 /**
  * This class is being used to do initialization for the REST application.
  * e.g. creating folder hierarchies needed by the application.
@@ -34,21 +33,21 @@
 @Slf4j
 public class StartupComponent implements CommandLineRunner {
 
-  private final AppProperties appProperties;
+    private final AppProperties appProperties;
 
-  public StartupComponent(AppProperties appProperties) {
-    this.appProperties = appProperties;
-  }
+    public StartupComponent(AppProperties appProperties) {
+        this.appProperties = appProperties;
+    }
 
-  @Override
-  public void run(String... args) throws Exception {
-    log.info("Running startup stuff");
-    log.info("Creating the config dir: {}", appProperties.getConfigPath());
-    Files.createDirectories(Paths.get(appProperties.getConfigPath()));
-    log.info("Done creating the config dir");
+    @Override
+    public void run(String... args) throws Exception {
+        log.info("Running startup stuff");
+        log.info("Creating the config dir: {}", appProperties.getConfigPath());
+        Files.createDirectories(Paths.get(appProperties.getConfigPath()));
+        log.info("Done creating the config dir");
 
-    log.info("Creating the pipeline dir: {}", appProperties.getPipelinesPath());
-    Files.createDirectories(Paths.get(appProperties.getPipelinesPath()));
-    log.info("Done creating the pipeline dir");
-  }
+        log.info("Creating the pipeline dir: {}", appProperties.getPipelinesPath());
+        Files.createDirectories(Paths.get(appProperties.getPipelinesPath()));
+        log.info("Done creating the pipeline dir");
+    }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/ApplicationConstants.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/ApplicationConstants.java
index b522af48..233612b1 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/ApplicationConstants.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/ApplicationConstants.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/ValidationException.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/ValidationException.java
index 41cb78f1..6157f8a7 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/ValidationException.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/exception/ValidationException.java
@@ -6,18 +6,21 @@
 @ResponseStatus(value = HttpStatus.BAD_REQUEST)
 public class ValidationException extends RuntimeException {
     public ValidationException() {
-        this(null,null);
+        this(null, null);
     }
+
     public ValidationException(String message) {
-        this(message,null);
+        this(message, null);
     }
+
     public ValidationException(Throwable cause) {
-        this(null,cause);
+        this(null, cause);
     }
+
     public ValidationException(String message, Throwable cause) {
-       super(message);
-       if (cause != null) {
-           this.initCause(cause);
-       }
+        super(message);
+        if (cause != null) {
+            this.initCause(cause);
+        }
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/CollectionsUtils.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/CollectionsUtils.java
index 5ac53ac0..db25e949 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/CollectionsUtils.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/CollectionsUtils.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/Utils.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/Utils.java
index 675bb739..f4a22d59 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/Utils.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/common/utils/Utils.java
@@ -1,21 +1,17 @@
 package com.cloudera.parserchains.queryservice.common.utils;
 
-import org.apache.commons.lang.StringUtils;
-
-import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeFormatterBuilder;
 import java.time.format.DateTimeParseException;
-import java.time.temporal.ChronoField;
 import java.time.temporal.TemporalAccessor;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import org.apache.commons.lang.StringUtils;
 
 
 public class Utils {
@@ -81,10 +77,8 @@ public class Utils {
      * Assume that any date more than 4 days in the future is in the past as per
      * SyslogUtils
      *
-     * @param candidate
-     *            The possible date.
-     * @param validPatterns
-     *            A list of SimpleDateFormat instances to try parsing with.
+     * @param candidate     The possible date.
+     * @param validPatterns A list of SimpleDateFormat instances to try parsing with.
      * @return A java.util.Date based on the parse result
      */
     public static Long parseData(String candidate, List validPatterns) {
@@ -94,7 +88,7 @@ public static Long parseData(String candidate, List validPatte
             for (SimpleDateFormat pattern : validPatterns) {
                 try {
                     DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder()
-                            .appendPattern(pattern.toPattern());
+                          .appendPattern(pattern.toPattern());
                     DateTimeFormatter formatter = formatterBuilder.toFormatter();
                     ZonedDateTime parsedValue = parseDateTimeWithDefaultTimezone(candidate, formatter);
                     return parsedValue.toInstant().toEpochMilli();
@@ -102,15 +96,15 @@ public static Long parseData(String candidate, List validPatte
                     // Continue to the next pattern
                 }
             }
-          return null;
+            return null;
         }
     }
 
     private static ZonedDateTime parseDateTimeWithDefaultTimezone(String candidate, DateTimeFormatter formatter) {
         TemporalAccessor temporalAccessor = formatter.parseBest(candidate, ZonedDateTime::from, LocalDateTime::from);
         return temporalAccessor instanceof ZonedDateTime
-                ? ((ZonedDateTime) temporalAccessor)
-                : ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault());
+              ? ((ZonedDateTime) temporalAccessor)
+              : ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault());
     }
 
     public static int compareLongs(Long a, Long b) {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AppProperties.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AppProperties.java
index 5c426920..5b748ec8 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AppProperties.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/AppProperties.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java
index 9e9c6aa2..7824e1ae 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java
@@ -6,6 +6,10 @@
 import com.cloudera.service.common.config.kafka.ClouderaKafkaProperties;
 import com.cloudera.service.common.request.RequestBody;
 import com.cloudera.service.common.response.ResponseBody;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.common.serialization.StringDeserializer;
@@ -31,11 +35,6 @@
 import org.springframework.kafka.support.serializer.JsonDeserializer;
 import org.springframework.kafka.support.serializer.JsonSerializer;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
 @Slf4j
 @EnableKafka
 @Configuration
@@ -44,98 +43,110 @@
 @EnableConfigurationProperties({ClouderaKafkaProperties.class})
 public class KafkaConfig {
 
-  /***
-   * Provides the default kafka properties for the consumers
-   * @return Default kafka properties for the consumers
-   */
-  @Bean
-  @Primary
-  public ClouderaKafkaProperties kafkaProperties() {
-    return new ClouderaKafkaProperties();
-  }
-
-  /***
-   * Provides a map with key=clusterId and value=ClouderaKafkaProperties
-   * @return Map with key of clusterId and value - kafkaProperties. This map is a mapping between clusterId and connection details for that cluster
-   */
-  @Bean(name = "kafka-external-cluster-map")
-  @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
-  @ConfigurationProperties("spring.kafka.external-clusters")
-  @ConditionalOnProperty(name = "spring.kafka.mock", matchIfMissing = true, havingValue = "false")
-  public Map replyKafkaPropertiesMap() {
-    return new HashMap<>();
-  }
-
-  /**
-   * Map of KafkaTemplates that with key=clusterId, which allows to send requests over Kafka to different clusters and
-   * get responses.
-   *
-   * @param replyKafkaPropertiesMap - replyKafkaPropertiesMap bean with properties for each cluster, injected by Spring
-   * @return Map with key of clusterId and value - kafkaTemplate that allows to send requests over Kafka to different clusters and get
-   * responses.
-   */
-  @Bean(name = "kafkaTemplatePool")
-  @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
-  @ConditionalOnProperty(name = "spring.kafka.mock", matchIfMissing = true, havingValue = "false")
-  public Map> kafkaTemplatePool(
-      @Qualifier("kafka-external-cluster-map") Map replyKafkaPropertiesMap) {
-    final Map> templatePool = new HashMap<>();
-
-    replyKafkaPropertiesMap.forEach((clusterId, kafkaProperties) -> {
-      final ProducerFactory producerFactory = producerFactory(kafkaProperties);
-      final ConsumerFactory consumerFactory = consumerFactory(kafkaProperties);
-      final GenericMessageListenerContainer replyContainer = replyContainer(consumerFactory,
-          kafkaProperties.getReplyTopic());
-
-      final ClouderaReplyingKafkaTemplate kafkaTemplate = replyingKafkaTemplate(producerFactory,
-          replyContainer, kafkaProperties.getRequestTopic());
-      kafkaTemplate.start();
-
-      templatePool.put(clusterId, kafkaTemplate);
-    });
-
-    return Collections.unmodifiableMap(templatePool);
-  }
-
-  @Bean
-  @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
-  @ConditionalOnProperty(name = "spring.kafka.mock", matchIfMissing = true, havingValue = "false")
-  public Set kafkaClustersSet(@Qualifier("kafka-external-cluster-map") Map replyKafkaPropertiesMap) {
-    return Collections.unmodifiableSet(replyKafkaPropertiesMap.keySet());
-  }
-  @Bean
-  @ConditionalOnProperty(name = "spring.kafka.mock")
-  public KafkaServiceInterface mockKafkaService() {
-    return new MockKafkaService();
-  }
-
-  @Bean
-  @ConditionalOnProperty(name = "spring.kafka.mock", matchIfMissing = true, havingValue = "false")
-  public KafkaServiceInterface kafkaService(@Qualifier("kafkaTemplatePool") Map> kafkaTemplatePool,
-                                            @Value("${kafka.reply.future.timeout:45}") Long replyFutureTimeout,
-                                            @Value("${kafka.reply.timeout:45}") Long kafkaTemplateTimeout) {
-    return new KafkaService(kafkaTemplatePool,replyFutureTimeout,kafkaTemplateTimeout);
-  }
-
-
-  private ProducerFactory producerFactory(ClouderaKafkaProperties kafkaProperties) {
-    return new DefaultKafkaProducerFactory<>(kafkaProperties.buildConsumerProperties(), new StringSerializer(), new JsonSerializer<>());
-  }
-
-  private ConsumerFactory consumerFactory(ClouderaKafkaProperties kafkaProperties) {
-    return new DefaultKafkaConsumerFactory<>(kafkaProperties.buildConsumerProperties(), new StringDeserializer(), new JsonDeserializer<>(ResponseBody.class));
-  }
-
-  private static ClouderaReplyingKafkaTemplate replyingKafkaTemplate(
-      ProducerFactory producerFactory, GenericMessageListenerContainer replyContainer,
-      String requestTopic) {
-    return new ClouderaReplyingKafkaTemplate<>(producerFactory, replyContainer, requestTopic);
-  }
-
-  private static GenericMessageListenerContainer replyContainer(
-      ConsumerFactory consumerFactory, String replyTopic) {
-    ContainerProperties containerProperties = new ContainerProperties(replyTopic);
-    return new ConcurrentMessageListenerContainer<>(consumerFactory, containerProperties);
-  }
+    /**
+     * Provides the default kafka properties for the consumers.
+     *
+     * @return Default kafka properties for the consumers
+     */
+    @Bean
+    @Primary
+    public ClouderaKafkaProperties kafkaProperties() {
+        return new ClouderaKafkaProperties();
+    }
+
+    /**
+     * Provides a map with key=clusterId and value=ClouderaKafkaProperties.
+     *
+     * @return Map with key of clusterId and value - kafkaProperties.
+     *       This map is a mapping between clusterId and connection details for that cluster
+     */
+    @Bean(name = "kafka-external-cluster-map")
+    @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
+    @ConfigurationProperties("spring.kafka.external-clusters")
+    @ConditionalOnProperty(name = "spring.kafka.mock", matchIfMissing = true, havingValue = "false")
+    public Map replyKafkaPropertiesMap() {
+        return new HashMap<>();
+    }
+
+    /**
+     * Map of KafkaTemplates that with key=clusterId, which allows to send requests over Kafka to different clusters and
+     * get responses.
+     *
+     * @param replyKafkaPropertiesMap - replyKafkaPropertiesMap bean with properties for each cluster, injected by Spring
+     * @return Map with key of clusterId and value - kafkaTemplate that allows to send requests over Kafka to different clusters and get
+     *       responses.
+     */
+    @Bean(name = "kafkaTemplatePool")
+    @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
+    @ConditionalOnProperty(name = "spring.kafka.mock", matchIfMissing = true, havingValue = "false")
+    public Map> kafkaTemplatePool(
+          @Qualifier("kafka-external-cluster-map") Map replyKafkaPropertiesMap) {
+        final Map> templatePool =
+              new HashMap<>();
+
+        replyKafkaPropertiesMap.forEach((clusterId, kafkaProperties) -> {
+            final ProducerFactory producerFactory = producerFactory(kafkaProperties);
+            final ConsumerFactory consumerFactory = consumerFactory(kafkaProperties);
+            final GenericMessageListenerContainer replyContainer = replyContainer(consumerFactory,
+                  kafkaProperties.getReplyTopic());
+
+            final ClouderaReplyingKafkaTemplate kafkaTemplate =
+                  replyingKafkaTemplate(producerFactory,
+                        replyContainer, kafkaProperties.getRequestTopic());
+            kafkaTemplate.start();
+
+            templatePool.put(clusterId, kafkaTemplate);
+        });
+
+        return Collections.unmodifiableMap(templatePool);
+    }
+
+    @Bean
+    @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
+    @ConditionalOnProperty(name = "spring.kafka.mock", matchIfMissing = true, havingValue = "false")
+    public Set kafkaClustersSet(
+          @Qualifier("kafka-external-cluster-map") Map replyKafkaPropertiesMap) {
+        return Collections.unmodifiableSet(replyKafkaPropertiesMap.keySet());
+    }
+
+    @Bean
+    @ConditionalOnProperty(name = "spring.kafka.mock")
+    public KafkaServiceInterface mockKafkaService() {
+        return new MockKafkaService();
+    }
+
+    @Bean
+    @ConditionalOnProperty(name = "spring.kafka.mock", matchIfMissing = true, havingValue = "false")
+    public KafkaServiceInterface kafkaService(
+          @Qualifier("kafkaTemplatePool")
+          Map> kafkaTemplatePool,
+          @Value("${kafka.reply.future.timeout:45}") Long replyFutureTimeout,
+          @Value("${kafka.reply.timeout:45}") Long kafkaTemplateTimeout) {
+        return new KafkaService(kafkaTemplatePool, replyFutureTimeout, kafkaTemplateTimeout);
+    }
+
+
+    private ProducerFactory producerFactory(ClouderaKafkaProperties kafkaProperties) {
+        return new DefaultKafkaProducerFactory<>(kafkaProperties.buildConsumerProperties(), new StringSerializer(),
+              new JsonSerializer<>());
+    }
+
+    private ConsumerFactory consumerFactory(ClouderaKafkaProperties kafkaProperties) {
+        return new DefaultKafkaConsumerFactory<>(kafkaProperties.buildConsumerProperties(), new StringDeserializer(),
+              new JsonDeserializer<>(ResponseBody.class));
+    }
+
+    private static ClouderaReplyingKafkaTemplate replyingKafkaTemplate(
+          ProducerFactory producerFactory,
+          GenericMessageListenerContainer replyContainer,
+          String requestTopic) {
+        return new ClouderaReplyingKafkaTemplate<>(producerFactory, replyContainer, requestTopic);
+    }
+
+    private static GenericMessageListenerContainer replyContainer(
+          ConsumerFactory consumerFactory, String replyTopic) {
+        ContainerProperties containerProperties = new ContainerProperties(replyTopic);
+        return new ConcurrentMessageListenerContainer<>(consumerFactory, containerProperties);
+    }
 
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java
index d93dace3..b8bfbf6d 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java
@@ -12,6 +12,12 @@
 
 package com.cloudera.parserchains.queryservice.controller;
 
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_CHAINS;
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_CHAINS_READ_URL;
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_INDEXING;
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TEST;
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL;
+
 import com.cloudera.parserchains.core.ChainLink;
 import com.cloudera.parserchains.core.InvalidParserException;
 import com.cloudera.parserchains.core.model.define.ParserChainSchema;
@@ -36,6 +42,14 @@
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.flink.core.fs.Path;
@@ -51,17 +65,6 @@
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.io.IOException;
-import java.net.URI;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.Supplier;
-
-import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.*;
-
 /**
  * The controller responsible for operations on parser chains.
  */
@@ -84,12 +87,12 @@ public class ChainController {
 
     @Operation(summary = "Retrieves all available parser chains.")
     @ApiResponse(responseCode = "200", description = "A list of all parser chains.", content = @Content(
-            mediaType = "application/json",
-            array = @ArraySchema(schema = @Schema(implementation = ParserChainSummary.class))))
+          mediaType = "application/json",
+          array = @ArraySchema(schema = @Schema(implementation = ParserChainSummary.class))))
     @GetMapping(value = API_CHAINS)
     public ResponseEntity> findAll(
-            @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
-            @RequestParam(name = "pipelineName", required = false) String pipelineName
+          @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
+          @RequestParam(name = "pipelineName", required = false) String pipelineName
     ) throws IOException {
         String configPath = getConfigPath(pipelineName);
         List configs = chainPersistenceService.findAll(Paths.get(configPath));
@@ -98,35 +101,36 @@ public ResponseEntity> findAll(
 
     @Operation(summary = "Creates a new parser chain.")
     @ApiResponse(responseCode = "200", description = "The parser chain was created.", content = @Content(
-            mediaType = "application/json",
-            schema = @Schema(implementation = ParserChainSchema.class)))
+          mediaType = "application/json",
+          schema = @Schema(implementation = ParserChainSchema.class)))
     @ApiResponse(responseCode = "404", description = "Unable to create a new parser chain.")
     @PostMapping(value = API_CHAINS)
     public ResponseEntity create(
-            @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
-            @RequestParam(name = "pipelineName", required = false) String pipelineName,
-            @Parameter(name = "parserChain", description = "The parser chain to create.", required = true)
-            @RequestBody ParserChainSchema chain) throws IOException {
+          @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
+          @RequestParam(name = "pipelineName", required = false) String pipelineName,
+          @Parameter(name = "parserChain", description = "The parser chain to create.", required = true)
+          @RequestBody ParserChainSchema chain) throws IOException {
         String configPath = getConfigPath(pipelineName);
         ParserChainSchema createdChain = chainPersistenceService.create(chain, Paths.get(configPath));
         if (createdChain == null) {
             return ResponseEntity.notFound().build();
         } else {
-            return ResponseEntity.created(URI.create(API_CHAINS_READ_URL.replace("{id}", createdChain.getId()))).body(createdChain);
+            return ResponseEntity.created(URI.create(API_CHAINS_READ_URL.replace("{id}", createdChain.getId())))
+                                 .body(createdChain);
         }
     }
 
     @Operation(summary = "Retrieves an existing parser chain.")
     @ApiResponse(responseCode = "200", description = "The parser chain with the given ID.", content = @Content(
-            mediaType = "application/json",
-            schema = @Schema(implementation = ParserChainSchema.class)))
+          mediaType = "application/json",
+          schema = @Schema(implementation = ParserChainSchema.class)))
     @ApiResponse(responseCode = "404", description = "The parser chain does not exist.")
     @GetMapping(value = API_CHAINS + "/{id}")
     public ResponseEntity read(
-            @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
-            @RequestParam(name = "pipelineName", required = false) String pipelineName,
-            @Parameter(name = "id", description = "The ID of the parser chain to retrieve.", required = true)
-            @PathVariable String id) throws IOException {
+          @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
+          @RequestParam(name = "pipelineName", required = false) String pipelineName,
+          @Parameter(name = "id", description = "The ID of the parser chain to retrieve.", required = true)
+          @PathVariable String id) throws IOException {
         String configPath = getConfigPath(pipelineName);
         ParserChainSchema chain = chainPersistenceService.read(id, Paths.get(configPath));
         if (chain == null) {
@@ -141,12 +145,12 @@ public ResponseEntity read(
     @ApiResponse(responseCode = "404", description = "The parser chain does not exist.")
     @PutMapping(value = API_CHAINS + "/{id}")
     public ResponseEntity update(
-            @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
-            @RequestParam(name = "pipelineName", required = false) String pipelineName,
-            @Parameter(name = "parserChain", description = "The new parser chain definition.", required = true)
-            @RequestBody ParserChainSchema chain,
-            @Parameter(name = "id", description = "The ID of the parser chain to update.")
-            @PathVariable String id) throws IOException {
+          @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
+          @RequestParam(name = "pipelineName", required = false) String pipelineName,
+          @Parameter(name = "parserChain", description = "The new parser chain definition.", required = true)
+          @RequestBody ParserChainSchema chain,
+          @Parameter(name = "id", description = "The ID of the parser chain to update.")
+          @PathVariable String id) throws IOException {
         String configPath = getConfigPath(pipelineName);
         try {
             ParserChainSchema updatedChain = chainPersistenceService.update(id, chain, Paths.get(configPath));
@@ -165,10 +169,10 @@ public ResponseEntity update(
     @ApiResponse(responseCode = "404", description = "The parser chain does not exist.")
     @DeleteMapping(value = API_CHAINS + "/{id}")
     public ResponseEntity delete(
-            @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
-            @RequestParam(name = "pipelineName", required = false) String pipelineName,
-            @Parameter(name = "id", description = "The ID of the parser chain to delete.", required = true)
-            @PathVariable String id) throws IOException {
+          @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
+          @RequestParam(name = "pipelineName", required = false) String pipelineName,
+          @Parameter(name = "id", description = "The ID of the parser chain to delete.", required = true)
+          @PathVariable String id) throws IOException {
         String configPath = getConfigPath(pipelineName);
         if (chainPersistenceService.delete(id, Paths.get(configPath))) {
             return ResponseEntity.noContent().build();
@@ -179,14 +183,14 @@ public ResponseEntity delete(
 
 
     @Operation(summary = "Loads table mappings for the indexing job.",
-            responses = {
-                    @ApiResponse(responseCode = "200", description = "The mapping file parsed successfully."),
-            })
+          responses = {
+                @ApiResponse(responseCode = "200", description = "The mapping file parsed successfully."),
+          })
     @PostMapping(value = API_INDEXING)
     public ResponseEntity> getMappingsFromPath(
-            @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
-            @RequestParam(name = "pipelineName", required = false) String pipelineName,
-            @RequestBody IndexMappingDescriptor body) throws IOException {
+          @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
+          @RequestParam(name = "pipelineName", required = false) String pipelineName,
+          @RequestBody IndexMappingDescriptor body) throws IOException {
         final String indexPath = getIndexingPath(body.getFilePath(), pipelineName);
 
         try {
@@ -207,9 +211,9 @@ public ResponseEntity> getMappingsFromPath(
 
     @PostMapping(API_INDEXING + "/new")
     public ResponseEntity saveMappingsToPath(
-            @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
-            @RequestParam(name = "pipelineName", required = false) String pipelineName,
-            @RequestBody OcsfIndexMappingDescriptor body) throws IOException {
+          @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
+          @RequestParam(name = "pipelineName", required = false) String pipelineName,
+          @RequestBody OcsfIndexMappingDescriptor body) throws IOException {
         final String indexPath = getIndexingPath(body.getFilePath(), pipelineName);
 
         try {
@@ -223,24 +227,24 @@ public ResponseEntity saveMappingsToPath(
 
     @Operation(summary = "Executes a parser chain to parse sample data.")
     @ApiResponse(responseCode = "200", description = "The result of parsing the message.", content = @Content(
-            mediaType = "application/json",
-            schema = @Schema(implementation = ChainTestResponse.class)))
+          mediaType = "application/json",
+          schema = @Schema(implementation = ChainTestResponse.class)))
     @PostMapping(value = API_PARSER_TEST)
     public ResponseEntity test(
-            @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
-            @RequestParam(name = "pipelineName", required = false) String pipelineName,
-            @Parameter(name = "testRun", description = "Describes the parser chain test to run.", required = true)
-            @RequestBody ChainTestRequest testRun) throws IOException {
+          @Parameter(name = "pipelineName", description = "The pipeline to execute request in.")
+          @RequestParam(name = "pipelineName", required = false) String pipelineName,
+          @Parameter(name = "testRun", description = "Describes the parser chain test to run.", required = true)
+          @RequestBody ChainTestRequest testRun) throws IOException {
         String configPath = getConfigPath(pipelineName);
 
         ParserChainSchema chain = testRun.getParserChainSchema();
         chain.setBasePath(configPath);
         ChainTestResponse results = new ChainTestResponse();
         testRun.getSampleData().getSource()
-                .stream()
-                .limit(MAX_SAMPLES_PER_TEST)
-                .map(sample -> doTest(chain, sample))
-                .forEach(results::addResult);
+               .stream()
+               .limit(MAX_SAMPLES_PER_TEST)
+               .map(sample -> doTest(chain, sample))
+               .forEach(results::addResult);
         return ResponseEntity.ok(results);
     }
 
@@ -260,11 +264,11 @@ private ParserResult doTest(ParserChainSchema schema, String textToParse) {
         } catch (InvalidParserException e) {
             log.info("The parser chain is invalid as constructed.", e);
             ResultLog log = ResultLogBuilder.error()
-                    .parserId(e.getBadParser().getLabel())
-                    .exception(e)
-                    .build();
+                                            .parserId(e.getBadParser().getLabel())
+                                            .exception(e)
+                                            .build();
             result = new ParserResult()
-                    .setLog(log);
+                  .setLog(log);
         }
         return result;
     }
@@ -286,9 +290,9 @@ private String getPipelinePath(String pipelineName, Supplier defaultPath
         }
 
         return Optional.ofNullable(pipelineService.findAll())
-                .map(pipelineMap -> pipelineMap.get(pipelineName))
-                .map(PipelineResult::getPath)
-                .map(Path::getPath)
-                .orElseGet(defaultPathSupplier);
+                       .map(pipelineMap -> pipelineMap.get(pipelineName))
+                       .map(PipelineResult::getPath)
+                       .map(Path::getPath)
+                       .orElseGet(defaultPathSupplier);
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java
index afb97ba1..124eeea5 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java
@@ -10,6 +10,8 @@
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import java.io.IOException;
+import java.util.List;
 import lombok.RequiredArgsConstructor;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -20,9 +22,6 @@
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
-import java.io.IOException;
-import java.util.List;
-
 /**
  * The controller responsible for operations with cluster to run and stop cyber jobs on the clusters.
  */
@@ -35,7 +34,7 @@ public class ClusterController {
 
     @Operation(description = "Retrieves information about all cluster services.")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "200", description = "A list of all clusters.")
+          @ApiResponse(responseCode = "200", description = "A list of all clusters.")
     })
     @GetMapping
     public List getAllServices() throws FailedAllClusterReponseException {
@@ -44,20 +43,20 @@ public List getAllServices() throws FailedAllClusterReponseExcepti
 
     @Operation(description = "Retrieves information about a cluster with specified id.")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "200", description = "A response with cluster information."),
-            @ApiResponse(responseCode = "404", description = "The cluster does not exist.")
+          @ApiResponse(responseCode = "200", description = "A response with cluster information."),
+          @ApiResponse(responseCode = "404", description = "The cluster does not exist.")
 
     })
     @GetMapping(value = "/{id}")
     public ResponseBody getClusterService(
-            @Parameter(name = "id", description = "The ID of the cluster to retrieve.", required = true)
-            @PathVariable("id") String clusterId) throws FailedClusterReponseException {
+          @Parameter(name = "id", description = "The ID of the cluster to retrieve.", required = true)
+          @PathVariable("id") String clusterId) throws FailedClusterReponseException {
         return clusterService.getClusterInfo(clusterId);
     }
 
     @Operation(description = "Retrieves information about all pipelines on all services.")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "200", description = "A list of all pipelines.")
+          @ApiResponse(responseCode = "200", description = "A list of all pipelines.")
     })
     @GetMapping(value = "/pipelines")
     public List getAllPipelines() throws FailedAllClusterReponseException {
@@ -66,43 +65,44 @@ public List getAllPipelines() throws FailedAllClusterReponseException
 
     @Operation(description = "Retrieves information about all pipelines on all services.")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "200", description = "A list of all pipelines.")
+          @ApiResponse(responseCode = "200", description = "A list of all pipelines.")
     })
     @PostMapping(value = "{clusterId}/pipelines")
     public ResponseBody createEmptyPipeline(
-            @RequestBody com.cloudera.service.common.request.RequestBody body,
-            @Parameter(name = "clusterId", description = "The ID of the cluster to update config on.", required = true)
-            @PathVariable("clusterId") String clusterId
+          @RequestBody com.cloudera.service.common.request.RequestBody body,
+          @Parameter(name = "clusterId", description = "The ID of the cluster to update config on.", required = true)
+          @PathVariable("clusterId") String clusterId
     ) throws FailedClusterReponseException {
         return clusterService.createEmptyPipeline(clusterId, body);
     }
 
     @Operation(description = "Retrieves information about all pipelines on all services.")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "200", description = "A list of all pipelines.")
+          @ApiResponse(responseCode = "200", description = "A list of all pipelines.")
     })
     @PostMapping(value = "{clusterId}/pipelines/{name}/start")
     public ResponseBody startPipeline(
-            @Parameter(name = "name", description = "The pipeline name to create empty pipeline.", required = true)
-            @PathVariable("name") String name,
-            @Parameter(name = "clusterId", description = "The ID of the cluster to update config on.", required = true)
-            @PathVariable("clusterId") String clusterId,
-            @RequestPart("payload") MultipartFile payload,
-            @RequestPart("body") com.cloudera.service.common.request.RequestBody body
+          @Parameter(name = "name", description = "The pipeline name to create empty pipeline.", required = true)
+          @PathVariable("name") String name,
+          @Parameter(name = "clusterId", description = "The ID of the cluster to update config on.", required = true)
+          @PathVariable("clusterId") String clusterId,
+          @RequestPart("payload") MultipartFile payload,
+          @RequestPart("body") com.cloudera.service.common.request.RequestBody body
     ) throws IOException, FailedClusterReponseException {
-        return clusterService.startPipelineJob(clusterId, name, body.getBranch(), body.getProfileName(), body.getJobs(), payload.getBytes());
+        return clusterService.startPipelineJob(clusterId, name, body.getBranch(), body.getProfileName(), body.getJobs(),
+              payload.getBytes());
     }
 
     @Operation(description = "Retrieves information about a pipeline on the cluster with specified id.")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "200", description = "A response with cluster information."),
-            @ApiResponse(responseCode = "404", description = "The cluster does not exist.")
+          @ApiResponse(responseCode = "200", description = "A response with cluster information."),
+          @ApiResponse(responseCode = "404", description = "The cluster does not exist.")
 
     })
     @GetMapping(value = "/{id}/pipelines")
     public List getClusterPipelines(
-            @Parameter(name = "id", description = "The ID of the cluster to retrieve.", required = true)
-            @PathVariable("id") String clusterId) throws FailedClusterReponseException {
+          @Parameter(name = "id", description = "The ID of the cluster to retrieve.", required = true)
+          @PathVariable("id") String clusterId) throws FailedClusterReponseException {
         return clusterService.getClusterPipelines(clusterService.getClusterInfo(clusterId));
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java
index 6a229462..4e089104 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java
@@ -3,16 +3,20 @@
  * file distributed with this work for additional information regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
  * License.  You may obtain a copy of the License at
- * 
      
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
      
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ *
+ * 
      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * 
      Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
  */
 
 package com.cloudera.parserchains.queryservice.controller;
 
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_FORM_CONFIG;
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TYPES;
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL;
+
 import com.cloudera.parserchains.core.model.define.ParserID;
 import com.cloudera.parserchains.queryservice.model.describe.ParserDescriptor;
 import com.cloudera.parserchains.queryservice.model.summary.ParserSummary;
@@ -22,20 +26,15 @@
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 import lombok.RequiredArgsConstructor;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_FORM_CONFIG;
-import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TYPES;
-import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL;
-
 /**
  * The controller responsible for operations on parsers.
  */
@@ -48,8 +47,8 @@ public class ParserController {
 
     @Operation(summary = "Retrieves all available parsers.")
     @ApiResponse(responseCode = "200", description = "A list of all parser types.", content = @Content(
-            mediaType = "application/json",
-            array = @ArraySchema(schema = @Schema(implementation = ParserSummary.class))))
+          mediaType = "application/json",
+          array = @ArraySchema(schema = @Schema(implementation = ParserSummary.class))))
     @GetMapping(value = API_PARSER_TYPES)
     public ResponseEntity> findAll() throws IOException {
         List types = parserDiscoveryService.findAll();
@@ -58,8 +57,8 @@ public ResponseEntity> findAll() throws IOException {
 
 
     @Operation(summary = "Describes the configuration parameters for all available parsers.")
-    @ApiResponse(responseCode = "200", description = "A map of parser types and their associated configuration parameters.", content = @Content(
-            mediaType = "application/json"))
+    @ApiResponse(responseCode = "200", description = "A map of parser types and their associated configuration parameters.",
+          content = @Content(mediaType = "application/json"))
     @ApiResponse(responseCode = "404", description = "Unable to retrieve.")
     @GetMapping(value = API_PARSER_FORM_CONFIG)
     public ResponseEntity> describeAll() {
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java
index 480a4cef..57e178ea 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
      
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
      
- * Unless required by applicable law or agreed to in writing, software
+ *
+ * 
      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * 
      Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
@@ -18,6 +18,9 @@
 
 package com.cloudera.parserchains.queryservice.controller;
 
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TEST_SAMPLES;
+import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL;
+
 import com.cloudera.parserchains.queryservice.config.AppProperties;
 import com.cloudera.parserchains.queryservice.model.describe.SampleFolderDescriptor;
 import com.cloudera.parserchains.queryservice.model.sample.ParserSample;
@@ -28,6 +31,9 @@
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
 import lombok.RequiredArgsConstructor;
 import org.springframework.http.ResponseEntity;
 import org.springframework.util.StringUtils;
@@ -38,13 +44,6 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.io.IOException;
-import java.net.URI;
-import java.util.List;
-
-import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.API_PARSER_TEST_SAMPLES;
-import static com.cloudera.parserchains.queryservice.common.ApplicationConstants.PARSER_CONFIG_BASE_URL;
-
 /**
  * The controller responsible for operations on parsers.
  */
@@ -58,13 +57,14 @@ public class ParserSampleController {
     private final AppProperties appProperties;
 
     @Operation(summary = "Retrieves all parser samples for the specified chain.",
-            responses = {
-                    @ApiResponse(responseCode = "200", description = "A list of all parser samples for the specified chain.")
-            })
+          responses = {
+                @ApiResponse(responseCode = "200", description = "A list of all parser samples for the specified chain.")
+          })
     @PostMapping(value = API_PARSER_TEST_SAMPLES + "/{id}")
-    public ResponseEntity> findAllById(@Parameter(name = "id", description = "The ID of the parser chain to retrieve samples for.", required = true)
-                                                   @PathVariable String id,
-                                                   @RequestBody SampleFolderDescriptor body) throws IOException {
+    public ResponseEntity> findAllById(
+          @Parameter(name = "id", description = "The ID of the parser chain to retrieve samples for.", required = true)
+          @PathVariable String id,
+          @RequestBody SampleFolderDescriptor body) throws IOException {
         String sampleFolderPath = getSampleFolderPath(body);
         List types = parserSampleService.findAllById(sampleFolderPath, id);
         if (types == null) {
@@ -75,25 +75,27 @@ public ResponseEntity> findAllById(@Parameter(name = "id", de
 
 
     @Operation(summary = "Create or replace parser chain sample list.",
-            responses = {
-                    @ApiResponse(responseCode = "204", description = "The parser chain list was created/replaced.", content = @Content(array = @ArraySchema(schema = @Schema(implementation = ParserSample.class)))),
-                    @ApiResponse(responseCode = "404", description = "The parser chain does not exist.")
-            })
+          responses = {
+                @ApiResponse(responseCode = "204", description = "The parser chain list was created/replaced.",
+                      content = @Content(array = @ArraySchema(schema = @Schema(implementation = ParserSample.class)))),
+                @ApiResponse(responseCode = "404", description = "The parser chain does not exist.")
+          })
     @PutMapping(value = API_PARSER_TEST_SAMPLES + "/{id}")
     public ResponseEntity> update(
-            @Parameter(name = "sampleList", description = "The new sample definition list.", required = true)
-            @RequestBody SampleFolderDescriptor body,
-            @Parameter(name = "id", description = "The ID of the parser chain sample to update.")
-            @PathVariable String id) {
+          @Parameter(name = "sampleList", description = "The new sample definition list.", required = true)
+          @RequestBody SampleFolderDescriptor body,
+          @Parameter(name = "id", description = "The ID of the parser chain sample to update.")
+          @PathVariable String id) {
         String sampleFolderPath = getSampleFolderPath(body);
         try {
-            List createdSampleList = parserSampleService.update(sampleFolderPath, id, body.getSampleList());
+            List createdSampleList =
+                  parserSampleService.update(sampleFolderPath, id, body.getSampleList());
             if (null == createdSampleList) {
                 return ResponseEntity.notFound().build();
             }
             return ResponseEntity
-                    .created(URI.create(API_PARSER_TEST_SAMPLES + "/" + id))
-                    .body(createdSampleList);
+                  .created(URI.create(API_PARSER_TEST_SAMPLES + "/" + id))
+                  .body(createdSampleList);
             // TODO: fix this exception handling
         } catch (IOException ioe) {
             throw new RuntimeException("Unable to create parser chain samples with id=" + id);
@@ -102,8 +104,8 @@ public ResponseEntity> update(
 
     private String getSampleFolderPath(SampleFolderDescriptor body) {
         String sampleFolderPath = StringUtils.hasText(body.getFolderPath())
-                ? body.getFolderPath()
-                : appProperties.getSampleFolderPath();
+              ? body.getFolderPath()
+              : appProperties.getSampleFolderPath();
         if (!sampleFolderPath.endsWith("/")) {
             sampleFolderPath = sampleFolderPath.concat("/");
         }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/PipelineController.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/PipelineController.java
index a09cfb88..bcf8eb23 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/PipelineController.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/PipelineController.java
@@ -64,7 +64,8 @@ public ResponseEntity> findAll() throws IOException {
     })
     @PostMapping("/{pipelineName}")
     public ResponseEntity> createPipeline(
-            @Parameter(description = "The name of the pipeline to create") @PathVariable String pipelineName) throws IOException {
+          @Parameter(description = "The name of the pipeline to create") @PathVariable String pipelineName)
+          throws IOException {
         PipelineResult newPipeline = pipelineService.createPipeline(pipelineName);
         if (newPipeline != null) {
             return findAll();
@@ -78,8 +79,8 @@ public ResponseEntity> createPipeline(
     })
     @PutMapping("/{pipelineName}")
     public ResponseEntity> renamePipeline(
-            @Parameter(description = "The current name of the pipeline to be renamed") @PathVariable String pipelineName,
-            @Parameter(description = "The new name for the pipeline") @RequestParam String newName) throws IOException {
+          @Parameter(description = "The current name of the pipeline to be renamed") @PathVariable String pipelineName,
+          @Parameter(description = "The new name for the pipeline") @RequestParam String newName) throws IOException {
         PipelineResult updatedPipeline = pipelineService.renamePipeline(pipelineName, newName);
         if (updatedPipeline != null) {
             return findAll();
@@ -93,7 +94,8 @@ public ResponseEntity> renamePipeline(
     })
     @DeleteMapping("/{pipelineName}")
     public ResponseEntity> deletePipeline(
-            @Parameter(description = "The name of the pipeline to be deleted") @PathVariable String pipelineName) throws IOException {
+          @Parameter(description = "The name of the pipeline to be deleted") @PathVariable String pipelineName)
+          throws IOException {
         boolean pipelineDeleted = pipelineService.deletePipeline(pipelineName);
         if (pipelineDeleted) {
             return findAll();
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/DocumentationPluginsManagerBootAdapter.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/DocumentationPluginsManagerBootAdapter.java
index 8fffdb51..fa7d2295 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/DocumentationPluginsManagerBootAdapter.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/DocumentationPluginsManagerBootAdapter.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/TypeNameExtractorBootAdapter.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/TypeNameExtractorBootAdapter.java
index ad222db9..49b62607 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/TypeNameExtractorBootAdapter.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/hack/TypeNameExtractorBootAdapter.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ParserDescriptor.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ParserDescriptor.java
index 0441360f..1950b87b 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ParserDescriptor.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/model/describe/ParserDescriptor.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java
index 721f801d..e95c522e 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java
@@ -9,17 +9,16 @@
 import com.cloudera.service.common.response.Pipeline;
 import com.cloudera.service.common.response.ResponseBody;
 import com.cloudera.service.common.response.ResponseType;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.springframework.stereotype.Service;
-
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.stereotype.Service;
 
 @Slf4j
 @Service
@@ -28,11 +27,14 @@ public class ClusterService {
     private final KafkaServiceInterface kafkaService;
 
     public List getAllClusterInfo() throws FailedAllClusterReponseException {
-        List> response = kafkaService.sendWithReply(RequestType.GET_ALL_CLUSTERS_SERVICE_REQUEST, RequestBody.builder().build());
+        List> response =
+              kafkaService.sendWithReply(RequestType.GET_ALL_CLUSTERS_SERVICE_REQUEST, RequestBody.builder().build());
         List failedResponses = response.stream()
-                .filter(pair -> pair.getKey() != null && ResponseType.GET_ALL_CLUSTERS_SERVICE_RESPONSE != pair.getKey())
-                .map(Pair::getValue)
-                .collect(Collectors.toList());
+                                                     .filter(pair -> pair.getKey() != null
+                                                                     && ResponseType.GET_ALL_CLUSTERS_SERVICE_RESPONSE
+                                                                        != pair.getKey())
+                                                     .map(Pair::getValue)
+                                                     .collect(Collectors.toList());
         if (!failedResponses.isEmpty()) {
             throw new FailedAllClusterReponseException(failedResponses);
         }
@@ -41,51 +43,55 @@ public List getAllClusterInfo() throws FailedAllClusterReponseExce
 
     public List getAllPipelines() throws FailedAllClusterReponseException {
         return getAllClusterInfo().stream()
-                .flatMap(responseBody -> getClusterPipelines(responseBody).stream())
-                .collect(Collectors.toList());
+                                  .flatMap(responseBody -> getClusterPipelines(responseBody).stream())
+                                  .collect(Collectors.toList());
     }
 
     public List getClusterPipelines(ResponseBody responseBody) {
         return responseBody.getJobs().stream().collect(ArrayList::new,
-                (acc, job) -> {
-                    Optional duplicatePipeline = acc.stream().filter(pipeline -> StringUtils.equalsIgnoreCase(pipeline.getName(), job.getJobPipeline())).findFirst();
-                    if (duplicatePipeline.isPresent()) {
-                        Pipeline pipeline = duplicatePipeline.get();
-                        if (isNewJobTimeLater(job, pipeline)) {
-                            pipeline.setDate(job.getStartTime());
-                        }
-                    } else {
-                        acc.add(Pipeline.builder()
-                                .name(job.getJobPipeline())
-                                .clusterName(responseBody.getClusterMeta().getName())
-                                .date(job.getStartTime())
-                                .userName(job.getUser())
-                                .build());
-                    }
-                }, ArrayList::addAll);
+              (acc, job) -> {
+                  Optional duplicatePipeline = acc.stream().filter(
+                        pipeline -> StringUtils.equalsIgnoreCase(pipeline.getName(), job.getJobPipeline())).findFirst();
+                  if (duplicatePipeline.isPresent()) {
+                      Pipeline pipeline = duplicatePipeline.get();
+                      if (isNewJobTimeLater(job, pipeline)) {
+                          pipeline.setDate(job.getStartTime());
+                      }
+                  } else {
+                      acc.add(Pipeline.builder()
+                                      .name(job.getJobPipeline())
+                                      .clusterName(responseBody.getClusterMeta().getName())
+                                      .date(job.getStartTime())
+                                      .userName(job.getUser())
+                                      .build());
+                  }
+              }, ArrayList::addAll);
     }
 
     public ResponseBody createEmptyPipeline(String clusterId, RequestBody body) throws FailedClusterReponseException {
         RequestBody requestBody = RequestBody.builder()
-                .pipelineName(body.getPipelineName())
-                .branch(body.getBranch())
-                .build();
-        Pair response = kafkaService.sendWithReply(RequestType.CREATE_EMPTY_PIPELINE, clusterId, requestBody);
+                                             .pipelineName(body.getPipelineName())
+                                             .branch(body.getBranch())
+                                             .build();
+        Pair response =
+              kafkaService.sendWithReply(RequestType.CREATE_EMPTY_PIPELINE, clusterId, requestBody);
         if (response.getKey() != ResponseType.CREATE_EMPTY_PIPELINE_RESPONSE) {
             throw new FailedClusterReponseException(response.getValue());
         }
         return response.getValue();
     }
 
-    public ResponseBody startPipelineJob(String clusterId, String pipeline, String branch, String profileName, List jobs, byte[] payload) throws FailedClusterReponseException {
-        Pair response = kafkaService.sendWithReply(RequestType.START_ARCHIVE_PIPELINE, clusterId, RequestBody
-                .builder()
-                .payload(Base64.getEncoder().encode(payload))
-                .branch(branch)
-                .profileName(profileName)
-                .jobs(jobs)
-                .pipelineName(pipeline)
-                .build());
+    public ResponseBody startPipelineJob(String clusterId, String pipeline, String branch, String profileName,
+                                         List jobs, byte[] payload) throws FailedClusterReponseException {
+        Pair response =
+              kafkaService.sendWithReply(RequestType.START_ARCHIVE_PIPELINE, clusterId, RequestBody
+                    .builder()
+                    .payload(Base64.getEncoder().encode(payload))
+                    .branch(branch)
+                    .profileName(profileName)
+                    .jobs(jobs)
+                    .pipelineName(pipeline)
+                    .build());
         if (response.getKey() != ResponseType.CREATE_EMPTY_PIPELINE_RESPONSE) {
             throw new FailedClusterReponseException(response.getValue());
         }
@@ -98,7 +104,8 @@ public ResponseBody getClusterInfo(String clusterId) throws FailedClusterReponse
     }
 
     public ResponseBody getClusterInfo(String clusterId, RequestBody body) throws FailedClusterReponseException {
-        Pair response = kafkaService.sendWithReply(RequestType.GET_CLUSTER_SERVICE_REQUEST, clusterId, body);
+        Pair response =
+              kafkaService.sendWithReply(RequestType.GET_CLUSTER_SERVICE_REQUEST, clusterId, body);
         if (response.getKey() != ResponseType.GET_CLUSTER_SERVICE_RESPONSE) {
             throw new FailedClusterReponseException(response.getValue());
         }
@@ -107,6 +114,6 @@ public ResponseBody getClusterInfo(String clusterId, RequestBody body) throws Fa
 
     private static boolean isNewJobTimeLater(Job job, Pipeline pipeline) {
         return Utils.compareLongs(Utils.parseData(job.getStartTime(), Utils.DATE_FORMATS),
-                (Utils.parseData(pipeline.getDate(), Utils.DATE_FORMATS))) == 1;
+              (Utils.parseData(pipeline.getDate(), Utils.DATE_FORMATS))) == 1;
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java
index c7b2c5d6..4304d2f4 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java
@@ -9,18 +9,6 @@
 import com.cloudera.service.common.response.ClusterMeta;
 import com.cloudera.service.common.response.ResponseBody;
 import com.cloudera.service.common.response.ResponseType;
-import lombok.AllArgsConstructor;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.kafka.core.KafkaOperations;
-import org.springframework.kafka.requestreply.RequestReplyFuture;
-import org.springframework.stereotype.Service;
-
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
@@ -29,6 +17,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.springframework.kafka.requestreply.RequestReplyFuture;
 
 @Slf4j
 @RequiredArgsConstructor
@@ -38,7 +32,8 @@ public class KafkaService implements KafkaServiceInterface {
     private final Long kafkaTemplateTimeout;
 
     public Pair sendWithReply(RequestType requestType, String clusterId, RequestBody body) {
-        final ClouderaReplyingKafkaTemplate kafkaTemplate = kafkaTemplatePool.get(clusterId);
+        final ClouderaReplyingKafkaTemplate kafkaTemplate =
+              kafkaTemplatePool.get(clusterId);
         if (kafkaTemplate == null) {
             log.error("Cluster not found with cluster id: '{}'", clusterId);
             throw new KafkaClusterNotFound("Cluster not found! with cluster id '" + clusterId + "'");
@@ -48,31 +43,41 @@ public Pair sendWithReply(RequestType requestType, S
 
     public List> sendWithReply(RequestType requestType, RequestBody body) {
         return kafkaTemplatePool.entrySet().stream()
-                .map(entry -> send(requestType, body, entry.getKey(), entry.getValue()))
-                .collect(Collectors.toList());
+                                .map(entry -> send(requestType, body, entry.getKey(), entry.getValue()))
+                                .collect(Collectors.toList());
     }
 
-    private Pair send(RequestType requestType, RequestBody body, String clusterId, ClouderaReplyingKafkaTemplate kafkaTemplate) {
-        ProducerRecord producerRecord = new ProducerRecord<>(kafkaTemplate.getRequestTopic(), requestType.name(), body);
-        RequestReplyFuture replyFuture = kafkaTemplate.sendAndReceive(producerRecord, Duration.ofSeconds(kafkaTemplateTimeout));
+    private Pair send(RequestType requestType, RequestBody body, String clusterId,
+                                                  ClouderaReplyingKafkaTemplate kafkaTemplate) {
+        ProducerRecord producerRecord =
+              new ProducerRecord<>(kafkaTemplate.getRequestTopic(), requestType.name(), body);
+        RequestReplyFuture replyFuture =
+              kafkaTemplate.sendAndReceive(producerRecord, Duration.ofSeconds(kafkaTemplateTimeout));
         try {
             ConsumerRecord consumerRecord = replyFuture.get(replyFutureTimeout, TimeUnit.SECONDS);
             if (consumerRecord == null) {
                 throw new KafkaException("Got no reply from kafka");
             }
             ResponseBody responseBody = consumerRecord.value();
-            return Pair.of(Utils.getEnumFromString(consumerRecord.key(), ResponseType.class, ResponseType::name), responseBody);
+            return Pair.of(Utils.getEnumFromString(consumerRecord.key(), ResponseType.class, ResponseType::name),
+                  responseBody);
         } catch (ExecutionException e) {
             // Handle exception
-            log.error("Exception thrown when attempting to retrieve the information from kafka. Message: '{}' \n Cause: '{}'", e.getMessage(), e.getCause().getMessage());
-            return Pair.of(null, ResponseBody.builder().clusterMeta(ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build());
+            log.error(
+                  "Exception thrown when attempting to retrieve the information from kafka. Message: '{}' \n Cause: '{}'",
+                  e.getMessage(), e.getCause().getMessage());
+            return Pair.of(null, ResponseBody.builder().clusterMeta(
+                  ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build());
         } catch (TimeoutException e) {
             // Handle exception
             log.error("Timeout while waiting for reply");
-            return Pair.of(null, ResponseBody.builder().clusterMeta(ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build());
+            return Pair.of(null, ResponseBody.builder().clusterMeta(
+                  ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build());
         } catch (InterruptedException e) {
-            log.warn("Kafka throws interruption exception. Message: '{}' Cause: '{}'", e.getMessage(), Optional.ofNullable(e.getCause()).map(Throwable::getMessage).orElse("Cause is empty"));
-            return Pair.of(null, ResponseBody.builder().clusterMeta(ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build());
+            log.warn("Kafka throws interruption exception. Message: '{}' Cause: '{}'", e.getMessage(),
+                  Optional.ofNullable(e.getCause()).map(Throwable::getMessage).orElse("Cause is empty"));
+            return Pair.of(null, ResponseBody.builder().clusterMeta(
+                  ClusterMeta.builder().clusterId(clusterId).clusterStatus("offline").build()).build());
         }
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaServiceInterface.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaServiceInterface.java
index 4914b7f1..4fa3001f 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaServiceInterface.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaServiceInterface.java
@@ -4,9 +4,8 @@
 import com.cloudera.service.common.request.RequestType;
 import com.cloudera.service.common.response.ResponseBody;
 import com.cloudera.service.common.response.ResponseType;
-import org.apache.commons.lang3.tuple.Pair;
-
 import java.util.List;
+import org.apache.commons.lang3.tuple.Pair;
 
 public interface KafkaServiceInterface {
     Pair sendWithReply(RequestType requestType, String clusterId, RequestBody body);
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/MockKafkaService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/MockKafkaService.java
index 19ba7cb8..dd7bee08 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/MockKafkaService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/MockKafkaService.java
@@ -7,114 +7,114 @@
 import com.cloudera.service.common.response.ResponseBody;
 import com.cloudera.service.common.response.ResponseType;
 import com.google.common.collect.Lists;
+import java.util.List;
+import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.JobStatus;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-import java.util.stream.Collectors;
 
 public class MockKafkaService implements KafkaServiceInterface {
     private final ClusterMeta cluster1 = ClusterMeta.builder()
-            .clusterId("1")
-            .name("Cluster1")
-            .flinkVersion("3.2.1")
-            .version("1.0.1")
-            .clusterStatus("online")
-            .build();
+                                                    .clusterId("1")
+                                                    .name("Cluster1")
+                                                    .flinkVersion("3.2.1")
+                                                    .version("1.0.1")
+                                                    .clusterStatus("online")
+                                                    .build();
     private final ClusterMeta cluster2 = ClusterMeta.builder()
-            .clusterId("2")
-            .name("Cluster2")
-            .flinkVersion("3.4.1")
-            .version("1.0.5")
-            .clusterStatus("online")
-            .build();
+                                                    .clusterId("2")
+                                                    .name("Cluster2")
+                                                    .flinkVersion("3.4.1")
+                                                    .version("1.0.5")
+                                                    .clusterStatus("online")
+                                                    .build();
     private final ClusterMeta cluster3 = ClusterMeta.builder()
-            .clusterId("3")
-            .name("Cluster3")
-            .flinkVersion("3.1.1")
-            .version("1.0.4")
-            .clusterStatus("offline")
-            .build();
+                                                    .clusterId("3")
+                                                    .name("Cluster3")
+                                                    .flinkVersion("3.1.1")
+                                                    .version("1.0.4")
+                                                    .clusterStatus("offline")
+                                                    .build();
 
     private final Job job1 = Job.builder()
-            .jobId(JobID.fromHexString("fd72014d4c864993a2e5a9287b4a9c5d"))
-            .jobName("job1")
-            .jobIdString("job1-id-string")
-            .jobBranch("main")
-            .jobState(JobStatus.RUNNING)
-            .jobFullName("main-job1")
-            .jobPipeline("cyb-33")
-            .startTime("2024-01-11 10:33:55")
-            .jobType(Job.JobType.PARSER)
-            .user("Cybersec")
-            .build();
+                                .jobId(JobID.fromHexString("fd72014d4c864993a2e5a9287b4a9c5d"))
+                                .jobName("job1")
+                                .jobIdString("job1-id-string")
+                                .jobBranch("main")
+                                .jobState(JobStatus.RUNNING)
+                                .jobFullName("main-job1")
+                                .jobPipeline("cyb-33")
+                                .startTime("2024-01-11 10:33:55")
+                                .jobType(Job.JobType.PARSER)
+                                .user("Cybersec")
+                                .build();
     private final Job job2 = Job.builder()
-            .jobId(JobID.fromHexString("ad72014d4c864993a2e5a9287b4a9c5d"))
-            .jobName("job2")
-            .jobIdString("job2-id-string")
-            .jobBranch("main")
-            .jobState(JobStatus.RUNNING)
-            .jobFullName("main-job2")
-            .jobPipeline("cyb-33")
-            .startTime("2024-01-14 14:33:55")
-            .jobType(Job.JobType.PARSER)
-            .user("Cybersec")
-            .build();
+                                .jobId(JobID.fromHexString("ad72014d4c864993a2e5a9287b4a9c5d"))
+                                .jobName("job2")
+                                .jobIdString("job2-id-string")
+                                .jobBranch("main")
+                                .jobState(JobStatus.RUNNING)
+                                .jobFullName("main-job2")
+                                .jobPipeline("cyb-33")
+                                .startTime("2024-01-14 14:33:55")
+                                .jobType(Job.JobType.PARSER)
+                                .user("Cybersec")
+                                .build();
     private final Job job3 = Job.builder()
-            .jobId(JobID.fromHexString("cd72014d4c864993a2e5a9287b4a9c5d"))
-            .jobName("job3")
-            .jobIdString("job3-id-string")
-            .jobBranch("main")
-            .jobState(JobStatus.RUNNING)
-            .jobFullName("main-job3")
-            .jobPipeline("cyb-43")
-            .startTime("2024-01-11 11:33:55")
-            .jobType(Job.JobType.PARSER)
-            .user("Cybersec")
-            .build();
+                                .jobId(JobID.fromHexString("cd72014d4c864993a2e5a9287b4a9c5d"))
+                                .jobName("job3")
+                                .jobIdString("job3-id-string")
+                                .jobBranch("main")
+                                .jobState(JobStatus.RUNNING)
+                                .jobFullName("main-job3")
+                                .jobPipeline("cyb-43")
+                                .startTime("2024-01-11 11:33:55")
+                                .jobType(Job.JobType.PARSER)
+                                .user("Cybersec")
+                                .build();
     private final Job job4 = Job.builder()
-            .jobId(JobID.fromHexString("bd72014d4c864993a2e5a9287b4a9c5d"))
-            .jobName("job4")
-            .jobIdString("job1-id-string")
-            .jobBranch("main")
-            .jobState(JobStatus.RUNNING)
-            .jobFullName("main-job4")
-            .jobPipeline("cyb-43")
-            .startTime("2024-01-15 15:33:55")
-            .jobType(Job.JobType.PARSER)
-            .user("Cybersec")
-            .build();
+                                .jobId(JobID.fromHexString("bd72014d4c864993a2e5a9287b4a9c5d"))
+                                .jobName("job4")
+                                .jobIdString("job1-id-string")
+                                .jobBranch("main")
+                                .jobState(JobStatus.RUNNING)
+                                .jobFullName("main-job4")
+                                .jobPipeline("cyb-43")
+                                .startTime("2024-01-15 15:33:55")
+                                .jobType(Job.JobType.PARSER)
+                                .user("Cybersec")
+                                .build();
 
     private final ResponseBody responseFromCluster1 = ResponseBody.builder()
-            .jobs(Lists.newArrayList(job1, job2, job4))
-            .clusterMeta(cluster1)
-            .build();
+                                                                  .jobs(Lists.newArrayList(job1, job2, job4))
+                                                                  .clusterMeta(cluster1)
+                                                                  .build();
     private final ResponseBody responseFromCluster2 = ResponseBody.builder()
-            .jobs(Lists.newArrayList(job1))
-            .clusterMeta(cluster2)
-            .build();
+                                                                  .jobs(Lists.newArrayList(job1))
+                                                                  .clusterMeta(cluster2)
+                                                                  .build();
     private final ResponseBody responseFromCluster3 = ResponseBody.builder()
-            .jobs(Lists.newArrayList(job3))
-            .clusterMeta(cluster3)
-            .build();
-    private final List responses = Lists.newArrayList(responseFromCluster1, responseFromCluster2, responseFromCluster3);
+                                                                  .jobs(Lists.newArrayList(job3))
+                                                                  .clusterMeta(cluster3)
+                                                                  .build();
+    private final List responses =
+          Lists.newArrayList(responseFromCluster1, responseFromCluster2, responseFromCluster3);
 
     @Override
     public Pair sendWithReply(RequestType requestType, String clusterId, RequestBody body) {
         return responses.stream()
-                .filter(response -> StringUtils.equalsIgnoreCase(response.getClusterMeta().getName(), clusterId))
-                .map(val -> Pair.of(ResponseType.GET_CLUSTER_SERVICE_RESPONSE, val))
-                .findFirst()
-                .orElse(Pair.of(ResponseType.GET_CLUSTER_SERVICE_RESPONSE, responseFromCluster1));
+                        .filter(
+                              response -> StringUtils.equalsIgnoreCase(response.getClusterMeta().getName(), clusterId))
+                        .map(val -> Pair.of(ResponseType.GET_CLUSTER_SERVICE_RESPONSE, val))
+                        .findFirst()
+                        .orElse(Pair.of(ResponseType.GET_CLUSTER_SERVICE_RESPONSE, responseFromCluster1));
     }
 
     @Override
     public List> sendWithReply(RequestType requestType, RequestBody body) {
         return responses.stream()
-                .map(val -> Pair.of(ResponseType.GET_ALL_CLUSTERS_SERVICE_RESPONSE, val))
-                .collect(Collectors.toList());
+                        .map(val -> Pair.of(ResponseType.GET_ALL_CLUSTERS_SERVICE_RESPONSE, val))
+                        .collect(Collectors.toList());
     }
 }
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultParserDiscoveryService.java b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultParserDiscoveryService.java
index da62bdcc..7c74d36f 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultParserDiscoveryService.java
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/impl/DefaultParserDiscoveryService.java
@@ -6,10 +6,10 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * 
      
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * 
      
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,