From 4dbd473c0356b17ad64e9c58964f7195e9b287e0 Mon Sep 17 00:00:00 2001 From: fritz-astronomer <80706212+fritz-astronomer@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:17:14 -0400 Subject: [PATCH] Deployed c086e7c with MkDocs version: 1.6.0 --- 404.html | 14 +- Rules_and_Rulesets/index.html | 860 +++----- Rules_and_Rulesets/rules/index.html | 18 +- Rules_and_Rulesets/rulesets/index.html | 1874 ++++++++++++----- Rules_and_Rulesets/template/index.html | 174 +- assets/javascripts/bundle.af256bd8.min.js | 29 + ....min.js.map => bundle.af256bd8.min.js.map} | 8 +- assets/javascripts/bundle.fe8b6f2b.min.js | 29 - cli/index.html | 65 +- index.html | 127 +- objects.inv | Bin 820 -> 819 bytes .../callbacks/index.html | 50 +- .../operators/index.html | 14 +- objects/Tasks/index.html | 77 +- objects/index.html | 42 +- objects/project/index.html | 742 ++++--- objects/{dags => workflow}/index.html | 229 +- origins/index.html | 56 +- search/search_index.json | 2 +- sitemap.xml | 30 +- sitemap.xml.gz | Bin 314 -> 316 bytes 21 files changed, 2476 insertions(+), 1964 deletions(-) create mode 100644 assets/javascripts/bundle.af256bd8.min.js rename assets/javascripts/{bundle.fe8b6f2b.min.js.map => bundle.af256bd8.min.js.map} (66%) delete mode 100644 assets/javascripts/bundle.fe8b6f2b.min.js rename objects/{dags => workflow}/index.html (92%) diff --git a/404.html b/404.html index d13be95..181cb59 100644 --- a/404.html +++ b/404.html @@ -12,7 +12,7 @@ - + @@ -398,7 +398,7 @@ - Rulesets + Translation @@ -501,11 +501,11 @@
load_filetype
-
-
-
-
xmltodict_parse
-
-
-
Rules
and the Rulesets
that contain them.
Rule
contains a python function that is evaluated and produces something
-(typically an Object) or nothingRule
is a python function that is evaluated and produces something
+(typically an Object) or nothingRuleset
is a collection of Rules
that are
evaluated in priority orderTranslationRuleset
is a collection of Rulesets
,
- relating to an Origin and FileType,
+ relating to an Origin and FileType
,
with a translate_fn
which determines how to apply the rulesets.Different Rules
are applied in different scenarios;
-such as for converting input to a DAG (@dag_rule
),
-or a specific Airflow Operator (@task_rule
),
-or for filtering entries from the input data
-(@dag_filter_rule
, @task_filter_rule
).
Different Rules
are applied in different scenarios, such as:
@dag_rule
),@task_rule
),@dag_filter_rule
, @task_filter_rule
).Tip
To map the following input @@ -839,6 +823,8 @@
This returns a
OrbiterBashOperator
, which will become an Airflow
@@ -884,16 +870,19 @@
Orbiter expects a folder containing text files which may have a structure like:
-The default translation function (orbiter.rules.rulesets.translate
) performs the following steps:
The standard translation function performs the following steps:
TranslationRuleset.file_type
- (.json
, .xml
, .yaml
, etc.) in the input folder. Load each file and turn it into a Python Dictionary..json
, .xml
, .yaml
, etc.) in the input folder.TranslationRuleset.dag_filter_ruleset
to filter down to entries that can translate to a DAG, in priority order.TranslationRuleset.dag_ruleset
,
-to convert the object to an OrbiterDAG
,
+to convert the object to an OrbiterDAG
,
in priority-order, stopping when the first rule returns a match.
If no rule returns a match, the entry is filtered.OrbiterTaskDependency
,
which are then added to each task in the
- OrbiterDAG
OrbiterDAG
TranslationRuleset.post_processing_ruleset
,
against the OrbiterProject
, which can make modifications after all
other rules have been applied.orbiter/rules/rulesets.py
99 -100 -101 -102 -103 -104 -105 -106 -107 -108 -109 -110 -111 -112 -113 -114 -115 -116 -117 -118 -119 -120 -121 -122 -123 -124 -125 -126 -127 -128 -129 -130 -131 -132 -133 -134 -135 -136 -137 -138 -139 -140 -141 -142 -143 -144 -145 -146 -147 -148 -149 -150 -151 -152 -153 -154 -155 -156 -157 -158 -159 -160 -161 -162 -163 -164 -165 -166 -167 -168 -169 -170 -171 -172 -173 -174 -175 -176 -177 -178 -179 -180 -181 -182 -183 -184 -185 -186 -187 -188 -189 -190 -191 -192 -193 -194 -195 -196 -197 -198 -199 -200 -201 -202 -203 -204 -205 -206 -207 -208 -209 -210 -211 -212 -213 +
-
-
-
-
-- orbiter.rules.rulesets.load_filetype - - -¶- - -
-
-
-
-Orbiter converts all file types into a Python dictionary "intermediate representation" form, -prior to any rulesets being applied. -
-
- Source code in
- |
Name | -Type | -Description | -
---|---|---|
input_str |
-
- str
- |
-
-
-
- The XML string to parse - |
-
Returns:
-Type | -Description | -
---|---|
- dict
- |
-
-
-
- The parsed XML - |
-
orbiter/rules/rulesets.py
642 -643 -644 -645 -646 -647 -648 -649 -650 -651 -652 -653 -654 -655 -656 -657 -658 -659 -660 -661 -662 -663 -664 -665 -666 -667 -668 -669 -670 -671 -672 -673 -674 -675 -676 -677 -678 -679 -680 -681 -682 -683 -684 -685 -686 -687 -688 -689 -690 -691 -692 -693 -694 -695 -696 -697 -698 -699 -700 -701 -702 -703 |
|
|
An @task_dependency_rule
decorator creates a TaskDependencyRule
,
-which takes an OrbiterDAG
+which takes an OrbiterDAG
and returns a list[OrbiterTaskDependency]
or None
An OrbiterDAG
An OrbiterDAG
A list of rules, which are evaluated to generate different types of output
-You must pass a Rule
(or dict
with the schema of Rule
)
-
>>> from orbiter.rules import rule
->>> @rule
-... def x(val):
-... return None
->>> Ruleset(ruleset=[x, {"rule": lambda: None}])
-... # doctest: +ELLIPSIS
-Ruleset(ruleset=[Rule(...), Rule(...)])
-
Note
-You can't pass non-Rules -
-Convert Python dictionary back to source string form, useful for testing
+FileType | +Conversion Method | +
---|---|
XML |
+xmltodict.unparse |
+
YAML |
+yaml.safe_dump |
+
JSON |
+json.dumps |
+
ruleset
input_dict
List[Rule | Callable[[Any], Any | None]]
+ dict
Returns:
+Type | +Description | +
---|---|
+ str
+ |
+
+
+
+ The string representation of the input_dict, in the file_type format + |
+
orbiter/rules/rulesets.py
apply(
- take_first: bool = False, **kwargs
-) -> List[Any] | Any
+¶
+
- Apply all rules in ruleset to a single item, in priority order, removing any None
results.
-A ruleset with one rule can produce up to one result
-
>>> from orbiter.rules import rule
-
->>> @rule
-... def gt_4(val):
-... return str(val) if val > 4 else None
->>> Ruleset(ruleset=[gt_4]).apply(val=5)
-['5']
-
-Many rules can produce many results, one for each rule.
-
>>> @rule
-... def gt_3(val):
-... return str(val) if val > 3 else None
->>> Ruleset(ruleset=[gt_4, gt_3]).apply(val=5)
-['5', '5']
-
-The take_first
flag will evaluate rules in the ruleset and return the first match
-
-If nothing matched, an empty list is returned
-
>>> @rule
-... def always_none(val):
-... return None
->>> @rule
-... def more_always_none(val):
-... return None
->>> Ruleset(ruleset=[always_none, more_always_none]).apply(val=5)
-[]
-
-If nothing matched, and take_first=True
, None
is returned
-
-
-Tip
-If no input is given, an error is returned
-
-
+ A generator that yields files with a specific extension(s) in a directory
@@ -1166,23 +1266,13 @@
- take_first
-
- bool
-
-
-
- only take the first (if any) result from the ruleset application
-
-
-
-
- kwargs
+ input_dir
+ Path
- key=val pairs to pass to the evaluated rule function
+ The directory to search in
@@ -1201,44 +1291,11 @@
- List[Any] | Any | None
-
-
-
- List of rules that evaluated to Any
(in priority order), or an empty list, or Any
(if take_first=True
)
-
-
-
-
-
-
-
-
Raises:
-