From ed5e68fb5b03cdc42d4a783acf6b3a5181a924b1 Mon Sep 17 00:00:00 2001 From: Yoni Davidson Date: Tue, 10 Aug 2021 20:13:07 +0300 Subject: [PATCH] Add organic intallations (#20) * simple solution for organic installs * return all the streams * make organic installs opt out * remove comment * add todo * add state Co-authored-by: yonidavidson --- tap_appsflyer/__init__.py | 134 +++++++++- .../schemas/raw_data/organic_installs.json | 251 ++++++++++++++++++ 2 files changed, 383 insertions(+), 2 deletions(-) create mode 100644 tap_appsflyer/schemas/raw_data/organic_installs.json diff --git a/tap_appsflyer/__init__.py b/tap_appsflyer/__init__.py index f2851ab..3b44808 100644 --- a/tap_appsflyer/__init__.py +++ b/tap_appsflyer/__init__.py @@ -22,7 +22,7 @@ CONFIG = { "app_id": None, - "api_token": None, + "api_token": None } @@ -31,6 +31,7 @@ ENDPOINTS = { "installs": "/export/{app_id}/installs_report/v5", + "organic_installs": "/export/{app_id}/organic_installs_report/v5", "in_app_events": "/export/{app_id}/in_app_events_report/v5" } @@ -298,6 +299,132 @@ def sync_installs(): utils.update_state(STATE, "installs", bookmark) singer.write_state(STATE) +def sync_organic_installs(): + + schema = load_schema("raw_data/organic_installs") + singer.write_schema("organic_installs", schema, [ + "event_time", + "event_name", + "appsflyer_id" + ]) + + # This order matters + fieldnames = ( + "attributed_touch_type", + "attributed_touch_time", + "install_time", + "event_time", + "event_name", + "event_value", + "event_revenue", + "event_revenue_currency", + "event_revenue_usd", + "event_source", + "is_receipt_validated", + "af_prt", + "media_source", + "af_channel", + "af_keywords", + "campaign", + "af_c_id", + "af_adset", + "af_adset_id", + "af_ad", + "af_ad_id", + "af_ad_type", + "af_siteid", + "af_sub_siteid", + "af_sub1", + "af_sub2", + "af_sub3", + "af_sub4", + "af_sub5", + "af_cost_model", + "af_cost_value", + "af_cost_currency", + "contributor1_af_prt", + "contributor1_media_source", + "contributor1_campaign", + "contributor1_touch_type", + "contributor1_touch_time", + "contributor2_af_prt", + "contributor2_media_source", + "contributor2_campaign", + "contributor2_touch_type", + "contributor2_touch_time", + "contributor3_af_prt", + "contributor3_media_source", + "contributor3_campaign", + "contributor3_touch_type", + "contributor3_touch_time", + "region", + "country_code", + "state", + "city", + "postal_code", + "dma", + "ip", + "wifi", + "operator", + "carrier", + "language", + "appsflyer_id", + "advertising_id", + "idfa", + "android_id", + "customer_user_id", + "imei", + "idfv", + "platform", + "device_type", + "os_version", + "app_version", + "sdk_version", + "app_id", + "app_name", + "bundle_id", + "is_retargeting", + "retargeting_conversion_type", + "af_attribution_lookback", + "af_reengagement_window", + "is_primary_attribution", + "user_agent", + "http_referrer", + "original_url", + ) + + from_datetime = get_start("organic_installs") + to_datetime = get_stop(from_datetime, datetime.datetime.now()) + + if to_datetime < from_datetime: + LOGGER.error("to_datetime (%s) is less than from_endtime (%s).", to_datetime, from_datetime) + return + + params = dict() + params["from"] = from_datetime.strftime("%Y-%m-%d %H:%M") + params["to"] = to_datetime.strftime("%Y-%m-%d %H:%M") + params["api_token"] = CONFIG["api_token"] + + url = get_url("organic_installs", app_id=CONFIG["app_id"]) + request_data = request(url, params) + + csv_data = RequestToCsvAdapter(request_data) + reader = csv.DictReader(csv_data, fieldnames) + + next(reader) # Skip the heading row + + bookmark = from_datetime + for i, row in enumerate(reader): + record = xform(row, schema) + singer.write_record("organic_installs", record) + # AppsFlyer returns records in order of most recent first. + if utils.strptime(record["event_time"]) > bookmark: + bookmark = utils.strptime(record["event_time"]) + + # Write out state + utils.update_state(STATE, "organic_installs", bookmark) + singer.write_state(STATE) + def sync_in_app_events(): @@ -438,6 +565,9 @@ def sync_in_app_events(): def get_streams_to_sync(streams, state): target_stream = state.get("this_stream") result = streams + if "organic_installs" in CONFIG: + if CONFIG["organic_installs"]: + result.append(Stream("organic_installs", sync_organic_installs)) if target_stream: result = list(itertools.dropwhile(lambda x: x.name != target_stream, streams)) if not result: @@ -462,7 +592,7 @@ def main(): args = utils.parse_args( [ "app_id", - "api_token", + "api_token" ]) CONFIG.update(args.config) diff --git a/tap_appsflyer/schemas/raw_data/organic_installs.json b/tap_appsflyer/schemas/raw_data/organic_installs.json new file mode 100644 index 0000000..4d92835 --- /dev/null +++ b/tap_appsflyer/schemas/raw_data/organic_installs.json @@ -0,0 +1,251 @@ +{ + "type": "object", + "properties": { + "attributed_touch_type": { + "type": ["null", "string"] + }, + "attributed_touch_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "install_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "af_prt": { + "type": ["null", "string"] + }, + "media_source": { + "type": ["null", "string"] + }, + "is_retargeting": { + "type": ["null", "boolean"] + }, + "retargeting_conversion_type": { + "type": ["null", "string"] + }, + "af_channel": { + "type": ["null", "string"] + }, + "af_keywords": { + "type": ["null", "string"] + }, + "campaign": { + "type": ["null", "string"] + }, + "af_c_id": { + "type": ["null", "string"] + }, + "af_adset": { + "type": ["null", "string"] + }, + "af_adset_id": { + "type": ["null", "string"] + }, + "af_ad": { + "type": ["null", "string"] + }, + "af_ad_id": { + "type": ["null", "string"] + }, + "af_ad_type": { + "type": ["null", "string"] + }, + "af_siteid": { + "type": ["null", "string"] + }, + "af_sub1": { + "type": ["null", "string"] + }, + "af_sub2": { + "type": ["null", "string"] + }, + "af_sub3": { + "type": ["null", "string"] + }, + "af_sub4": { + "type": ["null", "string"] + }, + "af_sub5": { + "type": ["null", "string"] + }, + "http_referrer": { + "type": ["null", "string"] + }, + "original_url": { + "type": ["null", "string"] + }, + "user_agent": { + "type": ["null", "string"] + }, + "af_cost_model": { + "type": ["null", "string"] + }, + "af_cost_value": { + "type": ["null", "string"] + }, + "af_cost_currency": { + "type": ["null", "string"] + }, + "contributor1_af_prt": { + "type": ["null", "string"] + }, + "contributor1_media_source": { + "type": ["null", "string"] + }, + "contributor1_campaign": { + "type": ["null", "string"] + }, + "contributor1_touch_type": { + "type": ["null", "string"] + }, + "contributor1_touch_time": { + "type": ["null", "string"] + }, + "contributor2_af_prt": { + "type": ["null", "string"] + }, + "contributor2_media_source": { + "type": ["null", "string"] + }, + "contributor2_campaign": { + "type": ["null", "string"] + }, + "contributor2_touch_type": { + "type": ["null", "string"] + }, + "contributor2_touch_time": { + "type": ["null", "string"] + }, + "contributor3_af_prt": { + "type": ["null", "string"] + }, + "contributor3_media_source": { + "type": ["null", "string"] + }, + "contributor3_campaign": { + "type": ["null", "string"] + }, + "contributor3_touch_type": { + "type": ["null", "string"] + }, + "contributor3_touch_time": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "ip": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "wifi": { + "type": ["null", "boolean"] + }, + "operator": { + "type": ["null", "string"] + }, + "carrier": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + }, + "appsflyer_id": { + "type": ["null", "string"] + }, + "advertising_id": { + "type": ["null", "string"] + }, + "idfa": { + "type": ["null", "string"] + }, + "android_id": { + "type": ["null", "string"] + }, + "customer_user_id": { + "type": ["null", "integer", "string"] + }, + "imei": { + "type": ["null", "string"] + }, + "idfv": { + "type": ["null", "string"] + }, + "platform": { + "type": ["null", "string"] + }, + "device_type": { + "type": ["null", "string"] + }, + "os_version": { + "type": ["null", "string"] + }, + "app_version": { + "type": ["null", "string"] + }, + "sdk_version": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "string"] + }, + "app_name": { + "type": ["null", "string"] + }, + "bundle_id": { + "type": ["null", "string"] + }, + "af_attribution_lookback": { + "type": ["null", "string"] + }, + "event_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "event_name": { + "type": ["null", "string"] + }, + "event_value": { + "type": ["null", "string"] + }, + "event_revenue": { + "type": ["null", "string"] + }, + "event_revenue_usd": { + "type": ["null", "string"] + }, + "event_revenue_currency": { + "type": ["null", "string"] + }, + "is_receipt_validated": { + "type": ["null", "boolean"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "dma": { + "type": ["null", "string"] + }, + "af_reengagement_window": { + "type": ["null", "string"] + }, + "event_source": { + "type": ["null", "string"] + }, + "is_primary_attribution": { + "type": ["null", "boolean"] + }, + "af_sub_siteid": { + "type": ["null", "string"] + } + } +}