add multi-project support

Author: SorinSV

dbt_project.yml

@@ -33,15 +33,24 @@ seeds:
 vars: 
   overbase_firebase:
     "OVERBASE:DONT_CARE": "MAKE_YAML_WORK" # optional
+    # "OVERBASE:SOURCES":
+    #   - {project_id: watermark-maker,
+    #     analytics_dataset_id: analytics_150733022,
+    #     events_table: events_*,
+    #     crashlytics_dataset_id: firebase_crashlytics,
+    #     crashlytics_table: com_kobe_watermarkmaker_*}
+    #   # add more projects here
+    #   - {project_id: logo-maker-35628,
+    #     analytics_dataset_id: analytics_153578578,
+    #     events_table: events_*,
+    #     crashlytics_dataset_id: firebase_crashlytics,
+    #     crashlytics_table: com_dave_logomaker_*}
+    
     # don't define variables here, because if they aren't overwritten in the client yml, then they will resolve to "None". 
     # Instead, use them in the package where you need them with them with the appropriate default value
     # Furthermore, whatever we use in sources.yml, we can't define in this package's dbt_project.yml, or else they won't be able to be overwritten by the client
     # Any mandatory variables you want to be enforced should go into overbase_mandatory_vars.sql
     # The comments liste here is just for our sanity & easier c/p in client configs
-    # "OVERBASE:FIREBASE_PROJECT_ID": "overbase" # mandatory, but used in sources.yml
-    # "OVERBASE:FIREBASE_ANALYTICS_DATASET_ID": "firebase_analytics_raw_test" # mandatory, but used in sources.yml
-    # "OVERBASE:FIREBASE_ANALYTICS_EVENTS_TABLE_NAME": "events_*" # optional, but used in sources.yml
-    # "OVERBASE:FIREBASE_ANALYTICS_EVENTS_INTRADAY_TABLE_NAME": "events_intraday_*" # optional, but used in sources.yml
     # "OVERBASE:FIREBASE_ANALYTICS_CUSTOM_INSTALL_EVENT": "" # optional e.g. attributed_first_open
     # "OVERBASE:CUSTOM_PLATFORM_PREPROCESSOR": "IF(app_info.version LIKE '%nexus', 'nexus', platform)"
     # "OVERBASE:FIREBASE_ANALYTICS_DEFAULT_INCREMENTAL_DAYS":  # optional

models/analytics/fb_analytics_events_raw.sql

@@ -57,7 +57,6 @@ SELECT    TIMESTAMP_MICROS(event_timestamp) as event_ts
         ) AS users_ltv
         , STRUCT<firebase_app_id STRING, stream_id STRING, advertising_id STRING>(
             LOWER(app_info.firebase_app_id), LOWER(stream_id), LOWER({{ null_if_length_zero('device.advertising_id') }})
-        ) as other_ids
         , {{ overbase_firebase.generate_date_timezone_struct('TIMESTAMP_MICROS(event_timestamp)') }} as event_dates
         , {{ overbase_firebase.generate_date_timezone_struct('TIMESTAMP_MICROS(user_first_touch_timestamp)') }} as install_dates
         , COUNT(1) OVER (PARTITION BY user_pseudo_id, event_bundle_sequence_id, event_name, event_timestamp, event_previous_timestamp) as duplicates_cnt
@@ -71,6 +70,4 @@ LEFT JOIN {{ref('ob_iso_country')}} as language_region_codes -- some language ha
 
 WHERE True 
 AND {{ overbase_firebase.analyticsTableSuffixFilter() }} -- already extended by 1 day compared to event_timestamp filter
-AND {{ overbase_firebase.analyticsDateFilterFor('DATE(TIMESTAMP_MICROS(event_timestamp))') }}
 
-QUALIFY ROW_NUMBER() OVER (PARTITION BY user_pseudo_id, event_bundle_sequence_id, event_name, event_timestamp, event_previous_timestamp) = 1

models/crashlytics/fb_crashlytics_events_raw.sql

@@ -12,7 +12,13 @@
 ) }}
 
 -- https://firebase.google.com/docs/crashlytics/bigquery-export#without_stack_traces
-SELECT    event_timestamp as event_ts
+
+{% set projects = var('OVERBASE:SOURCES', []) %}
+
+{% for p in projects %}
+{% if not loop.first %}UNION ALL{% endif %}
+SELECT    
+    '{{ p.project_id }}' as project_id,
     , DATE(event_timestamp) as event_date
     , received_timestamp as received_ts
     , installation_uuid as crashlytics_user_pseudo_id
@@ -77,8 +83,8 @@ SELECT    event_timestamp as event_ts
             ], "unity_metadata." )| join(", ") }}
           ) AS unity_metadata
         , COUNT(1) OVER (PARTITION BY installation_uuid, event_id, variant_id) as duplicates_cnt
-FROM {{ source("firebase_crashlytics", "events") }}  as events
-WHERE True 
-AND {{ overbase_firebase.crashlyticsTSFilterFor("event_timestamp") }}
-QUALIFY ROW_NUMBER() OVER (PARTITION BY installation_uuid, event_id, variant_id ORDER BY received_ts) = 1
 
+FROM {{ source('firebase_crashlytics__' ~ p.project_id, 'events') }}
+WHERE  {{ overbase_firebase.crashlyticsTSFilterFor("event_timestamp") }}
+QUALIFY ROW_NUMBER() OVER (PARTITION BY installation_uuid, event_id, variant_id ORDER BY received_ts) = 1
+{% endfor %}