Merge pull request #8 from TechArtists/multiple-project-source

Multiple project source

Author: SorinSV

README.md

@@ -1,5 +1,4 @@
 This library works by default for one Firebase/Google Cloud Project:
-
 # "TA:SOURCES":
     #   - {project_id: google_cloud_project_id,
     #     analytics_dataset_id: schema_id,
@@ -10,11 +9,11 @@ This library works by default for one Firebase/Google Cloud Project:
 Adding more project_ids and multiple dataset_ids for specific datasets is also possible.
 A few additional steps are required for multiple sources to be added as sources.
 
-1) TA:SOURCES_READY must be set to false (default).
+1) TA:SOURCES_MULTIPLE_PROJECTS_GENERATED must be set to false (default).
 2) add projects and datasets to the TA:SOURCES variable in your dbt project
 3) run the following command to generate sources for all projects: 
-dbt run-operation -q generate_firebase_sources > models/firebase_sources.yml 
-4) change TA:SOURCES_READY to true
+dbt run-operation -q generate_sources_multiple_projects > models/firebase_sources_multiple_projects.yml 
+4) change TA:SOURCES_MULTIPLE_PROJECTS_GENERATED to true
 # "TA:SOURCES":
     #   - {project_id: google_cloud_project_id,
     #     analytics_dataset_id: schema_id,

dbt_project.yml

@@ -33,7 +33,7 @@ seeds:
 vars: 
   ta_firebase:
     "TA:DONT_CARE": "MAKE_YAML_WORK" # optional
-    "TA:SOURCES_READY" : false
+    "TA:SOURCES_MULTIPLE_PROJECTS_GENERATED" : false
     # "TA:SOURCES":
     #   - {project_id: watermark-maker,
     #     analytics_dataset_id: analytics_150733022,

macros/custom_parameters.sql

@@ -16,11 +16,11 @@
 
 {# Array of tuples [
 (key_name, ta_type, rollup_type, extract_transformation, metric_rollup_transformation, struct_field_name, event_name_filter, null_dimension_event_name_filter, bigquery_type, how_to_extract_from_unnest)] 
-    ('ta_view_name', 'STRING', 'dimension', 'STRING', 'ta_ui_view_shown', 'LOWER(value.string_value)', 'ta_view_name_string')
+    ('ta_view_name', 'STRING', 'dimension', 'STRING', 'ta_ui_view_show', 'LOWER(value.string_value)', 'ta_view_name_string')
 #}
 {% macro get_event_parameter_tuples_all() -%}
-    {# ta_ui_view_shown & ta_ui_button_tapped #}
-    {%- set uiEventNameFilters = ["ta_ui_view_shown", "ta_ui_button_tapped", "ui_view_shown", "ui_button_tapped"] -%}
+    {# ta_ui_view_show & ta_ui_button_tapped #}
+    {%- set uiEventNameFilters = ["ta_ui_view_show", "ta_ui_button_tapped", "ui_view_show", "ui_button_tapped"] -%}
     {%- set builtin_parameters = [ 
             {"key_name":"view_name", "data_type":"STRING", "rollup_type":"dimension", "event_name_filter": uiEventNameFilters, "extract_transformation": "LOWER(TRIM(##))"} 
            ,{"key_name":"view_type", "data_type":"STRING", "rollup_type":"alsoForceNullDimension", "force_null_dimension_event_name_filter": uiEventNameFilters,  "extract_transformation": "LOWER(TRIM(##))"}

macros/generate_sources.sql

@@ -0,0 +1,70 @@
+{% macro generate_sources_multiple_projects(
+    var_name='TA:SOURCES',
+    source_prefix='firebase',
+    indent=2,
+    include_comments=true
+) %}
+  {% set sp = ' ' * indent %}
+  {% set nl = '\n' %}
+  {% set out = [] %}
+
+  {% set projects = var(var_name, []) %}
+  {% if projects is string %}{% set projects = fromjson(projects) %}{% endif %}
+
+  {% do out.append('version: 2') %}
+  {% do out.append('') %}
+  {% do out.append('sources:') %}
+
+  {% for p in projects %}
+    {% set pid = (p.get('project_id') or '') | trim %}
+    {% set events_table = (p.get('events_table') or 'events_*') | trim %}
+    {% set intraday_table = (p.get('intraday_table') or 'events_intraday_*') | trim %}
+    {% set cds = (p.get('crashlytics_dataset_id') or '') | trim %}
+    {% set crash_table = (p.get('crashlytics_table') or 'com_labpixies_flood_floodit_*') | trim %}
+
+    {# accept either analytics_dataset_ids (list) or analytics_dataset_id (string) #}
+    {% set ads_raw = p.get('analytics_dataset_ids') if p.get('analytics_dataset_ids') is not none else p.get('analytics_dataset_id') %}
+    {% if ads_raw is string %}
+      {% set ads_list = [ads_raw | trim] %}
+    {% elif ads_raw is iterable %}
+      {% set ads_list = ads_raw %}
+    {% else %}
+      {% set ads_list = [] %}
+    {% endif %}
+
+    {% for ads in ads_list %}
+      {% set ds = ads | trim %}
+      {% if ds != '' %}
+        {% if include_comments %}
+          {% do out.append(sp ~ "# Analytics for " ~ pid ~ " (" ~ ds ~ ")") %}
+        {% endif %}
+        {% do out.append(sp ~ "- name: " ~ source_prefix ~ "_analytics__" ~ pid ~ "__" ~ ds) %}
+        {% do out.append(sp ~ "  database: " ~ pid) %}
+        {% do out.append(sp ~ "  schema: " ~ ds) %}
+        {% do out.append(sp ~ "  tables:") %}
+        {% do out.append(sp ~ "    - name: events") %}
+        {% do out.append(sp ~ "      identifier: " ~ events_table) %}
+        {% do out.append(sp ~ "    - name: events_intraday") %}
+        {% do out.append(sp ~ "      identifier: " ~ intraday_table) %}
+        {% do out.append('') %}
+      {% endif %}
+    {% endfor %}
+
+    {% if cds != '' %}
+      {% if include_comments %}
+        {% do out.append(sp ~ "# Crashlytics for " ~ pid) %}
+      {% endif %}
+      {% do out.append(sp ~ "- name: " ~ source_prefix ~ "_crashlytics__" ~ pid) %}
+      {% do out.append(sp ~ "  database: " ~ pid) %}
+      {% do out.append(sp ~ "  schema: " ~ cds) %}
+      {% do out.append(sp ~ "  tables:") %}
+      {% do out.append(sp ~ "    - name: events") %}
+      {% do out.append(sp ~ "      identifier: " ~ crash_table) %}
+      {% do out.append('') %}
+    {% endif %}
+  {% endfor %}
+
+  {% set result = out | join(nl) | trim %}
+  {{ print(result) }}
+  {{ return(result) }}
+{% endmacro %}

macros/generate_sources_multiple_projects.sql

@@ -7,6 +7,7 @@
       "granularity": "day"
      },
      incremental_strategy = 'insert_overwrite',
+     lookback = 4
      require_partition_filter = true,
      cluster_by = ["event_name", "platform", "app_id"]
 ) }}
@@ -67,26 +68,26 @@ SELECT
 FROM 
 (
 {%- set projects = var('TA:SOURCES', []) -%}
-{%- set ready = var('TA:SOURCES_READY', false) -%}
+{%- set ready = var('TA:SOURCES_MULTIPLE_PROJECTS_GENERATED', false) -%}
 
  {%- set first = (projects[0] if projects and (projects[0] is mapping) else {}) -%}
-  {%- set pid0 = first.get('project_id', 'fallback_project') -%}
+  {%- set pid0 = first.get('project_id', 'single_project') -%}
   {%- set ads_raw0 = first.get('analytics_dataset_ids') if first.get('analytics_dataset_ids') is not none else first.get('analytics_dataset_id') -%}
   {%- if ads_raw0 is string -%}
     {%- set ds0 = ads_raw0 -%}
   {%- elif ads_raw0 is iterable and (ads_raw0 | length) > 0 -%}
     {%- set ds0 = ads_raw0[0] -%}
   {%- else -%}
-    {%- set ds0 = 'fallback_dataset' -%}
+    {%- set ds0 = 'single_project_dataset' -%}
   {%- endif -%}
 
  {%- if not ready -%}
-    -- FALLBACK: use the single parse-safe source until generated sources are ready
+    -- use the single project source if generated sources are not ready
     SELECT
       '{{ pid0 }}' as project_id,
       '{{ ds0 }}'  as dataset_id,
       *
-    FROM {{ source('firebase_analytics__fallback', 'events') }}
+    FROM {{ source('firebase_analytics__single_project', 'events') }}
     WHERE {{ ta_firebase.analyticsTableSuffixFilter() }}
     AND {{ ta_firebase.analyticsDateFilterFor('DATE(TIMESTAMP_MICROS(event_timestamp))') }}
  {%- else -%}

models/analytics/fb_analytics_events_raw.sql

@@ -9,13 +9,13 @@ models:
       - name: crashlytics_user_pseudo_id
         description: "`installation_uuid` from Firebase Crashlytics"
       - name: firebase_analytics_user_pseudo_id
-        description: "`fb_user_pseudo_id` custom key set in Firebase Crashlytics (by the Overbase SDK)"
+        description: "`fb_user_pseudo_id` custom key set in Firebase Crashlytics (by the TA SDK)"
       - name: user_id
-        description: "`app_user_id` custom key set in Firebase Crashlytics (by the Overbase SDK)"
+        description: "`app_user_id` custom key set in Firebase Crashlytics (by the TA SDK)"
       - name: app_id
         description: "iOS: bundle_identifier, Android: package_name"
       - name: reverse_app_id
-        description: "iOS: bundle_identifier, Android: package_name in reversed format. So 'com.overbase.myapp' becomes 'myapp.overbase.com'"
+        description: "iOS: bundle_identifier, Android: package_name in reversed format. So 'com.ta.myapp' becomes 'myapp.ta.com'"
       - name: event_id
         description: ""
       - name: platform

models/crashlytics/docs/fb_crashlytics_events_raw.yml

@@ -14,11 +14,11 @@
 -- https://firebase.google.com/docs/crashlytics/bigquery-export#without_stack_traces
 
 {% set projects = var('TA:SOURCES', []) %}
-{% set ready = var('TA:SOURCES_READY', false) %}
+{% set ready = var('TA:SOURCES_MULTIPLE_PROJECTS_GENERATED', false) %}
 
 {% set first = projects[0] if projects and (projects[0] is mapping) else {} %}
-{% set pid0 = first.get('project_id', 'fallback_project') %}
-{% set ds0  = first.get('crashlytics_dataset_id', 'fallback_crashlytics_dataset') %}
+{% set pid0 = first.get('project_id', 'single_project') %}
+{% set ds0  = first.get('crashlytics_dataset_id', 'single_project_crashlytics_dataset') %}
 
 {%- set ns = namespace(first=true) -%}
 
@@ -93,7 +93,7 @@
           ) AS unity_metadata
         , COUNT(1) OVER (PARTITION BY installation_uuid, event_id, variant_id) as duplicates_cnt
 
-    FROM {{ source('firebase_crashlytics__fallback', 'events') }}
+    FROM {{ source('firebase_crashlytics__single_project', 'events') }}
     WHERE {{ ta_firebase.crashlyticsTSFilterFor("event_timestamp") }}
     QUALIFY ROW_NUMBER() OVER (PARTITION BY crashlytics_user_pseudo_id, event_id, variant_id ORDER BY received_ts) = 1
 

models/crashlytics/fb_crashlytics_events_raw.sql

@@ -2,17 +2,17 @@ version: 2
 
 sources:
   # Firebase Analytics (safe default)
-  - name: "firebase_analytics__fallback"
+  - name: "firebase_analytics__single_project"
     database: "{{ var('TA:SOURCES','ta')[0]['project_id'] }}"
     schema: "{{ var('TA:SOURCES','firebase_analytics_raw_test')[0]['analytics_dataset_id'] }}"
     tables:
       - name: events
         identifier: "{{ var('TA:SOURCES','events_*')[0]['events_table'] }}"
 
   # Firebase Crashlytics (safe default)
-  - name: firebase_crashlytics__fallback
+  - name: "firebase_crashlytics__single_project"
     database: "{{ var('TA:SOURCES','ta')[0]['project_id'] }}"
-    schema: "{{ var('TA:SOURCES', 'firebase_crashlytics__fallback')[0]['crashlytics_dataset_id'] }}"
+    schema: "{{ var('TA:SOURCES', 'firebase_crashlytics__single_project')[0]['crashlytics_dataset_id'] }}"
     tables:
       - name: events
         identifier: "{{ var('TA:SOURCES','com_app_*')[0]['crashlytics_table'] }}"

models/safe_sources.yml models/sources_single_project.ymlmodels/safe_sources.yml renamed to models/sources_single_project.yml

@@ -13,16 +13,15 @@ WITH stg AS (
 
 src AS (
   {%- set projects = var('TA:SOURCES', []) -%}
-  {%- set ready    = var('TA:SOURCES_READY', false) -%}
+  {%- set ready    = var('TA:SOURCES_MULTIPLE_PROJECTS_GENERATED', false) -%}
   {%- set ns = namespace(first=true) -%}
 
   {%- if not ready -%}
     SELECT
       DATE(TIMESTAMP_MICROS(event_timestamp)) AS event_date,
       COUNT(*) AS cnt,
-      -- use first project's id if available; otherwise literal 'fallback'
-      {{ (projects[0]['project_id'] if projects and (projects[0] is mapping) and projects[0].get('project_id') else 'fallback') | tojson }} AS project_id
-    FROM {{ source('firebase_analytics__fallback', 'events') }}
+      {{ (projects[0]['project_id'] if projects and (projects[0] is mapping) and projects[0].get('project_id') else 'single_project') | tojson }} AS project_id
+    FROM {{ source('firebase_analytics__single_project', 'events') }}
     WHERE {{ ta_firebase.analyticsTestTableSuffixFilter(extend=3) }}
       AND {{ ta_firebase.analyticsTestDateFilter('DATE(TIMESTAMP_MICROS(event_timestamp))', extend=2) }}
       AND DATE(TIMESTAMP_MICROS(event_timestamp)) <= CURRENT_DATE() - 5