Release/1.8.0

.quickstart/quickstart.yml

@@ -9,7 +9,6 @@ table_variables:
     - FiscalCalendar
   netsuite2__multibook_accounting_enabled:
     - accountingBook
-    - AccountingBookSubsidiaries
 
   # Source
   netsuite2__using_account_types:
@@ -123,6 +122,11 @@ supported_vars:
     description: "Additional `TRANSACTION_LINE` columns to include in `netsuite2__transaction_details`"
     display_name: "Additional Transaction Line Columns"
 
+  transaction_accounting_lines_pass_through_columns:
+    type: dictionary
+    description: "Additional `TRANSACTION_ACCOUNTING_LINE` columns to include in `netsuite2__transaction_details`"
+    display_name: "Additional Transaction Accounting Line Columns"
+
   balance_sheet_transaction_detail_columns:
     type: list
     description: "Additional columns from `netsuite2__transaction_details` to include in `netsuite2__balance_sheet`"

CHANGELOG.md

@@ -1,3 +1,29 @@
+# dbt_netsuite v1.8.0
+
+[PR #210](https://github.com/fivetran/dbt_netsuite/pull/210) includes the following updates:
+
+## Schema/Data Change
+**1 total change • 1 possible breaking change**
+
+| Data Model(s) | Change type | Old | New | Notes |
+| ------------- | ----------- | --- | --- | ----- |
+| `stg_netsuite2__accounting_book_subsidiaries`<br>`stg_netsuite2__accounting_book_subsidiaries_tmp` | Removed model | | | Deprecated in v0.9.0 and unused in downstream models. Now fully sunset. |
+
+## Feature Update
+- Adds the `transaction_accounting_lines_pass_through_columns` variable, persisting chosen custom columns from the `TRANSACTION_ACCOUNTING_LINE` source table. These fields are brought into `netsuite2__transaction_details`. For more details, refer to the [Passing Through Additional Fields](README.md#passing-through-additional-fields) section of the README.
+
+## Bug Fix
+- Fixes null conversion amounts for top-level subsidiaries missing a self-conversion exchange rate; rate now defaults to 1. Updates `converted_amount` values in `netsuite2__transaction_details`, `netsuite2__income_statement`, and `netsuite2__balance_sheet`.
+
+> **Note**: A `--full-refresh` is recommended after upgrading.
+
+# dbt_netsuite v1.8.0-a1
+
+## Bug Fix
+- Fixes null conversion amounts for top-level subsidiaries missing a self-conversion exchange rate; rate now defaults to 1. Updates `converted_amount` values in `netsuite2__transaction_details`, `netsuite2__income_statement`, and `netsuite2__balance_sheet`.
+
+> **Note**: A `--full-refresh` is recommended after upgrading.
+
 # dbt_netsuite v1.7.1
 
 [PR #208](https://github.com/fivetran/dbt_netsuite/pull/208) includes the following updates:

README.md

@@ -5,7 +5,7 @@ This dbt package transforms data from Fivetran's Netsuite connector into analyti
 
 ## Resources
 
-- Number of materialized models¹: 92
+- Number of materialized models¹: 58 (Netsuite2)
 - Connector documentation
   - [Netsuite connector documentation](https://fivetran.com/docs/connectors/applications/netsuite-suiteanalytics#netsuitesuiteanalytics)
   - [Netsuite ERD](https://fivetran.com/docs/connectors/applications/netsuite-suiteanalytics#schemainformation)
@@ -99,7 +99,6 @@ To use this dbt package, you must have at least one Fivetran **Netsuite** (netsu
 - **Not required but recommended**:
   - accounttype
   - accountingbook
-  - accountingbooksubsidiary
   - accountingperiodfiscalcalendar
   - accountingbook
   - classification
@@ -126,7 +125,7 @@ Include the following netsuite package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/netsuite
-    version: [">=1.7.0", "<1.8.0"]
+    version: [">=1.8.0", "<1.9.0"]
 ```
 
 #### Databricks dispatch configuration
@@ -188,7 +187,7 @@ Your Netsuite connection may not sync every table that this package expects. If
 vars:
     # Features
     netsuite2__fiscal_calendar_enabled: true # False by default. Enable `fiscalcalendar` if you have a fiscal year starting on a month different than January.
-    netsuite2__multibook_accounting_enabled: true # False by default. Disable `accountingbooksubsidiary` and `accountingbook` if you are not using the Multi-Book Accounting feature
+    netsuite2__multibook_accounting_enabled: true # False by default. Disable `accountingbook` if you are not using the Multi-Book Accounting feature
 
     # Sources
     netsuite2__using_employees: false # True by default. Disable `employee` if you don't use employees.
@@ -259,6 +258,8 @@ vars:
         - name: "other_id"
           alias: "another_id"
           transform_sql: "cast(another_id as int64)"
+    transaction_accounting_lines_pass_through_columns: # Included in transaction_details model
+        - name: "accounting_custom_field"
     locations_pass_through_columns: # Included in transaction_details model
         - name: "location_custom_field"
     subsidiaries_pass_through_columns: # Included in transaction_details model

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'netsuite'
-version: '1.7.1'
+version: '1.8.0'
 require-dbt-version: [">=1.3.0", "<3.0.0"]
 
 models:
@@ -40,7 +40,6 @@ vars:
     netsuite_vendor_types: "{{ source('netsuite', 'vendor_types') }}"
     netsuite_vendors: "{{ source('netsuite', 'vendors') }}"
     netsuite2_account_types: "{{ source('netsuite2', 'account_type') }}"
-    netsuite2_accounting_book_subsidiaries: "{{ source('netsuite2', 'accounting_book_subsidiaries') }}"
     netsuite2_accounting_books: "{{ source('netsuite2', 'accounting_book') }}"
     netsuite2_accounting_period_fiscal_calendars: "{{ source('netsuite2', 'accounting_period_fiscal_calendars') }}"
     netsuite2_fiscal_calendar: "{{ source('netsuite2', 'fiscal_calendar') }}"
@@ -77,6 +76,7 @@ vars:
     subsidiaries_pass_through_columns: []
     transactions_pass_through_columns: []
     transaction_lines_pass_through_columns: []
+    transaction_accounting_lines_pass_through_columns: []
     balance_sheet_transaction_detail_columns: []
     income_statement_transaction_detail_columns: []
     consolidated_exchange_rates_pass_through_columns: []

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'netsuite_integration_tests'
-version: '1.7.1'
+version: '1.8.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -40,7 +40,6 @@ vars:
     netsuite2_account_identifier: "netsuite2_account_data"
     netsuite2_account_type_identifier: "netsuite2_account_type_data"
     netsuite2_accounting_book_identifier: "netsuite2_accounting_book_data"
-    netsuite2_accounting_book_subsidiaries_identifier: "netsuite2_accounting_book_sub_data"
     netsuite2_accounting_period_fiscal_calendars_identifier: "netsuite2_accounting_period_fiscal_cal_data"
     netsuite2_accounting_period_identifier: "netsuite2_accounting_period_data"
     netsuite2_classification_identifier: "netsuite2_classification_data"

integration_tests/seeds/netsuite2_tran_acct_line_data.csv

@@ -1,6 +1,6 @@
 accountingbook,transaction,transactionline,_fivetran_deleted,_fivetran_synced,account,amount,amountpaid,amountunpaid,credit,date_deleted,debit,exchangerate,netamount,overheadparentitem,paymentamountunused,paymentamountused,posting,amountlinked
-1,5914,5,FALSE,2022-07-10 22:40:48,133,,,,,,,1,,,,,T,0
-1,3,5,FALSE,2022-07-10 22:40:48,157,,,,,,,1,,,,,T,0
-1,302,4,FALSE,2022-07-10 22:40:48,118,,,,,,,1,,,,,T,0
-1,4458,0,FALSE,2022-07-10 22:40:48,,,,,,,,1,,,,,T,0
+1,5914,5,FALSE,2022-07-10 22:40:48,133,,,,,,,1,,1,,,T,0
+1,3,5,FALSE,2022-07-10 22:40:48,157,,,,,,,1,,2,,,T,0
+1,302,4,FALSE,2022-07-10 22:40:48,118,,,,,,,1,,3,,,T,0
+1,4458,0,FALSE,2022-07-10 22:40:48,,,,,,,,1,,4,,,T,0
 1,4533,0,FALSE,2022-07-10 22:40:48,,,,,,,,1,,,,,T,0

macros/staging/get_transactionaccountingline_columns.sql

@@ -46,6 +46,8 @@
     {"name": "transactionline", "datatype": dbt.type_int()}
 ] %}
 
+{{ fivetran_utils.add_pass_through_columns(columns, var('transaction_accounting_lines_pass_through_columns')) }}
+
 {{ return(columns) }}
 
 {% endmacro %}

models/docs.md

@@ -50,8 +50,6 @@
 
 {% docs vendor_categories_table %} A table containing categories and how they map to vendors. {% enddocs %}
 
-{% docs accounting_book_subsidiaries_table %} A table containing the various account books and the respective subsidiaries. {% enddocs %}
-
 {% docs accounting_period_fiscal_calendars_table %} A table containing the accounting fiscal calendar periods. {% enddocs %}
 
 {% docs account_types_table %} A table containing the various account types within Netsuite. {% enddocs %}

models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql

@@ -1,4 +1,5 @@
 {%- set multibook_accounting_enabled = var('netsuite2__multibook_accounting_enabled', false) -%}
+{%- set transaction_accounting_lines_pass_through_columns = var('transaction_accounting_lines_pass_through_columns', []) -%}
 
 {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }}
 
@@ -42,6 +43,8 @@ joined as (
         transaction_accounting_lines.unpaid_amount,
         transaction_accounting_lines.is_posting
 
+        {{ netsuite.persist_pass_through_columns(transaction_accounting_lines_pass_through_columns, identifier='transaction_accounting_lines') }}
+
     from transaction_lines
     left join transaction_accounting_lines
         on transaction_lines.transaction_line_id = transaction_accounting_lines.transaction_line_id
@@ -69,6 +72,8 @@ joined as (
         transaction_accounting_lines.unpaid_amount,
         transaction_accounting_lines.is_posting
 
+        {{ netsuite.persist_pass_through_columns(transaction_accounting_lines_pass_through_columns, identifier='transaction_accounting_lines') }}
+
     from transaction_lines
     left join transaction_accounting_lines
         on transaction_lines.transaction_line_id = transaction_accounting_lines.transaction_line_id

models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql

@@ -99,4 +99,4 @@ accountxperiod_exchange_rate_map as ( -- account table with exchange rate detail
 )
 
 select *
-from accountxperiod_exchange_rate_map
+from accountxperiod_exchange_rate_map

models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql

@@ -27,10 +27,20 @@ transaction_and_reporting_periods as (
 ), 
 
 accounts as (
-  select * 
+  select *
   from {{ ref('int_netsuite2__accounts') }}
 ),
 
+{% if using_exchange_rate %}
+subsidiaries as (
+  select
+    subsidiary_id,
+    source_relation,
+    parent_id
+  from {{ ref('stg_netsuite2__subsidiaries') }}
+),
+{% endif %}
+
 retained_earnings_accounts as (
   select
     source_relation,
@@ -73,8 +83,8 @@ transactions_in_every_calculation_period_w_exchange_rates as (
     transaction_and_reporting_periods.reporting_accounting_period_id
 
     {% if using_exchange_rate %}
-    , exchange_reporting_period.exchange_rate as exchange_rate_reporting_period
-    , exchange_transaction_period.exchange_rate as exchange_rate_transaction_period
+    , coalesce(exchange_reporting_period.exchange_rate, case when subsidiaries.parent_id is null then 1 end) as exchange_rate_reporting_period
+    , coalesce(exchange_transaction_period.exchange_rate, case when subsidiaries.parent_id is null then 1 end) as exchange_rate_transaction_period
     {% endif %}
 
     {% if using_to_subsidiary and using_exchange_rate %}
@@ -90,6 +100,10 @@ transactions_in_every_calculation_period_w_exchange_rates as (
       and transaction_and_reporting_periods.source_relation = transaction_lines_w_accounting_period.source_relation 
 
   {% if using_exchange_rate %}
+  left join subsidiaries
+    on subsidiaries.subsidiary_id = transaction_lines_w_accounting_period.subsidiary_id
+    and subsidiaries.source_relation = transaction_lines_w_accounting_period.source_relation
+
   left join accountxperiod_exchange_rate_map as exchange_reporting_period
     on exchange_reporting_period.accounting_period_id = transaction_and_reporting_periods.reporting_accounting_period_id
       and exchange_reporting_period.source_relation = transaction_and_reporting_periods.source_relation

models/netsuite2/netsuite2__transaction_details.sql

@@ -10,6 +10,7 @@
 {%- set subsidiaries_pass_through_columns = var('subsidiaries_pass_through_columns', []) -%}
 {%- set transactions_pass_through_columns = var('transactions_pass_through_columns', []) -%}
 {%- set transaction_lines_pass_through_columns = var('transaction_lines_pass_through_columns', []) -%}
+{%- set transaction_accounting_lines_pass_through_columns = var('transaction_accounting_lines_pass_through_columns', []) -%}
 {%- set lookback_window = var('lookback_window', 3) -%}
 
 {{
@@ -166,8 +167,12 @@ transaction_details as (
     {{ netsuite.persist_pass_through_columns(transactions_pass_through_columns, identifier='transactions') }}
 
     --The below script allows for transaction lines table pass through columns.
-
-    {{ netsuite.persist_pass_through_columns(transaction_lines_pass_through_columns, identifier='transaction_lines') }},
+
+    {{ netsuite.persist_pass_through_columns(transaction_lines_pass_through_columns, identifier='transaction_lines') }}
+
+    --The below script allows for transaction accounting lines table pass through columns.
+
+    {{ netsuite.persist_pass_through_columns(transaction_accounting_lines_pass_through_columns, identifier='transaction_lines') }},
 
     accounting_periods.ending_at as accounting_period_ending,
     accounting_periods.accounting_period_full_name,

models/netsuite2/staging/src_netsuite2.yml

@@ -30,23 +30,6 @@ sources:
           - name: longname
             description: The name of the account type.
 
-      - name: accounting_book_subsidiaries
-        identifier: "{{ var('netsuite2_accounting_book_subsidiaries_identifier', 'accountingbooksubsidiaries') }}"
-        description: "{{ doc('accounting_book_subsidiaries_table') }}"
-        config:
-          enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) and var('netsuite2_sources', []) == [] }}"
-        columns:
-          - name: _fivetran_id
-            description: "{{ doc('_fivetran_id') }}"
-          - name: _fivetran_synced
-            description: "{{ doc('_fivetran_synced') }}"
-          - name: accountingbook
-            description: "{{ doc('accounting_book_id') }}"
-          - name: status
-            description: The status of the accounting book subsidiary.
-          - name: subsidiary
-            description: "{{ doc('subsidiary_id') }}"
-
       - name: accounting_book
         identifier: "{{ var('netsuite2_accounting_book_identifier', 'accountingbook') }}"
         description: "{{ doc('accounting_books_table') }}"

models/netsuite2/staging/stg_netsuite2.yml

@@ -21,24 +21,6 @@ models:
       - name: type_name
         description: The name of the account type.
 
-  - name: stg_netsuite2__accounting_book_subsidiaries
-    description: "{{ doc('accounting_book_subsidiaries_table') }}"
-    columns:
-      - name: source_relation
-        description: "{{ doc('source_relation') }}"
-      - name: _fivetran_id
-        description: "{{ doc('_fivetran_id') }}"
-        tests: 
-          - not_null
-      - name: _fivetran_synced
-        description: "{{ doc('_fivetran_synced') }}"
-      - name: accounting_book_id
-        description: "{{ doc('accounting_book_id') }}"
-      - name: status
-        description: The status of the accounting book subsidiary.
-      - name: subsidiary_id
-        description: "{{ doc('subsidiary_id') }}"
-
   - name: stg_netsuite2__accounting_books
     description: "{{ doc('accounting_books_table') }}"
     columns: