setup dbt

logs/dbt.log

+16:44:30.818193 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fcec0ca6e20>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fcec1392910>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fcec13929a0>]}
+
+
+============================== 16:44:30.820908 | 81b85372-6b00-4d14-95b1-0767610d2640 ==============================
+16:44:30.820908 [info ] [MainThread]: Running with dbt=1.6.2
+16:44:30.821455 [debug] [MainThread]: running dbt with arguments {'printer_width': '80', 'indirect_selection': 'eager', 'log_cache_events': 'False', 'write_json': 'True', 'partial_parse': 'True', 'cache_selected_only': 'False', 'profiles_dir': '/home/sysprd/.dbt', 'fail_fast': 'False', 'debug': 'False', 'log_path': 'logs', 'version_check': 'True', 'warn_error': 'None', 'use_colors': 'True', 'use_experimental_parser': 'False', 'no_print': 'None', 'quiet': 'False', 'log_format': 'default', 'invocation_command': 'dbt init spark-test', 'introspect': 'True', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'target_path': 'None', 'static_parser': 'True', 'send_anonymous_usage_stats': 'True'}
+16:44:30.821881 [warn ] [MainThread]: [ConfigFolderDirectory]: Unable to parse dict {'dir': PosixPath('/home/sysprd/.dbt')}
+16:44:30.822211 [info ] [MainThread]: Creating dbt configuration folder at 
+16:44:38.080806 [error] [MainThread]: Encountered an error:
+
+16:44:38.082129 [error] [MainThread]: Traceback (most recent call last):
+  File "/home/sysprd/dbt-env/lib/python3.8/site-packages/dbt/cli/requires.py", line 87, in wrapper
+    result, success = func(*args, **kwargs)
+  File "/home/sysprd/dbt-env/lib/python3.8/site-packages/dbt/cli/requires.py", line 72, in wrapper
+    return func(*args, **kwargs)
+  File "/home/sysprd/dbt-env/lib/python3.8/site-packages/dbt/cli/main.py", line 468, in init
+    results = task.run()
+  File "/home/sysprd/dbt-env/lib/python3.8/site-packages/dbt/task/init.py", line 283, in run
+    project_name = self.get_valid_project_name()
+  File "/home/sysprd/dbt-env/lib/python3.8/site-packages/dbt/task/init.py", line 246, in get_valid_project_name
+    name = click.prompt("Enter a name for your project (letters, digits, underscore)")
+  File "/home/sysprd/dbt-env/lib/python3.8/site-packages/click/termui.py", line 164, in prompt
+    value = prompt_func(prompt)
+  File "/home/sysprd/dbt-env/lib/python3.8/site-packages/click/termui.py", line 147, in prompt_func
+    raise Abort() from None
+click.exceptions.Abort
+
+16:44:38.082902 [debug] [MainThread]: Command `dbt init` failed at 16:44:38.082764 after 7.27 seconds
+16:44:38.083296 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fcec0ca6e20>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fcec133ed90>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fcec133ea00>]}
+16:44:38.083710 [debug] [MainThread]: Flushing usage events
+16:44:46.129430 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7faacb944fa0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7faacc031b20>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7faacc0319d0>]}
+
+
+============================== 16:44:46.132115 | 1784c8cc-909e-4f6c-9084-a5761df44efa ==============================
+16:44:46.132115 [info ] [MainThread]: Running with dbt=1.6.2
+16:44:46.132622 [debug] [MainThread]: running dbt with arguments {'printer_width': '80', 'indirect_selection': 'eager', 'write_json': 'True', 'log_cache_events': 'False', 'partial_parse': 'True', 'cache_selected_only': 'False', 'warn_error': 'None', 'debug': 'False', 'version_check': 'True', 'log_path': 'logs', 'fail_fast': 'False', 'profiles_dir': '/home/sysprd/.dbt', 'use_colors': 'True', 'use_experimental_parser': 'False', 'no_print': 'None', 'quiet': 'False', 'log_format': 'default', 'invocation_command': 'dbt init spark_test', 'introspect': 'True', 'static_parser': 'True', 'target_path': 'None', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'send_anonymous_usage_stats': 'True'}
+16:44:46.133361 [debug] [MainThread]: Starter project path: /home/sysprd/dbt-env/lib/python3.8/site-packages/dbt/include/starter_project
+16:44:46.135751 [info ] [MainThread]: 
+Your new dbt project "spark_test" was created!
+
+For more information on how to configure the profiles.yml file,
+please consult the dbt documentation here:
+
+  https://docs.getdbt.com/docs/configure-your-profile
+
+One more thing:
+
+Need help? Don't hesitate to reach out to us via GitHub issues or on Slack:
+
+  https://community.getdbt.com/
+
+Happy modeling!
+
+16:44:46.136117 [info ] [MainThread]: Setting up your profile.
+16:53:07.981847 [info ] [MainThread]: Profile spark_test written to /home/sysprd/.dbt/profiles.yml using target's profile_template.yml and your supplied values. Run 'dbt debug' to validate the connection.
+16:53:07.982733 [debug] [MainThread]: Command `dbt init` succeeded at 16:53:07.982580 after 501.86 seconds
+16:53:07.983184 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7faacb944fa0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7faacd951490>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7faacbfddd00>]}
+16:53:07.983616 [debug] [MainThread]: Flushing usage events
+16:53:13.546827 [debug] [MainThread]: Error sending anonymous usage statistics. Disabling tracking for this execution. If you wish to permanently disable tracking, see: https://docs.getdbt.com/reference/global-configs#send-anonymous-usage-stats.
+16:53:23.387880 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fc14e5cddc0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fc14eca0760>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fc14eca0550>]}
+
+
+============================== 16:53:23.390589 | 8028ce6d-58f4-47bd-ac64-5ec77483d740 ==============================
+16:53:23.390589 [info ] [MainThread]: Running with dbt=1.6.2
+16:53:23.391126 [debug] [MainThread]: running dbt with arguments {'printer_width': '80', 'indirect_selection': 'eager', 'log_cache_events': 'False', 'write_json': 'True', 'partial_parse': 'True', 'cache_selected_only': 'False', 'warn_error': 'None', 'version_check': 'True', 'fail_fast': 'False', 'log_path': 'logs', 'debug': 'False', 'profiles_dir': '/home/sysprd/.dbt', 'use_colors': 'True', 'use_experimental_parser': 'False', 'no_print': 'None', 'quiet': 'False', 'log_format': 'default', 'static_parser': 'True', 'introspect': 'True', 'invocation_command': 'dbt debug', 'target_path': 'None', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'send_anonymous_usage_stats': 'True'}
+16:53:23.391575 [info ] [MainThread]: dbt version: 1.6.2
+16:53:23.391929 [info ] [MainThread]: python version: 3.8.10
+16:53:23.392255 [info ] [MainThread]: python path: /home/sysprd/dbt-env/bin/python3
+16:53:23.392563 [info ] [MainThread]: os info: Linux-5.4.0-144-generic-x86_64-with-glibc2.29
+16:53:23.457703 [info ] [MainThread]: Using profiles dir at /home/sysprd/.dbt
+16:53:23.458171 [info ] [MainThread]: Using profiles.yml file at /home/sysprd/.dbt/profiles.yml
+16:53:23.458546 [info ] [MainThread]: Using dbt_project.yml file at /home/sysprd/dbt-test/dbt_project.yml
+16:53:23.458992 [info ] [MainThread]: adapter type: spark
+16:53:23.459326 [info ] [MainThread]: adapter version: 1.6.0
+16:53:23.459658 [info ] [MainThread]: Configuration:
+16:53:23.460026 [info ] [MainThread]:   profiles.yml file [OK found and valid]
+16:53:23.460356 [info ] [MainThread]:   dbt_project.yml file [ERROR not found]
+16:53:23.460657 [info ] [MainThread]: Required dependencies:
+16:53:23.460999 [debug] [MainThread]: Executing "git --help"
+16:53:23.466116 [debug] [MainThread]: STDOUT: "b"usage: git [--version] [--help] [-C <path>] [-c <name>=<value>]\n           [--exec-path[=<path>]] [--html-path] [--man-path] [--info-path]\n           [-p | --paginate | -P | --no-pager] [--no-replace-objects] [--bare]\n           [--git-dir=<path>] [--work-tree=<path>] [--namespace=<name>]\n           <command> [<args>]\n\nThese are common Git commands used in various situations:\n\nstart a working area (see also: git help tutorial)\n   clone             Clone a repository into a new directory\n   init              Create an empty Git repository or reinitialize an existing one\n\nwork on the current change (see also: git help everyday)\n   add               Add file contents to the index\n   mv                Move or rename a file, a directory, or a symlink\n   restore           Restore working tree files\n   rm                Remove files from the working tree and from the index\n   sparse-checkout   Initialize and modify the sparse-checkout\n\nexamine the history and state (see also: git help revisions)\n   bisect            Use binary search to find the commit that introduced a bug\n   diff              Show changes between commits, commit and working tree, etc\n   grep              Print lines matching a pattern\n   log               Show commit logs\n   show              Show various types of objects\n   status            Show the working tree status\n\ngrow, mark and tweak your common history\n   branch            List, create, or delete branches\n   commit            Record changes to the repository\n   merge             Join two or more development histories together\n   rebase            Reapply commits on top of another base tip\n   reset             Reset current HEAD to the specified state\n   switch            Switch branches\n   tag               Create, list, delete or verify a tag object signed with GPG\n\ncollaborate (see also: git help workflows)\n   fetch             Download objects and refs from another repository\n   pull              Fetch from and integrate with another repository or a local branch\n   push              Update remote refs along with associated objects\n\n'git help -a' and 'git help -g' list available subcommands and some\nconcept guides. See 'git help <command>' or 'git help <concept>'\nto read about a specific subcommand or concept.\nSee 'git help git' for an overview of the system.\n""
+16:53:23.466827 [debug] [MainThread]: STDERR: "b''"
+16:53:23.467207 [info ] [MainThread]:  - git [OK found]
+
+16:53:23.467550 [info ] [MainThread]: Connection:
+16:53:23.467942 [info ] [MainThread]:   host: 10.130.8.223
+16:53:23.468264 [info ] [MainThread]:   port: 9083
+16:53:23.468568 [info ] [MainThread]:   cluster: None
+16:53:23.468866 [info ] [MainThread]:   endpoint: None
+16:53:23.469200 [info ] [MainThread]:   schema: test
+16:53:23.469512 [info ] [MainThread]:   organization: 0
+16:53:23.470028 [info ] [MainThread]: Registered adapter: spark=1.6.0
+16:53:23.470848 [debug] [MainThread]: Acquiring new spark connection 'debug'
+16:53:23.471242 [debug] [MainThread]: Using spark connection "debug"
+16:53:23.471573 [debug] [MainThread]: On debug: select 1 as id
+16:53:23.472153 [debug] [MainThread]: Opening a new connection, currently in state init
+16:53:23.473272 [debug] [MainThread]: Spark adapter: Error while running:
+select 1 as id
+16:53:23.473659 [debug] [MainThread]: Spark adapter: Database Error
+  failed to connect
+16:53:23.474060 [info ] [MainThread]:   Connection test: [ERROR]
+
+16:53:23.474397 [info ] [MainThread]: 1 check failed:
+16:53:23.474811 [info ] [MainThread]: Project loading failed for the following reason:
+ project path </home/sysprd/dbt-test/dbt_project.yml> not found
+
+16:53:23.475563 [debug] [MainThread]: Command `dbt debug` failed at 16:53:23.475402 after 0.10 seconds
+16:53:23.475954 [debug] [MainThread]: Connection 'debug' was properly closed.
+16:53:23.476348 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [<snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fc14e5cddc0>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fc14c356280>, <snowplow_tracker.self_describing_json.SelfDescribingJson object at 0x7fc14c356880>]}
+16:53:23.476832 [debug] [MainThread]: Flushing usage events
+16:53:28.742622 [debug] [MainThread]: Error sending anonymous usage statistics. Disabling tracking for this execution. If you wish to permanently disable tracking, see: https://docs.getdbt.com/reference/global-configs#send-anonymous-usage-stats.

spark_test/.gitignore

+
+target/
+dbt_packages/
+logs/

spark_test/README.md

+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

spark_test/dbt_project.yml

+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'spark_test'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'spark_test'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{ config(...) }}` macro.
+models:
+  spark_test:
+    # Config indicated by + and applies to all files under models/example/
+    example:
+      +materialized: view

spark_test/models/example/my_first_dbt_model.sql

+
+/*
+    Welcome to your first dbt model!
+    Did you know that you can also configure models directly within SQL files?
+    This will override configurations stated in dbt_project.yml
+
+    Try changing "table" to "view" below
+*/
+
+{{ config(materialized='table') }}
+
+with source_data as (
+
+    select 1 as id
+    union all
+    select null as id
+
+)
+
+select *
+from source_data
+
+/*
+    Uncomment the line below to remove records with null `id` values
+*/
+
+-- where id is not null

spark_test/models/example/my_second_dbt_model.sql

+
+-- Use the `ref` function to select from other models
+
+select *
+from {{ ref('my_first_dbt_model') }}
+where id = 1

spark_test/models/example/schema.yml

+
+version: 2
+
+models:
+  - name: my_first_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        tests:
+          - unique
+          - not_null
+
+  - name: my_second_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        tests:
+          - unique
+          - not_null