Verified Commit be14eaac authored by Douwe Maan's avatar Douwe Maan
Browse files

Add all taps currently discoverable in Meltano

parent c53c67fa
Pipeline #295354131 passed with stage
in 58 seconds
name: Google Ads
type: tap
tap_type: api # TODO: Find single name instead of having both tap_type and target_type
source: # TODO: Better name that can stand in for "source" and "destination", replaces tap/target_type
source: # TODO: Better name that can stand in for "source" and "destination", replaces target_type
type: api
name: Google Adwords API
url: https://developers.google.com/adwords/api/
......@@ -21,7 +20,7 @@ variants:
- state
# TODO: `select` is not appropriate for hub, but necessary in Meltano because the default selection
# of `*.*` which selects all streams and properties is invalid.
# It'd be better to mark these with `selected-by-default: true` in `tap-adwords`'s catalog'
# It'd be better to mark the other properties with `selected-by-default: false` in `tap-adwords`'s catalog
# Meltano doesn't support that yet, though: https://gitlab.com/meltano/meltano/-/issues/2556
select:
- campaigns.*
......
name: BigQuery
type: tap
source:
type: database
name: Google BigQuery data warehouse
url: https://cloud.google.com/bigquery
description: BigQuery data warehouse
singer_name: tap-bigquery
variants:
- name: anelendata
primary_variant: true
maintainer:
name: Anelen
link: https://anelen.co
maintenance_status: Active
repo: https://github.com/anelendata/tap-bigquery
pip_url: tap-bigquery
capabilities:
- catalog
- discover
- state
settings_group_validation:
- ['streams', 'start_datetime', 'credentials_path']
settings:
- name: streams
kind: array
description: Array holding objects describing streams (tables) to extract, with `name`, `table`, `columns`, `datetime_key`, and `filters` keys. See docs for details.
- name: credentials_path # TODO: This is not a real setting, so it wouldn't show up in the `config.json` JSON schema. We should probably make it a real setting!
# The tap doesn't know this setting, but will read the GOOGLE_APPLICATION_CREDENTIALS env var.
env_aliases: [GOOGLE_APPLICATION_CREDENTIALS]
meltano_default: $MELTANO_PROJECT_ROOT/client_secrets.json
description: Fully qualified path to `client_secrets.json` for your service account.
- name: start_datetime
kind: date_iso8601
description: Determines how much historical data will be extracted. Please be aware that the larger the time period and amount of data, the longer the initial extraction can be expected to take.
- name: end_datetime
kind: date_iso8601
description: Date up to when historical data will be extracted.
- name: limit
kind: integer
description: Limits the number of records returned in each stream, applied as a limit in the query.
- name: start_always_inclusive
kind: boolean
value: true
description: When replicating incrementally, disable to only select records whose `datetime_key` is greater than the maximum value replicated in the last run, by excluding records whose timestamps match exactly. This could cause records to be missed that were created after the last run finished, but during the same second and with the same timestamp.
name: Bing Ads
type: tap
source:
type: api
name: Bing Ads API
url: https://docs.microsoft.com/en-us/advertising/guides/
description: Advertising Platform
singer_name: tap-bing-ads
variants:
- name: singer-io
primary_variant: true
maintainer:
name: Stitch
link: https://www.stitchdata.com/
maintenance_status: Unresponsive
repo: https://github.com/singer-io/tap-bing-ads
pip_url: tap-bing-ads
capabilities:
- catalog
- discover
- state
# TODO: `metadata` is not appropriate for hub, but necessary in Meltano because the default selection
# of `*.*` which selects all streams and properties is invalid. (See not on fieldExclusions in readme)
# It'd be better to mark these with `selected-by-default: false` in `tap-bing-ads`'s catalog
# Meltano doesn't support that yet, though: https://gitlab.com/meltano/meltano/-/issues/2556
metadata:
ad_group_performance_report:
AbsoluteTopImpressionSharePercent:
inclusion: available
selected: false
AbsoluteTopImpressionRatePercent:
inclusion: available
selected: false
AbsoluteTopImpressionShareLostToBudgetPercent:
inclusion: available
selected: false
AbsoluteTopImpressionShareLostToRankPercent:
inclusion: available
selected: false
AudienceImpressionLostToBudgetPercent:
inclusion: available
selected: false
AudienceImpressionLostToRankPercent:
inclusion: available
selected: false
AudienceImpressionSharePercent:
inclusion: available
selected: false
ClickSharePercent:
inclusion: available
selected: false
ExactMatchImpressionSharePercent:
inclusion: available
selected: false
ImpressionLostToAdRelevancePercent:
inclusion: available
selected: false
ImpressionLostToBidPercent:
inclusion: available
selected: false
ImpressionLostToBudgetPercent:
inclusion: available
selected: false
ImpressionLostToExpectedCtrPercent:
inclusion: available
selected: false
ImpressionLostToRankPercent:
inclusion: available
selected: false
ImpressionLostToRankAggPercent:
inclusion: available
selected: false
ImpressionSharePercent:
inclusion: available
selected: false
TopImpressionRatePercent:
inclusion: available
selected: false
TopImpressionShareLostToBudgetPercent:
inclusion: available
selected: false
TopImpressionShareLostToRankPercent:
inclusion: available
selected: false
TopImpressionSharePercent:
inclusion: available
selected: false
campaign_performance_report:
AbsoluteTopImpressionSharePercent:
inclusion: available
selected: false
AbsoluteTopImpressionRatePercent:
inclusion: available
selected: false
AbsoluteTopImpressionShareLostToBudgetPercent:
inclusion: available
selected: false
AbsoluteTopImpressionShareLostToRankPercent:
inclusion: available
selected: false
AudienceImpressionLostToBudgetPercent:
inclusion: available
selected: false
AudienceImpressionLostToRankPercent:
inclusion: available
selected: false
AudienceImpressionSharePercent:
inclusion: available
selected: false
ClickSharePercent:
inclusion: available
selected: false
ExactMatchImpressionSharePercent:
inclusion: available
selected: false
ImpressionLostToAdRelevancePercent:
inclusion: available
selected: false
ImpressionLostToBidPercent:
inclusion: available
selected: false
ImpressionLostToBudgetPercent:
inclusion: available
selected: false
ImpressionLostToExpectedCtrPercent:
inclusion: available
selected: false
ImpressionLostToRankPercent:
inclusion: available
selected: false
ImpressionLostToRankAggPercent:
inclusion: available
selected: false
ImpressionSharePercent:
inclusion: available
selected: false
TopImpressionRatePercent:
inclusion: available
selected: false
TopImpressionShareLostToBudgetPercent:
inclusion: available
selected: false
TopImpressionShareLostToRankPercent:
inclusion: available
selected: false
TopImpressionSharePercent:
inclusion: available
selected: false
settings_group_validation:
- ['customer_id', 'account_ids', 'oauth_client_id', 'oauth_client_secret', 'refresh_token', 'developer_token', 'start_date']
settings:
- name: developer_token
env: OAUTH_BING_ADS_DEVELOPER_TOKEN
kind: password
- name: oauth_client_id
env: OAUTH_BING_ADS_CLIENT_ID
label: OAuth Client ID
kind: password
- name: oauth_client_secret
env: OAUTH_BING_ADS_CLIENT_SECRET
label: OAuth Client Secret
kind: password
- name: refresh_token
kind: password
- name: customer_id
label: Customer ID
- name: account_ids
label: Account ID(s)
- name: start_date
description: Determines how much historical data will be extracted. Please be aware that the larger the time period and amount of data, the longer the initial extraction can be expected to take.
kind: date_iso8601
name: Chargebee
type: tap
source:
type: api
name: Chargebee API
url: https://apidocs.chargebee.com/docs/api
description: Subscription billing software
singer_name: tap-chargebee
variants:
- name: hotgluexyz
primary_variant: true
maintainer:
name: Hotglue
url: https://hotglue.xyz/
maintenance_status: Active
repo: https://github.com/hotgluexyz/tap-chargebee
pip_url: git+https://github.com/hotgluexyz/tap-chargebee.git
capabilities:
- properties
- discover
- state
settings_group_validation:
- - api_key
- site
- product_catalog
- start_date
settings:
- name: api_key
label: API Key
kind: password
- name: site
label: Chargebee Site
- name: product_catalog
label: Chargebee Product Catalog
- name: start_date
kind: date_iso8601
description: Determines how much historical data will be extracted. Please be
aware that the larger the time period and amount of data, the longer the initial
extraction can be expected to take.
- name: select_fields_by_default
kind: boolean
value: true
description: Select by default any new fields discovered in Quickbooks objects
- name: state_message_threshold
kind: integer
value: 1000
description: Generate a STATE message every N records
- name: max_workers
kind: integer
value: 8
label: Maximum number of threads to use
name: Comma Separated Values (CSV)
type: tap
source:
type: file
name: Comma Separated Values (CSV) file
url: https://en.wikipedia.org/wiki/Comma-separated_values
description: Generic data extractor of CSV (comma separated value) files
singer_name: tap-csv
variants:
- name: meltano
primary_variant: true
maintainer:
name: Meltano Community
maintenance_status: Active
repo: https://gitlab.com/meltano/tap-csv
pip_url: git+https://gitlab.com/meltano/tap-csv.git
capabilities:
- discover
- state
settings_group_validation:
- ['files']
- ['csv_files_definition']
settings:
- name: files
kind: array
description: Array of objects with `entity`, `file`, and `keys` keys
- name: csv_files_definition
env: TAP_CSV_FILES_DEFINITION
label: CSV Files Definition
description: Project-relative path to JSON file holding array of objects with `entity`, `file`, and `keys` keys
placeholder: Ex. files-def.json
documentation: https://gitlab.com/meltano/tap-csv#run
name: Facebook Ads
type: tap
source:
type: api
name: Facebook Ads API
url: https://developers.facebook.com/docs/marketing-apis
description: Advertising Platform
singer_name: tap-facebook
variants:
- name: meltano
primary_variant: true
maintainer:
name: Meltano Community
maintenance_status: Active
repo: https://gitlab.com/meltano/tap-facebook
pip_url: git+https://gitlab.com/meltano/tap-facebook.git
capabilities:
- properties
- discover
- state
settings_group_validation:
- ['account_id', 'access_token', 'start_date']
settings:
- name: account_id
label: Account ID
placeholder: Ex. 123456789012345
description: Your Facebook Ads Account ID
- name: access_token
label: Access Token
placeholder: Ex. *****************
description: User Token generated by Facebook OAuth handshake
kind: oauth
oauth:
provider: facebook
- name: start_date
kind: date_iso8601
description: Determines how much historical data will be extracted. Please be aware that the larger the time period and amount of data, the longer the initial extraction can be expected to take.
- name: end_date
kind: date_iso8601
description: Date up to when historical data will be extracted.
- name: insights_buffer_days
kind: integer
value: 0
label: Ads Insights Buffer Days
description: How many Days before the Start Date to fetch Ads Insights for
name: Fastly
type: tap
source:
type: api
name: Fastly API
url: https://developer.fastly.com/reference/api/
description: Edge cloud computing services provider
singer_name: tap-fastly
variants:
- name: meltano
primary_variant: true
maintainer:
name: Meltano Community
maintenance_status: Active
repo: https://gitlab.com/meltano/tap-fastly
pip_url: 'git+https://gitlab.com/meltano/tap-fastly.git'
capabilities:
- catalog
- discover
- state
settings:
- name: api_token
label: API Token
kind: password
placeholder: Ex. *****************
- name: start_date
label: Start Date
kind: date_iso8601
name: GitLab
type: tap
source:
type: api
name: GitLab API
url: https://docs.gitlab.com/ee/api/
description: Single application for the entire DevOps lifecycle
singer_name: tap-gitlab
variants:
- name: meltano
primary_variant: true
maintainer:
name: Meltano Community
maintenance_status: Active
repo: https://gitlab.com/meltano/tap-gitlab
pip_url: 'git+https://gitlab.com/meltano/tap-gitlab.git'
capabilities:
- catalog
- discover
- state
settings_group_validation:
- ['api_url', 'groups', 'start_date']
- ['api_url', 'projects', 'start_date']
settings:
- name: api_url
label: GitLab Instance
value: 'https://gitlab.com'
description: GitLab API/instance URL. When an API path is omitted, `/api/v4/` is assumed.
protected: true
- name: private_token
env_aliases: [GITLAB_API_TOKEN]
kind: password
value: ""
label: Access Token
description: GitLab personal access token or other API token.
placeholder: Ex. *****************
- name: groups
env_aliases: [GITLAB_API_GROUPS]
value: ''
label: Groups
description: Space-separated names of groups to extract data from. Leave empty and provide a project name if you'd like to pull data from a project in a personal user namespace.
placeholder: Ex. my-organization
- name: projects
env_aliases: [GITLAB_API_PROJECTS]
value: ''
label: Project
description: Space-separated `namespace/project` paths of projects to extract data from. Leave empty and provide a group name to extract data from all group projects.
placeholder: Ex. my-organization/project-1
- name: ultimate_license
env_aliases: [GITLAB_API_ULTIMATE_LICENSE]
kind: boolean
value: false
description: Enable to pull in extra data (like Epics, Epic Issues and other entities) only available to GitLab Ultimate and GitLab.com Gold accounts.
- name: fetch_merge_request_commits
kind: boolean
value: false
description: For each Merge Request, also fetch the MR's commits and create the join table `merge_request_commits` with the Merge Request and related Commit IDs. This can slow down extraction considerably because of the many API calls required.
- name: fetch_pipelines_extended
kind: boolean
value: false
description: For every Pipeline, also fetch extended details of each of these pipelines. This can slow down extraction considerably because of the many API calls required.
- name: start_date
env_aliases: [GITLAB_API_START_DATE]
kind: date_iso8601
description: Determines how much historical data will be extracted. Please be aware that the larger the time period and amount of data, the longer the initial extraction can be expected to take.
name: Google Analytics
type: tap
source:
type: api
name: Google Analytics Reporting API
url: https://developers.google.com/analytics/devguides/reporting/core/v4/
description: App and website analytics platform hosted by Google
singer_name: tap-google-analytics
variants:
- name: meltano
primary_variant: true
maintainer:
name: Meltano Community
maintenance_status: Active
repo: https://gitlab.com/meltano/tap-google-analytics
pip_url: 'git+https://gitlab.com/meltano/tap-google-analytics.git'
capabilities:
- catalog
- discover
settings_group_validation:
- ['key_file_location', 'view_id', 'start_date']
- ['oauth_credentials.client_id', 'oauth_credentials.client_secret', 'oauth_credentials.access_token', 'oauth_credentials.refresh_token', 'view_id', 'start_date']
settings:
- name: key_file_location
env_aliases: [GOOGLE_ANALYTICS_API_CLIENT_SECRETS]
kind: file
value: $MELTANO_PROJECT_ROOT/client_secrets.json
label: Client Secrets
placeholder: Ex. client_secrets.json
- name: oauth_credentials.client_id
env_aliases: [GOOGLE_ANALYTICS_API_OAUTH_CLIENT_ID]
kind: password
label: OAuth Client ID
- name: oauth_credentials.client_secret
env_aliases: [GOOGLE_ANALYTICS_API_OAUTH_CLIENT_SECRET]
kind: password
label: OAuth Client Secret
- name: oauth_credentials.access_token
env_aliases: [GOOGLE_ANALYTICS_API_OAUTH_ACCESS_TOKEN]
kind: password
label: OAuth Access Token
- name: oauth_credentials.refresh_token
env_aliases: [GOOGLE_ANALYTICS_API_OAUTH_REFRESH_TOKEN]
kind: password
label: OAuth Refresh Token
- name: view_id
env_aliases: [GOOGLE_ANALYTICS_API_VIEW_ID]
label: View ID
placeholder: Ex. 198343027
- name: reports
env_aliases: [GOOGLE_ANALYTICS_API_REPORTS]
label: Reports
placeholder: Ex. my_report_definition.json
- name: start_date
env_aliases: [GOOGLE_ANALYTICS_API_START_DATE]
kind: date_iso8601
description: This property determines how much historical data will be extracted. Please be aware that the larger the time period and amount of data, the longer the initial extraction can be expected to take.
- name: end_date
env_aliases: [GOOGLE_ANALYTICS_API_END_DATE]
kind: date_iso8601
description: Date up to when historical data will be extracted.
name: Sage Intacct
type: tap
source:
type: api
name: Sage Intacct API
url: https://developer.intacct.com/api/
description: Financial management software
singer_name: tap-intacct
variants:
- name: hotgluexyz
primary_variant: true
maintainer:
name: Hotglue
url: https://hotglue.xyz/
maintenance_status: Active
repo: https://github.com/hotgluexyz/tap-intacct
pip_url: git+https://github.com/hotgluexyz/tap-intacct.git
capabilities:
- properties
- discover
- state
settings_group_validation:
- - company_id
- sender_id
- sender_password
- user_id
- user_password
- start_date
settings:
- name: company_id
label: Company Id
- name: sender_id
label: Intacct Sender Id
- name: sender_password
label: Intacct Sender Password
kind: password
- name: user_id
label: Intacct User Id
- name: user_password
label: Intacct User Password
kind: password
- name: start_date
kind: date_iso8601
description: Determines how much historical data will be extracted. Please be
aware that the larger the time period and amount of data, the longer the initial
extraction can be expected to take.
- name: select_fields_by_default
kind: boolean
value: true
description: Select by default any new fields discovered in Quickbooks objects
- name: state_message_threshold
kind: integer
value: 1000
description: Generate a STATE message every N records
- name: max_workers
kind: integer
value: 8
label: Maximum number of threads to use
name: Marketo
type: tap
source:
type: api
name: Marketo API
url: https://developers.marketo.com/rest-api/
description: Marketing automation for account-based marketing
singer_name: tap-marketo
variants:
- name: meltano
primary_variant: true
maintainer:
name: Meltano Community
maintenance_status: Active
repo: https://gitlab.com/meltano/tap-marketo
pip_url: 'git+https://gitlab.com/meltano/tap-marketo.git'
capabilities: []
settings:
- name: endpoint
- name: identity
- name: client_id
label: Client ID
- name: client_secret
kind: password
- name: start_date
description: Determines how much historical data will be extracted. Please be aware that the larger the time period and amount of data, the longer the initial extraction can be expected to take.
kind: date_iso8601