Hub JSON API endpoint for discovery, plus detailed API for plugin defs

Pat Nadolnyrequested to merge
discovery_endpoint into main

Closes: #224 (closed)

This creates an endpoint meltano/api/v1/plugins/index which returns and index of all the available plugins, their default and the list of variants. Additionally theres an index for each plugin type following this convention meltano/api/v1/plugins/<plugin_type>/index.

Click to expand 
{
    "utilities": {
        "great_expectations": {
            "logo_url": null,
            "default_variant": "great-expectations",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/utilities/great_expectations--great-expectations"
                }
            }
        },
        "sqlfluff": {
            "logo_url": null,
            "default_variant": "sqlfluff",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/utilities/sqlfluff--sqlfluff"
                }
            }
        }
    },
    "orchestrators": {
        "airflow": {
            "logo_url": null,
            "default_variant": "apache",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/orchestrators/airflow--apache"
                }
            }
        }
    },
    "transformers": {
        "dbt": {
            "logo_url": null,
            "default_variant": "dbt-labs",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/transformers/dbt--dbt-labs"
                }
            }
        }
    },
    "files": {
        "great_expectations": {
            "logo_url": null,
            "default_variant": "meltano",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/files/great_expectations--meltano"
                }
            }
        },
        "gitlab-ci": {
            "logo_url": null,
            "default_variant": "meltano",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/files/gitlab-ci--meltano"
                }
            }
        },
        "airflow": {
            "logo_url": null,
            "default_variant": "meltano",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/files/airflow--meltano"
                }
            }
        },
        "docker-compose": {
            "logo_url": null,
            "default_variant": "meltano",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/files/docker-compose--meltano"
                }
            }
        },
        "dbt": {
            "logo_url": null,
            "default_variant": "meltano",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/files/dbt--meltano"
                }
            }
        },
        "docker": {
            "logo_url": null,
            "default_variant": "meltano",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/files/docker--meltano"
                }
            }
        }
    },
    "mappers": {
        "transform-field": {
            "logo_url": null,
            "default_variant": "transferwise",
            "variants": {
                "transferwise": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/mappers/transform-field--transferwise"
                }
            }
        },
        "meltano-map-transformer": {
            "logo_url": null,
            "default_variant": "meltano",
            "variants": {
                "variant_name": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/mappers/meltano-map-transformer--meltano"
                }
            }
        }
    },
    "extractors": {
        "tap-chargebee": {
            "logo_url": "http://localhost:4000/assets/logos/taps/chargebee.png",
            "default_variant": "hotgluexyz",
            "variants": {
                "hotgluexyz": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-chargebee--hotgluexyz"
                }
            }
        },
        "tap-stripe": {
            "logo_url": "http://localhost:4000/assets/logos/taps/stripe.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-stripe--singer-io"
                },
                "prratek": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-stripe--prratek"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-stripe--meltano"
                }
            }
        },
        "tap-google-sheets": {
            "logo_url": "http://localhost:4000/assets/logos/taps/google-sheets.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-google-sheets--singer-io"
                }
            }
        },
        "tap-facebook": {
            "logo_url": "http://localhost:4000/assets/logos/taps/facebook.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-facebook--singer-io"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-facebook--meltano"
                }
            }
        },
        "tap-pendo": {
            "logo_url": "http://localhost:4000/assets/logos/taps/pendo.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-pendo--singer-io"
                }
            }
        },
        "tap-csv": {
            "logo_url": "http://localhost:4000/assets/logos/taps/csv.png",
            "default_variant": "meltanolabs",
            "variants": {
                "meltanolabs": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-csv--meltanolabs"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-csv--meltano"
                }
            }
        },
        "tap-twilio": {
            "logo_url": "http://localhost:4000/assets/logos/taps/twilio.png",
            "default_variant": "transferwise",
            "variants": {
                "transferwise": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-twilio--transferwise"
                }
            }
        },
        "tap-mongodb": {
            "logo_url": "http://localhost:4000/assets/logos/taps/mongodb.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-mongodb--singer-io"
                }
            }
        },
        "tap-mysql": {
            "logo_url": "http://localhost:4000/assets/logos/taps/mysql.png",
            "default_variant": "transferwise",
            "variants": {
                "transferwise": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-mysql--transferwise"
                }
            }
        },
        "tap-slack": {
            "logo_url": "http://localhost:4000/assets/logos/taps/slack.png",
            "default_variant": "mashey",
            "variants": {
                "mashey": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-slack--mashey"
                }
            }
        },
        "tap-bigquery": {
            "logo_url": "http://localhost:4000/assets/logos/taps/bigquery.png",
            "default_variant": "anelendata",
            "variants": {
                "anelendata": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-bigquery--anelendata"
                }
            }
        },
        "tap-marketo": {
            "logo_url": "http://localhost:4000/assets/logos/taps/marketo.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-marketo--singer-io"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-marketo--meltano"
                }
            }
        },
        "tap-google-analytics": {
            "logo_url": "http://localhost:4000/assets/logos/taps/google-analytics.png",
            "default_variant": "meltanolabs",
            "variants": {
                "meltanolabs": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-google-analytics--meltanolabs"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-google-analytics--meltano"
                }
            }
        },
        "tap-hubspot": {
            "logo_url": "http://localhost:4000/assets/logos/taps/hubspot.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-hubspot--singer-io"
                }
            }
        },
        "tap-github": {
            "logo_url": "http://localhost:4000/assets/logos/taps/github.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-github--singer-io"
                }
            }
        },
        "tap-postgres": {
            "logo_url": "http://localhost:4000/assets/logos/taps/postgres.png",
            "default_variant": "transferwise",
            "variants": {
                "transferwise": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-postgres--transferwise"
                }
            }
        },
        "tap-gitlab": {
            "logo_url": "http://localhost:4000/assets/logos/taps/gitlab.png",
            "default_variant": "meltanolabs",
            "variants": {
                "meltanolabs": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-gitlab--meltanolabs"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-gitlab--meltano"
                }
            }
        },
        "tap-carbon-intensity": {
            "logo_url": "http://localhost:4000/assets/logos/taps/carbon-intensity.png",
            "default_variant": "meltano",
            "variants": {
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-carbon-intensity--meltano"
                }
            }
        },
        "tap-solarvista": {
            "logo_url": "http://localhost:4000/assets/logos/taps/solarvista.png",
            "default_variant": "matatika",
            "variants": {
                "matatika": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-solarvista--matatika"
                }
            }
        },
        "tap-quickbooks": {
            "logo_url": "http://localhost:4000/assets/logos/taps/quickbooks.png",
            "default_variant": "hotgluexyz",
            "variants": {
                "hotgluexyz": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-quickbooks--hotgluexyz"
                }
            }
        },
        "tap-jira": {
            "logo_url": "http://localhost:4000/assets/logos/taps/jira.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-jira--singer-io"
                }
            }
        },
        "tap-bing-ads": {
            "logo_url": "http://localhost:4000/assets/logos/taps/bing-ads.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-bing-ads--singer-io"
                }
            }
        },
        "tap-zendesk": {
            "logo_url": "http://localhost:4000/assets/logos/taps/zendesk.png",
            "default_variant": "twilio-labs",
            "variants": {
                "twilio-labs": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-zendesk--twilio-labs"
                },
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-zendesk--singer-io"
                }
            }
        },
        "tap-salesforce": {
            "logo_url": "http://localhost:4000/assets/logos/taps/salesforce.png",
            "default_variant": "meltano",
            "variants": {
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-salesforce--meltano"
                }
            }
        },
        "tap-recharge": {
            "logo_url": "http://localhost:4000/assets/logos/taps/recharge.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-recharge--singer-io"
                }
            }
        },
        "tap-ask-nicely": {
            "logo_url": "http://localhost:4000/assets/logos/taps/ask-nicely.png",
            "default_variant": "mashey",
            "variants": {
                "mashey": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-ask-nicely--mashey"
                }
            }
        },
        "tap-fastly": {
            "logo_url": "http://localhost:4000/assets/logos/taps/fastly.png",
            "default_variant": "splitio",
            "variants": {
                "splitio": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-fastly--splitio"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-fastly--meltano"
                }
            }
        },
        "tap-adwords": {
            "logo_url": "http://localhost:4000/assets/logos/taps/adwords.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-adwords--singer-io"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-adwords--meltano"
                }
            }
        },
        "tap-shopify": {
            "logo_url": "http://localhost:4000/assets/logos/taps/shopify.png",
            "default_variant": "singer-io",
            "variants": {
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-shopify--singer-io"
                }
            }
        },
        "tap-intacct": {
            "logo_url": "http://localhost:4000/assets/logos/taps/intacct.png",
            "default_variant": "hotgluexyz",
            "variants": {
                "hotgluexyz": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-intacct--hotgluexyz"
                }
            }
        },
        "tap-zoom": {
            "logo_url": "http://localhost:4000/assets/logos/taps/zoom.png",
            "default_variant": "mashey",
            "variants": {
                "mashey": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-zoom--mashey"
                }
            }
        },
        "tap-spreadsheets-anywhere": {
            "logo_url": "http://localhost:4000/assets/logos/taps/spreadsheets-anywhere.png",
            "default_variant": "ets",
            "variants": {
                "ets": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/extractors/tap-spreadsheets-anywhere--ets"
                }
            }
        }
    },
    "loaders": {
        "target-csv": {
            "logo_url": "http://localhost:4000/assets/logos/targets/csv.png",
            "default_variant": "hotgluexyz",
            "variants": {
                "hotgluexyz": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-csv--hotgluexyz"
                },
                "singer-io": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-csv--singer-io"
                }
            }
        },
        "target-sqlite": {
            "logo_url": "http://localhost:4000/assets/logos/targets/sqlite.png",
            "default_variant": "meltanolabs",
            "variants": {
                "meltanolabs": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-sqlite--meltanolabs"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-sqlite--meltano"
                }
            }
        },
        "target-bigquery": {
            "logo_url": "http://localhost:4000/assets/logos/targets/bigquery.png",
            "default_variant": "adswerve",
            "variants": {
                "adswerve": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-bigquery--adswerve"
                }
            }
        },
        "target-postgres": {
            "logo_url": "http://localhost:4000/assets/logos/targets/postgres.png",
            "default_variant": "transferwise",
            "variants": {
                "transferwise": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-postgres--transferwise"
                },
                "datamill-co": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-postgres--datamill-co"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-postgres--meltano"
                }
            }
        },
        "target-jsonl": {
            "logo_url": "http://localhost:4000/assets/logos/targets/jsonl.png",
            "default_variant": "andyh1203",
            "variants": {
                "andyh1203": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-jsonl--andyh1203"
                }
            }
        },
        "target-redshift": {
            "logo_url": "http://localhost:4000/assets/logos/targets/redshift.png",
            "default_variant": "transferwise",
            "variants": {
                "transferwise": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-redshift--transferwise"
                }
            }
        },
        "target-snowflake": {
            "logo_url": "http://localhost:4000/assets/logos/targets/snowflake.png",
            "default_variant": "transferwise",
            "variants": {
                "transferwise": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-snowflake--transferwise"
                },
                "datamill-co": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-snowflake--datamill-co"
                },
                "meltano": {
                    "ref": "http://localhost:4000/meltano/api/v1/plugins/loaders/target-snowflake--meltano"
                }
            }
        }
    }
}

and then it adds a path for each variant definition like meltano/api/v1/plugins/extractors/tap-google-analytics--meltanolabs would return:

Click to expand 
{
    "name": "tap-google-analytics",
    "label": "Google Analytics",
    "description": "App and website analytics platform hosted by Google",
    "namespace": "tap_google_analytics",
    "logo_url": "http://localhost:4000/assets/logos/taps/google-analytics.png",
    "variant": "meltanolabs",
    "docs": "https://hub.meltano.com/extractors/google-analytics.html",
    "repo": "https://github.com/MeltanoLabs/tap-google-analytics",
    "pip_url": "git+https://github.com/MeltanoLabs/tap-google-analytics.git",
    "capabilities": [
        "catalog",
        "discover",
        "state"
    ],
    "settings_group_validation": [
        [
            "key_file_location",
            "view_id",
            "start_date"
        ],
        [
            "client_secrets",
            "view_id",
            "start_date"
        ],
        [
            "oauth_credentials.client_id",
            "oauth_credentials.client_secret",
            "oauth_credentials.access_token",
            "oauth_credentials.refresh_token",
            "view_id",
            "start_date"
        ]
    ],
    "settings": [
        {
            "name": "key_file_location",
            "kind": "file",
            "description": "A file that contains the Google Analytics client secrets json.",
            "value": "$MELTANO_PROJECT_ROOT/client_secrets.json",
            "label": "Client Secrets File Location",
            "placeholder": "Ex. client_secrets.json"
        },
        {
            "name": "client_secrets",
            "description": "An object that contains the Google Analytics client secrets.",
            "kind": "object",
            "label": "Client Secrets JSON",
            "placeholder": "Ex. client_secrets.json"
        },
        {
            "name": "oauth_credentials.client_id",
            "description": "The Google Analytics oauth client ID.",
            "kind": "password",
            "label": "OAuth Client ID"
        },
        {
            "name": "oauth_credentials.client_secret",
            "description": "The Google Analytics oauth client secret.",
            "kind": "password",
            "label": "OAuth Client Secret"
        },
        {
            "name": "oauth_credentials.access_token",
            "description": "The Google Analytics oauth access token.",
            "kind": "password",
            "label": "OAuth Access Token"
        },
        {
            "name": "oauth_credentials.refresh_token",
            "kind": "password",
            "description": "The Google Analytics oauth refresh token.",
            "label": "OAuth Refresh Token"
        },
        {
            "name": "view_id",
            "description": "The ID for the view to fetch data from.",
            "label": "View ID",
            "placeholder": "Ex. 198343027"
        },
        {
            "name": "reports",
            "description": "The reports definition of which fields to retrieve from the view.",
            "label": "Reports",
            "placeholder": "Ex. my_report_definition.json"
        },
        {
            "name": "start_date",
            "label": "Start Date",
            "kind": "date_iso8601",
            "description": "This property determines how much historical data will be extracted. Please be aware that the larger the time period and amount of data, the longer the initial extraction can be expected to take."
        },
        {
            "name": "end_date",
            "label": "End Date",
            "kind": "date_iso8601",
            "description": "Date up to when historical data will be extracted."
        }
    ]
}

Also this:

  • refreshes the hubs plugin definitions that are commited to the repo based on discovery.yml. Right now the git files are just a sample for testing and the real files are built during deployment since it needs to refresh from discovery.yml endpoint daily.
  • adds a variant to each plugin that didnt contain one (re: !226 (comment 898117809))
  • splits the discovery.yml schema for each plugin
  • as part of CI, asserts that the output of the API build process passes the JSON schemas for each plugin type

I'm happy to change the format, endpoint urls, etc. but just wanted to get the ball rolling on how we could do this via the API logic vs translating all the source files.

Edited by Pat Nadolny

Merge request reports