Re-enable indexing on GKE logs with reduced schema
detailed error:
```
{"type":"illegal_argument_exception","reason":"field expansion matches too many fields, limit: 1024, got: 1629"}}}]},"status":400}
```
see: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-settings.html for more background
this config option cannot be adjusted through the API:
```
"persistent setting [indices.query.bool.max_clause_count], not dynamically updateable"
```
we need to bring down the number of fields in the indices
---
Here is an example request/response:
<details>
```
{
"version": true,
"size": 500,
"sort": [
{
"json.time": {
"order": "desc",
"unmapped_type": "boolean"
}
}
],
"_source": {
"excludes": []
},
"aggs": {
"2": {
"date_histogram": {
"field": "json.time",
"fixed_interval": "30s",
"time_zone": "UTC",
"min_doc_count": 1
}
}
},
"stored_fields": [
"*"
],
"script_fields": {
"controller_and_action": {
"script": {
"source": "doc['json.controller.keyword'] + \"#\" + doc['json.action.keyword']",
"lang": "painless"
}
}
},
"docvalue_fields": [
{
"field": "@timestamp",
"format": "date_time"
},
{
"field": "json.expiry_from",
"format": "date_time"
},
{
"field": "json.expiry_to",
"format": "date_time"
},
{
"field": "json.extra.bucket.start",
"format": "date_time"
},
{
"field": "json.extra.bucket.stop",
"format": "date_time"
},
{
"field": "json.extra.commits.timestamp",
"format": "date_time"
},
{
"field": "json.extra.created_after",
"format": "date_time"
},
{
"field": "json.extra.created_at",
"format": "date_time"
},
{
"field": "json.extra.created_before",
"format": "date_time"
},
{
"field": "json.extra.due_date",
"format": "date_time"
},
{
"field": "json.extra.head_commit.timestamp",
"format": "date_time"
},
{
"field": "json.extra.pull_request.base.repo.created_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.base.repo.pushed_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.base.repo.updated_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.closed_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.created_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.head.repo.created_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.head.repo.pushed_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.head.repo.updated_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.merged_at",
"format": "date_time"
},
{
"field": "json.extra.pull_request.updated_at",
"format": "date_time"
},
{
"field": "json.extra.raw_response.created_on",
"format": "date_time"
},
{
"field": "json.extra.raw_response.updated_on",
"format": "date_time"
},
{
"field": "json.extra.repository.updated_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.bucket.start",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.bucket.stop",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.commits.timestamp",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.head_commit.timestamp",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.base.repo.created_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.base.repo.pushed_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.base.repo.updated_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.closed_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.created_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.head.repo.created_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.head.repo.pushed_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.head.repo.updated_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.merged_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.pull_request.updated_at",
"format": "date_time"
},
{
"field": "json.extra.request_forgery_protection.repository.updated_at",
"format": "date_time"
},
{
"field": "json.time",
"format": "date_time"
},
{
"field": "publish_time",
"format": "date_time"
}
],
"query": {
"bool": {
"must": [],
"filter": [
{
"multi_match": {
"type": "best_fields",
"query": "ExternalDiffUploader",
"lenient": true
}
},
{
"match_phrase": {
"json.controller": {
"query": "Projects::MergeRequests::DiffsController"
}
}
},
{
"range": {
"json.time": {
"format": "strict_date_optional_time",
"gte": "2020-03-27T12:00:00.000Z",
"lte": "2020-03-27T12:30:00.000Z"
}
}
}
],
"should": [],
"must_not": []
}
},
"highlight": {
"pre_tags": [
"@kibana-highlighted-field@"
],
"post_tags": [
"@/kibana-highlighted-field@"
],
"fields": {
"*": {}
},
"fragment_size": 2147483647
}
}
```
</details>
Response:
<details>
```
{
"took": 5158,
"timed_out": false,
"_shards": {
"total": 762,
"successful": 750,
"skipped": 750,
"failed": 12,
"failures": [
{
"shard": 0,
"index": "pubsub-rails-inf-gprd-001925",
"node": "jmnNQegZRWOO0aJBFjnZew",
"reason": {
"type": "query_shard_exception",
"reason": "failed to create query: {\n \"bool\" : {\n \"filter\" : [\n {\n \"multi_match\" : {\n \"query\" : \"ExternalDiffUploader\",\n \"fields\" : [ ],\n \"type\" : \"best_fields\",\n \"operator\" : \"OR\",\n \"slop\" : 0,\n \"prefix_length\" : 0,\n \"max_expansions\" : 50,\n \"lenient\" : true,\n \"zero_terms_query\" : \"NONE\",\n \"auto_generate_synonyms_phrase_query\" : true,\n \"fuzzy_transpositions\" : true,\n \"boost\" : 1.0\n }\n },\n {\n \"match_phrase\" : {\n \"json.controller\" : {\n \"query\" : \"Projects::MergeRequests::DiffsController\",\n \"slop\" : 0,\n \"zero_terms_query\" : \"NONE\",\n \"boost\" : 1.0\n }\n }\n },\n {\n \"range\" : {\n \"json.time\" : {\n \"from\" : \"2020-03-27T12:00:00.000Z\",\n \"to\" : \"2020-03-27T12:30:00.000Z\",\n \"include_lower\" : true,\n \"include_upper\" : true,\n \"format\" : \"strict_date_optional_time\",\n \"boost\" : 1.0\n }\n }\n }\n ],\n \"adjust_pure_negative\" : true,\n \"boost\" : 1.0\n }\n}",
"index_uuid": "HunDEJAFRKieC7kFcif7zw",
"index": "pubsub-rails-inf-gprd-001925",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "field expansion matches too many fields, limit: 1024, got: 1470"
}
}
},
{
"shard": 0,
"index": "pubsub-rails-inf-gprd-001926",
"node": "Nce627z_R7aRVIjH1JkAog",
"reason": {
"type": "query_shard_exception",
"reason": "failed to create query: {\n \"bool\" : {\n \"filter\" : [\n {\n \"multi_match\" : {\n \"query\" : \"ExternalDiffUploader\",\n \"fields\" : [ ],\n \"type\" : \"best_fields\",\n \"operator\" : \"OR\",\n \"slop\" : 0,\n \"prefix_length\" : 0,\n \"max_expansions\" : 50,\n \"lenient\" : true,\n \"zero_terms_query\" : \"NONE\",\n \"auto_generate_synonyms_phrase_query\" : true,\n \"fuzzy_transpositions\" : true,\n \"boost\" : 1.0\n }\n },\n {\n \"match_phrase\" : {\n \"json.controller\" : {\n \"query\" : \"Projects::MergeRequests::DiffsController\",\n \"slop\" : 0,\n \"zero_terms_query\" : \"NONE\",\n \"boost\" : 1.0\n }\n }\n },\n {\n \"range\" : {\n \"json.time\" : {\n \"from\" : \"2020-03-27T12:00:00.000Z\",\n \"to\" : \"2020-03-27T12:30:00.000Z\",\n \"include_lower\" : true,\n \"include_upper\" : true,\n \"format\" : \"strict_date_optional_time\",\n \"boost\" : 1.0\n }\n }\n }\n ],\n \"adjust_pure_negative\" : true,\n \"boost\" : 1.0\n }\n}",
"index_uuid": "URp08IJpRjKQ6kRnKFJQ8w",
"index": "pubsub-rails-inf-gprd-001926",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "field expansion matches too many fields, limit: 1024, got: 1136"
}
}
}
]
},
"hits": {
"total": 0,
"max_score": 0,
"hits": []
}
}
```
</details>
issue