Elasticsearch Index Optimization

# Elasticsearch Index Optimization

You are an expert in Elasticsearch for building fast, scalable search applications with optimized indexing and query performance.

## Key Principles
- Design index templates for consistent mappings
- Use aliases for zero-downtime reindexing
- Optimize field types and analyzers
- Tune refresh intervals for performance
- Manage shard allocation strategically

## Index Template Design
```json
PUT _index_template/products-template
{
  "index_patterns": ["products-*"],
  "priority": 100,
  "template": {
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 1,
      "refresh_interval": "5s",
      "index.mapping.total_fields.limit": 2000,
      "analysis": {
        "analyzer": {
          "product_analyzer": {
            "type": "custom",
            "tokenizer": "standard",
            "filter": [
              "lowercase",
              "asciifolding",
              "word_delimiter_graph",
              "english_stemmer",
              "edge_ngram_filter"
            ]
          },
          "search_analyzer": {
            "type": "custom",
            "tokenizer": "standard",
            "filter": ["lowercase", "asciifolding", "english_stemmer"]
          }
        },
        "filter": {
          "english_stemmer": {
            "type": "stemmer",
            "language": "english"
          },
          "edge_ngram_filter": {
            "type": "edge_ngram",
            "min_gram": 2,
            "max_gram": 15
          }
        }
      }
    },
    "mappings": {
      "dynamic": "strict",
      "properties": {
        "id": { "type": "keyword" },
        "name": {
          "type": "text",
          "analyzer": "product_analyzer",
          "search_analyzer": "search_analyzer",
          "fields": {
            "keyword": { "type": "keyword" },
            "suggest": {
              "type": "completion",
              "contexts": [
                { "name": "category", "type": "category" }
              ]
            }
          }
        },
        "description": {
          "type": "text",
          "analyzer": "product_analyzer"
        },
        "category": {
          "type": "keyword",
          "fields": {
            "text": { "type": "text" }
          }
        },
        "price": { "type": "scaled_float", "scaling_factor": 100 },
        "stock": { "type": "integer" },
        "rating": { "type": "float" },
        "created_at": { "type": "date" },
        "tags": { "type": "keyword" },
        "attributes": {
          "type": "nested",
          "properties": {
            "name": { "type": "keyword" },
            "value": { "type": "keyword" }
          }
        }
      }
    }
  }
}
```

## Alias-Based Index Management
```json
// Create new index with timestamp
PUT products-2024-01-15
{
  "aliases": {
    "products-write": {},
    "products-read": {}
  }
}

// Zero-downtime reindex
POST _reindex?wait_for_completion=false
{
  "source": { "index": "products-2024-01-01" },
  "dest": { "index": "products-2024-01-15" }
}

// Switch alias atomically
POST _aliases
{
  "actions": [
    { "remove": { "index": "products-2024-01-01", "alias": "products-read" }},
    { "add": { "index": "products-2024-01-15", "alias": "products-read" }}
  ]
}
```

## Optimized Queries
```json
// Multi-match with boosting
GET products-read/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "multi_match": {
            "query": "wireless headphones",
            "fields": ["name^3", "description", "tags^2"],
            "type": "best_fields",
            "fuzziness": "AUTO",
            "prefix_length": 2
          }
        }
      ],
      "filter": [
        { "term": { "category": "electronics" }},
        { "range": { "price": { "gte": 50, "lte": 500 }}},
        { "range": { "stock": { "gt": 0 }}}
      ],
      "should": [
        { "term": { "tags": { "value": "bestseller", "boost": 2 }}},
        { "range": { "rating": { "gte": 4.5, "boost": 1.5 }}}
      ]
    }
  },
  "highlight": {
    "fields": {
      "name": { "number_of_fragments": 0 },
      "description": { "fragment_size": 150, "number_of_fragments": 3 }
    },
    "pre_tags": ["<em>"],
    "post_tags": ["</em>"]
  },
  "aggs": {
    "categories": {
      "terms": { "field": "category", "size": 20 }
    },
    "price_ranges": {
      "range": {
        "field": "price",
        "ranges": [
          { "to": 50 },
          { "from": 50, "to": 100 },
          { "from": 100, "to": 500 },
          { "from": 500 }
        ]
      }
    },
    "avg_rating": { "avg": { "field": "rating" }}
  },
  "sort": [
    { "_score": "desc" },
    { "rating": "desc" },
    { "created_at": "desc" }
  ],
  "from": 0,
  "size": 20
}
```

## Bulk Indexing
```python
from elasticsearch import Elasticsearch, helpers

es = Elasticsearch(["http://localhost:9200"])

def bulk_index_products(products):
    actions = [
        {
            "_index": "products-write",
            "_id": product["id"],
            "_source": product
        }
        for product in products
    ]
    
    success, failed = helpers.bulk(
        es,
        actions,
        chunk_size=500,
        request_timeout=60,
        raise_on_error=False,
        raise_on_exception=False
    )
    
    return success, failed

# Parallel bulk for large datasets
helpers.parallel_bulk(
    es,
    actions,
    thread_count=4,
    chunk_size=500,
    queue_size=4
)
```

## Performance Tuning
```json
// Optimize for indexing speed
PUT products-write/_settings
{
  "index": {
    "refresh_interval": "30s",
    "number_of_replicas": 0,
    "translog.durability": "async",
    "translog.sync_interval": "30s"
  }
}

// Restore for search performance
PUT products-write/_settings
{
  "index": {
    "refresh_interval": "1s",
    "number_of_replicas": 1,
    "translog.durability": "request"
  }
}

// Force merge for read-heavy indices
POST products-read/_forcemerge?max_num_segments=1
```

## Monitoring Queries
```json
// Slow query log
PUT products-read/_settings
{
  "index.search.slowlog.threshold.query.warn": "5s",
  "index.search.slowlog.threshold.query.info": "2s",
  "index.search.slowlog.threshold.fetch.warn": "1s"
}

// Index stats
GET products-read/_stats

// Shard allocation
GET _cat/shards/products-*?v&h=index,shard,prirep,state,docs,store,node
```

## Best Practices
- Use keyword fields for exact matches and aggregations
- Avoid dynamic mapping in production
- Set appropriate shard count (target 20-40GB per shard)
- Use index lifecycle management (ILM)
- Profile queries with _profile API
- Implement search-as-you-type with completion suggester