Querier and Disributor are failing and throwing below error if the
ingester_client:
grpc_client_config:
grpc_compression: "snappy"
`ts=2026-04-27T15:43:56.315233069Z caller=scheduler_processor.go:106 level=error msg="error contacting scheduler" err="rpc error: code = Canceled desc = context canceled" addr=100.64.19.87:9095
ts=2026-04-27T15:43:56.31527908Z caller=scheduler_processor.go:106 level=error msg="error contacting scheduler" err="rpc error: code = Canceled desc = context canceled" addr=100.64.19.87:9095
ts=2026-04-27T15:43:56.31529213Z caller=scheduler_processor.go:106 level=error msg="error contacting scheduler" err="rpc error: code = Canceled desc = context canceled" addr=100.64.19.87:9095
ts=2026-04-27T15:44:06.577433399Z caller=memberlist_client.go:580 level=info msg="joined memberlist cluster" reached_nodes=36
ts=2026-04-27T15:44:07.803571061Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=1.032293ms
ts=2026-04-27T15:44:07.80573017Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=2.307791ms
ts=2026-04-27T15:44:07.80574204Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=2.920726ms
ts=2026-04-27T15:44:07.80620126Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=1.656287ms
ts=2026-04-27T15:44:07.808768638Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.36.220:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=938.973µs merge_duration=810ns response_time=2.337193ms
ts=2026-04-27T15:44:07.808818899Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=861.021µs merge_duration=870ns response_time=2.621369ms
ts=2026-04-27T15:44:07.80889587Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.36.220:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=887.212µs merge_duration=880ns response_time=3.852186ms
ts=2026-04-27T15:44:07.808967392Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=831µs merge_duration=750ns response_time=2.535377ms
ts=2026-04-27T15:44:07.809150216Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=1.022833ms
ts=2026-04-27T15:44:07.811167792Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.36.220:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=934.112µs merge_duration=880ns response_time=1.79348ms
ts=2026-04-27T15:44:07.811690373Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=28.307646ms
ts=2026-04-27T15:44:07.813538025Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=881.102µs merge_duration=1.19µs response_time=1.672758ms
ts=2026-04-27T15:44:07.897994383Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.35.229:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=2 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=44.868716ms merge_duration=1.681µs response_time=91.929156ms
ts=2026-04-27T15:44:08.180555124Z caller=spanlogger.go:87 x-cortex-request-id=09b275fa-b98e-4cc1-8157-bfb0a2fc16d5 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=433.48µs merge_duration=1.63µs response_time=1.448133ms
ts=2026-04-27T15:44:08.180557434Z caller=spanlogger.go:87 x-cortex-request-id=09b275fa-b98e-4cc1-8157-bfb0a2fc16d5 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=464.441µs merge_duration=1.19µs response_time=1.408572ms
ts=2026-04-27T15:44:08.180710047Z caller=spanlogger.go:87 x-cortex-request-id=09b275fa-b98e-4cc1-8157-bfb0a2fc16d5 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.36.220:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=622.689µs merge_duration=1.25µs response_time=1.634517ms
ts=2026-04-27T15:44:08.211036239Z caller=spanlogger.go:87 x-cortex-request-id=09b275fa-b98e-4cc1-8157-bfb0a2fc16d5 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.35.229:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=1 postings_touched_size_sum=3 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=3 get_all_duration=9.998765ms merge_duration=1.2µs response_time=31.872347ms
panic: not implemented, use WriteTo instead
goroutine 774 [running]:
github.com/thanos-io/thanos/pkg/extgrpc/snappy.(*reader).Read(0xc001658f48?, {0xa71346?, 0x371e740?, 0x8000?})
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/extgrpc/snappy/snappy.go:72 +0x25
io.(*LimitedReader).Read(0xc001b01608, {0xc0015aa000?, 0xc000623008?, 0xc001b01608?})
/usr/local/go/src/io/io.go:479 +0x43
google.golang.org/grpc/mem.ReadAll({0x4125820, 0xc001b01608}, {0x414e6c8, 0xc0006d4940})
/__w/cortex/cortex/vendor/google.golang.org/grpc/mem/buffer_slice.go:260 +0x245
google.golang.org/grpc.decompress({0x41626d0, 0xc0003ad310}, {0xc001002ee0?, 0x90000c001659158?, 0xa78c00?}, {0x0?, 0x0?}, 0x1900000, {0x414e6c8, 0xc0006d4940})
/__w/cortex/cortex/vendor/google.golang.org/grpc/rpc_util.go:879 +0x3a5
google.golang.org/grpc.recvAndDecompress(0xc001d09da0, {0x4128ee0, 0xc001d0dc70}, {0x0, 0x0}, 0x1900000, 0x0, {0x41626d0, 0xc0003ad310}, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/rpc_util.go:835 +0x2b7
google.golang.org/grpc.recv(0x35ed500?, {0x7f3c16d90978, 0xc000359080}, {0x4128ee0?, 0xc001d0dc70?}, {0x0?, 0x0?}, {0x368d780, 0xc001d09a10}, 0x1900000, ...)
/__w/cortex/cortex/vendor/google.golang.org/grpc/rpc_util.go:902 +0xab
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00100ec30, {0x368d780, 0xc001d09a10}, 0x1000000000000?)
/__w/cortex/cortex/vendor/google.golang.org/grpc/stream.go:1127 +0x2e5
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
/__w/cortex/cortex/vendor/google.golang.org/grpc/stream.go:970 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc000aa4fc0, 0xc001659550, 0xc001659540)
/__w/cortex/cortex/vendor/google.golang.org/grpc/stream.go:790 +0x142
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000aa4fc0, {0x368d780?, 0xc001d09a10?})
/__w/cortex/cortex/vendor/google.golang.org/grpc/stream.go:969 +0x155
google.golang.org/grpc.invoke({0x4166208?, 0xc001d09bf0?}, {0x3b53281?, 0xc001659650?}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0x3778855e631cd301?, {0xc001d030c0, ...})
/__w/cortex/cortex/vendor/google.golang.org/grpc/call.go:73 +0xc4
github.com/cortexproject/cortex/pkg/util/grpcclient.Instrument.PrometheusGRPCUnaryInstrumentation.func1({0x4166208, 0xc001d09bf0}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0x3d27280, ...)
/__w/cortex/cortex/pkg/util/middleware/grpc.go:20 +0x104
github.com/cortexproject/cortex/pkg/util/grpcclient.(*Config).DialOption.ChainUnaryClient.func6.1({0x4166208?, 0xc001d09bf0?}, {0x3b53281?, 0xffffffffffffffff?}, {0x368d600?, 0xc001d03080?}, {0x368d780?, 0xc001d09a10?}, 0xc001d09b30?, {0xc001d030c0, ...})
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:113 +0x91
github.com/weaveworks/common/middleware.ClientUserHeaderInterceptor({0x4166208?, 0xc001d09b60?}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0xc001d09a40, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:17 +0xd7
github.com/cortexproject/cortex/pkg/util/grpcclient.(*Config).DialOption.ChainUnaryClient.func6.1({0x4166208?, 0xc001d09b60?}, {0x3b53281?, 0xc000db9c0b?}, {0x368d600?, 0xc001d03080?}, {0x368d780?, 0xc001d09a10?}, 0xc001607890?, {0xc001d030c0, ...})
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:113 +0x91
github.com/opentracing-contrib/go-grpc.OpenTracingClientInterceptor.func1({0x4166208, 0xc001d099e0}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0xc001d09a70, ...)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/client.go:63 +0x44d
github.com/cortexproject/cortex/pkg/util/grpcclient.(*Config).DialOption.ChainUnaryClient.func6.1({0x4166208?, 0xc001d099e0?}, {0x3b53281?, 0x1607ac8?}, {0x368d600?, 0xc001d03080?}, {0x368d780?, 0xc001d09a10?}, 0x7f3c16c1b201?, {0xc001d030c0, ...})
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:113 +0x91
github.com/cortexproject/cortex/pkg/util/grpcutil.HTTPHeaderPropagationClientInterceptor({0x4166208?, 0xc001d099e0?}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0xc001d09aa0, ...)
/__w/cortex/cortex/pkg/util/grpcutil/util.go:72 +0xc8
github.com/cortexproject/cortex/pkg/util/grpcclient.(*Config).DialOption.ChainUnaryClient.func6({0x4166208, 0xc001d099e0}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0x3d27280, ...)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:116 +0x1d3
google.golang.org/grpc.(*ClientConn).Invoke(0xc00067f008, {0x4166208?, 0xc001d099e0?}, {0x3b53281?, 0x1c?}, {0x368d600?, 0xc001d03080?}, {0x368d780?, 0xc001d09a10?}, {0xc000d97d90, ...})
/__w/cortex/cortex/vendor/google.golang.org/grpc/call.go:35 +0x205
google.golang.org/grpc/health/grpc_health_v1.(*healthClient).Check(0xc000acb280, {0x4166208, 0xc001d099e0}, 0xc001d03080, {0x0, 0x0, 0x4111801?})
/__w/cortex/cortex/vendor/google.golang.org/grpc/health/grpc_health_v1/health_grpc.pb.go:90 +0x165
github.com/cortexproject/cortex/pkg/ring/client.healthCheck({0x7f3c16e84d00, 0xc001105180}, 0x2?)
/__w/cortex/cortex/pkg/ring/client/pool.go:197 +0xb5
github.com/cortexproject/cortex/pkg/ring/client.(*Pool).cleanUnhealthy(0xc0009019e0)
/__w/cortex/cortex/pkg/ring/client/pool.go:182 +0x86
github.com/cortexproject/cortex/pkg/ring/client.(*Pool).iteration(0xc0009019e0, {0xc000807b40?, 0x0?})
/__w/cortex/cortex/pkg/ring/client/pool.go:80 +0x28
github.com/cortexproject/cortex/pkg/ring/client.NewPool.NewTimerService.func1({0x4166240, 0xc000807b30})
/__w/cortex/cortex/pkg/util/services/services.go:33 +0x105
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006a0dc0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1af
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 in goroutine 772
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x105`
Querier and Disributor are failing and throwing below error if the
ingester_client:
grpc_client_config:
grpc_compression: "snappy"
`ts=2026-04-27T15:43:56.315233069Z caller=scheduler_processor.go:106 level=error msg="error contacting scheduler" err="rpc error: code = Canceled desc = context canceled" addr=100.64.19.87:9095
ts=2026-04-27T15:43:56.31527908Z caller=scheduler_processor.go:106 level=error msg="error contacting scheduler" err="rpc error: code = Canceled desc = context canceled" addr=100.64.19.87:9095
ts=2026-04-27T15:43:56.31529213Z caller=scheduler_processor.go:106 level=error msg="error contacting scheduler" err="rpc error: code = Canceled desc = context canceled" addr=100.64.19.87:9095
ts=2026-04-27T15:44:06.577433399Z caller=memberlist_client.go:580 level=info msg="joined memberlist cluster" reached_nodes=36
ts=2026-04-27T15:44:07.803571061Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=1.032293ms
ts=2026-04-27T15:44:07.80573017Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=2.307791ms
ts=2026-04-27T15:44:07.80574204Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=2.920726ms
ts=2026-04-27T15:44:07.80620126Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=1.656287ms
ts=2026-04-27T15:44:07.808768638Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.36.220:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=938.973µs merge_duration=810ns response_time=2.337193ms
ts=2026-04-27T15:44:07.808818899Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=861.021µs merge_duration=870ns response_time=2.621369ms
ts=2026-04-27T15:44:07.80889587Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.36.220:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=887.212µs merge_duration=880ns response_time=3.852186ms
ts=2026-04-27T15:44:07.808967392Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=831µs merge_duration=750ns response_time=2.535377ms
ts=2026-04-27T15:44:07.809150216Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=1.022833ms
ts=2026-04-27T15:44:07.811167792Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.36.220:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=934.112µs merge_duration=880ns response_time=1.79348ms
ts=2026-04-27T15:44:07.811690373Z caller=loader.go:148 level=info msg="loaded bucket index" user=env-2a-2025052 duration=28.307646ms
ts=2026-04-27T15:44:07.813538025Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=881.102µs merge_duration=1.19µs response_time=1.672758ms
ts=2026-04-27T15:44:07.897994383Z caller=spanlogger.go:87 x-cortex-request-id=659e14aa-02b5-4991-923c-c69669576357 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.35.229:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=2 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=44.868716ms merge_duration=1.681µs response_time=91.929156ms
ts=2026-04-27T15:44:08.180555124Z caller=spanlogger.go:87 x-cortex-request-id=09b275fa-b98e-4cc1-8157-bfb0a2fc16d5 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=433.48µs merge_duration=1.63µs response_time=1.448133ms
ts=2026-04-27T15:44:08.180557434Z caller=spanlogger.go:87 x-cortex-request-id=09b275fa-b98e-4cc1-8157-bfb0a2fc16d5 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.33.23:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=464.441µs merge_duration=1.19µs response_time=1.408572ms
ts=2026-04-27T15:44:08.180710047Z caller=spanlogger.go:87 x-cortex-request-id=09b275fa-b98e-4cc1-8157-bfb0a2fc16d5 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.36.220:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=0 postings_touched_size_sum=0 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=0 get_all_duration=622.689µs merge_duration=1.25µs response_time=1.634517ms
ts=2026-04-27T15:44:08.211036239Z caller=spanlogger.go:87 x-cortex-request-id=09b275fa-b98e-4cc1-8157-bfb0a2fc16d5 org_id=env-2a-2025052 level=info msg="store gateway series request stats" instance=100.64.35.229:9095 queryable_chunk_bytes_fetched=0 queryable_data_bytes_fetched=0 blocks_queried=1 series_merged_count=0 chunks_merged_count=0 postings_touched=1 postings_touched_size_sum=3 postings_to_fetch=0 postings_fetched=0 postings_fetch_count=0 postings_fetched_size_sum=0 series_touched=0 series_touched_size_sum=0 series_fetched=0 series_fetch_count=0 series_fetched_size_sum=0 chunks_touched=0 chunks_touched_size_sum=0 chunks_fetched=0 chunks_fetch_count=0 chunks_fetched_size_sum=0 data_downloaded_size_sum=3 get_all_duration=9.998765ms merge_duration=1.2µs response_time=31.872347ms
panic: not implemented, use WriteTo instead
goroutine 774 [running]:
github.com/thanos-io/thanos/pkg/extgrpc/snappy.(*reader).Read(0xc001658f48?, {0xa71346?, 0x371e740?, 0x8000?})
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/extgrpc/snappy/snappy.go:72 +0x25
io.(*LimitedReader).Read(0xc001b01608, {0xc0015aa000?, 0xc000623008?, 0xc001b01608?})
/usr/local/go/src/io/io.go:479 +0x43
google.golang.org/grpc/mem.ReadAll({0x4125820, 0xc001b01608}, {0x414e6c8, 0xc0006d4940})
/__w/cortex/cortex/vendor/google.golang.org/grpc/mem/buffer_slice.go:260 +0x245
google.golang.org/grpc.decompress({0x41626d0, 0xc0003ad310}, {0xc001002ee0?, 0x90000c001659158?, 0xa78c00?}, {0x0?, 0x0?}, 0x1900000, {0x414e6c8, 0xc0006d4940})
/__w/cortex/cortex/vendor/google.golang.org/grpc/rpc_util.go:879 +0x3a5
google.golang.org/grpc.recvAndDecompress(0xc001d09da0, {0x4128ee0, 0xc001d0dc70}, {0x0, 0x0}, 0x1900000, 0x0, {0x41626d0, 0xc0003ad310}, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/rpc_util.go:835 +0x2b7
google.golang.org/grpc.recv(0x35ed500?, {0x7f3c16d90978, 0xc000359080}, {0x4128ee0?, 0xc001d0dc70?}, {0x0?, 0x0?}, {0x368d780, 0xc001d09a10}, 0x1900000, ...)
/__w/cortex/cortex/vendor/google.golang.org/grpc/rpc_util.go:902 +0xab
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00100ec30, {0x368d780, 0xc001d09a10}, 0x1000000000000?)
/__w/cortex/cortex/vendor/google.golang.org/grpc/stream.go:1127 +0x2e5
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
/__w/cortex/cortex/vendor/google.golang.org/grpc/stream.go:970 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc000aa4fc0, 0xc001659550, 0xc001659540)
/__w/cortex/cortex/vendor/google.golang.org/grpc/stream.go:790 +0x142
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000aa4fc0, {0x368d780?, 0xc001d09a10?})
/__w/cortex/cortex/vendor/google.golang.org/grpc/stream.go:969 +0x155
google.golang.org/grpc.invoke({0x4166208?, 0xc001d09bf0?}, {0x3b53281?, 0xc001659650?}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0x3778855e631cd301?, {0xc001d030c0, ...})
/__w/cortex/cortex/vendor/google.golang.org/grpc/call.go:73 +0xc4
github.com/cortexproject/cortex/pkg/util/grpcclient.Instrument.PrometheusGRPCUnaryInstrumentation.func1({0x4166208, 0xc001d09bf0}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0x3d27280, ...)
/__w/cortex/cortex/pkg/util/middleware/grpc.go:20 +0x104
github.com/cortexproject/cortex/pkg/util/grpcclient.(*Config).DialOption.ChainUnaryClient.func6.1({0x4166208?, 0xc001d09bf0?}, {0x3b53281?, 0xffffffffffffffff?}, {0x368d600?, 0xc001d03080?}, {0x368d780?, 0xc001d09a10?}, 0xc001d09b30?, {0xc001d030c0, ...})
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:113 +0x91
github.com/weaveworks/common/middleware.ClientUserHeaderInterceptor({0x4166208?, 0xc001d09b60?}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0xc001d09a40, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:17 +0xd7
github.com/cortexproject/cortex/pkg/util/grpcclient.(*Config).DialOption.ChainUnaryClient.func6.1({0x4166208?, 0xc001d09b60?}, {0x3b53281?, 0xc000db9c0b?}, {0x368d600?, 0xc001d03080?}, {0x368d780?, 0xc001d09a10?}, 0xc001607890?, {0xc001d030c0, ...})
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:113 +0x91
github.com/opentracing-contrib/go-grpc.OpenTracingClientInterceptor.func1({0x4166208, 0xc001d099e0}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0xc001d09a70, ...)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/client.go:63 +0x44d
github.com/cortexproject/cortex/pkg/util/grpcclient.(*Config).DialOption.ChainUnaryClient.func6.1({0x4166208?, 0xc001d099e0?}, {0x3b53281?, 0x1607ac8?}, {0x368d600?, 0xc001d03080?}, {0x368d780?, 0xc001d09a10?}, 0x7f3c16c1b201?, {0xc001d030c0, ...})
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:113 +0x91
github.com/cortexproject/cortex/pkg/util/grpcutil.HTTPHeaderPropagationClientInterceptor({0x4166208?, 0xc001d099e0?}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0xc001d09aa0, ...)
/__w/cortex/cortex/pkg/util/grpcutil/util.go:72 +0xc8
github.com/cortexproject/cortex/pkg/util/grpcclient.(*Config).DialOption.ChainUnaryClient.func6({0x4166208, 0xc001d099e0}, {0x3b53281, 0x1c}, {0x368d600, 0xc001d03080}, {0x368d780, 0xc001d09a10}, 0xc00067f008, 0x3d27280, ...)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:116 +0x1d3
google.golang.org/grpc.(*ClientConn).Invoke(0xc00067f008, {0x4166208?, 0xc001d099e0?}, {0x3b53281?, 0x1c?}, {0x368d600?, 0xc001d03080?}, {0x368d780?, 0xc001d09a10?}, {0xc000d97d90, ...})
/__w/cortex/cortex/vendor/google.golang.org/grpc/call.go:35 +0x205
google.golang.org/grpc/health/grpc_health_v1.(*healthClient).Check(0xc000acb280, {0x4166208, 0xc001d099e0}, 0xc001d03080, {0x0, 0x0, 0x4111801?})
/__w/cortex/cortex/vendor/google.golang.org/grpc/health/grpc_health_v1/health_grpc.pb.go:90 +0x165
github.com/cortexproject/cortex/pkg/ring/client.healthCheck({0x7f3c16e84d00, 0xc001105180}, 0x2?)
/__w/cortex/cortex/pkg/ring/client/pool.go:197 +0xb5
github.com/cortexproject/cortex/pkg/ring/client.(*Pool).cleanUnhealthy(0xc0009019e0)
/__w/cortex/cortex/pkg/ring/client/pool.go:182 +0x86
github.com/cortexproject/cortex/pkg/ring/client.(*Pool).iteration(0xc0009019e0, {0xc000807b40?, 0x0?})
/__w/cortex/cortex/pkg/ring/client/pool.go:80 +0x28
github.com/cortexproject/cortex/pkg/ring/client.NewPool.NewTimerService.func1({0x4166240, 0xc000807b30})
/__w/cortex/cortex/pkg/util/services/services.go:33 +0x105
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006a0dc0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:190 +0x1af
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 in goroutine 772
/__w/cortex/cortex/pkg/util/services/basic_service.go:119 +0x105`