|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! TableScan for full table scan. |
| 19 | +//! |
| 20 | +//! Reference: [pypaimon.read.table_scan.TableScan](https://github.com/apache/paimon/blob/release-1.3/paimon-python/pypaimon/read/table_scan.py) |
| 21 | +//! and [FullStartingScanner](https://github.com/apache/paimon/blob/release-1.3/paimon-python/pypaimon/read/scanner/full_starting_scanner.py). |
| 22 | +
|
| 23 | +use super::Table; |
| 24 | +use crate::io::FileIO; |
| 25 | +use crate::spec::{BinaryRow, FileKind, ManifestEntry, Snapshot}; |
| 26 | +use crate::table::source::{DataSplitBuilder, Plan}; |
| 27 | +use crate::table::SnapshotManager; |
| 28 | +use crate::Error; |
| 29 | +use std::collections::{HashMap, HashSet}; |
| 30 | + |
| 31 | +/// Path segment for manifest directory under table. |
| 32 | +const MANIFEST_DIR: &str = "manifest"; |
| 33 | + |
| 34 | +/// Reads a manifest list file (Avro) and returns manifest file metas. |
| 35 | +async fn read_manifest_list( |
| 36 | + file_io: &FileIO, |
| 37 | + table_path: &str, |
| 38 | + list_name: &str, |
| 39 | +) -> crate::Result<Vec<crate::spec::ManifestFileMeta>> { |
| 40 | + let path = format!( |
| 41 | + "{}/{}/{}", |
| 42 | + table_path.trim_end_matches('/'), |
| 43 | + MANIFEST_DIR, |
| 44 | + list_name |
| 45 | + ); |
| 46 | + let input = file_io.new_input(&path)?; |
| 47 | + if !input.exists().await? { |
| 48 | + return Ok(Vec::new()); |
| 49 | + } |
| 50 | + let bytes = input.read().await?; |
| 51 | + crate::spec::from_avro_bytes::<crate::spec::ManifestFileMeta>(&bytes) |
| 52 | +} |
| 53 | + |
| 54 | +/// Reads all manifest entries for a snapshot (base + delta manifest lists, then each manifest file). |
| 55 | +async fn read_all_manifest_entries( |
| 56 | + file_io: &FileIO, |
| 57 | + table_path: &str, |
| 58 | + snapshot: &Snapshot, |
| 59 | +) -> crate::Result<Vec<ManifestEntry>> { |
| 60 | + let mut manifest_files = |
| 61 | + read_manifest_list(file_io, table_path, snapshot.base_manifest_list()).await?; |
| 62 | + let delta = read_manifest_list(file_io, table_path, snapshot.delta_manifest_list()).await?; |
| 63 | + manifest_files.extend(delta); |
| 64 | + |
| 65 | + let manifest_path_prefix = format!("{}/{}", table_path.trim_end_matches('/'), MANIFEST_DIR); |
| 66 | + let mut all_entries = Vec::new(); |
| 67 | + // todo: consider use multiple-threads read manifest |
| 68 | + for meta in manifest_files { |
| 69 | + let path = format!("{}/{}", manifest_path_prefix, meta.file_name()); |
| 70 | + let entries = crate::spec::Manifest::read(file_io, &path).await?; |
| 71 | + all_entries.extend(entries); |
| 72 | + } |
| 73 | + Ok(all_entries) |
| 74 | +} |
| 75 | + |
| 76 | +/// Merges add/delete manifest entries: keeps only ADD entries whose (partition, bucket, file_name) is not in DELETE set. |
| 77 | +fn merge_manifest_entries(entries: Vec<ManifestEntry>) -> Vec<ManifestEntry> { |
| 78 | + let mut deleted = HashSet::new(); |
| 79 | + let mut added = Vec::new(); |
| 80 | + for e in entries { |
| 81 | + // follow python code to use partition, bucket, filename as duplicator |
| 82 | + let key = ( |
| 83 | + e.partition().to_vec(), |
| 84 | + e.bucket(), |
| 85 | + e.file().file_name.clone(), |
| 86 | + ); |
| 87 | + match e.kind() { |
| 88 | + FileKind::Add => added.push(e), |
| 89 | + FileKind::Delete => { |
| 90 | + deleted.insert(key); |
| 91 | + } |
| 92 | + } |
| 93 | + } |
| 94 | + added |
| 95 | + .into_iter() |
| 96 | + .filter(|e| { |
| 97 | + !deleted.contains(&( |
| 98 | + e.partition().to_vec(), |
| 99 | + e.bucket(), |
| 100 | + e.file().file_name.clone(), |
| 101 | + )) |
| 102 | + }) |
| 103 | + .collect() |
| 104 | +} |
| 105 | + |
| 106 | +/// TableScan for full table scan (no incremental, no predicate). |
| 107 | +/// |
| 108 | +/// Reference: [pypaimon.read.table_scan.TableScan](https://github.com/apache/paimon/blob/master/paimon-python/pypaimon/read/table_scan.py) |
| 109 | +#[derive(Debug, Clone)] |
| 110 | +pub struct TableScan { |
| 111 | + table: Table, |
| 112 | +} |
| 113 | + |
| 114 | +impl TableScan { |
| 115 | + pub fn new(table: Table) -> Self { |
| 116 | + Self { table } |
| 117 | + } |
| 118 | + |
| 119 | + /// Plan the full scan: read latest snapshot, manifest list, manifest entries, then build one DataSplit per (partition, bucket). |
| 120 | + pub async fn plan(&self) -> crate::Result<Plan> { |
| 121 | + let file_io = self.table.file_io(); |
| 122 | + let table_path = self.table.location(); |
| 123 | + let snapshot_manager = SnapshotManager::new(file_io.clone(), table_path.to_string()); |
| 124 | + |
| 125 | + let snapshot = match snapshot_manager.get_latest_snapshot().await? { |
| 126 | + Some(s) => s, |
| 127 | + None => return Ok(Plan::new(Vec::new())), |
| 128 | + }; |
| 129 | + Self::plan_snapshot(snapshot, file_io, table_path).await |
| 130 | + } |
| 131 | + |
| 132 | + pub async fn plan_snapshot( |
| 133 | + snapshot: Snapshot, |
| 134 | + file_io: &FileIO, |
| 135 | + table_path: &str, |
| 136 | + ) -> crate::Result<Plan> { |
| 137 | + let entries = read_all_manifest_entries(file_io, table_path, &snapshot).await?; |
| 138 | + let entries = merge_manifest_entries(entries); |
| 139 | + if entries.is_empty() { |
| 140 | + return Ok(Plan::new(Vec::new())); |
| 141 | + } |
| 142 | + |
| 143 | + // Group by (partition, bucket). Key = (partition_bytes, bucket). |
| 144 | + let mut groups: HashMap<(Vec<u8>, i32), Vec<ManifestEntry>> = HashMap::new(); |
| 145 | + for e in entries { |
| 146 | + let key = (e.partition().to_vec(), e.bucket()); |
| 147 | + groups.entry(key).or_default().push(e); |
| 148 | + } |
| 149 | + |
| 150 | + let snapshot_id = snapshot.id(); |
| 151 | + let base_path = table_path; |
| 152 | + let mut splits = Vec::new(); |
| 153 | + |
| 154 | + for ((_partition, bucket), group_entries) in groups { |
| 155 | + let total_buckets = group_entries |
| 156 | + .first() |
| 157 | + .map(|e| e.total_buckets()) |
| 158 | + .ok_or_else(|| Error::UnexpectedError { |
| 159 | + message: format!("Manifest entry group for bucket {bucket} is empty, cannot determine total_buckets"), |
| 160 | + source: None, |
| 161 | + })?; |
| 162 | + let mut data_files = Vec::new(); |
| 163 | + |
| 164 | + // currently, only group by splits by bucket |
| 165 | + // todo: consider use binpack to generate split |
| 166 | + for manifest_entry in group_entries { |
| 167 | + let ManifestEntry { file, .. } = manifest_entry; |
| 168 | + data_files.push(file); |
| 169 | + } |
| 170 | + |
| 171 | + // todo: consider partitioned table |
| 172 | + let bucket_path = format!("{base_path}/bucket-{bucket}"); |
| 173 | + let partition = BinaryRow::new(0); |
| 174 | + |
| 175 | + let split = DataSplitBuilder::new() |
| 176 | + .with_snapshot(snapshot_id) |
| 177 | + .with_partition(partition) |
| 178 | + .with_bucket(bucket) |
| 179 | + .with_bucket_path(bucket_path) |
| 180 | + .with_total_buckets(total_buckets) |
| 181 | + .with_data_files(data_files) |
| 182 | + .build()?; |
| 183 | + splits.push(split); |
| 184 | + } |
| 185 | + Ok(Plan::new(splits)) |
| 186 | + } |
| 187 | +} |
0 commit comments