class IOOperations:
@staticmethod
def read_fasta(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
) -> pl.DataFrame:
"""
Read a FASTA file into a DataFrame.
Parameters:
path: The path to the FASTA file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the FASTA file. If not specified, it will be detected automatically based on the file extension. BGZF and GZIP compressions are supported ('bgz', 'gz').
projection_pushdown: Enable column projection pushdown optimization. When True, only requested columns are processed at the DataFusion execution level, improving performance and reducing memory usage.
!!! Example
```shell
wget https://www.ebi.ac.uk/ena/browser/api/fasta/BK006935.2?download=true -O /tmp/test.fasta
```
```python
import polars_bio as pb
pb.read_fasta("/tmp/test.fasta").limit(1)
```
```shell
shape: (1, 3)
┌─────────────────────────┬─────────────────────────────────┬─────────────────────────────────┐
│ name ┆ description ┆ sequence │
│ --- ┆ --- ┆ --- │
│ str ┆ str ┆ str │
╞═════════════════════════╪═════════════════════════════════╪═════════════════════════════════╡
│ ENA|BK006935|BK006935.2 ┆ TPA_inf: Saccharomyces cerevis… ┆ CCACACCACACCCACACACCCACACACCAC… │
└─────────────────────────┴─────────────────────────────────┴─────────────────────────────────┘
```
"""
return IOOperations.scan_fasta(
path,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
compression_type,
projection_pushdown,
).collect()
@staticmethod
def scan_fasta(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
) -> pl.LazyFrame:
"""
Lazily read a FASTA file into a LazyFrame.
Parameters:
path: The path to the FASTA file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the FASTA file. If not specified, it will be detected automatically based on the file extension. BGZF and GZIP compressions are supported ('bgz', 'gz').
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
!!! Example
```shell
wget https://www.ebi.ac.uk/ena/browser/api/fasta/BK006935.2?download=true -O /tmp/test.fasta
```
```python
import polars_bio as pb
pb.scan_fasta("/tmp/test.fasta").limit(1).collect()
```
```shell
shape: (1, 3)
┌─────────────────────────┬─────────────────────────────────┬─────────────────────────────────┐
│ name ┆ description ┆ sequence │
│ --- ┆ --- ┆ --- │
│ str ┆ str ┆ str │
╞═════════════════════════╪═════════════════════════════════╪═════════════════════════════════╡
│ ENA|BK006935|BK006935.2 ┆ TPA_inf: Saccharomyces cerevis… ┆ CCACACCACACCCACACACCCACACACCAC… │
└─────────────────────────┴─────────────────────────────────┴─────────────────────────────────┘
```
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
fasta_read_options = FastaReadOptions(
object_storage_options=object_storage_options
)
read_options = ReadOptions(fasta_read_options=fasta_read_options)
return _read_file(path, InputFormat.Fasta, read_options, projection_pushdown)
@staticmethod
def read_vcf(
path: str,
info_fields: Union[list[str], None] = None,
format_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
samples: Union[list[str], None] = None,
genotype_encoding_raw: bool = True,
) -> pl.DataFrame:
"""
Read a VCF file into a DataFrame.
!!! hint "Parallelism & Indexed Reads"
Indexed parallel reads and predicate pushdown are automatic when a TBI/CSI index
is present. See [File formats support](/polars-bio/features/#file-formats-support),
[Indexed reads](/polars-bio/features/#indexed-reads-predicate-pushdown),
and [Automatic parallel partitioning](/polars-bio/features/#automatic-parallel-partitioning) for details.
Parameters:
path: The path to the VCF file.
info_fields: List of INFO field names to include. If *None*, all INFO fields from the VCF header are included by default. Use this to limit fields for better performance.
format_fields: List of FORMAT field names to include (per-sample genotype data). If *None*, all FORMAT fields are included by default. For **single-sample** VCFs, FORMAT fields are top-level columns (e.g., `GT`, `DP`). For **multi-sample** VCFs, FORMAT data is exposed as a nested `genotypes` column (`struct<GT: list, DP: list, ...>`) with sample names in `meta["header"]["sample_names"]`.
samples: Optional list of sample names to include from the VCF header. Matching is exact and case-sensitive. Missing sample names are skipped with a warning. The output follows the requested sample order.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the VCF file. If not specified, it will be detected automatically..
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
predicate_pushdown: Enable predicate pushdown using index files (TBI/CSI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.vcf.gz.tbi`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
genotype_encoding_raw: If True, output GT as raw typed allele calls. If False, output VCF-style GT strings.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
!!! Example "Reading VCF with INFO and FORMAT fields"
```python
import polars_bio as pb
# Read VCF with both INFO and FORMAT fields
df = pb.read_vcf(
"sample.vcf.gz",
info_fields=["END"], # INFO field
format_fields=["GT", "DP", "GQ"] # FORMAT fields
)
# Single-sample VCF: FORMAT fields are top-level columns (GT, DP, GQ)
print(df.select(["chrom", "start", "ref", "alt", "END", "GT", "DP", "GQ"]))
# Output:
# shape: (10, 8)
# ┌───────┬───────┬─────┬─────┬──────┬─────┬─────┬─────┐
# │ chrom ┆ start ┆ ref ┆ alt ┆ END ┆ GT ┆ DP ┆ GQ │
# │ str ┆ u32 ┆ str ┆ str ┆ i32 ┆ str ┆ i32 ┆ i32 │
# ╞═══════╪═══════╪═════╪═════╪══════╪═════╪═════╪═════╡
# │ 1 ┆ 10009 ┆ A ┆ . ┆ null ┆ 0/0 ┆ 10 ┆ 27 │
# │ 1 ┆ 10015 ┆ A ┆ . ┆ null ┆ 0/0 ┆ 17 ┆ 35 │
# └───────┴───────┴─────┴─────┴──────┴─────┴─────┴─────┘
# Multi-sample VCF: FORMAT data is nested in "genotypes"
df = pb.read_vcf("multisample.vcf", format_fields=["GT", "DP"])
print(df.select(["chrom", "start", "genotypes"]))
```
"""
lf = IOOperations.scan_vcf(
path=path,
info_fields=info_fields,
format_fields=format_fields,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
projection_pushdown=projection_pushdown,
predicate_pushdown=predicate_pushdown,
use_zero_based=use_zero_based,
samples=samples,
)
# Get metadata before collecting (polars-config-meta doesn't preserve through collect)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
# Set metadata on the collected DataFrame
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_vcf(
path: str,
info_fields: Union[list[str], None] = None,
format_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
samples: Union[list[str], None] = None,
) -> pl.LazyFrame:
"""
Lazily read a VCF file into a LazyFrame.
!!! hint "Parallelism & Indexed Reads"
Indexed parallel reads and predicate pushdown are automatic when a TBI/CSI index
is present. See [File formats support](/polars-bio/features/#file-formats-support),
[Indexed reads](/polars-bio/features/#indexed-reads-predicate-pushdown),
and [Automatic parallel partitioning](/polars-bio/features/#automatic-parallel-partitioning) for details.
Parameters:
path: The path to the VCF file.
info_fields: List of INFO field names to include. If *None*, all INFO fields from the VCF header are included by default. Use this to limit fields for better performance.
format_fields: List of FORMAT field names to include (per-sample genotype data). If *None*, all FORMAT fields are included by default. For **single-sample** VCFs, FORMAT fields are top-level columns (e.g., `GT`, `DP`). For **multi-sample** VCFs, FORMAT data is exposed as a nested `genotypes` column (`struct<GT: list, DP: list, ...>`) with sample names in `meta["header"]["sample_names"]`.
samples: Optional list of sample names to include from the VCF header. Matching is exact and case-sensitive. Missing sample names are skipped with a warning. The output follows the requested sample order.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the VCF file. If not specified, it will be detected automatically..
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
predicate_pushdown: Enable predicate pushdown using index files (TBI/CSI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.vcf.gz.tbi`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
!!! Example "Lazy scanning VCF with INFO and FORMAT fields"
```python
import polars_bio as pb
# Lazily scan VCF with both INFO and FORMAT fields
lf = pb.scan_vcf(
"sample.vcf.gz",
info_fields=["END"], # INFO field
format_fields=["GT", "DP", "GQ"] # FORMAT fields
)
# Apply filters and collect only what's needed
df = lf.filter(pl.col("DP") > 20).select(
["chrom", "start", "ref", "alt", "GT", "DP", "GQ"]
).collect()
# Single-sample VCF: FORMAT fields are top-level columns (GT, DP, GQ)
# Multi-sample VCF: FORMAT data is nested in "genotypes"
```
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
# Upstream VCF reader projects all INFO fields by default when info_fields is None.
initial_info_fields = info_fields
zero_based = _resolve_zero_based(use_zero_based)
vcf_read_options = VcfReadOptions(
info_fields=initial_info_fields,
format_fields=format_fields,
samples=samples,
object_storage_options=object_storage_options,
zero_based=zero_based,
)
read_options = ReadOptions(vcf_read_options=vcf_read_options)
return _read_file(
path,
InputFormat.Vcf,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_vcf_zarr(
path: str,
info_fields: Union[list[str], None] = None,
format_fields: Union[list[str], None] = None,
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
samples: Union[list[str], None] = None,
genotype_encoding_raw: bool = True,
) -> pl.DataFrame:
"""
Read a local VCF Zarr store into a DataFrame.
Parameters:
path: The path to the VCF Zarr store directory.
info_fields: Optional list of INFO field names to include. If None, local INFO arrays are discovered automatically. Use [] to disable INFO fields.
format_fields: Optional list of FORMAT field names to include. If None, local FORMAT arrays are discovered automatically. Use [] to disable FORMAT fields.
projection_pushdown: Enable column projection pushdown at the DataFusion level.
predicate_pushdown: Enable predicate pushdown at the DataFusion level.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None, uses the global configuration.
samples: Optional list of sample names to include.
genotype_encoding_raw: If True, output GT as raw typed allele calls. If False, output VCF-style GT strings.
"""
lf = IOOperations.scan_vcf_zarr(
path=path,
info_fields=info_fields,
format_fields=format_fields,
projection_pushdown=projection_pushdown,
predicate_pushdown=predicate_pushdown,
use_zero_based=use_zero_based,
samples=samples,
genotype_encoding_raw=genotype_encoding_raw,
)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_vcf_zarr(
path: str,
info_fields: Union[list[str], None] = None,
format_fields: Union[list[str], None] = None,
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
samples: Union[list[str], None] = None,
genotype_encoding_raw: bool = True,
) -> pl.LazyFrame:
"""
Lazily read a local VCF Zarr store into a LazyFrame.
Parameters:
path: The path to the VCF Zarr store directory.
info_fields: Optional list of INFO field names to include. If None, local INFO arrays are discovered automatically. Use [] to disable INFO fields.
format_fields: Optional list of FORMAT field names to include. If None, local FORMAT arrays are discovered automatically. Use [] to disable FORMAT fields.
projection_pushdown: Enable column projection pushdown at the DataFusion level.
predicate_pushdown: Enable predicate pushdown at the DataFusion level.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None, uses the global configuration.
samples: Optional list of sample names to include.
genotype_encoding_raw: If True, output GT as raw typed allele calls. If False, output VCF-style GT strings.
"""
zero_based = _resolve_zero_based(use_zero_based)
vcf_zarr_read_options = VcfZarrReadOptions(
info_fields=info_fields,
format_fields=format_fields,
samples=samples,
zero_based=zero_based,
genotype_encoding_raw=genotype_encoding_raw,
)
read_options = ReadOptions(vcf_zarr_read_options=vcf_zarr_read_options)
return _read_file(
path,
InputFormat.VcfZarr,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_gff(
path: str,
attr_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.DataFrame:
"""
Read a GFF file into a DataFrame.
Parameters:
path: The path to the GFF file.
attr_fields: List of attribute field names to extract as separate columns. If *None*, attributes will be kept as a nested structure. Use this to extract specific attributes like 'ID', 'gene_name', 'gene_type', etc. as direct columns for easier access.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the GFF file. If not specified, it will be detected automatically..
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
predicate_pushdown: Enable predicate pushdown using index files (TBI/CSI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.gff.gz.tbi`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
lf = IOOperations.scan_gff(
path,
attr_fields,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
compression_type,
projection_pushdown,
predicate_pushdown,
use_zero_based,
)
# Get metadata before collecting (polars-config-meta doesn't preserve through collect)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
# Set metadata on the collected DataFrame
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_gff(
path: str,
attr_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.LazyFrame:
"""
Lazily read a GFF file into a LazyFrame.
Parameters:
path: The path to the GFF file.
attr_fields: List of attribute field names to extract as separate columns. If *None*, attributes will be kept as a nested structure. Use this to extract specific attributes like 'ID', 'gene_name', 'gene_type', etc. as direct columns for easier access.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large-scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the GFF file. If not specified, it will be detected automatically.
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
predicate_pushdown: Enable predicate pushdown using index files (TBI/CSI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.gff.gz.tbi`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
zero_based = _resolve_zero_based(use_zero_based)
gff_read_options = GffReadOptions(
attr_fields=attr_fields,
object_storage_options=object_storage_options,
zero_based=zero_based,
)
read_options = ReadOptions(gff_read_options=gff_read_options)
_store_py_object_storage_options(read_options, object_storage_options)
return _read_file(
path,
InputFormat.Gff,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_gtf(
path: str,
attr_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.DataFrame:
"""
Read a GTF file into a DataFrame.
GTF (Gene Transfer Format) shares the same 9-column structure as GFF but uses
different attribute syntax (``key "value"`` vs GFF's ``key=value``).
Parameters:
path: The path to the GTF file.
attr_fields: List of attribute field names to extract as separate columns.
If *None*, attributes will be kept as a nested structure.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large-scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the GTF file. If not specified, it will be detected automatically.
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
predicate_pushdown: Enable predicate pushdown using index files (TBI/CSI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.gtf.gz.tbi`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
lf = IOOperations.scan_gtf(
path,
attr_fields,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
compression_type,
projection_pushdown,
predicate_pushdown,
use_zero_based,
)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_gtf(
path: str,
attr_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.LazyFrame:
"""
Lazily read a GTF file into a LazyFrame.
GTF (Gene Transfer Format) shares the same 9-column structure as GFF but uses
different attribute syntax (``key "value"`` vs GFF's ``key=value``).
Parameters:
path: The path to the GTF file.
attr_fields: List of attribute field names to extract as separate columns.
If *None*, attributes will be kept as a nested structure.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large-scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the GTF file. If not specified, it will be detected automatically.
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
predicate_pushdown: Enable predicate pushdown using index files (TBI/CSI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.gtf.gz.tbi`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
zero_based = _resolve_zero_based(use_zero_based)
gtf_read_options = GtfReadOptions(
attr_fields=attr_fields,
object_storage_options=object_storage_options,
zero_based=zero_based,
)
read_options = ReadOptions(gtf_read_options=gtf_read_options)
_store_py_object_storage_options(read_options, object_storage_options)
return _read_file(
path,
InputFormat.Gtf,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_bam(
path: str,
tag_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
infer_tag_types: bool = True,
infer_tag_sample_size: int = 100,
tag_type_hints: Optional[list[str]] = None,
) -> pl.DataFrame:
"""
Read a BAM file into a DataFrame.
!!! hint "Parallelism & Indexed Reads"
Indexed parallel reads and predicate pushdown are automatic when a BAI/CSI index
is present. See [File formats support](/polars-bio/features/#file-formats-support),
[Indexed reads](/polars-bio/features/#indexed-reads-predicate-pushdown),
and [Automatic parallel partitioning](/polars-bio/features/#automatic-parallel-partitioning) for details.
Parameters:
path: The path to the BAM file.
tag_fields: List of BAM tag names to include as columns (e.g., ["NM", "MD", "AS"]). If None, no optional tags are parsed (default). Common tags include: NM (edit distance), MD (mismatch string), AS (alignment score), XS (secondary alignment score), RG (read group), CB (cell barcode), UB (UMI barcode).
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large-scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large-scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
predicate_pushdown: Enable predicate pushdown using index files (BAI/CSI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.bam.bai`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
infer_tag_types: If True (default), sample the file to auto-detect types for custom/unknown tags. This prevents integer tags from being decoded as ASCII characters.
infer_tag_sample_size: Number of records to sample for tag type inference (default: 100).
tag_type_hints: Explicit SAM-style type hints for tags (e.g., ["pt:i", "ML:B:C", "FZ:B:S"]). Used as fallback when inference is disabled or a tag is not found in sampled records. Supported forms: TAG:TYPE, TAG:B, or TAG:B:SUBTYPE where TYPE is one of A, c, C, s, S, i, I, f, Z, H and SUBTYPE is one of c, C, s, S, i, I, f.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
lf = IOOperations.scan_bam(
path,
tag_fields,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
projection_pushdown,
predicate_pushdown,
use_zero_based,
infer_tag_types,
infer_tag_sample_size,
tag_type_hints,
)
# Get metadata before collecting (polars-config-meta doesn't preserve through collect)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
# Set metadata on the collected DataFrame
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_bam(
path: str,
tag_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
infer_tag_types: bool = True,
infer_tag_sample_size: int = 100,
tag_type_hints: Optional[list[str]] = None,
) -> pl.LazyFrame:
"""
Lazily read a BAM file into a LazyFrame.
!!! hint "Parallelism & Indexed Reads"
Indexed parallel reads and predicate pushdown are automatic when a BAI/CSI index
is present. See [File formats support](/polars-bio/features/#file-formats-support),
[Indexed reads](/polars-bio/features/#indexed-reads-predicate-pushdown),
and [Automatic parallel partitioning](/polars-bio/features/#automatic-parallel-partitioning) for details.
Parameters:
path: The path to the BAM file.
tag_fields: List of BAM tag names to include as columns (e.g., ["NM", "MD", "AS"]). If None, no optional tags are parsed (default). Common tags include: NM (edit distance), MD (mismatch string), AS (alignment score), XS (secondary alignment score), RG (read group), CB (cell barcode), UB (UMI barcode).
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
predicate_pushdown: Enable predicate pushdown using index files (BAI/CSI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.bam.bai`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
infer_tag_types: If True (default), sample the file to auto-detect types for custom/unknown tags. This prevents integer tags from being decoded as ASCII characters.
infer_tag_sample_size: Number of records to sample for tag type inference (default: 100).
tag_type_hints: Explicit SAM-style type hints for tags (e.g., ["pt:i", "ML:B:C", "FZ:B:S"]). Used as fallback when inference is disabled or a tag is not found in sampled records. Supported forms: TAG:TYPE, TAG:B, or TAG:B:SUBTYPE where TYPE is one of A, c, C, s, S, i, I, f, Z, H and SUBTYPE is one of c, C, s, S, i, I, f.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type="auto",
)
zero_based = _resolve_zero_based(use_zero_based)
if tag_type_hints is not None:
_validate_tag_type_hints(tag_type_hints)
tag_type_hints = _normalize_read_tag_type_hints(tag_type_hints)
bam_read_options = BamReadOptions(
object_storage_options=object_storage_options,
zero_based=zero_based,
tag_fields=tag_fields,
infer_tag_types=infer_tag_types,
infer_tag_sample_size=infer_tag_sample_size,
tag_type_hints=tag_type_hints,
)
read_options = ReadOptions(bam_read_options=bam_read_options)
return _read_file(
path,
InputFormat.Bam,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_cram(
path: str,
reference_path: str = None,
tag_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
infer_tag_types: bool = True,
infer_tag_sample_size: int = 100,
tag_type_hints: Optional[list[str]] = None,
) -> pl.DataFrame:
"""
Read a CRAM file into a DataFrame.
!!! hint "Parallelism & Indexed Reads"
Indexed parallel reads and predicate pushdown are automatic when a CRAI index
is present. See [File formats support](/polars-bio/features/#file-formats-support),
[Indexed reads](/polars-bio/features/#indexed-reads-predicate-pushdown),
and [Automatic parallel partitioning](/polars-bio/features/#automatic-parallel-partitioning) for details.
Parameters:
path: The path to the CRAM file (local or cloud storage: S3, GCS, Azure Blob).
reference_path: Optional path to external FASTA reference file (**local path only**, cloud storage not supported). If not provided, the CRAM file must contain embedded reference sequences. The FASTA file must have an accompanying index file (.fai) in the same directory. Create the index using: `samtools faidx reference.fasta`
tag_fields: List of CRAM tag names to include as columns (e.g., ["NM", "MD", "AS"]). If None, no optional tags are parsed (default). Common tags include: NM (edit distance), MD (mismatch string), AS (alignment score), XS (secondary alignment score), RG (read group), CB (cell barcode), UB (UMI barcode).
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
projection_pushdown: Enable column projection pushdown optimization. When True, only requested columns are processed at the DataFusion execution level, improving performance and reducing memory usage.
predicate_pushdown: Enable predicate pushdown using index files (CRAI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.cram.crai`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
infer_tag_types: If True (default), sample the file to auto-detect types for custom/unknown tags. This prevents integer tags from being decoded as ASCII characters.
infer_tag_sample_size: Number of records to sample for tag type inference (default: 100).
tag_type_hints: Explicit SAM-style type hints for tags (e.g., ["pt:i", "ML:B:C", "FZ:B:S"]). Used as fallback when inference is disabled or a tag is not found in sampled records. Supported forms: TAG:TYPE, TAG:B, or TAG:B:SUBTYPE where TYPE is one of A, c, C, s, S, i, I, f, Z, H and SUBTYPE is one of c, C, s, S, i, I, f.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
!!! warning "Known Limitation: MD and NM Tags"
Due to a limitation in the underlying noodles-cram library, **MD (mismatch descriptor) and NM (edit distance) tags are not accessible** from CRAM files, even when stored in the file. These tags can be seen with samtools but are not exposed through the noodles-cram record.data() interface.
Other optional tags (RG, MQ, AM, OQ, etc.) work correctly. This issue is tracked at: https://github.com/biodatageeks/datafusion-bio-formats/issues/54
**Workaround**: Use BAM format if MD/NM tags are required for your analysis.
!!! example "Using External Reference"
```python
import polars_bio as pb
# Read CRAM with external reference
df = pb.read_cram(
"/path/to/file.cram",
reference_path="/path/to/reference.fasta"
)
```
!!! example "Public CRAM File Example"
Download and read a public CRAM file from 42basepairs:
```bash
# Download the CRAM file and reference
wget https://42basepairs.com/download/s3/gatk-test-data/wgs_cram/NA12878_20k_hg38/NA12878.cram
wget https://storage.googleapis.com/genomics-public-data/resources/broad/hg38/v0/Homo_sapiens_assembly38.fasta
# Create FASTA index (required)
samtools faidx Homo_sapiens_assembly38.fasta
```
```python
import polars_bio as pb
# Read first 5 reads from the CRAM file
df = pb.scan_cram(
"NA12878.cram",
reference_path="Homo_sapiens_assembly38.fasta"
).limit(5).collect()
print(df.select(["name", "chrom", "start", "end", "cigar"]))
```
!!! example "Creating CRAM with Embedded Reference"
To create a CRAM file with embedded reference using samtools:
```bash
samtools view -C -o output.cram --output-fmt-option embed_ref=1 input.bam
```
Returns:
A Polars DataFrame with the following schema:
- name: Read name (String)
- chrom: Chromosome/contig name (String)
- start: Alignment start position, 1-based (UInt32)
- end: Alignment end position, 1-based (UInt32)
- flags: SAM flags (UInt32)
- cigar: CIGAR string (String)
- mapping_quality: Mapping quality (UInt32)
- mate_chrom: Mate chromosome/contig name (String)
- mate_start: Mate alignment start position, 1-based (UInt32)
- sequence: Read sequence (String)
- quality_scores: Base quality scores (String)
"""
lf = IOOperations.scan_cram(
path,
reference_path,
tag_fields,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
projection_pushdown,
predicate_pushdown,
use_zero_based,
infer_tag_types,
infer_tag_sample_size,
tag_type_hints,
)
# Get metadata before collecting (polars-config-meta doesn't preserve through collect)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
# Set metadata on the collected DataFrame
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_cram(
path: str,
reference_path: str = None,
tag_fields: Union[list[str], None] = None,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
infer_tag_types: bool = True,
infer_tag_sample_size: int = 100,
tag_type_hints: Optional[list[str]] = None,
) -> pl.LazyFrame:
"""
Lazily read a CRAM file into a LazyFrame.
!!! hint "Parallelism & Indexed Reads"
Indexed parallel reads and predicate pushdown are automatic when a CRAI index
is present. See [File formats support](/polars-bio/features/#file-formats-support),
[Indexed reads](/polars-bio/features/#indexed-reads-predicate-pushdown),
and [Automatic parallel partitioning](/polars-bio/features/#automatic-parallel-partitioning) for details.
Parameters:
path: The path to the CRAM file (local or cloud storage: S3, GCS, Azure Blob).
reference_path: Optional path to external FASTA reference file (**local path only**, cloud storage not supported). If not provided, the CRAM file must contain embedded reference sequences. The FASTA file must have an accompanying index file (.fai) in the same directory. Create the index using: `samtools faidx reference.fasta`
tag_fields: List of CRAM tag names to include as columns (e.g., ["NM", "MD", "AS"]). If None, no optional tags are parsed (default). Common tags include: NM (edit distance), MD (mismatch string), AS (alignment score), XS (secondary alignment score), RG (read group), CB (cell barcode), UB (UMI barcode).
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
projection_pushdown: Enable column projection pushdown optimization. When True, only requested columns are processed at the DataFusion execution level, improving performance and reducing memory usage.
predicate_pushdown: Enable predicate pushdown using index files (CRAI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.cram.crai`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates like `.str.contains()` or OR logic are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
infer_tag_types: If True (default), sample the file to auto-detect types for custom/unknown tags. This prevents integer tags from being decoded as ASCII characters.
infer_tag_sample_size: Number of records to sample for tag type inference (default: 100).
tag_type_hints: Explicit SAM-style type hints for tags (e.g., ["pt:i", "ML:B:C", "FZ:B:S"]). Used as fallback when inference is disabled or a tag is not found in sampled records. Supported forms: TAG:TYPE, TAG:B, or TAG:B:SUBTYPE where TYPE is one of A, c, C, s, S, i, I, f, Z, H and SUBTYPE is one of c, C, s, S, i, I, f.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
!!! warning "Known Limitation: MD and NM Tags"
Due to a limitation in the underlying noodles-cram library, **MD (mismatch descriptor) and NM (edit distance) tags are not accessible** from CRAM files, even when stored in the file. These tags can be seen with samtools but are not exposed through the noodles-cram record.data() interface.
Other optional tags (RG, MQ, AM, OQ, etc.) work correctly. This issue is tracked at: https://github.com/biodatageeks/datafusion-bio-formats/issues/54
**Workaround**: Use BAM format if MD/NM tags are required for your analysis.
!!! example "Using External Reference"
```python
import polars_bio as pb
# Lazy scan CRAM with external reference
lf = pb.scan_cram(
"/path/to/file.cram",
reference_path="/path/to/reference.fasta"
)
# Apply transformations and collect
df = lf.filter(pl.col("chrom") == "chr1").collect()
```
!!! example "Public CRAM File Example"
Download and read a public CRAM file from 42basepairs:
```bash
# Download the CRAM file and reference
wget https://42basepairs.com/download/s3/gatk-test-data/wgs_cram/NA12878_20k_hg38/NA12878.cram
wget https://storage.googleapis.com/genomics-public-data/resources/broad/hg38/v0/Homo_sapiens_assembly38.fasta
# Create FASTA index (required)
samtools faidx Homo_sapiens_assembly38.fasta
```
```python
import polars_bio as pb
import polars as pl
# Lazy scan and filter for chromosome 20 reads
df = pb.scan_cram(
"NA12878.cram",
reference_path="Homo_sapiens_assembly38.fasta"
).filter(
pl.col("chrom") == "chr20"
).select(
["name", "chrom", "start", "end", "mapping_quality"]
).limit(10).collect()
print(df)
```
!!! example "Creating CRAM with Embedded Reference"
To create a CRAM file with embedded reference using samtools:
```bash
samtools view -C -o output.cram --output-fmt-option embed_ref=1 input.bam
```
Returns:
A Polars LazyFrame with the following schema:
- name: Read name (String)
- chrom: Chromosome/contig name (String)
- start: Alignment start position, 1-based (UInt32)
- end: Alignment end position, 1-based (UInt32)
- flags: SAM flags (UInt32)
- cigar: CIGAR string (String)
- mapping_quality: Mapping quality (UInt32)
- mate_chrom: Mate chromosome/contig name (String)
- mate_start: Mate alignment start position, 1-based (UInt32)
- sequence: Read sequence (String)
- quality_scores: Base quality scores (String)
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type="auto",
)
zero_based = _resolve_zero_based(use_zero_based)
if tag_type_hints is not None:
_validate_tag_type_hints(tag_type_hints)
tag_type_hints = _normalize_read_tag_type_hints(tag_type_hints)
cram_read_options = CramReadOptions(
reference_path=reference_path,
object_storage_options=object_storage_options,
zero_based=zero_based,
tag_fields=tag_fields,
infer_tag_types=infer_tag_types,
infer_tag_sample_size=infer_tag_sample_size,
tag_type_hints=tag_type_hints,
)
read_options = ReadOptions(cram_read_options=cram_read_options)
return _read_file(
path,
InputFormat.Cram,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def describe_bam(
path: str,
sample_size: int = 100,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
use_zero_based: Optional[bool] = None,
) -> pl.DataFrame:
"""
Get schema information for a BAM file with automatic tag discovery.
Samples the first N records to discover all available tags and their types.
Returns detailed schema information including column names, data types,
nullability, category (standard/tag), SAM type, and descriptions.
Parameters:
path: The path to the BAM file.
sample_size: Number of records to sample for tag discovery (default: 100).
Use higher values for more comprehensive tag discovery.
chunk_size: The size in MB of a chunk when reading from object storage.
concurrent_fetches: The number of concurrent fetches when reading from object storage.
allow_anonymous: Whether to allow anonymous access to object storage.
enable_request_payer: Whether to enable request payer for object storage.
max_retries: The maximum number of retries for reading the file.
timeout: The timeout in seconds for reading the file.
compression_type: The compression type of the file. If "auto" (default), compression is detected automatically.
use_zero_based: If True, output 0-based coordinates. If False, 1-based coordinates.
Returns:
DataFrame with columns:
- column_name: Name of the column/field
- data_type: Arrow data type (e.g., "Utf8", "Int32")
- nullable: Whether the field can be null
- category: "core" for fixed columns, "tag" for optional SAM tags
- sam_type: SAM type code (e.g., "Z", "i") for tags, null for core columns
- description: Human-readable description of the field
Example:
```python
import polars_bio as pb
# Auto-discover all tags present in the file
schema = pb.describe_bam("file.bam", sample_size=100)
print(schema)
# Output:
# shape: (15, 6)
# ┌─────────────┬───────────┬──────────┬──────────┬──────────┬──────────────────────┐
# │ column_name ┆ data_type ┆ nullable ┆ category ┆ sam_type ┆ description │
# │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │
# │ str ┆ str ┆ bool ┆ str ┆ str ┆ str │
# ╞═════════════╪═══════════╪══════════╪══════════╪══════════╪══════════════════════╡
# │ name ┆ Utf8 ┆ true ┆ core ┆ null ┆ Query name │
# │ chrom ┆ Utf8 ┆ true ┆ core ┆ null ┆ Reference name │
# │ ... ┆ ... ┆ ... ┆ ... ┆ ... ┆ ... │
# │ NM ┆ Int32 ┆ true ┆ tag ┆ i ┆ Edit distance │
# │ AS ┆ Int32 ┆ true ┆ tag ┆ i ┆ Alignment score │
# └─────────────┴───────────┴──────────┴──────────┴──────────┴──────────────────────┘
```
"""
# Build object storage options
object_storage_options = PyObjectStorageOptions(
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
# Resolve zero_based setting
zero_based = _resolve_zero_based(use_zero_based)
# Call Rust function with tag auto-discovery (tag_fields=None)
df = py_describe_bam(
ctx, # PyBioSessionContext
path,
object_storage_options,
zero_based,
None, # tag_fields=None enables auto-discovery
sample_size,
)
# Convert DataFusion DataFrame to Polars DataFrame
return pl.from_arrow(df.to_arrow_table())
@staticmethod
def describe_cram(
path: str,
reference_path: str = None,
sample_size: int = 100,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
use_zero_based: Optional[bool] = None,
) -> pl.DataFrame:
"""
Get schema information for a CRAM file with automatic tag discovery.
Samples the first N records to discover all available tags and their types.
Returns detailed schema information including column names, data types,
nullability, category (core/tag), SAM type, and descriptions.
Parameters:
path: The path to the CRAM file.
reference_path: Optional path to external FASTA reference file.
sample_size: Number of records to sample for tag discovery (default: 100).
chunk_size: The size in MB of a chunk when reading from object storage.
concurrent_fetches: The number of concurrent fetches when reading from object storage.
allow_anonymous: Whether to allow anonymous access to object storage.
enable_request_payer: Whether to enable request payer for object storage.
max_retries: The maximum number of retries for reading the file.
timeout: The timeout in seconds for reading the file.
compression_type: The compression type of the file. If "auto" (default), compression is detected automatically.
use_zero_based: If True, output 0-based coordinates. If False, 1-based coordinates.
Returns:
DataFrame with columns:
- column_name: Name of the column/field
- data_type: Arrow data type (e.g., "Utf8", "Int32")
- nullable: Whether the field can be null
- category: "core" for fixed columns, "tag" for optional SAM tags
- sam_type: SAM type code (e.g., "Z", "i") for tags, null for core columns
- description: Human-readable description of the field
!!! warning "Known Limitation: MD and NM Tags"
Due to a limitation in the underlying noodles-cram library, **MD (mismatch descriptor) and NM (edit distance) tags are not discoverable** from CRAM files, even when stored. Automatic tag discovery will not include MD/NM tags. Other optional tags (RG, MQ, AM, OQ, etc.) are discovered correctly. See: https://github.com/biodatageeks/datafusion-bio-formats/issues/54
Example:
```python
import polars_bio as pb
# Auto-discover all tags present in the file
schema = pb.describe_cram("file.cram", sample_size=100)
print(schema)
# Filter to see only tag columns
tags = schema.filter(schema["category"] == "tag")
print(tags["column_name"])
```
"""
# Build object storage options
object_storage_options = PyObjectStorageOptions(
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
# Resolve zero_based setting
zero_based = _resolve_zero_based(use_zero_based)
# Call Rust function with tag auto-discovery (tag_fields=None)
df = py_describe_cram(
ctx,
path,
reference_path,
object_storage_options,
zero_based,
None, # tag_fields=None enables auto-discovery
sample_size,
)
# Convert DataFusion DataFrame to Polars DataFrame
return pl.from_arrow(df.to_arrow_table())
@staticmethod
def read_fastq(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
) -> pl.DataFrame:
"""
Read a FASTQ file into a DataFrame.
!!! hint "Parallelism & Compression"
See [File formats support](/polars-bio/features/#file-formats-support),
[Compression](/polars-bio/features/#compression),
and [Automatic parallel partitioning](/polars-bio/features/#automatic-parallel-partitioning) for details on parallel reads and supported compression types.
Parameters:
path: The path to the FASTQ file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the FASTQ file. If not specified, it will be detected automatically based on the file extension. BGZF and GZIP compressions are supported ('bgz', 'gz').
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
"""
return IOOperations.scan_fastq(
path,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
compression_type,
projection_pushdown,
).collect()
@staticmethod
def scan_fastq(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
) -> pl.LazyFrame:
"""
Lazily read a FASTQ file into a LazyFrame.
!!! hint "Parallelism & Compression"
See [File formats support](/polars-bio/features/#file-formats-support),
[Compression](/polars-bio/features/#compression),
and [Automatic parallel partitioning](/polars-bio/features/#automatic-parallel-partitioning) for details on parallel reads and supported compression types.
Parameters:
path: The path to the FASTQ file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the FASTQ file. If not specified, it will be detected automatically based on the file extension. BGZF and GZIP compressions are supported ('bgz', 'gz').
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
fastq_read_options = FastqReadOptions(
object_storage_options=object_storage_options,
)
read_options = ReadOptions(fastq_read_options=fastq_read_options)
return _read_file(path, InputFormat.Fastq, read_options, projection_pushdown)
@staticmethod
def read_pairs(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.DataFrame:
"""
Read a Pairs (Hi-C) file into a DataFrame.
The Pairs format (4DN project) stores chromatin contact data with columns:
readID, chr1, pos1, chr2, pos2, strand1, strand2.
!!! hint "Parallelism & Indexed Reads"
Indexed parallel reads and predicate pushdown are automatic when a TBI index
is present. See [File formats support](/polars-bio/features/#file-formats-support)
and [Indexed reads](/polars-bio/features/#indexed-reads-predicate-pushdown) for details.
Parameters:
path: The path to the Pairs file (.pairs, .pairs.gz, .pairs.bgz).
chunk_size: The size in MB of a chunk when reading from an object store.
concurrent_fetches: The number of concurrent fetches when reading from an object store.
allow_anonymous: Whether to allow anonymous access to object storage.
enable_request_payer: Whether to enable request payer for object storage.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type. If not specified, it will be detected automatically.
projection_pushdown: Enable column projection pushdown to optimize query performance.
predicate_pushdown: Enable predicate pushdown using index files (TBI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.pairs.gz.tbi`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
lf = IOOperations.scan_pairs(
path,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
compression_type,
projection_pushdown,
predicate_pushdown,
use_zero_based,
)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_pairs(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.LazyFrame:
"""
Lazily read a Pairs (Hi-C) file into a LazyFrame.
The Pairs format (4DN project) stores chromatin contact data with columns:
readID, chr1, pos1, chr2, pos2, strand1, strand2.
!!! hint "Parallelism & Indexed Reads"
Indexed parallel reads and predicate pushdown are automatic when a TBI index
is present. See [File formats support](/polars-bio/features/#file-formats-support)
and [Indexed reads](/polars-bio/features/#indexed-reads-predicate-pushdown) for details.
Parameters:
path: The path to the Pairs file (.pairs, .pairs.gz, .pairs.bgz).
chunk_size: The size in MB of a chunk when reading from an object store.
concurrent_fetches: The number of concurrent fetches when reading from an object store.
allow_anonymous: Whether to allow anonymous access to object storage.
enable_request_payer: Whether to enable request payer for object storage.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type. If not specified, it will be detected automatically.
projection_pushdown: Enable column projection pushdown to optimize query performance.
predicate_pushdown: Enable predicate pushdown using index files (TBI) for efficient region-based filtering. Index files are auto-discovered (e.g., `file.pairs.gz.tbi`). Only simple predicates are pushed down (equality, comparisons, IN); complex predicates are filtered client-side. Correctness is always guaranteed.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
zero_based = _resolve_zero_based(use_zero_based)
pairs_read_options = PairsReadOptions(
object_storage_options=object_storage_options,
zero_based=zero_based,
)
read_options = ReadOptions(pairs_read_options=pairs_read_options)
return _read_file(
path,
InputFormat.Pairs,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_bed(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.DataFrame:
"""
Read a BED file into a DataFrame.
Parameters:
path: The path to the BED file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the BED file. If not specified, it will be detected automatically based on the file extension. BGZF compressions is supported ('bgz').
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! Note
Only **BED4** format is supported. It extends the basic BED format (BED3) by adding a name field, resulting in four columns: chromosome, start position, end position, and name.
Also unlike other text formats, **GZIP** compression is not supported.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
lf = IOOperations.scan_bed(
path,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
compression_type,
projection_pushdown,
use_zero_based,
)
# Get metadata before collecting (polars-config-meta doesn't preserve through collect)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
# Set metadata on the collected DataFrame
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_bed(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.LazyFrame:
"""
Lazily read a BED file into a LazyFrame.
Parameters:
path: The path to the BED file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the BED file. If not specified, it will be detected automatically based on the file extension. BGZF compressions is supported ('bgz').
projection_pushdown: Enable column projection pushdown to optimize query performance by only reading the necessary columns at the DataFusion level.
use_zero_based: If True, output 0-based half-open coordinates. If False, output 1-based closed coordinates. If None (default), uses the global configuration `datafusion.bio.coordinate_system_zero_based`.
!!! Note
Only **BED4** format is supported. It extends the basic BED format (BED3) by adding a name field, resulting in four columns: chromosome, start position, end position, and name.
Also unlike other text formats, **GZIP** compression is not supported.
!!! note
By default, coordinates are output in **1-based closed** format. Use `use_zero_based=True` or set `pb.set_option(pb.POLARS_BIO_COORDINATE_SYSTEM_ZERO_BASED, True)` for 0-based half-open coordinates.
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
zero_based = _resolve_zero_based(use_zero_based)
bed_read_options = BedReadOptions(
object_storage_options=object_storage_options,
zero_based=zero_based,
)
read_options = ReadOptions(bed_read_options=bed_read_options)
return _read_file(
path,
InputFormat.Bed,
read_options,
projection_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_bigwig(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.DataFrame:
"""
Read a BigWig file into a DataFrame.
BigWig rows are exposed as ``chrom``, ``start``, ``end``, and ``value``.
Parameters:
path: The path to the BigWig file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the BigWig file. If not specified, it will be detected automatically based on the file extension.
projection_pushdown: Enable column projection pushdown optimization. When True, only requested columns are processed at the DataFusion execution level, improving performance and reducing memory usage.
predicate_pushdown: Enable predicate pushdown on the genomic coordinate columns so range filters are evaluated at the DataFusion execution level.
use_zero_based: Coordinate system override. BigWig is natively 0-based half-open; set to *False* to emit 1-based closed coordinates, or *None* to use the global default.
"""
lf = IOOperations.scan_bigwig(
path,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
compression_type,
projection_pushdown,
predicate_pushdown,
use_zero_based,
)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_bigwig(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
) -> pl.LazyFrame:
"""
Lazily read a BigWig file into a LazyFrame.
BigWig is natively 0-based half-open. Set ``use_zero_based=False`` to emit
1-based closed coordinates.
Parameters:
path: The path to the BigWig file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the BigWig file. If not specified, it will be detected automatically based on the file extension.
projection_pushdown: Enable column projection pushdown optimization. When True, only requested columns are processed at the DataFusion execution level, improving performance and reducing memory usage.
predicate_pushdown: Enable predicate pushdown on the genomic coordinate columns so range filters are evaluated at the DataFusion execution level.
use_zero_based: Coordinate system override. BigWig is natively 0-based half-open; set to *False* to emit 1-based closed coordinates, or *None* to use the global default.
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
zero_based = _resolve_zero_based(use_zero_based)
bigwig_read_options = BigWigReadOptions(
object_storage_options=object_storage_options,
zero_based=zero_based,
)
read_options = ReadOptions(bigwig_read_options=bigwig_read_options)
return _read_file(
path,
InputFormat.BigWig,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_bigbed(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
schema: str = "auto",
) -> pl.DataFrame:
"""
Read a BigBed file into a DataFrame.
``schema="auto"`` uses supported autoSQL fields when available.
``schema="rest"`` exposes the raw trailing fields in ``rest``.
Parameters:
path: The path to the BigBed file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the BigBed file. If not specified, it will be detected automatically based on the file extension.
projection_pushdown: Enable column projection pushdown optimization. When True, only requested columns are processed at the DataFusion execution level, improving performance and reducing memory usage.
predicate_pushdown: Enable predicate pushdown on the genomic coordinate columns so range filters are evaluated at the DataFusion execution level.
use_zero_based: Coordinate system override. BigBed is natively 0-based half-open; set to *False* to emit 1-based closed coordinates, or *None* to use the global default.
schema: Schema mode. ``"auto"`` exposes the supported autoSQL fields when available; ``"rest"`` exposes the raw trailing fields in a single ``rest`` column.
"""
lf = IOOperations.scan_bigbed(
path,
chunk_size,
concurrent_fetches,
allow_anonymous,
enable_request_payer,
max_retries,
timeout,
compression_type,
projection_pushdown,
predicate_pushdown,
use_zero_based,
schema,
)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_bigbed(
path: str,
chunk_size: int = 8,
concurrent_fetches: int = 1,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
max_retries: int = 5,
timeout: int = 300,
compression_type: str = "auto",
projection_pushdown: bool = True,
predicate_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
schema: str = "auto",
) -> pl.LazyFrame:
"""
Lazily read a BigBed file into a LazyFrame.
BigBed is natively 0-based half-open. Set ``use_zero_based=False`` to emit
1-based closed coordinates.
Parameters:
path: The path to the BigBed file.
chunk_size: The size in MB of a chunk when reading from an object store. The default is 8 MB. For large scale operations, it is recommended to increase this value to 64.
concurrent_fetches: [GCS] The number of concurrent fetches when reading from an object store. The default is 1. For large scale operations, it is recommended to increase this value to 8 or even more.
allow_anonymous: [GCS, AWS S3] Whether to allow anonymous access to object storage.
enable_request_payer: [AWS S3] Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
max_retries: The maximum number of retries for reading the file from object storage.
timeout: The timeout in seconds for reading the file from object storage.
compression_type: The compression type of the BigBed file. If not specified, it will be detected automatically based on the file extension.
projection_pushdown: Enable column projection pushdown optimization. When True, only requested columns are processed at the DataFusion execution level, improving performance and reducing memory usage.
predicate_pushdown: Enable predicate pushdown on the genomic coordinate columns so range filters are evaluated at the DataFusion execution level.
use_zero_based: Coordinate system override. BigBed is natively 0-based half-open; set to *False* to emit 1-based closed coordinates, or *None* to use the global default.
schema: Schema mode. ``"auto"`` exposes the supported autoSQL fields when available; ``"rest"`` exposes the raw trailing fields in a single ``rest`` column.
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=chunk_size,
concurrent_fetches=concurrent_fetches,
max_retries=max_retries,
timeout=timeout,
compression_type=compression_type,
)
zero_based = _resolve_zero_based(use_zero_based)
bigbed_read_options = BigBedReadOptions(
object_storage_options=object_storage_options,
zero_based=zero_based,
schema=_normalize_bigbed_schema_mode(schema),
)
read_options = ReadOptions(bigbed_read_options=bigbed_read_options)
return _read_file(
path,
InputFormat.BigBed,
read_options,
projection_pushdown,
predicate_pushdown,
zero_based=zero_based,
)
@staticmethod
def read_table(path: str, schema: Dict = None, **kwargs) -> pl.DataFrame:
"""
Read a tab-delimited (i.e. BED) file into a Polars DataFrame.
Tries to be compatible with Bioframe's [read_table](https://bioframe.readthedocs.io/en/latest/guide-io.html)
but faster. Schema should follow the Bioframe's schema [format](https://github.com/open2c/bioframe/blob/2b685eebef393c2c9e6220dcf550b3630d87518e/bioframe/io/schemas.py#L174).
Parameters:
path: The path to the file.
schema: Schema should follow the Bioframe's schema [format](https://github.com/open2c/bioframe/blob/2b685eebef393c2c9e6220dcf550b3630d87518e/bioframe/io/schemas.py#L174).
"""
return IOOperations.scan_table(path, schema, **kwargs).collect()
@staticmethod
def scan_table(path: str, schema: Dict = None, **kwargs) -> pl.LazyFrame:
"""
Lazily read a tab-delimited (i.e. BED) file into a Polars LazyFrame.
Tries to be compatible with Bioframe's [read_table](https://bioframe.readthedocs.io/en/latest/guide-io.html)
but faster and lazy. Schema should follow the Bioframe's schema [format](https://github.com/open2c/bioframe/blob/2b685eebef393c2c9e6220dcf550b3630d87518e/bioframe/io/schemas.py#L174).
Parameters:
path: The path to the file.
schema: Schema should follow the Bioframe's schema [format](https://github.com/open2c/bioframe/blob/2b685eebef393c2c9e6220dcf550b3630d87518e/bioframe/io/schemas.py#L174).
"""
df = pl.scan_csv(path, separator="\t", has_header=False, **kwargs)
if schema is not None:
columns = SCHEMAS[schema]
if len(columns) != len(df.collect_schema()):
raise ValueError(
f"Schema incompatible with the input. Expected {len(columns)} columns in a schema, got {len(df.collect_schema())} in the input data file. Please provide a valid schema."
)
for i, c in enumerate(columns):
df = df.rename({f"column_{i + 1}": c})
return df
@staticmethod
def describe_vcf(
path: str,
allow_anonymous: bool = True,
enable_request_payer: bool = False,
compression_type: str = "auto",
) -> pl.DataFrame:
"""
Describe VCF INFO schema.
Parameters:
path: The path to the VCF file.
allow_anonymous: Whether to allow anonymous access to object storage (GCS and S3 supported).
enable_request_payer: Whether to enable request payer for object storage. This is useful for reading files from AWS S3 buckets that require request payer.
compression_type: The compression type of the VCF file. If not specified, it will be detected automatically..
"""
object_storage_options = PyObjectStorageOptions(
allow_anonymous=allow_anonymous,
enable_request_payer=enable_request_payer,
chunk_size=8,
concurrent_fetches=1,
max_retries=1,
timeout=10,
compression_type=compression_type,
)
return py_describe_vcf(ctx, path, object_storage_options).to_polars()
@staticmethod
def describe_vcf_zarr(path: str) -> pl.DataFrame:
"""
Describe VCF Zarr INFO and FORMAT schema.
Parameters:
path: The path to the local VCF Zarr store directory.
"""
return py_describe_vcf_zarr(ctx, path).to_polars()
@staticmethod
def from_polars(name: str, df: Union[pl.DataFrame, pl.LazyFrame]) -> None:
"""
Register a Polars DataFrame as a DataFusion table.
Parameters:
name: The name of the table.
df: The Polars DataFrame.
"""
reader = (
df.to_arrow()
if isinstance(df, pl.DataFrame)
else df.collect().to_arrow().to_reader()
)
py_from_polars(ctx, name, reader)
@staticmethod
def write_vcf(
df: Union[pl.DataFrame, pl.LazyFrame],
path: str,
) -> int:
"""
Write a DataFrame to VCF format.
Coordinate system is automatically read from DataFrame metadata (set during
read_vcf). Compression is auto-detected from the file extension.
Parameters:
df: The DataFrame or LazyFrame to write.
path: The output file path. Compression is auto-detected from extension
(.vcf.bgz for BGZF, .vcf.gz for GZIP, .vcf for uncompressed).
Returns:
The number of rows written.
!!! Example "Writing VCF files"
```python
import polars_bio as pb
# Read a VCF file
df = pb.read_vcf("input.vcf")
# Write to uncompressed VCF
pb.write_vcf(df, "output.vcf")
# Write to BGZF-compressed VCF
pb.write_vcf(df, "output.vcf.bgz")
# Write to GZIP-compressed VCF
pb.write_vcf(df, "output.vcf.gz")
```
"""
return _write_file(df, path, OutputFormat.Vcf)
@staticmethod
def sink_vcf(
lf: pl.LazyFrame,
path: str,
) -> None:
"""
Streaming write a LazyFrame to VCF format.
This method executes the LazyFrame immediately and writes the results
to the specified path. Unlike `write_vcf`, it doesn't return the row count.
Coordinate system is automatically read from LazyFrame metadata (set during
scan_vcf). Compression is auto-detected from the file extension.
Parameters:
lf: The LazyFrame to write.
path: The output file path. Compression is auto-detected from extension
(.vcf.bgz for BGZF, .vcf.gz for GZIP, .vcf for uncompressed).
!!! Example "Streaming write VCF"
```python
import polars_bio as pb
# Lazy read and filter, then sink to VCF
lf = pb.scan_vcf("large_input.vcf").filter(pl.col("qual") > 30)
pb.sink_vcf(lf, "filtered_output.vcf.bgz")
```
"""
_write_file(lf, path, OutputFormat.Vcf)
@staticmethod
def write_fasta(
df: Union[pl.DataFrame, pl.LazyFrame],
path: str,
) -> int:
"""
Write a DataFrame to FASTA format.
Compression is auto-detected from the file extension.
Parameters:
df: The DataFrame or LazyFrame to write. Must have columns:
- name: Sequence name/identifier
- sequence: DNA/RNA sequence
Optional: description (added after name on header line)
path: The output file path. Compression is auto-detected from extension
(.fasta.bgz for BGZF, .fasta.gz/.fa.gz for GZIP, .fasta/.fa for uncompressed).
Returns:
The number of rows written.
!!! Example "Writing FASTA files"
```python
import polars_bio as pb
# Read a FASTA file
df = pb.read_fasta("input.fasta")
# Write to uncompressed FASTA
pb.write_fasta(df, "output.fasta")
# Write to GZIP-compressed FASTA
pb.write_fasta(df, "output.fasta.gz")
```
"""
return _write_file(df, path, OutputFormat.Fasta)
@staticmethod
def sink_fasta(
lf: pl.LazyFrame,
path: str,
) -> None:
"""
Streaming write a LazyFrame to FASTA format.
Compression is auto-detected from the file extension.
Parameters:
lf: The LazyFrame to write.
path: The output file path. Compression is auto-detected from extension
(.fasta.bgz for BGZF, .fasta.gz/.fa.gz for GZIP, .fasta/.fa for uncompressed).
!!! Example "Streaming write FASTA"
```python
import polars_bio as pb
# Lazy read, filter, then sink
lf = pb.scan_fasta("large_input.fasta.gz")
pb.sink_fasta(lf.limit(1000), "sample_output.fasta")
```
"""
_write_file(lf, path, OutputFormat.Fasta)
@staticmethod
def write_fastq(
df: Union[pl.DataFrame, pl.LazyFrame],
path: str,
) -> int:
"""
Write a DataFrame to FASTQ format.
Compression is auto-detected from the file extension.
Parameters:
df: The DataFrame or LazyFrame to write. Must have columns:
- name: Read name/identifier
- sequence: DNA sequence
- quality_scores: Quality scores string
Optional: description (added after name on header line)
path: The output file path. Compression is auto-detected from extension
(.fastq.bgz for BGZF, .fastq.gz for GZIP, .fastq for uncompressed).
Returns:
The number of rows written.
!!! Example "Writing FASTQ files"
```python
import polars_bio as pb
# Read a FASTQ file
df = pb.read_fastq("input.fastq")
# Write to uncompressed FASTQ
pb.write_fastq(df, "output.fastq")
# Write to GZIP-compressed FASTQ
pb.write_fastq(df, "output.fastq.gz")
```
"""
return _write_file(df, path, OutputFormat.Fastq)
@staticmethod
def sink_fastq(
lf: pl.LazyFrame,
path: str,
) -> None:
"""
Streaming write a LazyFrame to FASTQ format.
Compression is auto-detected from the file extension.
Parameters:
lf: The LazyFrame to write.
path: The output file path. Compression is auto-detected from extension
(.fastq.bgz for BGZF, .fastq.gz for GZIP, .fastq for uncompressed).
!!! Example "Streaming write FASTQ"
```python
import polars_bio as pb
# Lazy read, filter by quality, then sink
lf = pb.scan_fastq("large_input.fastq.gz")
pb.sink_fastq(lf.limit(1000), "sample_output.fastq")
```
"""
_write_file(lf, path, OutputFormat.Fastq)
@staticmethod
def write_bam(
df: Union[pl.DataFrame, pl.LazyFrame],
path: str,
sort_on_write: bool = False,
tag_type_overrides: Optional[dict[str, str]] = None,
) -> int:
"""
Write a DataFrame to BAM/SAM format.
Compression is auto-detected from file extension:
- .sam → Uncompressed SAM (plain text)
- .bam → BGZF-compressed BAM
For CRAM format, use `write_cram()` instead.
Parameters:
df: DataFrame or LazyFrame with 11 core BAM columns + optional tag columns
path: Output file path (.bam or .sam)
sort_on_write: If True, sort records by (chrom, start) and set header SO:coordinate.
If False (default), set header SO:unsorted.
tag_type_overrides: Optional exact SAM tag type specifications for ambiguous
or newly created tag columns, e.g. {"tp": "A", "XH": "H", "ML": "B:C"}.
Overrides take precedence over preserved source metadata and Arrow dtype inference.
Returns:
Number of rows written
!!! Example "Write BAM files"
```python
import polars_bio as pb
df = pb.read_bam("input.bam", tag_fields=["NM", "AS"])
pb.write_bam(df, "output.bam")
pb.write_bam(df, "output.sam")
```
"""
return _write_bam_file(
df,
path,
OutputFormat.Bam,
None,
sort_on_write=sort_on_write,
tag_type_overrides=tag_type_overrides,
)
@staticmethod
def sink_bam(
lf: pl.LazyFrame,
path: str,
sort_on_write: bool = False,
tag_type_overrides: Optional[dict[str, str]] = None,
) -> None:
"""
Streaming write a LazyFrame to BAM/SAM format.
For CRAM format, use `sink_cram()` instead.
Parameters:
lf: LazyFrame to write
path: Output file path (.bam or .sam)
sort_on_write: If True, sort records by (chrom, start) and set header SO:coordinate.
If False (default), set header SO:unsorted.
tag_type_overrides: Optional exact SAM tag type specifications for ambiguous
or newly created tag columns, e.g. {"tp": "A", "XH": "H", "ML": "B:C"}.
Overrides take precedence over preserved source metadata and Arrow dtype inference.
!!! Example "Streaming write BAM"
```python
import polars_bio as pb
lf = pb.scan_bam("input.bam").filter(pl.col("mapping_quality") > 20)
pb.sink_bam(lf, "filtered.bam")
```
"""
_write_bam_file(
lf,
path,
OutputFormat.Bam,
None,
sort_on_write=sort_on_write,
tag_type_overrides=tag_type_overrides,
)
@staticmethod
def read_sam(
path: str,
tag_fields: Union[list[str], None] = None,
projection_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
infer_tag_types: bool = True,
infer_tag_sample_size: int = 100,
tag_type_hints: Optional[list[str]] = None,
) -> pl.DataFrame:
"""
Read a SAM file into a DataFrame.
SAM (Sequence Alignment/Map) is the plain-text counterpart of BAM.
This function reuses the BAM reader, which auto-detects the format
from the file extension.
Parameters:
path: The path to the SAM file.
tag_fields: List of SAM tag names to include as columns (e.g., ["NM", "MD", "AS"]).
If None, no optional tags are parsed (default).
projection_pushdown: Enable column projection pushdown to optimize query performance.
use_zero_based: If True, output 0-based half-open coordinates.
If False, output 1-based closed coordinates.
If None (default), uses the global configuration.
infer_tag_types: If True (default), sample the file to auto-detect types for custom/unknown tags.
infer_tag_sample_size: Number of records to sample for tag type inference (default: 100).
tag_type_hints: Explicit SAM-style type hints for tags (e.g., ["pt:i", "ML:B:C", "FZ:B:S"]). Supported forms: TAG:TYPE, TAG:B, or TAG:B:SUBTYPE where TYPE is one of A, c, C, s, S, i, I, f, Z, H and SUBTYPE is one of c, C, s, S, i, I, f.
!!! note
By default, coordinates are output in **1-based closed** format.
"""
lf = IOOperations.scan_sam(
path,
tag_fields,
projection_pushdown,
use_zero_based,
infer_tag_types,
infer_tag_sample_size,
tag_type_hints,
)
zero_based = lf.config_meta.get_metadata().get("coordinate_system_zero_based")
df = lf.collect()
if zero_based is not None:
set_coordinate_system(df, zero_based)
return df
@staticmethod
def scan_sam(
path: str,
tag_fields: Union[list[str], None] = None,
projection_pushdown: bool = True,
use_zero_based: Optional[bool] = None,
infer_tag_types: bool = True,
infer_tag_sample_size: int = 100,
tag_type_hints: Optional[list[str]] = None,
) -> pl.LazyFrame:
"""
Lazily read a SAM file into a LazyFrame.
SAM (Sequence Alignment/Map) is the plain-text counterpart of BAM.
This function reuses the BAM reader, which auto-detects the format
from the file extension.
Parameters:
path: The path to the SAM file.
tag_fields: List of SAM tag names to include as columns (e.g., ["NM", "MD", "AS"]).
If None, no optional tags are parsed (default).
projection_pushdown: Enable column projection pushdown to optimize query performance.
use_zero_based: If True, output 0-based half-open coordinates.
If False, output 1-based closed coordinates.
If None (default), uses the global configuration.
infer_tag_types: If True (default), sample the file to auto-detect types for custom/unknown tags.
infer_tag_sample_size: Number of records to sample for tag type inference (default: 100).
tag_type_hints: Explicit SAM-style type hints for tags (e.g., ["pt:i", "ML:B:C", "FZ:B:S"]). Supported forms: TAG:TYPE, TAG:B, or TAG:B:SUBTYPE where TYPE is one of A, c, C, s, S, i, I, f, Z, H and SUBTYPE is one of c, C, s, S, i, I, f.
!!! note
By default, coordinates are output in **1-based closed** format.
"""
zero_based = _resolve_zero_based(use_zero_based)
if tag_type_hints is not None:
_validate_tag_type_hints(tag_type_hints)
tag_type_hints = _normalize_read_tag_type_hints(tag_type_hints)
bam_read_options = BamReadOptions(
zero_based=zero_based,
tag_fields=tag_fields,
infer_tag_types=infer_tag_types,
infer_tag_sample_size=infer_tag_sample_size,
tag_type_hints=tag_type_hints,
)
read_options = ReadOptions(bam_read_options=bam_read_options)
return _read_file(
path,
InputFormat.Sam,
read_options,
projection_pushdown,
zero_based=zero_based,
)
@staticmethod
def describe_sam(
path: str,
sample_size: int = 100,
use_zero_based: Optional[bool] = None,
) -> pl.DataFrame:
"""
Get schema information for a SAM file with automatic tag discovery.
Samples the first N records to discover all available tags and their types.
Reuses the BAM describe logic, which auto-detects SAM from the file extension.
Parameters:
path: The path to the SAM file.
sample_size: Number of records to sample for tag discovery (default: 100).
use_zero_based: If True, output 0-based coordinates. If False, 1-based coordinates.
Returns:
DataFrame with columns: column_name, data_type, nullable, category, sam_type, description
"""
zero_based = _resolve_zero_based(use_zero_based)
df = py_describe_bam(
ctx,
path,
None,
zero_based,
None,
sample_size,
)
return pl.from_arrow(df.to_arrow_table())
@staticmethod
def write_sam(
df: Union[pl.DataFrame, pl.LazyFrame],
path: str,
sort_on_write: bool = False,
tag_type_overrides: Optional[dict[str, str]] = None,
) -> int:
"""
Write a DataFrame to SAM format (plain text).
Parameters:
df: DataFrame or LazyFrame with 11 core BAM/SAM columns + optional tag columns
path: Output file path (.sam)
sort_on_write: If True, sort records by (chrom, start) and set header SO:coordinate.
If False (default), set header SO:unsorted.
tag_type_overrides: Optional exact SAM tag type specifications for ambiguous
or newly created tag columns, e.g. {"tp": "A", "XH": "H", "ML": "B:C"}.
Overrides take precedence over preserved source metadata and Arrow dtype inference.
Returns:
Number of rows written
!!! Example "Write SAM files"
```python
import polars_bio as pb
df = pb.read_bam("input.bam", tag_fields=["NM", "AS"])
pb.write_sam(df, "output.sam")
```
"""
return _write_bam_file(
df,
path,
OutputFormat.Sam,
None,
sort_on_write=sort_on_write,
tag_type_overrides=tag_type_overrides,
)
@staticmethod
def sink_sam(
lf: pl.LazyFrame,
path: str,
sort_on_write: bool = False,
tag_type_overrides: Optional[dict[str, str]] = None,
) -> None:
"""
Streaming write a LazyFrame to SAM format (plain text).
Parameters:
lf: LazyFrame to write
path: Output file path (.sam)
sort_on_write: If True, sort records by (chrom, start) and set header SO:coordinate.
If False (default), set header SO:unsorted.
tag_type_overrides: Optional exact SAM tag type specifications for ambiguous
or newly created tag columns, e.g. {"tp": "A", "XH": "H", "ML": "B:C"}.
Overrides take precedence over preserved source metadata and Arrow dtype inference.
!!! Example "Streaming write SAM"
```python
import polars_bio as pb
lf = pb.scan_bam("input.bam").filter(pl.col("mapping_quality") > 20)
pb.sink_sam(lf, "filtered.sam")
```
"""
_write_bam_file(
lf,
path,
OutputFormat.Sam,
None,
sort_on_write=sort_on_write,
tag_type_overrides=tag_type_overrides,
)
@staticmethod
def write_cram(
df: Union[pl.DataFrame, pl.LazyFrame],
path: str,
reference_path: str,
sort_on_write: bool = False,
) -> int:
"""
Write a DataFrame to CRAM format.
CRAM uses reference-based compression, storing only differences from the
reference sequence. This achieves 30-60% better compression than BAM.
Parameters:
df: DataFrame or LazyFrame with 11 core BAM columns + optional tag columns
path: Output CRAM file path
reference_path: Path to reference FASTA file (required). The reference must
contain all sequences referenced by the alignment data.
sort_on_write: If True, sort records by (chrom, start) and set header SO:coordinate.
If False (default), set header SO:unsorted.
Returns:
Number of rows written
!!! warning "Known Limitation: MD and NM Tags"
Due to a limitation in the underlying noodles-cram library, **MD and NM tags cannot be read back from CRAM files** after writing, even though they are written to the file. If you need MD/NM tags for downstream analysis, use BAM format instead. Other optional tags (RG, MQ, AM, OQ, AS, etc.) work correctly. See: https://github.com/biodatageeks/datafusion-bio-formats/issues/54
!!! Example "Write CRAM files"
```python
import polars_bio as pb
df = pb.read_bam("input.bam", tag_fields=["NM", "AS"])
# Write CRAM with reference (required)
pb.write_cram(df, "output.cram", reference_path="reference.fasta")
# For sorted output
pb.write_cram(df, "output.cram", reference_path="reference.fasta", sort_on_write=True)
```
"""
return _write_bam_file(
df, path, OutputFormat.Cram, reference_path, sort_on_write=sort_on_write
)
@staticmethod
def sink_cram(
lf: pl.LazyFrame,
path: str,
reference_path: str,
sort_on_write: bool = False,
) -> None:
"""
Streaming write a LazyFrame to CRAM format.
CRAM uses reference-based compression, storing only differences from the
reference sequence. This method streams data without materializing all
rows in memory.
Parameters:
lf: LazyFrame to write
path: Output CRAM file path
reference_path: Path to reference FASTA file (required). The reference must
contain all sequences referenced by the alignment data.
sort_on_write: If True, sort records by (chrom, start) and set header SO:coordinate.
If False (default), set header SO:unsorted.
!!! warning "Known Limitation: MD and NM Tags"
Due to a limitation in the underlying noodles-cram library, **MD and NM tags cannot be read back from CRAM files** after writing, even though they are written to the file. If you need MD/NM tags for downstream analysis, use BAM format instead. Other optional tags (RG, MQ, AM, OQ, AS, etc.) work correctly. See: https://github.com/biodatageeks/datafusion-bio-formats/issues/54
!!! Example "Streaming write CRAM"
```python
import polars_bio as pb
import polars as pl
lf = pb.scan_bam("large_input.bam")
lf = lf.filter(pl.col("mapping_quality") > 30)
# Write CRAM with reference (required)
pb.sink_cram(lf, "filtered.cram", reference_path="reference.fasta")
# For sorted output
pb.sink_cram(lf, "filtered.cram", reference_path="reference.fasta", sort_on_write=True)
```
"""
_write_bam_file(
lf, path, OutputFormat.Cram, reference_path, sort_on_write=sort_on_write
)