lance

• lance package
  • Subpackages
    • lance.cuvs package
      • Submodules
      • lance.cuvs.kmeans module
      • Module contents
    • lance.ray package
      • Submodules
      • lance.ray.sink module
      • Module contents
    • lance.tf package
      • Submodules
      • lance.tf.data module
      • lance.tf.tfrecord module
      • Module contents
    • lance.torch package
      • Submodules
      • lance.torch.async_dataset module
      • lance.torch.bench_utils module
      • lance.torch.data module
      • lance.torch.dist module
      • lance.torch.distance module
      • lance.torch.kmeans module
      • Module contents
  • Submodules
  • lance.arrow module
    • BFloat16Array
      • BFloat16Array.from_numpy()
      • BFloat16Array.to_numpy()
    • BFloat16Type
      • BFloat16Type.to_pandas_dtype()
    • EncodedImageType
    • FixedShapeImageTensorType
    • ImageArray
      • ImageArray.from_array()
    • ImageScalar
      • ImageScalar.as_py()
    • ImageURIArray
      • ImageURIArray.from_uris()
      • ImageURIArray.read_uris()
    • ImageURIType
    • bfloat16_array()
    • cast()
  • lance.blob module
    • BlobColumn
    • BlobFile
      • BlobFile.close()
      • BlobFile.closed
      • BlobFile.readable()
      • BlobFile.readall()
      • BlobFile.readinto()
      • BlobFile.seek()
      • BlobFile.seekable()
      • BlobFile.size()
      • BlobFile.tell()
    • BlobIterator
  • lance.commit module
    • CommitConflictError
  • lance.conftest module
  • lance.dataset module
    • DatasetOptimizer
      • DatasetOptimizer.compact_files()
      • DatasetOptimizer.optimize_indices()
    • DatasetStats
      • DatasetStats.num_deleted_rows
      • DatasetStats.num_fragments
      • DatasetStats.num_small_files
    • LanceDataset
      • LanceDataset.add_columns()
      • LanceDataset.alter_columns()
      • LanceDataset.checkout_version()
      • LanceDataset.cleanup_old_versions()
      • LanceDataset.commit()
      • LanceDataset.count_rows()
      • LanceDataset.create_index()
      • LanceDataset.create_scalar_index()
      • LanceDataset.data_storage_version
      • LanceDataset.delete()
      • LanceDataset.drop_columns()
      • LanceDataset.get_fragment()
      • LanceDataset.get_fragments()
      • LanceDataset.has_index
      • LanceDataset.head()
      • LanceDataset.index_statistics()
      • LanceDataset.insert()
      • LanceDataset.join()
      • LanceDataset.lance_schema
      • LanceDataset.latest_version
      • LanceDataset.list_indices()
      • LanceDataset.merge()
      • LanceDataset.merge_insert()
      • LanceDataset.migrate_manifest_paths_v2()
      • LanceDataset.optimize
      • LanceDataset.partition_expression
      • LanceDataset.replace_schema()
      • LanceDataset.restore()
      • LanceDataset.sample()
      • LanceDataset.scanner()
      • LanceDataset.schema
      • LanceDataset.session()
      • LanceDataset.stats
      • LanceDataset.tags
      • LanceDataset.take()
      • LanceDataset.take_blobs()
      • LanceDataset.to_batches()
      • LanceDataset.to_table()
      • LanceDataset.update()
      • LanceDataset.uri
      • LanceDataset.validate()
      • LanceDataset.version
      • LanceDataset.versions()
    • LanceOperation
      • LanceOperation.Append
      • LanceOperation.BaseOperation
      • LanceOperation.CreateIndex
      • LanceOperation.Delete
      • LanceOperation.Merge
      • LanceOperation.Overwrite
      • LanceOperation.Restore
      • LanceOperation.Rewrite
      • LanceOperation.RewriteGroup
      • LanceOperation.RewrittenIndex
    • LanceScanner
      • LanceScanner.count_rows()
      • LanceScanner.dataset_schema
      • LanceScanner.explain_plan()
      • LanceScanner.from_batches()
      • LanceScanner.from_dataset()
      • LanceScanner.from_fragment()
      • LanceScanner.head()
      • LanceScanner.projected_schema
      • LanceScanner.scan_batches()
      • LanceScanner.take()
      • LanceScanner.to_batches()
      • LanceScanner.to_reader()
      • LanceScanner.to_table()
    • LanceStats
      • LanceStats.dataset_stats()
      • LanceStats.index_stats()
    • MergeInsertBuilder
      • MergeInsertBuilder.execute()
      • MergeInsertBuilder.when_matched_update_all()
      • MergeInsertBuilder.when_not_matched_by_source_delete()
      • MergeInsertBuilder.when_not_matched_insert_all()
    • ScannerBuilder
      • ScannerBuilder.apply_defaults()
      • ScannerBuilder.batch_readahead()
      • ScannerBuilder.batch_size()
      • ScannerBuilder.columns()
      • ScannerBuilder.fast_search()
      • ScannerBuilder.filter()
      • ScannerBuilder.fragment_readahead()
      • ScannerBuilder.full_text_search()
      • ScannerBuilder.io_buffer_size()
      • ScannerBuilder.late_materialization()
      • ScannerBuilder.limit()
      • ScannerBuilder.nearest()
      • ScannerBuilder.offset()
      • ScannerBuilder.prefilter()
      • ScannerBuilder.scan_in_order()
      • ScannerBuilder.to_scanner()
      • ScannerBuilder.use_scalar_index()
      • ScannerBuilder.use_stats()
      • ScannerBuilder.with_fragments()
      • ScannerBuilder.with_row_address()
      • ScannerBuilder.with_row_id()
    • Tags
      • Tags.create()
      • Tags.delete()
      • Tags.list()
      • Tags.update()
    • write_dataset()
  • lance.debug module
    • format_fragment()
    • format_manifest()
    • format_schema()
    • list_transactions()
  • lance.dependencies module
  • lance.file module
    • LanceBufferDescriptor
      • LanceBufferDescriptor.position
      • LanceBufferDescriptor.size
    • LanceColumnMetadata
      • LanceColumnMetadata.column_buffers
      • LanceColumnMetadata.pages
    • LanceFileMetadata
      • LanceFileMetadata.columns
      • LanceFileMetadata.global_buffers
      • LanceFileMetadata.major_version
      • LanceFileMetadata.minor_version
      • LanceFileMetadata.num_column_metadata_bytes
      • LanceFileMetadata.num_data_bytes
      • LanceFileMetadata.num_global_buffer_bytes
      • LanceFileMetadata.num_rows
      • LanceFileMetadata.schema
    • LanceFileReader
      • LanceFileReader.metadata()
      • LanceFileReader.read_all()
      • LanceFileReader.read_global_buffer()
      • LanceFileReader.read_range()
      • LanceFileReader.take_rows()
    • LanceFileWriter
      • LanceFileWriter.add_global_buffer()
      • LanceFileWriter.add_schema_metadata()
      • LanceFileWriter.close()
      • LanceFileWriter.write_batch()
    • LancePageMetadata
      • LancePageMetadata.buffers
      • LancePageMetadata.encoding
  • lance.fragment module
    • FragmentMetadata
      • FragmentMetadata.data_files()
      • FragmentMetadata.deletion_file()
      • FragmentMetadata.from_json()
      • FragmentMetadata.from_metadata()
      • FragmentMetadata.id
      • FragmentMetadata.to_json()
    • LanceFragment
      • LanceFragment.count_rows()
      • LanceFragment.create()
      • LanceFragment.create_from_file()
      • LanceFragment.data_files()
      • LanceFragment.delete()
      • LanceFragment.deletion_file()
      • LanceFragment.fragment_id
      • LanceFragment.head()
      • LanceFragment.merge_columns()
      • LanceFragment.metadata
      • LanceFragment.num_deletions
      • LanceFragment.partition_expression
      • LanceFragment.physical_rows
      • LanceFragment.physical_schema
      • LanceFragment.scanner()
      • LanceFragment.schema
      • LanceFragment.take()
      • LanceFragment.to_batches()
      • LanceFragment.to_table()
    • write_fragments()
  • lance.hf module
    • HuggingFaceConverter
      • HuggingFaceConverter.to_pytorch()
  • lance.indices module
    • IndicesBuilder
      • IndicesBuilder.assign_ivf_partitions()
      • IndicesBuilder.load_shuffled_vectors()
      • IndicesBuilder.shuffle_transformed_vectors()
      • IndicesBuilder.train_ivf()
      • IndicesBuilder.train_pq()
      • IndicesBuilder.transform_vectors()
    • IvfModel
      • IvfModel.centroids
      • IvfModel.distance_type
      • IvfModel.load()
      • IvfModel.num_partitions
      • IvfModel.save()
    • PqModel
      • PqModel.codebook
      • PqModel.dimension
      • PqModel.load()
      • PqModel.num_subvectors
      • PqModel.save()
  • lance.optimize module
    • Compaction
      • Compaction.commit()
      • Compaction.execute()
      • Compaction.plan()
    • CompactionMetrics
      • CompactionMetrics.files_added
      • CompactionMetrics.files_removed
      • CompactionMetrics.fragments_added
      • CompactionMetrics.fragments_removed
    • CompactionOptions
      • CompactionOptions.batch_size
      • CompactionOptions.materialize_deletions
      • CompactionOptions.materialize_deletions_threadhold
      • CompactionOptions.max_bytes_per_file
      • CompactionOptions.max_rows_per_group
      • CompactionOptions.num_threads
      • CompactionOptions.target_rows_per_fragment
    • CompactionPlan
      • CompactionPlan.from_json()
      • CompactionPlan.json()
      • CompactionPlan.num_tasks()
      • CompactionPlan.read_version
      • CompactionPlan.tasks
    • CompactionTask
      • CompactionTask.execute()
      • CompactionTask.fragments
      • CompactionTask.from_json()
      • CompactionTask.json()
      • CompactionTask.read_version
    • RewriteResult
      • RewriteResult.from_json()
      • RewriteResult.json()
      • RewriteResult.metrics
      • RewriteResult.new_fragments
      • RewriteResult.original_fragments
      • RewriteResult.read_version
  • lance.progress module
    • FileSystemFragmentWriteProgress
      • FileSystemFragmentWriteProgress.PROGRESS_EXT
      • FileSystemFragmentWriteProgress.begin()
      • FileSystemFragmentWriteProgress.complete()
    • FragmentWriteProgress
      • FragmentWriteProgress.begin()
      • FragmentWriteProgress.complete()
    • NoopFragmentWriteProgress
      • NoopFragmentWriteProgress.begin()
      • NoopFragmentWriteProgress.complete()
  • lance.sampler module
    • FragmentSampler
      • FragmentSampler.iter_fragments()
    • FullScanSampler
      • FullScanSampler.iter_fragments()
    • Sampler
    • ShardedBatchSampler
      • ShardedBatchSampler.from_torch()
    • ShardedFragmentSampler
      • ShardedFragmentSampler.from_torch()
      • ShardedFragmentSampler.iter_fragments()
    • maybe_sample()
  • lance.schema module
    • LanceSchema
      • LanceSchema.from_pyarrow()
      • LanceSchema.to_pyarrow()
    • json_to_schema()
    • schema_to_json()
  • lance.tracing module
    • lance_trace_to_chrome()
    • trace_to_chrome()
  • lance.types module
  • lance.udf module
    • BatchUDF
    • BatchUDFCheckpoint
      • BatchUDFCheckpoint.BatchInfo
      • BatchUDFCheckpoint.cleanup()
      • BatchUDFCheckpoint.get_batch()
      • BatchUDFCheckpoint.get_fragment()
      • BatchUDFCheckpoint.insert_batch()
      • BatchUDFCheckpoint.insert_fragment()
    • batch_udf()
    • normalize_transform()
  • lance.util module
    • HNSW
      • HNSW.build()
      • HNSW.to_lance_file()
      • HNSW.vectors()
    • KMeans
      • KMeans.centroids
      • KMeans.fit()
      • KMeans.predict()
    • sanitize_ts()
    • td_to_micros()
    • validate_vector_index()
  • lance.vector module
    • compute_partitions()
    • compute_pq_codes()
    • one_pass_assign_ivf_pq_on_accelerator()
    • one_pass_train_ivf_pq_on_accelerator()
    • train_ivf_centroids_on_accelerator()
    • train_pq_codebook_on_accelerator()
    • vec_to_table()
  • Module contents
    • BlobColumn
    • BlobFile
      • BlobFile.close()
      • BlobFile.closed
      • BlobFile.readable()
      • BlobFile.readall()
      • BlobFile.readinto()
      • BlobFile.seek()
      • BlobFile.seekable()
      • BlobFile.size()
      • BlobFile.tell()
    • FragmentMetadata
      • FragmentMetadata.data_files()
      • FragmentMetadata.deletion_file()
      • FragmentMetadata.from_json()
      • FragmentMetadata.from_metadata()
      • FragmentMetadata.id
      • FragmentMetadata.to_json()
    • LanceDataset
      • LanceDataset.add_columns()
      • LanceDataset.alter_columns()
      • LanceDataset.checkout_version()
      • LanceDataset.cleanup_old_versions()
      • LanceDataset.commit()
      • LanceDataset.count_rows()
      • LanceDataset.create_index()
      • LanceDataset.create_scalar_index()
      • LanceDataset.data_storage_version
      • LanceDataset.delete()
      • LanceDataset.drop_columns()
      • LanceDataset.get_fragment()
      • LanceDataset.get_fragments()
      • LanceDataset.has_index
      • LanceDataset.head()
      • LanceDataset.index_statistics()
      • LanceDataset.insert()
      • LanceDataset.join()
      • LanceDataset.lance_schema
      • LanceDataset.latest_version
      • LanceDataset.list_indices()
      • LanceDataset.merge()
      • LanceDataset.merge_insert()
      • LanceDataset.migrate_manifest_paths_v2()
      • LanceDataset.optimize
      • LanceDataset.partition_expression
      • LanceDataset.replace_schema()
      • LanceDataset.restore()
      • LanceDataset.sample()
      • LanceDataset.scanner()
      • LanceDataset.schema
      • LanceDataset.session()
      • LanceDataset.stats
      • LanceDataset.tags
      • LanceDataset.take()
      • LanceDataset.take_blobs()
      • LanceDataset.to_batches()
      • LanceDataset.to_table()
      • LanceDataset.update()
      • LanceDataset.uri
      • LanceDataset.validate()
      • LanceDataset.version
      • LanceDataset.versions()
    • LanceFragment
      • LanceFragment.count_rows()
      • LanceFragment.create()
      • LanceFragment.create_from_file()
      • LanceFragment.data_files()
      • LanceFragment.delete()
      • LanceFragment.deletion_file()
      • LanceFragment.fragment_id
      • LanceFragment.head()
      • LanceFragment.merge_columns()
      • LanceFragment.metadata
      • LanceFragment.num_deletions
      • LanceFragment.partition_expression
      • LanceFragment.physical_rows
      • LanceFragment.physical_schema
      • LanceFragment.scanner()
      • LanceFragment.schema
      • LanceFragment.take()
      • LanceFragment.to_batches()
      • LanceFragment.to_table()
    • LanceOperation
      • LanceOperation.Append
      • LanceOperation.BaseOperation
      • LanceOperation.CreateIndex
      • LanceOperation.Delete
      • LanceOperation.Merge
      • LanceOperation.Overwrite
      • LanceOperation.Restore
      • LanceOperation.Rewrite
      • LanceOperation.RewriteGroup
      • LanceOperation.RewrittenIndex
    • LanceScanner
      • LanceScanner.count_rows()
      • LanceScanner.dataset_schema
      • LanceScanner.explain_plan()
      • LanceScanner.from_batches()
      • LanceScanner.from_dataset()
      • LanceScanner.from_fragment()
      • LanceScanner.head()
      • LanceScanner.projected_schema
      • LanceScanner.scan_batches()
      • LanceScanner.take()
      • LanceScanner.to_batches()
      • LanceScanner.to_reader()
      • LanceScanner.to_table()
    • MergeInsertBuilder
      • MergeInsertBuilder.execute()
      • MergeInsertBuilder.when_matched_update_all()
      • MergeInsertBuilder.when_not_matched_by_source_delete()
      • MergeInsertBuilder.when_not_matched_insert_all()
    • batch_udf()
    • dataset()
    • json_to_schema()
    • schema_to_json()
    • write_dataset()