Remove libprotobuf dep (apache#527)

jdye64 · web-flow · commit da6c183ebb67 · 2023-10-26T12:24:43.000-06:00
* Remove libprotobuf as a build time dependency of Conda

* Add substrait feature which enables building with substrait features

* Add datafusion._internal/substrait to the pyproject.toml file so substrait will be built for pytests when using pip

* Rename feature to just substrait

* Add protoc since libprotobuf was removed from conda requirements file

* Refactor substrait module being initialized as a function

* Reintroduce libprotobuf as build dependency for conda build
diff --git a/Cargo.toml b/Cargo.toml
@@ -31,6 +31,7 @@ include = ["/src", "/datafusion", "/LICENSE.txt", "pyproject.toml", "Cargo.toml"
 [features]
 default = ["mimalloc"]
 protoc = [ "datafusion-substrait/protoc" ]
+substrait = ["dep:datafusion-substrait"]
 
 [dependencies]
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
@@ -41,7 +42,7 @@ datafusion-common = { version = "32.0.0", features = ["pyarrow"] }
 datafusion-expr = { version = "32.0.0" }
 datafusion-optimizer = { version = "32.0.0" }
 datafusion-sql = { version = "32.0.0" }
-datafusion-substrait = { version = "32.0.0" }
+datafusion-substrait = { version = "32.0.0", optional = true }
 prost = "0.11"
 prost-types = "0.11"
 uuid = { version = "1.3", features = ["v4"] }
diff --git a/pyproject.toml b/pyproject.toml
@@ -62,3 +62,4 @@ include = [
 exclude = [".github/**", "ci/**", ".asf.yaml"]
 # Require Cargo.lock is up to date
 locked = true
+features = ["substrait"]
diff --git a/src/lib.rs b/src/lib.rs
@@ -25,6 +25,8 @@ pub use datafusion_common;
 pub use datafusion_expr;
 pub use datafusion_optimizer;
 pub use datafusion_sql;
+
+#[cfg(feature = "substrait")]
 pub use datafusion_substrait;
 
 #[allow(clippy::borrow_deref_ref)]
@@ -48,6 +50,8 @@ mod pyarrow_filter_expression;
 mod record_batch;
 pub mod sql;
 pub mod store;
+
+#[cfg(feature = "substrait")]
 pub mod substrait;
 #[allow(clippy::borrow_deref_ref)]
 mod udaf;
@@ -108,9 +112,16 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> {
     m.add_submodule(store)?;
 
     // Register substrait as a submodule
+    #[cfg(feature = "substrait")]
+    setup_substrait_module(py, m)?;
+
+    Ok(())
+}
+
+#[cfg(feature = "substrait")]
+fn setup_substrait_module(py: Python, m: &PyModule) -> PyResult<()> {
     let substrait = PyModule::new(py, "substrait")?;
     substrait::init_module(substrait)?;
     m.add_submodule(substrait)?;
-
     Ok(())
 }
