Attempt at progressively feeding the Session to bypass type checking in

a sane way.

Author: Narsil

onnxruntime/src/error.rs

@@ -120,16 +120,16 @@ pub enum OrtError {
 #[derive(Error, Debug)]
 pub enum NonMatchingDimensionsError {
     /// Number of inputs from model does not match number of inputs from inference call
-    #[error("Non-matching number of inputs: {inference_input_count:?} for input vs {model_input_count:?} for model (inputs: {inference_input:?}, model: {model_input:?})")]
+    #[error("Non-matching number of inputs: {inference_input_count:?} for input vs {model_input_count:?}")]
     InputsCount {
         /// Number of input dimensions used by inference call
         inference_input_count: usize,
         /// Number of input dimensions defined in model
         model_input_count: usize,
-        /// Input dimensions used by inference call
-        inference_input: Vec<Vec<usize>>,
-        /// Input dimensions defined in model
-        model_input: Vec<Vec<Option<u32>>>,
+        // Input dimensions used by inference call
+        // inference_input: Vec<Vec<usize>>,
+        // Input dimensions defined in model
+        // model_input: Vec<Vec<Option<u32>>>,
     },
 }
 

onnxruntime/src/session.rs

@@ -215,12 +215,14 @@ impl<'a> SessionBuilder<'a> {
         let outputs = (0..num_output_nodes)
             .map(|i| dangerous::extract_output(session_ptr, allocator_ptr, i))
             .collect::<Result<Vec<Output>>>()?;
+        let input_ort_values = Vec::with_capacity(num_output_nodes as usize);
 
         Ok(Session {
             env: self.env,
             session_ptr,
             allocator_ptr,
             memory_info,
+            input_ort_values,
             inputs,
             outputs,
         })
@@ -271,12 +273,14 @@ impl<'a> SessionBuilder<'a> {
         let outputs = (0..num_output_nodes)
             .map(|i| dangerous::extract_output(session_ptr, allocator_ptr, i))
             .collect::<Result<Vec<Output>>>()?;
+        let input_ort_values = Vec::with_capacity(num_output_nodes as usize);
 
         Ok(Session {
             env: self.env,
             session_ptr,
             allocator_ptr,
             memory_info,
+            input_ort_values,
             inputs,
             outputs,
         })
@@ -290,6 +294,7 @@ pub struct Session<'a> {
     session_ptr: *mut sys::OrtSession,
     allocator_ptr: *mut sys::OrtAllocator,
     memory_info: MemoryInfo,
+    input_ort_values: Vec<*const sys::OrtValue>,
     /// Information about the ONNX's inputs as stored in loaded file
     pub inputs: Vec<Input>,
     /// Information about the ONNX's outputs as stored in loaded file
@@ -357,6 +362,26 @@ impl<'a> Drop for Session<'a> {
 }
 
 impl<'a> Session<'a> {
+    /// Somedoc
+    pub fn feed<'s, 't, 'm, TIn, D>(&'s mut self, input_array: Array<TIn, D>) -> Result<()>
+    where
+        TIn: TypeToTensorElementDataType + Debug + Clone,
+        D: ndarray::Dimension,
+        'm: 't, // 'm outlives 't (memory info outlives tensor)
+        's: 'm, // 's outlives 'm (session outlives memory info)
+    {
+        self.validate_input_shapes(&input_array)?;
+        // The C API expects pointers for the arrays (pointers to C-arrays)
+        let input_ort_tensor: OrtTensor<TIn, D> =
+            OrtTensor::from_array(&self.memory_info, self.allocator_ptr, input_array)?;
+
+        let input_ort_value: *const sys::OrtValue = input_ort_tensor.c_ptr as *const sys::OrtValue;
+        std::mem::forget(input_ort_tensor);
+        self.input_ort_values.push(input_ort_value);
+
+        Ok(())
+    }
+
     /// Run the input data through the ONNX graph, performing inference.
     ///
     /// Note that ONNX models can have multiple inputs; a `Vec<_>` is thus
@@ -371,9 +396,46 @@ impl<'a> Session<'a> {
         'm: 't, // 'm outlives 't (memory info outlives tensor)
         's: 'm, // 's outlives 'm (session outlives memory info)
     {
-        self.validate_input_shapes(&input_arrays)?;
-
+        input_arrays
+            .into_iter()
+            .for_each(|input_array| self.feed(input_array).unwrap());
+        self.inner_run()
+    }
+    /// Run the input data through the ONNX graph, performing inference.
+    ///
+    /// Note that ONNX models can have multiple inputs; a `Vec<_>` is thus
+    /// used for the input data here.
+    pub fn inner_run<'s, 't, 'm>(
+        &'s mut self,
+        // input_arrays: Vec<Array<TIn, D>>,
+    ) -> Result<Vec<DynOrtTensor<'m, ndarray::IxDyn>>>
+    where
+        'm: 't, // 'm outlives 't (memory info outlives tensor)
+        's: 'm, // 's outlives 'm (session outlives memory info)
+    {
         // Build arguments to Run()
+        if self.input_ort_values.len() != self.inputs.len() {
+            error!(
+                "Non-matching number of inputs: {} (inference) vs {} (model)",
+                self.input_ort_values.len(),
+                self.inputs.len()
+            );
+            return Err(OrtError::NonMatchingDimensions(
+                NonMatchingDimensionsError::InputsCount {
+                    inference_input_count: 0,
+                    model_input_count: 0,
+                    // inference_input: input_arrays
+                    //     .iter()
+                    //     .map(|input_array| input_array.shape().to_vec())
+                    //     .collect(),
+                    // model_input: self
+                    //     .inputs
+                    //     .iter()
+                    //     .map(|input| input.dimensions.clone())
+                    //     .collect(),
+                },
+            ));
+        }
 
         let input_names: Vec<String> = self.inputs.iter().map(|input| input.name.clone()).collect();
         let input_names_cstring: Vec<CString> = input_names
@@ -403,33 +465,22 @@ impl<'a> Session<'a> {
         let mut output_tensor_ptrs: Vec<*mut sys::OrtValue> =
             vec![std::ptr::null_mut(); self.outputs.len()];
 
-        // The C API expects pointers for the arrays (pointers to C-arrays)
-        let input_ort_tensors: Vec<OrtTensor<TIn, D>> = input_arrays
-            .into_iter()
-            .map(|input_array| {
-                OrtTensor::from_array(&self.memory_info, self.allocator_ptr, input_array)
-            })
-            .collect::<Result<Vec<OrtTensor<TIn, D>>>>()?;
-        let input_ort_values: Vec<*const sys::OrtValue> = input_ort_tensors
-            .iter()
-            .map(|input_array_ort| input_array_ort.c_ptr as *const sys::OrtValue)
-            .collect();
-
         let run_options_ptr: *const sys::OrtRunOptions = std::ptr::null();
 
         let status = unsafe {
             g_ort().Run.unwrap()(
                 self.session_ptr,
                 run_options_ptr,
                 input_names_ptr.as_ptr(),
-                input_ort_values.as_ptr(),
-                input_ort_values.len() as u64, // C API expects a u64, not isize
+                self.input_ort_values.as_ptr(),
+                self.input_ort_values.len() as u64, // C API expects a u64, not isize
                 output_names_ptr.as_ptr(),
                 output_names_ptr.len() as u64, // C API expects a u64, not isize
                 output_tensor_ptrs.as_mut_ptr(),
             )
         };
         status_to_result(status).map_err(OrtError::Run)?;
+        self.input_ort_values.iter().for_each(std::mem::drop);
 
         let memory_info_ref = &self.memory_info;
         let outputs: Result<Vec<DynOrtTensor<ndarray::Dim<ndarray::IxDynImpl>>>> =
@@ -494,7 +545,7 @@ impl<'a> Session<'a> {
     //     Tensor::from_array(self, array)
     // }
 
-    fn validate_input_shapes<TIn, D>(&mut self, input_arrays: &[Array<TIn, D>]) -> Result<()>
+    fn validate_input_shapes<TIn, D>(&mut self, input_array: &Array<TIn, D>) -> Result<()>
     where
         TIn: TypeToTensorElementDataType + Debug + Clone,
         D: ndarray::Dimension,
@@ -504,62 +555,41 @@ impl<'a> Session<'a> {
         // Make sure all dimensions match (except dynamic ones)
 
         // Verify length of inputs
-        if input_arrays.len() != self.inputs.len() {
-            error!(
-                "Non-matching number of inputs: {} (inference) vs {} (model)",
-                input_arrays.len(),
-                self.inputs.len()
-            );
-            return Err(OrtError::NonMatchingDimensions(
-                NonMatchingDimensionsError::InputsCount {
-                    inference_input_count: 0,
-                    model_input_count: 0,
-                    inference_input: input_arrays
-                        .iter()
-                        .map(|input_array| input_array.shape().to_vec())
-                        .collect(),
-                    model_input: self
-                        .inputs
-                        .iter()
-                        .map(|input| input.dimensions.clone())
-                        .collect(),
-                },
-            ));
-        }
 
         // Verify length of each individual inputs
-        let inputs_different_length = input_arrays
-            .iter()
-            .zip(self.inputs.iter())
-            .any(|(l, r)| l.shape().len() != r.dimensions.len());
-        if inputs_different_length {
+        let current_input = self.input_ort_values.len();
+        if current_input > self.inputs.len() {
             error!(
-                "Different input lengths: {:?} vs {:?}",
-                self.inputs, input_arrays
+                "Attempting to feed too many inputs, expecting {:?} inputs",
+                self.inputs.len()
             );
             panic!(
-                "Different input lengths: {:?} vs {:?}",
-                self.inputs, input_arrays
+                "Attempting to feed too many inputs, expecting {:?} inputs",
+                self.inputs.len()
             );
         }
+        let input = &self.inputs[current_input];
+        if input_array.shape().len() != input.dimensions().count() {
+            error!("Different input lengths: {:?} vs {:?}", input, input_array);
+            panic!("Different input lengths: {:?} vs {:?}", input, input_array);
+        }
 
-        // Verify shape of each individual inputs
-        let inputs_different_shape = input_arrays.iter().zip(self.inputs.iter()).any(|(l, r)| {
-            let l_shape = l.shape();
-            let r_shape = r.dimensions.as_slice();
-            l_shape.iter().zip(r_shape.iter()).any(|(l2, r2)| match r2 {
-                Some(r3) => *r3 as usize != *l2,
-                None => false, // None means dynamic size; in that case shape always match
-            })
+        let l = input_array;
+        let r = input;
+        let l_shape = l.shape();
+        let r_shape = r.dimensions.as_slice();
+        let inputs_different_shape = l_shape.iter().zip(r_shape.iter()).any(|(l2, r2)| match r2 {
+            Some(r3) => *r3 as usize != *l2,
+            None => false, // None means dynamic size; in that case shape always match
         });
         if inputs_different_shape {
             error!(
                 "Different input lengths: {:?} vs {:?}",
-                self.inputs, input_arrays
+                self.inputs, input_array
             );
             panic!(
                 "Different input lengths: {:?} vs {:?}",
-                self.inputs, input_arrays
+                self.inputs, input_array
             );
         }