[Impeller Scene] Import animation data (#38583)

* [Impeller Scene] Import animation data

* Cast

* Use correct length modifier

* Fix translation import

* Use ++ instead of fetch_add

Author: bdero

impeller/fixtures/two_triangles.glb

@@ -51,7 +51,15 @@ Color ToColor(const fb::Color& c) {
 /// Impeller -> Flatbuffers
 ///
 
-std::unique_ptr<fb::Matrix> ToFBMatrix(const Matrix& m) {
+fb::Matrix ToFBMatrix(const Matrix& m) {
+  auto array = std::array<Scalar, 16>{m.m[0],  m.m[1],  m.m[2],  m.m[3],   //
+                                      m.m[4],  m.m[5],  m.m[6],  m.m[7],   //
+                                      m.m[8],  m.m[9],  m.m[10], m.m[11],  //
+                                      m.m[12], m.m[13], m.m[14], m.m[15]};
+  return fb::Matrix(array);
+}
+
+std::unique_ptr<fb::Matrix> ToFBMatrixUniquePtr(const Matrix& m) {
   auto array = std::array<Scalar, 16>{m.m[0],  m.m[1],  m.m[2],  m.m[3],   //
                                       m.m[4],  m.m[5],  m.m[6],  m.m[7],   //
                                       m.m[8],  m.m[9],  m.m[10], m.m[11],  //

impeller/scene/importer/conversions.cc

@@ -35,7 +35,9 @@ Color ToColor(const fb::Color& c);
 /// Impeller -> Flatbuffers
 ///
 
-std::unique_ptr<fb::Matrix> ToFBMatrix(const Matrix& m);
+fb::Matrix ToFBMatrix(const Matrix& m);
+
+std::unique_ptr<fb::Matrix> ToFBMatrixUniquePtr(const Matrix& m);
 
 fb::Vec2 ToFBVec2(const Vector2 v);
 

impeller/scene/importer/conversions.h

@@ -202,8 +202,8 @@ static void ProcessNode(const tinygltf::Model& gltf,
   if (in_node.translation.size() == 3) {
     transform = transform * Matrix::MakeTranslation(
                                 {static_cast<Scalar>(in_node.translation[0]),
-                                 static_cast<Scalar>(in_node.translation[0]),
-                                 static_cast<Scalar>(in_node.translation[0])});
+                                 static_cast<Scalar>(in_node.translation[1]),
+                                 static_cast<Scalar>(in_node.translation[2])});
   }
   if (in_node.rotation.size() == 4) {
     transform = transform * Matrix::MakeRotation(Quaternion(
@@ -226,7 +226,7 @@ static void ProcessNode(const tinygltf::Model& gltf,
     }
     transform = ToMatrix(in_node.matrix);
   }
-  out_node.transform = ToFBMatrix(transform);
+  out_node.transform = ToFBMatrixUniquePtr(transform);
 
   //---------------------------------------------------------------------------
   /// Static meshes.
@@ -242,13 +242,42 @@ static void ProcessNode(const tinygltf::Model& gltf,
       out_node.mesh_primitives.push_back(std::move(mesh_primitive));
     }
   }
+
+  //---------------------------------------------------------------------------
+  /// Skin.
+  ///
+
+  if (WithinRange(in_node.skin, gltf.skins.size())) {
+    auto& skin = gltf.skins[in_node.skin];
+
+    auto ipskin = std::make_unique<fb::SkinT>();
+    ipskin->joints = skin.joints;
+    {
+      std::vector<fb::Matrix> matrices;
+      auto& matrix_accessor = gltf.accessors[skin.inverseBindMatrices];
+      auto& matrix_view = gltf.bufferViews[matrix_accessor.bufferView];
+      auto& matrix_buffer = gltf.buffers[matrix_view.buffer];
+      for (size_t matrix_i = 0; matrix_i < matrix_accessor.count; matrix_i++) {
+        auto* s = reinterpret_cast<const float*>(
+            matrix_buffer.data.data() + matrix_view.byteOffset +
+            matrix_accessor.ByteStride(matrix_view) * matrix_i);
+        Matrix m(s[0], s[1], s[2], s[3],    //
+                 s[4], s[5], s[6], s[7],    //
+                 s[8], s[9], s[10], s[11],  //
+                 s[12], s[13], s[14], s[15]);
+        matrices.push_back(ToFBMatrix(m));
+      }
+      ipskin->inverse_bind_matrices = std::move(matrices);
+    }
+    ipskin->skeleton = skin.skeleton;
+    out_node.skin = std::move(ipskin);
+  }
 }
 
 static void ProcessTexture(const tinygltf::Model& gltf,
                            const tinygltf::Texture& in_texture,
                            fb::TextureT& out_texture) {
-  if (in_texture.source < 0 ||
-      in_texture.source >= static_cast<int>(gltf.images.size())) {
+  if (!WithinRange(in_texture.source, gltf.images.size())) {
     return;
   }
   auto& image = gltf.images[in_texture.source];
@@ -283,6 +312,128 @@ static void ProcessTexture(const tinygltf::Model& gltf,
   out_texture.uri = image.uri;
 }
 
+static void ProcessAnimation(const tinygltf::Model& gltf,
+                             const tinygltf::Animation& in_animation,
+                             fb::AnimationT& out_animation) {
+  out_animation.name = in_animation.name;
+
+  std::vector<std::unique_ptr<impeller::fb::ChannelT>> channels;
+  for (auto& in_channel : in_animation.channels) {
+    auto out_channel = std::make_unique<fb::ChannelT>();
+
+    out_channel->node = in_channel.target_node;
+    auto& sampler = in_animation.samplers[in_channel.sampler];
+
+    /// Keyframe times.
+    auto& times_accessor = gltf.accessors[sampler.input];
+    if (times_accessor.count <= 0) {
+      continue;  // Nothing to record.
+    }
+    {
+      auto& times_bufferview = gltf.bufferViews[times_accessor.bufferView];
+      auto& times_buffer = gltf.buffers[times_bufferview.buffer];
+      if (times_accessor.componentType != TINYGLTF_COMPONENT_TYPE_FLOAT) {
+        std::cerr << "Unexpected component type \""
+                  << times_accessor.componentType
+                  << "\" for animation channel times accessor. Skipping."
+                  << std::endl;
+        continue;
+      }
+      if (times_accessor.type != TINYGLTF_TYPE_SCALAR) {
+        std::cerr << "Unexpected type \"" << times_accessor.type
+                  << "\" for animation channel times accessor. Skipping."
+                  << std::endl;
+        continue;
+      }
+      for (size_t time_i = 0; time_i < times_accessor.count; time_i++) {
+        const float* time_p = reinterpret_cast<const float*>(
+            times_buffer.data.data() + times_bufferview.byteOffset +
+            times_accessor.ByteStride(times_bufferview) * time_i);
+        out_channel->timeline.push_back(*time_p);
+      }
+    }
+
+    /// Keyframe values.
+    auto& values_accessor = gltf.accessors[sampler.output];
+    if (values_accessor.count != times_accessor.count) {
+      std::cerr << "Mismatch between time and value accessors for animation "
+                   "channel. Skipping."
+                << std::endl;
+      continue;
+    }
+    {
+      auto& values_bufferview = gltf.bufferViews[values_accessor.bufferView];
+      auto& values_buffer = gltf.buffers[values_bufferview.buffer];
+      if (values_accessor.componentType != TINYGLTF_COMPONENT_TYPE_FLOAT) {
+        std::cerr << "Unexpected component type \""
+                  << values_accessor.componentType
+                  << "\" for animation channel values accessor. Skipping."
+                  << std::endl;
+        continue;
+      }
+      if (in_channel.target_path == "translation") {
+        if (values_accessor.type != TINYGLTF_TYPE_VEC3) {
+          std::cerr << "Unexpected type \"" << values_accessor.type
+                    << "\" for animation channel \"translation\" accessor. "
+                       "Skipping."
+                    << std::endl;
+          continue;
+        }
+        fb::TranslationKeyframesT keyframes;
+        for (size_t value_i = 0; value_i < values_accessor.count; value_i++) {
+          const float* value_p = reinterpret_cast<const float*>(
+              values_buffer.data.data() + values_bufferview.byteOffset +
+              values_accessor.ByteStride(values_bufferview) * value_i);
+          keyframes.values.push_back(
+              fb::Vec3(value_p[0], value_p[1], value_p[2]));
+        }
+        out_channel->keyframes.Set(std::move(keyframes));
+      } else if (in_channel.target_path == "rotation") {
+        if (values_accessor.type != TINYGLTF_TYPE_VEC4) {
+          std::cerr << "Unexpected type \"" << values_accessor.type
+                    << "\" for animation channel \"rotation\" accessor. "
+                       "Skipping."
+                    << std::endl;
+          continue;
+        }
+        fb::RotationKeyframesT keyframes;
+        for (size_t value_i = 0; value_i < values_accessor.count; value_i++) {
+          const float* value_p = reinterpret_cast<const float*>(
+              values_buffer.data.data() + values_bufferview.byteOffset +
+              values_accessor.ByteStride(values_bufferview) * value_i);
+          keyframes.values.push_back(
+              fb::Vec4(value_p[0], value_p[1], value_p[2], value_p[3]));
+        }
+        out_channel->keyframes.Set(std::move(keyframes));
+      } else if (in_channel.target_path == "scale") {
+        if (values_accessor.type != TINYGLTF_TYPE_VEC3) {
+          std::cerr << "Unexpected type \"" << values_accessor.type
+                    << "\" for animation channel \"scale\" accessor. "
+                       "Skipping."
+                    << std::endl;
+          continue;
+        }
+        fb::ScaleKeyframesT keyframes;
+        for (size_t value_i = 0; value_i < values_accessor.count; value_i++) {
+          const float* value_p = reinterpret_cast<const float*>(
+              values_buffer.data.data() + values_bufferview.byteOffset +
+              values_accessor.ByteStride(values_bufferview) * value_i);
+          keyframes.values.push_back(
+              fb::Vec3(value_p[0], value_p[1], value_p[2]));
+        }
+        out_channel->keyframes.Set(std::move(keyframes));
+      } else {
+        std::cerr << "Unsupported animation channel target path \""
+                  << in_channel.target_path << "\". Skipping." << std::endl;
+        continue;
+      }
+    }
+
+    channels.push_back(std::move(out_channel));
+  }
+  out_animation.channels = std::move(channels);
+}
+
 bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) {
   tinygltf::Model gltf;
 
@@ -319,6 +470,13 @@ bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) {
     out_scene.nodes.push_back(std::move(node));
   }
 
+  for (size_t animation_i = 0; animation_i < gltf.animations.size();
+       animation_i++) {
+    auto animation = std::make_unique<fb::AnimationT>();
+    ProcessAnimation(gltf, gltf.animations[animation_i], *animation);
+    out_scene.animations.push_back(std::move(animation));
+  }
+
   return true;
 }
 

impeller/scene/importer/importer_gltf.cc

@@ -91,7 +91,7 @@ TEST(ImporterTest, CanParseSkinnedGLTF) {
   ASSERT_VECTOR3_NEAR(normal, Vector3(0, 0, 1));
 
   Vector4 tangent = ToVector4(vertex.vertex().tangent());
-  ASSERT_VECTOR4_NEAR(tangent, Vector4(0, 0, 0, 1));
+  ASSERT_VECTOR4_NEAR(tangent, Vector4(1, 0, 0, -1));
 
   Vector2 texture_coords = ToVector2(vertex.vertex().texture_coords());
   ASSERT_POINT_NEAR(texture_coords, Vector2(0, 1));
@@ -104,6 +104,19 @@ TEST(ImporterTest, CanParseSkinnedGLTF) {
 
   Vector4 weights = ToVector4(vertex.weights());
   ASSERT_COLOR_NEAR(weights, Vector4(1, 0, 0, 0));
+
+  ASSERT_EQ(scene.animations.size(), 2u);
+  ASSERT_EQ(scene.animations[0]->name, "Idle");
+  ASSERT_EQ(scene.animations[1]->name, "Metronome");
+  ASSERT_EQ(scene.animations[1]->channels.size(), 6u);
+  auto& channel = scene.animations[1]->channels[4];
+  ASSERT_EQ(channel->keyframes.type, fb::Keyframes::RotationKeyframes);
+  auto* keyframes = channel->keyframes.AsRotationKeyframes();
+  ASSERT_EQ(keyframes->values.size(), 40u);
+  ASSERT_VECTOR4_NEAR(ToVector4(keyframes->values[0]),
+                      Vector4(0.653281, 0.270598, -0.270598, 0.653281));
+  ASSERT_VECTOR4_NEAR(ToVector4(keyframes->values[10]),
+                      Vector4(0.425122, 0.565041, -0.565041, 0.425122));
 }
 
 }  // namespace testing

impeller/scene/importer/importer_unittests.cc

@@ -164,7 +164,7 @@ table Animation {
 }
 
 table Skin {
-  joints: [int];  // Index into `Scene`->`nodes`.
+  joints: [int];  // Indices into `Scene`->`nodes`.
   inverse_bind_matrices: [Matrix];
   /// The root joint of the skeleton.
   skeleton: int;  // Index into `Scene`->`nodes`.
@@ -180,14 +180,14 @@ struct Matrix {
 
 table Node {
   name: string;
-  children: [int];  // Index into `Scene`->`nodes`.
+  children: [int];  // Indices into `Scene`->`nodes`.
   transform: Matrix;
   mesh_primitives: [MeshPrimitive];
   skin: Skin;
 }
 
 table Scene {
-  children: [int];  // Index into `Scene`->`nodes`.
+  children: [int];  // Indices into `Scene`->`nodes`.
   nodes: [Node];
   textures: [Texture];  // Textures may be reused across different materials.
   animations: [Animation];

impeller/scene/importer/scene.fbs

@@ -4,9 +4,12 @@
 
 #include "impeller/scene/node.h"
 
+#include <inttypes.h>
+#include <atomic>
 #include <memory>
 
 #include "flutter/fml/logging.h"
+#include "impeller/base/strings.h"
 #include "impeller/base/validation.h"
 #include "impeller/geometry/matrix.h"
 #include "impeller/scene/importer/conversions.h"
@@ -18,6 +21,8 @@
 namespace impeller {
 namespace scene {
 
+static std::atomic_uint64_t kNextNodeID = 0;
+
 std::shared_ptr<Node> Node::MakeFromFlatbuffer(
     const fml::Mapping& ipscene_mapping,
     Allocator& allocator) {
@@ -155,8 +160,7 @@ void Node::UnpackFromFlatbuffer(
     const std::vector<std::shared_ptr<Node>>& scene_nodes,
     const std::vector<std::shared_ptr<Texture>>& textures,
     Allocator& allocator) {
-  /// Transform.
-
+  name_ = source_node.name()->str();
   SetLocalTransform(importer::ToMatrix(*source_node.transform()));
 
   /// Meshes.
@@ -190,7 +194,7 @@ void Node::UnpackFromFlatbuffer(
   }
 }
 
-Node::Node() = default;
+Node::Node() : name_(SPrintF("__node%" PRIu64, kNextNodeID++)){};
 
 Node::~Node() = default;
 
@@ -202,6 +206,26 @@ Node::Node(Node&& node) = default;
 
 Node& Node::operator=(Node&& node) = default;
 
+const std::string& Node::GetName() const {
+  return name_;
+}
+
+void Node::SetName(const std::string& new_name) {
+  name_ = new_name;
+}
+
+std::shared_ptr<Node> Node::FindNodeByName(const std::string& name) const {
+  for (auto& child : children_) {
+    if (child->GetName() == name) {
+      return child;
+    }
+    if (auto found = child->FindNodeByName(name)) {
+      return found;
+    }
+  }
+  return nullptr;
+}
+
 void Node::SetLocalTransform(Matrix transform) {
   local_transform_ = transform;
 }

impeller/scene/node.cc

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 #include <vector>
 
 #include "flutter/fml/macros.h"
@@ -33,6 +34,11 @@ class Node final {
   Node(Node&& node);
   Node& operator=(Node&& node);
 
+  const std::string& GetName() const;
+  void SetName(const std::string& new_name);
+
+  std::shared_ptr<Node> FindNodeByName(const std::string& name) const;
+
   void SetLocalTransform(Matrix transform);
   Matrix GetLocalTransform() const;
 
@@ -57,6 +63,7 @@ class Node final {
       const std::vector<std::shared_ptr<Texture>>& textures,
       Allocator& allocator);
 
+  std::string name_;
   bool is_root_ = false;
   Node* parent_ = nullptr;
   std::vector<std::shared_ptr<Node>> children_;