Handle input packages (#15)

Handle input packages by adding an `Input` field to the `Integration` struct that contains the metadata (e.g. fields, pipelines, samples events) from an input type package 

Closes #5

Author: efd6

fleetpkg.go

@@ -30,6 +30,7 @@ import (
 type Integration struct {
 	Build       *BuildManifest         `json:"build,omitempty" yaml:"build,omitempty"`
 	Manifest    Manifest               `json:"manifest,omitempty" yaml:"manifest,omitempty"`
+	Input       *DataStream            `json:"input,omitempty" yaml:"input,omitempty"`
 	DataStreams map[string]*DataStream `json:"data_streams,omitempty" yaml:"data_streams,omitempty"`
 	Changelog   Changelog              `json:"changelog,omitempty" yaml:"changelog,omitempty"`
 
@@ -436,7 +437,6 @@ func Read(path string) (*Integration, error) {
 	}
 	integration.Manifest.sourceFile = sourceFile
 	annotateFileMetadata(integration.Manifest.sourceFile, &integration.Manifest)
-
 	sourceFile = filepath.Join(path, "changelog.yml")
 	if err := readYAML(sourceFile, &integration.Changelog, true); err != nil {
 		return nil, err
@@ -455,21 +455,31 @@ func Read(path string) (*Integration, error) {
 		integration.Build.sourceFile = sourceFile
 	}
 
-	dataStreams, err := filepath.Glob(filepath.Join(path, "data_stream/*/manifest.yml"))
-	if err != nil {
-		return nil, err
+	var dataStreams []string
+	if integration.Manifest.Type == "input" {
+		dataStreams = []string{filepath.Join(path, "manifest.yml")}
+	} else {
+		var err error
+		dataStreams, err = filepath.Glob(filepath.Join(path, "data_stream/*/manifest.yml"))
+		if err != nil {
+			return nil, err
+		}
 	}
 	for _, manifestPath := range dataStreams {
 		ds := &DataStream{
 			sourceDir: filepath.Dir(manifestPath),
 		}
-		integration.DataStreams[filepath.Base(ds.sourceDir)] = ds
+		if integration.Manifest.Type == "input" {
+			integration.Input = ds
+		} else {
+			integration.DataStreams[filepath.Base(ds.sourceDir)] = ds
 
-		if err := readYAML(manifestPath, &ds.Manifest, true); err != nil {
-			return nil, err
+			if err := readYAML(manifestPath, &ds.Manifest, true); err != nil {
+				return nil, err
+			}
+			ds.Manifest.sourceFile = manifestPath
+			annotateFileMetadata(ds.Manifest.sourceFile, &ds.Manifest)
 		}
-		ds.Manifest.sourceFile = manifestPath
-		annotateFileMetadata(ds.Manifest.sourceFile, &ds.Manifest)
 
 		pipelines, err := filepath.Glob(filepath.Join(ds.sourceDir, "elasticsearch/ingest_pipeline/*.yml"))
 		if err != nil {

testdata/cel.golden.json

@@ -362,6 +362,100 @@
       "github": "elastic/security-external-integrations"
     }
   },
+  "input": {
+    "manifest": {},
+    "sample_event": {
+      "event": {
+        "@timestamp": "2023-03-23T00:04:52.561Z",
+        "agent": {
+          "ephemeral_id": "0b7c0c2b-32aa-4b65-95a4-d34416b8a7eb",
+          "id": "8c8782fa-cd5b-4ae8-94a0-ee8e3ea9a8df",
+          "name": "docker-fleet-agent",
+          "type": "filebeat",
+          "version": "8.6.1"
+        },
+        "data_stream": {
+          "dataset": "cel.cel",
+          "namespace": "ep",
+          "type": "logs"
+        },
+        "ecs": {
+          "version": "8.0.0"
+        },
+        "elastic_agent": {
+          "id": "8c8782fa-cd5b-4ae8-94a0-ee8e3ea9a8df",
+          "snapshot": false,
+          "version": "8.6.1"
+        },
+        "event": {
+          "dataset": "cel.cel"
+        },
+        "input": {
+          "type": "cel"
+        },
+        "message": "success",
+        "tags": [
+          "forwarded"
+        ]
+      }
+    },
+    "fields": {
+      "input.yml": {
+        "fields": [
+          {
+            "name": "@timestamp",
+            "external": "ecs"
+          },
+          {
+            "name": "ecs.version",
+            "external": "ecs"
+          },
+          {
+            "name": "message",
+            "external": "ecs"
+          },
+          {
+            "name": "input.name",
+            "type": "constant_keyword"
+          },
+          {
+            "name": "input.type",
+            "type": "keyword"
+          },
+          {
+            "name": "data_stream.type",
+            "type": "constant_keyword",
+            "external": "ecs"
+          },
+          {
+            "name": "data_stream.dataset",
+            "type": "constant_keyword",
+            "external": "ecs"
+          },
+          {
+            "name": "data_stream.namespace",
+            "type": "constant_keyword",
+            "external": "ecs"
+          },
+          {
+            "name": "event.module",
+            "type": "constant_keyword",
+            "value": "cel",
+            "external": "ecs"
+          },
+          {
+            "name": "event.dataset",
+            "type": "constant_keyword",
+            "external": "ecs"
+          },
+          {
+            "name": "tags",
+            "external": "ecs"
+          }
+        ]
+      }
+    }
+  },
   "changelog": {
     "releases": [
       {

testdata/cel.golden.yml

@@ -297,6 +297,64 @@ manifest:
           show_user: false
   owner:
     github: elastic/security-external-integrations
+input:
+  sample_event:
+    event:
+      '@timestamp': "2023-03-23T00:04:52.561Z"
+      agent:
+        ephemeral_id: 0b7c0c2b-32aa-4b65-95a4-d34416b8a7eb
+        id: 8c8782fa-cd5b-4ae8-94a0-ee8e3ea9a8df
+        name: docker-fleet-agent
+        type: filebeat
+        version: 8.6.1
+      data_stream:
+        dataset: cel.cel
+        namespace: ep
+        type: logs
+      ecs:
+        version: 8.0.0
+      elastic_agent:
+        id: 8c8782fa-cd5b-4ae8-94a0-ee8e3ea9a8df
+        snapshot: false
+        version: 8.6.1
+      event:
+        dataset: cel.cel
+      input:
+        type: cel
+      message: success
+      tags:
+        - forwarded
+  fields:
+    input.yml:
+      fields:
+        - name: '@timestamp'
+          external: ecs
+        - name: ecs.version
+          external: ecs
+        - name: message
+          external: ecs
+        - name: input.name
+          type: constant_keyword
+        - name: input.type
+          type: keyword
+        - name: data_stream.type
+          type: constant_keyword
+          external: ecs
+        - name: data_stream.dataset
+          type: constant_keyword
+          external: ecs
+        - name: data_stream.namespace
+          type: constant_keyword
+          external: ecs
+        - name: event.module
+          type: constant_keyword
+          value: cel
+          external: ecs
+        - name: event.dataset
+          type: constant_keyword
+          external: ecs
+        - name: tags
+          external: ecs
 changelog:
   releases:
     - version: 0.3.0