feat: add Elasticsearch transform support

fields_test.go

@@ -59,7 +59,7 @@ func TestReadFields(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	const expected = 81
+	const expected = 154
 	if len(allFields) != expected {
 		t.Fatalf("got %d, want %d fields", len(allFields), expected)
 	}

fleetpkg.go

@@ -32,6 +32,7 @@ type Integration struct {
 	Manifest    Manifest               `json:"manifest,omitempty" yaml:"manifest,omitempty"`
 	Input       *DataStream            `json:"input,omitempty" yaml:"input,omitempty"`
 	DataStreams map[string]*DataStream `json:"data_streams,omitempty" yaml:"data_streams,omitempty"`
+	Transforms  map[string]*Transform  `json:"transforms,omitempty" yaml:"transforms,omitempty"`
 	Changelog   Changelog              `json:"changelog,omitempty" yaml:"changelog,omitempty"`
 
 	sourceFile string
@@ -243,6 +244,7 @@ func (c *Conditions) UnmarshalYAML(value *yaml.Node) error {
 	if err := value.Decode(&pc); err != nil {
 		return err
 	}
+	*c = Conditions(pc.conditions)
 
 	if pc.Kibana.Version != "" {
 		c.Kibana.Version = pc.Kibana.Version
@@ -338,6 +340,11 @@ type PolicyTemplate struct {
 	TemplatePath    string           `json:"template_path,omitempty" yaml:"template_path,omitempty"`
 	Vars            []Var            `json:"vars,omitempty" yaml:"vars,omitempty"` // Policy template level variables.
 	DeploymentModes *DeploymentModes `json:"deployment_modes,omitempty" yaml:"deployment_modes,omitempty"`
+
+	// Indicate if this package is capable of satisfying FIPS requirements. Set
+	// to false if it uses any input that cannot be configured to use FIPS
+	// cryptography. Defaults to true.
+	FIPSCompatible *bool `json:"fips_compatible,omitempty" yaml:"fips_compatible,omitempty"`
 }
 
 // DeploymentModes options. The deployment mode refers to the mode used to deploy the Elastic Agents running this policy.
@@ -523,9 +530,9 @@ func (p *Processor) MarshalJSON() ([]byte, error) {
 	})
 }
 
-// Read reads the Fleet integration at the specified path. The path should
-// point to the directory containing the integration's main manifest.yml.
-func Read(path string) (*Integration, error) {
+// Read an integration package from a directory.
+// The path must be the root of the integration package.
+func Read(path string, options ...Option) (*Integration, error) {
 	integration := &Integration{
 		DataStreams: map[string]*DataStream{},
 		sourceFile:  path,
@@ -634,6 +641,13 @@ func Read(path string) (*Integration, error) {
 		}
 	}
 
+	// Read elasticsearch transforms.
+	var err error
+	integration.Transforms, err = readTransforms(filepath.Join(path, "elasticsearch", "transform"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to read transforms for '%s': %w", path, err)
+	}
+
 	return integration, nil
 }
 
@@ -670,3 +684,53 @@ func readJSON(path string, v any, strict bool) error {
 	}
 	return nil
 }
+
+func readTransforms(path string) (map[string]*Transform, error) {
+	dirs, err := os.ReadDir(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	transforms := map[string]*Transform{}
+	for _, de := range dirs {
+		if !de.IsDir() {
+			continue
+		}
+
+		transformPath := filepath.Join(path, de.Name())
+		transform, err := readTransform(transformPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read transform from %s: %w", transformPath, err)
+		}
+
+		transforms[de.Name()] = transform
+	}
+
+	if len(transforms) == 0 {
+		return nil, nil
+	}
+	return transforms, nil
+}
+
+func readTransform(path string) (*Transform, error) {
+	transform := Transform{sourceDir: path}
+
+	if err := readYAML(filepath.Join(path, "transform.yml"), &transform.Transform, false); err != nil {
+		return nil, err
+	}
+
+	if err := readYAML(filepath.Join(path, "manifest.yml"), &transform.Manifest, false); err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	var err error
+	transform.Fields, err = ReadFields(filepath.Join(path, "fields/*.yml"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &transform, nil
+}

testdata/packages/ti_abusech/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: "[email protected]"

testdata/packages/ti_abusech/_dev/build/docs/README.md

@@ -0,0 +1,61 @@
+# AbuseCH integration
+
+This integration is for [AbuseCH](https://urlhaus.abuse.ch/) logs. It includes the following datasets for retrieving indicators from the AbuseCH API:
+
+- `url` dataset: Supports URL based indicators from AbuseCH API.
+- `malware` dataset: Supports Malware based indicators from AbuseCH API.
+- `malwarebazaar` dataset: Supports indicators from the MalwareBazaar from AbuseCH.
+- `threatfox` dataset: Supports indicators from AbuseCH Threat Fox API.
+
+## Note:
+
+AbuseCH requires using an `Auth Key` (API Key) in the requests for authentication.
+Requests without authentication will be denied by the API.
+
+More details on this topic can be found [here](https://abuse.ch/blog/community-first/).
+
+## Agentless Enabled Integration
+
+Agentless integrations allow you to collect data without having to manage Elastic Agent in your cloud. They make manual agent deployment unnecessary, so you can focus on your data instead of the agent that collects it. For more information, refer to [Agentless integrations](https://www.elastic.co/guide/en/serverless/current/security-agentless-integrations.html) and the [Agentless integrations FAQ](https://www.elastic.co/guide/en/serverless/current/agentless-integration-troubleshooting.html).
+
+Agentless deployments are only supported in Elastic Serverless and Elastic Cloud environments.  This functionality is in beta and is subject to change. Beta features are not subject to the support SLA of official GA features.
+
+## Expiration of Indicators of Compromise (IOCs)
+All AbuseCH datasets now support indicator expiration. For `URL` dataset, a full list of active indicators are ingested every interval. For other datasets namely `Malware`, `MalwareBazaar`, and `ThreatFox`, the indicators are expired after duration `IOC Expiration Duration` configured in the integration setting. An [Elastic Transform](https://www.elastic.co/guide/en/elasticsearch/reference/current/transforms.html) is created for every source index to facilitate only active indicators be available to the end users. Each transform creates a destination index named `logs-ti_abusech_latest.dest_*` which only contains active and unexpired indicators. The indiator match rules and dashboards are updated to list only active indicators.
+Destinations indices are aliased to `logs-ti_abusech_latest.<datastream_name>`.
+
+| Source Datastream                  | Destination Index Pattern                        | Destination Alias                       |
+|:-----------------------------------|:-------------------------------------------------|-----------------------------------------|
+| `logs-ti_abusech.url-*`            | `logs-ti_abusech_latest.dest_url-*`              | `logs-ti_abusech_latest.url`            |
+| `logs-ti_abusech.malware-*`        | `logs-ti_abusech_latest.dest_malware-*`          | `logs-ti_abusech_latest.malware`        |
+| `logs-ti_abusech.malwarebazaar-*`  | `logs-ti_abusech_latest.dest_malwarebazaar-*`    | `logs-ti_abusech_latest.malwarebazaar`  |
+| `logs-ti_abusech.threatfox-*`      | `logs-ti_abusech_latest.dest_threatfox-*`        | `logs-ti_abusech_latest.threatfox`      |
+
+### ILM Policy
+To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_abusech.<datastream_name>-*` are allowed to contain duplicates from each polling interval. ILM policy `logs-ti_abusech.<datastream_name>-default_policy` is added to these source indices so it doesn't lead to unbounded growth. This means data in these source indices will be deleted after `5 days` from ingested date. 
+
+## Logs
+
+### URL
+
+The AbuseCH URL data_stream retrieves full list of active threat intelligence indicators every interval from the Active Indicators URL database dump `https://urlhaus.abuse.ch/downloads/json/`.
+
+{{fields "url"}}
+
+### Malware
+
+The AbuseCH malware data_stream retrieves threat intelligence indicators from the payload API endpoint `https://urlhaus-api.abuse.ch/v1/payloads/recent/`.
+
+{{fields "malware"}}
+
+### MalwareBazaar
+
+The AbuseCH malwarebazaar data_stream retrieves threat intelligence indicators from the MalwareBazaar API endpoint `https://mb-api.abuse.ch/api/v1/`.
+
+{{fields "malwarebazaar"}}
+
+### Threat Fox
+
+The AbuseCH threatfox data_stream retrieves threat intelligence indicators from the Threat Fox API endpoint `https://threatfox-api.abuse.ch/api/v1/`.
+
+{{fields "threatfox"}}

testdata/packages/ti_abusech/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,26 @@
+version: "2.3"
+services:
+  abusech:
+    image: docker.elastic.co/observability/stream:v0.18.0
+    ports:
+      - 8080
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8080
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml
+  abusech-threatfox:
+    image: docker.elastic.co/observability/stream:v0.18.0
+    ports:
+      - 8081
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8081
+    command:
+      - http-server
+      - --addr=:8081
+      - --config=/files/config-threatfox.yml

testdata/packages/ti_abusech/_dev/deploy/docker/files/config-threatfox.yml

@@ -0,0 +1,37 @@
+rules:
+  - path: /api/v1/
+    methods: ["POST"]
+    request_headers:
+      Content-Type: "application/json"
+      Auth-Key: "test_auth_key"
+    body:
+      query: "get_iocs"
+      days: /[1-7]+/
+    responses:
+      - status_code: 200
+        body: |-
+          {
+            "query_status": "ok",
+            "data": [
+              {"id":"841537","ioc":"wizzy.hopto.org","threat_type":"botnet_cc","threat_type_desc":"Indicator that identifies a botnet command&control server (C&C)","ioc_type":"domain","ioc_type_desc":"Domain that is used for botnet Command&control (C&C)","malware":"win.asyncrat","malware_printable":"AsyncRAT","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.asyncrat","confidence_level":100,"first_seen":"2022-08-05 19:43:08 UTC","last_seen":null,"reference":"https://tria.ge/220805-w57pxsgae2","reporter":"AndreGironda","tags":["asyncrat"]},
+              {"id":"839586","ioc":"872ff530d50579ae6bdc7cb4d658324b1d0e7a3e","threat_type":"payload","threat_type_desc":"Indicator that identifies a malware sample (payload)","ioc_type":"sha1_hash","ioc_type_desc":"SHA1 hash of a malware sample (payload)","malware":"win.vidar","malware_printable":"Vidar","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.vidar","confidence_level":75,"first_seen":"2022-07-25 22:27:09 UTC","last_seen":null,"reference":"","reporter":"crep1x","tags":["Vidar"]},
+              {"id":"839587","ioc":"a3b5c6d7e8f9g0h1i2j3k4l5m6n7o8p9q0r1s2t3","threat_type":"payload","threat_type_desc":"Indicator that identifies a malware sample (payload)","ioc_type":"sha1_hash","ioc_type_desc":"SHA1 hash of a malware sample (payload)","malware":"win.redline","malware_printable":"RedLine","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.redline","confidence_level":80,"first_seen":"2025-03-15 10:15:00 UTC","last_seen":null,"reference":"","reporter":"cyberhunter","tags":["RedLine"]}
+            ]
+          }
+  - path: /api/v1/
+    methods: ["POST"]
+    request_headers:
+      Content-Type: "application/json"
+    body:
+      query: "get_iocs"
+      days: /[1-7]+/
+    responses:
+      - status_code: 200
+        body: |-
+          {
+            "query_status": "ok",
+            "data": [
+              {"id":"841537","ioc":"wizzy.hopto.org","threat_type":"botnet_cc","threat_type_desc":"Indicator that identifies a botnet command&control server (C&C)","ioc_type":"domain","ioc_type_desc":"Domain that is used for botnet Command&control (C&C)","malware":"win.asyncrat","malware_printable":"AsyncRAT","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.asyncrat","confidence_level":100,"first_seen":"2022-08-05 19:43:08 UTC","last_seen":null,"reference":"https://tria.ge/220805-w57pxsgae2","reporter":"AndreGironda","tags":["asyncrat"]},
+              {"id":"839586","ioc":"872ff530d50579ae6bdc7cb4d658324b1d0e7a3e","threat_type":"payload","threat_type_desc":"Indicator that identifies a malware sample (payload)","ioc_type":"sha1_hash","ioc_type_desc":"SHA1 hash of a malware sample (payload)","malware":"win.vidar","malware_printable":"Vidar","malware_alias":null,"malware_malpedia":"https://malpedia.caad.fkie.fraunhofer.de/details/win.vidar","confidence_level":75,"first_seen":"2022-07-25 22:27:09 UTC","last_seen":null,"reference":"","reporter":"crep1x","tags":["Vidar"]}
+            ]
+          }