feat: proxy more config options for rdkafka

[floydspace]

closes #16

here is the way to define those particular options:

import { NodeRuntime } from "@effect/platform-node";
import { Effect, Logger, LogLevel } from "effect";
import { ConfluentRdKafkaInstance, Producer } from "../src";

const program = Producer.sendScoped({
  topic: "test-topic",
  messages: [{ value: "Hello, effect-kafka user!" }, { value: "How are you, effect-kafka user?" }],
}).pipe(
  Producer.withProducerOptions({
    idempotent: false,
    partitioner: "consistent_random",
    queueBuffering: {
      maxMessages: 1000000,
      maxKbytes: 1048576,
      maxMs: 100,
    },
    batching: {
      maxMessages: 100000,
      maxBytes: 4194304,
    },
    stickyPartitioning: {
      lingerMs: 25,
    },
  }),
);

const KafkaLive = ConfluentRdKafkaInstance.layer({
  "metadata.broker.list": "localhost:19092",
  // "fetch.max.bytes": 4194304, // ConsumerOptions.maxBytes
  // "max.partition.fetch.bytes": 1048576, // ConsumerOptions.maxBytesPerPartition
  // "queue.buffering.max.kbytes": 1048576, // ProducerOptions.queueBuffering.maxKbytes
  // "queue.buffering.max.messages": 1000000, // ProducerOptions.queueBuffering.maxMessages
  "message.max.bytes": 4194304,
  // "batch.size": "4194304", // ProducerOptions.batching.maxBytes
  // "batch.num.messages": "100000", // ProducerOptions.batching.maxMessages
  // "linger.ms": "100", // ProducerOptions.queueBuffering.maxMs
  // "sticky.partitioning.linger.ms": "25", // ProducerOptions.stickyPartitioning.lingerMs
  // "enable.idempotence": "false", // ProducerOptions.idempotent
  "max.in.flight.requests.per.connection": 1000000,
  // "partitioner": "consistent_random", // ProducerOptions.partitioner
});
const MainLive = Effect.scoped(program).pipe(Effect.provide(KafkaLive), Logger.withMinimumLogLevel(LogLevel.Debug));

NodeRuntime.runMain(MainLive);

[changeset-bot]

🦋 Changeset detected

Latest commit: 6e88d17

The changes in this PR will be included in the next version bump.

This PR includes changesets to release 1 package

Name	Type
effect-kafka	Patch

Not sure what this means? Click here to learn what changesets are.

Click here if you're a maintainer who wants to add another changeset to this PR