Skip to main content

ApiServerConfig

FieldTypeDefaultDescription
authAuthConfig
{
“jwtSecret”: "",
“providers”: [],
“didEncryption”: {
“enabled”: false
},
“passwordPolicy”: {
“minNewPasswordLength”: 8
},
“allowAnonymous”: true
}
Authentication & authorization
databaseDatabaseConfig
{
“provider”: “InMemory”
}
Database
secretsEncryptionSecretsEncryptionConfig
{
“enabled”: false
}
Secrets encryption configuration
engineEngineConfig
{
“runtime”: “Podman”,
“networkNs”: “Private”,
“startTimeout”: ”30s”,
“shutdownTimeout”: “5s”,
“images”: {
“spark”: “ghcr.io/kamu-data/engine-spark:0.23.1-spark_3.5.0”,
“flink”: “ghcr.io/kamu-data/engine-flink:0.18.2-flink_1.16.0-scala_2.12-java8”,
“datafusion”: “ghcr.io/kamu-data/engine-datafusion:0.9.0”,
“risingwave”: “ghcr.io/kamu-data/engine-risingwave:0.3.0”
},
“datafusionEmbedded”: {
“base”: {
“datafusion.catalog.default_catalog”: “kamu”,
“datafusion.catalog.default_schema”: “kamu”,
“datafusion.catalog.information_schema”: “true”,
“datafusion.sql_parser.enable_ident_normalization”: “false”
},
“ingest”: {
“datafusion.execution.target_partitions”: “1”
},
“batchQuery”: {},
“compaction”: {
“datafusion.execution.target_partitions”: “1”
},
“useLegacyArrowBufferEncoding”: false
}
}
Ingest and transform engines
protocolProtocolConfig
{
“ipfs”: {
“httpGateway”: “http://localhost:8080/”,
“preResolveDnslink”: true
},
“flightSql”: {
“allowAnonymous”: true,
“authedSessionExpirationTimeout”: “30m”,
“authedSessionInactivityTimeout”: “5s”,
“anonSessionExpirationTimeout”: “5m”,
“anonSessionInactivityTimeout”: “5s”
}
}
Protocols
runtimeRuntimeConfig{}Tokio runtime
repoRepoConfig
{
“caching”: {
“registryCacheEnabled”: false
}
}
Dataset repository
uploadRepoUploadRepoConfig
{
“maxFileSizeMb”: 50,
“storage”: {
“kind”: “Local”
}
}
File upload repository
urlUrlConfig
{
“baseUrlPlatform”: “http://localhost:4200/”,
“baseUrlRest”: “http://localhost:8080/”,
“baseUrlFlightsql”: “grpc://localhost:50050”
}
External URLs
flowSystemFlowSystemConfig
{
“flowAgent”: {
“awaitingStepSecs”: 1,
“mandatoryThrottlingPeriodSecs”: 60,
“defaultRetryPolicies”: {}
},
“flowSystemEventAgent”: {
“minDebounceInterval”: “100ms”,
“maxListeningTimeout”: “2m”,
“batchSize”: 100
},
“taskAgent”: {
“taskCheckingIntervalSecs”: 1
}
}
Configuration for the flow system
webhooksWebhooksConfig
{
“maxConsecutiveFailures”: 5,
“deliveryTimeoutSecs”: 10,
“secretEncryptionEnabled”: false
}
Configuration for webhooks
sourceSourceConfig
{
“targetRecordsPerSlice”: 10000,
“mqtt”: {
“brokerIdleTimeoutMs”: 1000
},
“ethereum”: {
“rpcEndpoints”: [],
“getLogsBlockStride”: 100000,
“commitAfterBlocksScanned”: 1000000,
“useBlockTimestampFallback”: false
}
}
Ingestion’s sources
outboxOutboxAgentConfig
{
“minDebounceInterval”: “100ms”,
“maxListeningTimeout”: “2m”,
“batchSize”: 100
}
Outbox agent configuration
emailEmailConfig
{
“senderAddress”: "",
“gateway”: {
“kind”: “Dummy”
}
}
Email gateway configuration
identityIdentityConfig{}UNSTABLE: Identity configuration
searchSearchConfig
{
“indexer”: {
“incrementalIndexing”: false,
“clearOnStart”: false
},
“embeddingsChunker”: {
“kind”: “Simple”,
“splitSections”: false,
“splitParagraphs”: false
},
“embeddingsEncoder”: {
“kind”: “OpenAi”,
“modelName”: “text-embedding-ada-002”,
“dimensions”: 1536
},
“repo”: {
“kind”: “Dummy”
},
“semanticSearchThresholdScore”: 0.0
}
Search configuration
quotaQuotaConfig
{
“account”: {}
}
Default quotas configured by type
extraExtraConfig
{
“graphql”: {}
}
Experimental and temporary module configuration

AuthConfig

FieldTypeDefaultDescription
jwtSecretstring""
providersarray[]
didEncryptionDidSecretEncryptionConfig
{
“enabled”: false
}
passwordPolicyPasswordPolicyConfig
{
“minNewPasswordLength”: 8
}
allowAnonymousbooleantrue

AuthProviderConfig

Variants
Github
Password

AuthProviderConfig::Github

FieldTypeDefaultDescription
clientIdstring
clientSecretstring
kindstring

AuthProviderConfig::Password

FieldTypeDefaultDescription
accountsarray[]
kindstring

AccountConfig

The declarative account configuration used to register an account if one does not already exist. To update an existing account, either id or private_key must be specified.
FieldTypeDefaultDescription
idAccountIDnullMay be omitted in favor of private_key.
privateKeyPrivateKeynullOptional ed25519 private key. When set, id is derived from it (and must match id if both are present).
accountNameAccountName
passwordPassword
emailEmail
displayNamestringnullAuto-derived from account_name if omitted
accountTypeAccountType”User”
providerstring”password”
providerIdentityKeystringnullAuto-derived from account_name if omitted
avatarUrlstringnull
registeredAtstringnull
propertiesarray[]
treatDatasetsAsPublicbooleanfalse

AccountID

Base type: string

PrivateKey

Base type: string

AccountName

Base type: string

Password

Base type: string

Email

Base type: string

AccountType

Variants
User
Organization

AccountPropertyName

Variants
CanProvisionAccounts
Admin

DidSecretEncryptionConfig

FieldTypeDefaultDescription
enabledbooleanfalse
encryptionKeystringnullThe encryption key must be a 32-character alphanumeric string, which includes both uppercase and lowercase Latin letters (A-Z, a-z) and digits (0-9).To generate, use:

PasswordPolicyConfig

FieldTypeDefaultDescription
minNewPasswordLengthinteger8

DatabaseConfig

Variants
InMemory
Sqlite
Postgres

DatabaseConfig::InMemory

FieldTypeDefaultDescription
providerstring

DatabaseConfig::Sqlite

FieldTypeDefaultDescription
databasePathstring
providerstring

DatabaseConfig::Postgres

FieldTypeDefaultDescription
credentialsPolicyDatabaseCredentialsPolicyConfig
databaseNamestring
hoststring
portintegernull
maxConnectionsintegernull
maxLifetimeSecsintegernull
acquireTimeoutSecsintegernull
providerstring

DatabaseCredentialsPolicyConfig

FieldTypeDefaultDescription
sourceDatabaseCredentialSourceConfig
rotationFrequencyInMinutesintegernull

DatabaseCredentialSourceConfig

Variants
RawPassword
AwsSecret

DatabaseCredentialSourceConfig::RawPassword

FieldTypeDefaultDescription
userNamestring
rawPasswordstring
kindstring

DatabaseCredentialSourceConfig::AwsSecret

FieldTypeDefaultDescription
secretNamestring
kindstring

SecretsEncryptionConfig

FieldTypeDefaultDescription
enabledbooleanfalse
encryptionKeystringnullRepresents the encryption key for secrets. This field is required if enabled is true or None.The encryption key must be a 32-character alphanumeric string, which includes both uppercase and lowercase Latin letters (A-Z, a-z) and digits (0-9).To generate use:

EngineConfig

FieldTypeDefaultDescription
maxConcurrencyintegernullMaximum number of engine operations that can be performed concurrently
runtimeContainerRuntimeType”Podman”Type of the runtime to use when running the data processing engines
networkNsNetworkNamespaceType”Private”Type of the networking namespace (relevant when running in container environments)
startTimeoutDurationString”30s”Timeout for starting an engine container
shutdownTimeoutDurationString”5s”Timeout for waiting the engine container to stop gracefully
imagesEngineImagesConfig
{
“spark”: “ghcr.io/kamu-data/engine-spark:0.23.1-spark_3.5.0”,
“flink”: “ghcr.io/kamu-data/engine-flink:0.18.2-flink_1.16.0-scala_2.12-java8”,
“datafusion”: “ghcr.io/kamu-data/engine-datafusion:0.9.0”,
“risingwave”: “ghcr.io/kamu-data/engine-risingwave:0.3.0”
}
UNSTABLE: Default engine images
datafusionEmbeddedEngineConfigDatafusion
{
“base”: {
“datafusion.catalog.default_catalog”: “kamu”,
“datafusion.catalog.default_schema”: “kamu”,
“datafusion.catalog.information_schema”: “true”,
“datafusion.sql_parser.enable_ident_normalization”: “false”
},
“ingest”: {
“datafusion.execution.target_partitions”: “1”
},
“batchQuery”: {},
“compaction”: {
“datafusion.execution.target_partitions”: “1”
},
“useLegacyArrowBufferEncoding”: false
}
Embedded Datafusion engine configuration

ContainerRuntimeType

Variants
Docker
Podman

NetworkNamespaceType

Corresponds to podman’s containers.conf::netns We podman is used inside containers (e.g. podman-in-docker or podman-in-k8s) it usually runs uses host network namespace.
Variants
Private
Host

DurationString

Base type: string

EngineImagesConfig

FieldTypeDefaultDescription
sparkstring”ghcr.io/kamu-data/engine-spark:0.23.1-spark_3.5.0”UNSTABLE: Spark engine image
flinkstring”ghcr.io/kamu-data/engine-flink:0.18.2-flink_1.16.0-scala_2.12-java8”UNSTABLE: Flink engine image
datafusionstring”ghcr.io/kamu-data/engine-datafusion:0.9.0”UNSTABLE: Datafusion engine image
risingwavestring”ghcr.io/kamu-data/engine-risingwave:0.3.0”UNSTABLE: RisingWave engine image

EngineConfigDatafusion

FieldTypeDefaultDescription
baseobject
{
“datafusion.catalog.default_catalog”: “kamu”,
“datafusion.catalog.default_schema”: “kamu”,
“datafusion.catalog.information_schema”: “true”,
“datafusion.sql_parser.enable_ident_normalization”: “false”
}
Base configuration options See: <https://datafusion.apache.org/user-guide/configs.html>
ingestobject
{
“datafusion.execution.target_partitions”: “1”
}
Ingest-specific overrides to the base config
batchQueryobject{}Batch query-specific overrides to the base config
compactionobject
{
“datafusion.execution.target_partitions”: “1”
}
Compaction-specific overrides to the base config
useLegacyArrowBufferEncodingbooleanfalseMakes arrow batches use contiguous Binary and Utf8 encodings instead of more modern BinaryView and Utf8View. This is only needed for compatibility with some older libraries that don’t yet support them.See: kamu-node#277

ProtocolConfig

FieldTypeDefaultDescription
ipfsIpfsConfig
{
“httpGateway”: “http://localhost:8080/”,
“preResolveDnslink”: true
}
IPFS configuration
flightSqlFlightSqlConfig
{
“allowAnonymous”: true,
“authedSessionExpirationTimeout”: “30m”,
“authedSessionInactivityTimeout”: “5s”,
“anonSessionExpirationTimeout”: “5m”,
“anonSessionInactivityTimeout”: “5s”
}
FlightSQL configuration

IpfsConfig

FieldTypeDefaultDescription
httpGatewaystring”http://localhost:8080/”HTTP Gateway URL to use for downloads. For safety, it defaults to http://localhost:8080 - a local IPFS daemon. If you don’t have IPFS installed, you can set this URL to one of the public gateways like https://ipfs.io. List of public gateways can be found here: https://ipfs.github.io/public-gateway-checker/
preResolveDnslinkbooleantrueWhether kamu should pre-resolve IPNS DNSLink names using DNS or leave it to the Gateway.

FlightSqlConfig

FieldTypeDefaultDescription
allowAnonymousbooleantrueWhether clients can authenticate as ‘anonymous’ user
authedSessionExpirationTimeoutDurationString”30m”Time after which FlightSQL client session will be forgotten and client will have to re-authroize (for authenticated clients)
authedSessionInactivityTimeoutDurationString”5s”Time after which FlightSQL session context will be released to free the resources (for authenticated clients)
anonSessionExpirationTimeoutDurationString”5m”Time after which FlightSQL client session will be forgotten and client will have to re-authroize (for anonymous clients)
anonSessionInactivityTimeoutDurationString”5s”Time after which FlightSQL session context will be released to free the resources (for anonymous clients)

RuntimeConfig

FieldTypeDefaultDescription
workerThreadsintegernull
maxBlockingThreadsintegernull
threadStackSizeintegernull

RepoConfig

FieldTypeDefaultDescription
repoUrlUrlOrPathnull
cachingRepoCachingConfig
{
“registryCacheEnabled”: false
}
dataBlocksPageSizeintegernull

UrlOrPath

Base type: string

RepoCachingConfig

FieldTypeDefaultDescription
registryCacheEnabledbooleanfalse
metadataLocalFsCachePathstringnull

UploadRepoConfig

FieldTypeDefaultDescription
maxFileSizeMbinteger50
storageUploadRepoStorageConfig
{
“kind”: “Local”
}

UploadRepoStorageConfig

Variants
S3
Local

UploadRepoStorageConfig::S3

FieldTypeDefaultDescription
bucketS3Urlstring
kindstring

UploadRepoStorageConfig::Local

FieldTypeDefaultDescription
kindstring

UrlConfig

FieldTypeDefaultDescription
baseUrlPlatformUrlOrPath”http://localhost:4200/“
baseUrlRestUrlOrPath”http://localhost:8080/“
baseUrlFlightsqlUrlOrPath”grpc://localhost:50050”

FlowSystemConfig

FieldTypeDefaultDescription
flowAgentFlowAgentConfig
{
“awaitingStepSecs”: 1,
“mandatoryThrottlingPeriodSecs”: 60,
“defaultRetryPolicies”: {}
}
flowSystemEventAgentFlowSystemEventAgentConfig
{
“minDebounceInterval”: “100ms”,
“maxListeningTimeout”: “2m”,
“batchSize”: 100
}
taskAgentTaskAgentConfig
{
“taskCheckingIntervalSecs”: 1
}

FlowAgentConfig

FieldTypeDefaultDescription
awaitingStepSecsinteger1
mandatoryThrottlingPeriodSecsinteger60
defaultRetryPoliciesobject{}

RetryPolicyConfig

FieldTypeDefaultDescription
maxAttemptsintegernull
minDelaySecsintegernull
backoffTypeRetryPolicyConfigBackoffTypenull

RetryPolicyConfigBackoffType

Variants
Fixed
Linear
Exponential
ExponentialWithJitter

FlowSystemEventAgentConfig

FieldTypeDefaultDescription
minDebounceIntervalDurationString”100ms”
maxListeningTimeoutDurationString”2m”
batchSizeinteger100

TaskAgentConfig

FieldTypeDefaultDescription
taskCheckingIntervalSecsinteger1

WebhooksConfig

FieldTypeDefaultDescription
maxConsecutiveFailuresinteger5
deliveryTimeoutSecsinteger10
secretEncryptionEnabledbooleanfalse
secretEncryptionKeystringnullRepresents the encryption key for the webhooks secret. This field is required if secret_encryption_enabled is true or None.The encryption key must be a 32-character alphanumeric string, which includes both uppercase and lowercase Latin letters (A-Z, a-z) and digits (0-9).

Example

let config = WebhooksConfig { … secret_encryption_enabled: Some(true), encryption_key: Some(String::from(“aBcDeFgHiJkLmNoPqRsTuVwXyZ012345”)) }; ```

SourceConfig

FieldTypeDefaultDescription
targetRecordsPerSliceinteger10000Target number of records after which we will stop consuming from the resumable source and commit data, leaving the rest for the next iteration. This ensures that one data slice doesn’t become too big.
mqttMqttSourceConfig
{
“brokerIdleTimeoutMs”: 1000
}
MQTT-specific configuration
ethereumEthereumSourceConfig
{
“rpcEndpoints”: [],
“getLogsBlockStride”: 100000,
“commitAfterBlocksScanned”: 1000000,
“useBlockTimestampFallback”: false
}
Ethereum-specific configuration

MqttSourceConfig

FieldTypeDefaultDescription
brokerIdleTimeoutMsinteger1000Time in milliseconds to wait for MQTT broker to send us some data after which we will consider that we have “caught up” and end the polling loop.

EthereumSourceConfig

FieldTypeDefaultDescription
rpcEndpointsarray[]Default RPC endpoints to use if source does not specify one explicitly.
getLogsBlockStrideinteger100000Default number of blocks to scan within one query to eth_getLogs RPC endpoint.
commitAfterBlocksScannedinteger1000000Forces iteration to stop after the specified number of blocks were scanned even if we didn’t reach the target record number. This is useful to not lose a lot of scanning progress in case of an RPC error.
useBlockTimestampFallbackbooleanfalseMany providers don’t yet return blockTimestamp from eth_getLogs RPC endpoint and in such cases block_timestamp column will be null. If you enable this fallback the library will perform additional call to eth_getBlock to populate the timestam, but this may result in significant performance penalty when fetching many log records.See: ethereum/execution-apis#295

EthRpcEndpoint

FieldTypeDefaultDescription
chainIdinteger
chainNamestring
nodeUrlstring

OutboxAgentConfig

FieldTypeDefaultDescription
minDebounceIntervalDurationString”100ms”
maxListeningTimeoutDurationString”2m”
batchSizeinteger100

EmailConfig

FieldTypeDefaultDescription
senderAddressstring
senderNamestringnull
gatewayEmailConfigGateway

EmailConfigGateway

Variants
Dummy
Postmark

EmailConfigGateway::Dummy

FieldTypeDefaultDescription
kindstring

EmailConfigGateway::Postmark

FieldTypeDefaultDescription
apiKeystring
kindstring

IdentityConfig

Private keys are used to sign API responses. Supported algorithms: ed25519, secp256k1.
FieldTypeDefaultDescription
ed25519PrivateKeyPrivateKeynullRoot private key that corresponds to the authority and is used to sign responses.To generate, use:
or
secp256k1PrivateKeySecp256k1SignernullSecp256k1 private key used to sign EIP-712 typed data.To generate, use:
or
or

Secp256k1Signer

Base type: string

SearchConfig

FieldTypeDefaultDescription
indexerSearchIndexerConfig
{
“incrementalIndexing”: false,
“clearOnStart”: false
}
Indexer configuration
embeddingsChunkerEmbeddingsChunkerConfig
{
“kind”: “Simple”,
“splitSections”: false,
“splitParagraphs”: false
}
Embeddings chunker configuration
embeddingsEncoderEmbeddingsEncoderConfig
{
“kind”: “OpenAi”,
“modelName”: “text-embedding-ada-002”,
“dimensions”: 1536
}
Embeddings encoder configuration
repoSearchRepositoryConfig
{
“kind”: “Dummy”
}
Search repository configuration
semanticSearchThresholdScorenumber0.0

SearchIndexerConfig

FieldTypeDefaultDescription
incrementalIndexingbooleanfalseWhether incremental indexing is enabled
clearOnStartbooleanfalse

EmbeddingsChunkerConfig

Variants
Simple

EmbeddingsChunkerConfig::Simple

FieldTypeDefaultDescription
splitSectionsbooleanfalse
splitParagraphsbooleanfalse
kindstring

EmbeddingsEncoderConfig

Variants
OpenAi
Dummy

EmbeddingsEncoderConfig::OpenAi

FieldTypeDefaultDescription
urlstringnull
apiKeystringnull
modelNamestring”text-embedding-ada-002”
dimensionsinteger1536
kindstring

EmbeddingsEncoderConfig::Dummy

FieldTypeDefaultDescription
kindstring

SearchRepositoryConfig

Variants
Dummy
Elasticsearch

SearchRepositoryConfig::Dummy

FieldTypeDefaultDescription
kindstring

SearchRepositoryConfig::Elasticsearch

FieldTypeDefaultDescription
urlstring”http://localhost:9200”
passwordstringnull
caCertPemPathstringnull
indexPrefixstring”kamu-node”
timeoutSecsinteger30
enableCompressionbooleanfalse
embeddingDimensionsinteger1536
kindstring

QuotaConfig

FieldTypeDefaultDescription
accountQuotaAccountConfig{}

QuotaAccountConfig

FieldTypeDefaultDescription
defaultStorageLimitInBytesintegernull

ExtraConfig

FieldTypeDefaultDescription
graphqlGqlConfig{}

GqlConfig

FieldTypeDefaultDescription
Last modified on June 29, 2026