> ## Documentation Index > Fetch the complete documentation index at: https://docs.kamu.dev/llms.txt > Use this file to discover all available pages before exploring further. # Config Reference ## `ApiServerConfig`
FieldTypeDefaultDescription
auth AuthConfig 
\{
      "allowAnonymous": true,
      "didEncryption": \{
      "enabled": false
      },
      "jwtSecret": "",
      "passwordPolicy": \{
      "minNewPasswordLength": 8
      },
      "providers": \[]
      }
 Authentication & authorization
database DatabaseConfig 
\{
      "provider": "InMemory"
      }
 Database
datasetEnvVars DatasetEnvVarsConfig 
\{
      "enabled": false
      }
 Dataset environment variable feature
email EmailConfig 
\{
      "gateway": \{
      "kind": "Dummy"
      },
      "senderAddress": ""
      }
 Email gateway configuration
engine EngineConfig 
\{
      "datafusionEmbedded": \{
      "base": \{
      "datafusion.catalog.default\_catalog": "kamu",
      "datafusion.catalog.default\_schema": "kamu",
      "datafusion.catalog.information\_schema": "true",
      "datafusion.sql\_parser.enable\_ident\_normalization": "false"
      },
      "batchQuery": \{},
      "compaction": \{
      "datafusion.execution.target\_partitions": "1"
      },
      "ingest": \{
      "datafusion.execution.target\_partitions": "1"
      },
      "useLegacyArrowBufferEncoding": false
      },
      "images": \{
      "datafusion": "ghcr.io/kamu-data/engine-datafusion:0.9.0",
      "flink": "ghcr.io/kamu-data/engine-flink:0.18.2-flink\_1.16.0-scala\_2.12-java8",
      "risingwave": "ghcr.io/kamu-data/engine-risingwave:0.2.0-risingwave\_1.7.0-alpha",
      "spark": "ghcr.io/kamu-data/engine-spark:0.23.1-spark\_3.5.0"
      },
      "networkNs": "Private",
      "runtime": "Podman",
      "shutdownTimeout": "5s",
      "startTimeout": "30s"
      }
 Ingest and transform engines
extra ExtraConfig 
\{
      "graphql": \{}
      }
 Experimental and temporary module configuration
flowSystem FlowSystemConfig 
\{
      "flowAgent": \{
      "awaitingStepSecs": 1,
      "defaultRetryPolicies": \{},
      "mandatoryThrottlingPeriodSecs": 60
      },
      "flowSystemEventAgent": \{
      "batchSize": 100,
      "maxListeningTimeoutMs": 60000,
      "minDebounceIntervalMs": 100
      },
      "taskAgent": \{
      "taskCheckingIntervalSecs": 1
      }
      }
 Configuration for the flow system
identity IdentityConfig null UNSTABLE: Identity configuration
outbox OutboxConfig 
\{
      "awaitingStepSecs": 1,
      "batchSize": 20
      }
 Outbox configuration
protocol ProtocolConfig 
\{
      "flightSql": \{
      "allowAnonymous": true,
      "anonSessionExpirationTimeout": "5m",
      "anonSessionInactivityTimeout": "5s",
      "authedSessionExpirationTimeout": "30m",
      "authedSessionInactivityTimeout": "5s"
      },
      "ipfs": \{
      "httpGateway": "http\://localhost:8080/",
      "preResolveDnslink": true
      }
      }
 Protocols
quota QuotaConfig 
\{
      "account": \{}
      }
 Default quotas configured by type
repo RepoConfig 
\{
      "caching": \{
      "registryCacheEnabled": false
      }
      }
 Dataset repository
runtime RuntimeConfig \{} Tokio runtime
search SearchConfig 
\{
      "embeddingsChunker": \{
      "kind": "Simple",
      "splitParagraphs": false,
      "splitSections": false
      },
      "embeddingsEncoder": \{
      "dimensions": 1536,
      "kind": "OpenAi",
      "modelName": "text-embedding-ada-002"
      },
      "indexer": \{
      "clearOnStart": false,
      "incrementalIndexing": false
      },
      "repo": \{
      "kind": "Dummy"
      },
      "semanticSearchThresholdScore": 0.0
      }
 Search configuration
source SourceConfig 
\{
      "ethereum": \{
      "commitAfterBlocksScanned": 1000000,
      "getLogsBlockStride": 100000,
      "rpcEndpoints": \[],
      "useBlockTimestampFallback": false
      },
      "mqtt": \{
      "brokerIdleTimeoutMs": 1000
      },
      "targetRecordsPerSlice": 10000
      }
 Ingestion's sources
uploadRepo UploadRepoConfig 
\{
      "maxFileSizeMb": 50,
      "storage": \{
      "kind": "Local"
      }
      }
 File upload repository
url UrlConfig 
\{
      "baseUrlFlightsql": "grpc://localhost:50050",
      "baseUrlPlatform": "http\://localhost:4200/",
      "baseUrlRest": "http\://localhost:8080/"
      }
 External URLs
webhooks WebhooksConfig 
\{
      "deliveryTimeoutSecs": 10,
      "maxConsecutiveFailures": 5,
      "secretEncryptionEnabled": false
      }
 Configuration for webhooks
## `AccountConfig`
FieldTypeDefaultDescription
accountName AccountName
accountType AccountType "User"
avatarUrl string null
displayName string null Auto-derived from `account_name` if omitted
email Email
id AccountID null Auto-derived from `account_name` if omitted
password Password
properties array \[]
provider string "password"
registeredAt string null
treatDatasetsAsPublic boolean false
## `AccountID` Base type: `string` ## `AccountName` Base type: `string` ## `AccountPropertyName`
Variants
CanProvisionAccounts
Admin
## `AccountType`
Variants
User
Organization
## `AuthConfig`
FieldTypeDefaultDescription
allowAnonymous boolean true
didEncryption DidSecretEncryptionConfig 
\{
      "enabled": false
      }
jwtSecret string ""
passwordPolicy PasswordPolicyConfig 
\{
      "minNewPasswordLength": 8
      }
providers array \[]
## `AuthProviderConfig`
Variants
Github
Password
## `AuthProviderConfig::Github`
FieldTypeDefaultDescription
clientId string
clientSecret string
kind string
## `AuthProviderConfig::Password`
FieldTypeDefaultDescription
accounts array \[]
kind string
## `ContainerRuntimeType`
Variants
Docker
Podman
## `DatabaseConfig`
Variants
InMemory
Sqlite
Postgres
## `DatabaseConfig::InMemory`
FieldTypeDefaultDescription
provider string
## `DatabaseConfig::Sqlite`
FieldTypeDefaultDescription
databasePath string
provider string
## `DatabaseConfig::Postgres`
FieldTypeDefaultDescription
acquireTimeoutSecs integer null
credentialsPolicy DatabaseCredentialsPolicyConfig
databaseName string
host string
maxConnections integer null
maxLifetimeSecs integer null
port integer null
provider string
## `DatabaseCredentialSourceConfig`
Variants
RawPassword
AwsSecret
AwsIamToken
## `DatabaseCredentialSourceConfig::RawPassword`
FieldTypeDefaultDescription
kind string
rawPassword string
userName string
## `DatabaseCredentialSourceConfig::AwsSecret`
FieldTypeDefaultDescription
kind string
secretName string
## `DatabaseCredentialSourceConfig::AwsIamToken`
FieldTypeDefaultDescription
kind string
userName string
## `DatabaseCredentialsPolicyConfig`
FieldTypeDefaultDescription
rotationFrequencyInMinutes integer null
source DatabaseCredentialSourceConfig
## `DatasetEnvVarsConfig`
FieldTypeDefaultDescription
enabled boolean false
encryptionKey string null Represents the encryption key for the dataset env vars. This field is required if `enabled` is `true` or `None`. The encryption key must be a 32-character alphanumeric string, which includes both uppercase and lowercase Latin letters (A-Z, a-z) and digits (0-9). To generate use: ```sh theme={null} tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 32; echo ```
## `DidSecretEncryptionConfig`
FieldTypeDefaultDescription
enabled boolean false
encryptionKey string null The encryption key must be a 32-character alphanumeric string, which includes both uppercase and lowercase Latin letters (A-Z, a-z) and digits (0-9). To generate use: ```sh theme={null} tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 32; echo ```
## `DurationString` Base type: `string` ## `Email` Base type: `string` ## `EmailConfig`
FieldTypeDefaultDescription
gateway EmailConfigGateway
senderAddress string
senderName string null
## `EmailConfigGateway`
Variants
Dummy
Postmark
## `EmailConfigGateway::Dummy`
FieldTypeDefaultDescription
kind string
## `EmailConfigGateway::Postmark`
FieldTypeDefaultDescription
apiKey string
kind string
## `EmbeddingsChunkerConfig`
Variants
Simple
## `EmbeddingsChunkerConfig::Simple`
FieldTypeDefaultDescription
kind string
splitParagraphs boolean false
splitSections boolean false
## `EmbeddingsEncoderConfig`
Variants
OpenAi
Dummy
## `EmbeddingsEncoderConfig::OpenAi`
FieldTypeDefaultDescription
apiKey string null
dimensions integer 1536
kind string
modelName string "text-embedding-ada-002"
url string null
## `EmbeddingsEncoderConfig::Dummy`
FieldTypeDefaultDescription
kind string
## `EngineConfig`
FieldTypeDefaultDescription
datafusionEmbedded EngineConfigDatafusion 
\{
      "base": \{
      "datafusion.catalog.default\_catalog": "kamu",
      "datafusion.catalog.default\_schema": "kamu",
      "datafusion.catalog.information\_schema": "true",
      "datafusion.sql\_parser.enable\_ident\_normalization": "false"
      },
      "batchQuery": \{},
      "compaction": \{
      "datafusion.execution.target\_partitions": "1"
      },
      "ingest": \{
      "datafusion.execution.target\_partitions": "1"
      },
      "useLegacyArrowBufferEncoding": false
      }
 Embedded Datafusion engine configuration
images EngineImagesConfig 
\{
      "datafusion": "ghcr.io/kamu-data/engine-datafusion:0.9.0",
      "flink": "ghcr.io/kamu-data/engine-flink:0.18.2-flink\_1.16.0-scala\_2.12-java8",
      "risingwave": "ghcr.io/kamu-data/engine-risingwave:0.2.0-risingwave\_1.7.0-alpha",
      "spark": "ghcr.io/kamu-data/engine-spark:0.23.1-spark\_3.5.0"
      }
 UNSTABLE: Default engine images
maxConcurrency integer null Maximum number of engine operations that can be performed concurrently
networkNs NetworkNamespaceType "Private" Type of the networking namespace (relevant when running in container environments)
runtime ContainerRuntimeType "Podman" Type of the runtime to use when running the data processing engines
shutdownTimeout DurationString "5s" Timeout for waiting the engine container to stop gracefully
startTimeout DurationString "30s" Timeout for starting an engine container
## `EngineConfigDatafusion`
FieldTypeDefaultDescription
base object 
\{
      "datafusion.catalog.default\_catalog": "kamu",
      "datafusion.catalog.default\_schema": "kamu",
      "datafusion.catalog.information\_schema": "true",
      "datafusion.sql\_parser.enable\_ident\_normalization": "false"
      }
 Base configuration options See: ``
batchQuery object \{} Batch query-specific overrides to the base config
compaction object 
\{
      "datafusion.execution.target\_partitions": "1"
      }
 Compaction-specific overrides to the base config
ingest object 
\{
      "datafusion.execution.target\_partitions": "1"
      }
 Ingest-specific overrides to the base config
useLegacyArrowBufferEncoding boolean false Makes arrow batches use contiguous `Binary` and `Utf8` encodings instead of more modern `BinaryView` and `Utf8View`. This is only needed for compatibility with some older libraries that don't yet support them. See: [kamu-node#277](https://github.com/kamu-data/kamu-node/issues/277)
## `EngineImagesConfig`
FieldTypeDefaultDescription
datafusion string "ghcr.io/kamu-data/engine-datafusion:0.9.0" UNSTABLE: Datafusion engine image
flink string "ghcr.io/kamu-data/engine-flink:0.18.2-flink\_1.16.0-scala\_2.12-java8" UNSTABLE: Flink engine image
risingwave string "ghcr.io/kamu-data/engine-risingwave:0.2.0-risingwave\_1.7.0-alpha" UNSTABLE: RisingWave engine image
spark string "ghcr.io/kamu-data/engine-spark:0.23.1-spark\_3.5.0" UNSTABLE: Spark engine image
## `EthRpcEndpoint`
FieldTypeDefaultDescription
chainId integer
chainName string
nodeUrl string
## `EthereumSourceConfig`
FieldTypeDefaultDescription
commitAfterBlocksScanned integer 1000000 Forces iteration to stop after the specified number of blocks were scanned even if we didn't reach the target record number. This is useful to not lose a lot of scanning progress in case of an RPC error.
getLogsBlockStride integer 100000 Default number of blocks to scan within one query to `eth_getLogs` RPC endpoint.
rpcEndpoints array \[] Default RPC endpoints to use if source does not specify one explicitly.
useBlockTimestampFallback boolean false Many providers don't yet return `blockTimestamp` from `eth_getLogs` RPC endpoint and in such cases `block_timestamp` column will be `null`. If you enable this fallback the library will perform additional call to `eth_getBlock` to populate the timestam, but this may result in significant performance penalty when fetching many log records. See: [ethereum/execution-apis#295](https://github.com/ethereum/execution-apis/issues/295)
## `ExtraConfig`
FieldTypeDefaultDescription
graphql GqlConfig \{}
## `FlightSqlConfig`
FieldTypeDefaultDescription
allowAnonymous boolean true Whether clients can authenticate as 'anonymous' user
anonSessionExpirationTimeout DurationString "5m" Time after which FlightSQL client session will be forgotten and client will have to re-authroize (for anonymous clients)
anonSessionInactivityTimeout DurationString "5s" Time after which FlightSQL session context will be released to free the resources (for anonymous clients)
authedSessionExpirationTimeout DurationString "30m" Time after which FlightSQL client session will be forgotten and client will have to re-authroize (for authenticated clients)
authedSessionInactivityTimeout DurationString "5s" Time after which FlightSQL session context will be released to free the resources (for authenticated clients)
## `FlowAgentConfig`
FieldTypeDefaultDescription
awaitingStepSecs integer 1
defaultRetryPolicies object \{}
mandatoryThrottlingPeriodSecs integer 60
## `FlowSystemConfig`
FieldTypeDefaultDescription
flowAgent FlowAgentConfig 
\{
      "awaitingStepSecs": 1,
      "defaultRetryPolicies": \{},
      "mandatoryThrottlingPeriodSecs": 60
      }
flowSystemEventAgent FlowSystemEventAgentConfig 
\{
      "batchSize": 100,
      "maxListeningTimeoutMs": 60000,
      "minDebounceIntervalMs": 100
      }
taskAgent TaskAgentConfig 
\{
      "taskCheckingIntervalSecs": 1
      }
## `FlowSystemEventAgentConfig`
FieldTypeDefaultDescription
batchSize integer 100
maxListeningTimeoutMs integer 60000
minDebounceIntervalMs integer 100
## `GqlConfig`
FieldTypeDefaultDescription
## `IdentityConfig`
FieldTypeDefaultDescription
privateKey PrivateKey null Private key used to sign API responses. Currently only `ed25519` keys are supported. To generate use: dd if=/dev/urandom bs=1 count=32 status=none | base64 -w0 | tr '+/' '-\_' | tr -d '=' | (echo -n u && cat) The command above: * reads 32 random bytes * base64-encodes them * converts default base64 encoding to base64url and removes padding * prepends a multibase prefix
## `IpfsConfig`
FieldTypeDefaultDescription
httpGateway string "http\://localhost:8080/" HTTP Gateway URL to use for downloads. For safety, it defaults to `http://localhost:8080` - a local IPFS daemon. If you don't have IPFS installed, you can set this URL to one of the public gateways like `https://ipfs.io`. List of public gateways can be found here: `https://ipfs.github.io/public-gateway-checker/`
preResolveDnslink boolean true Whether kamu should pre-resolve IPNS DNSLink names using DNS or leave it to the Gateway.
## `MqttSourceConfig`
FieldTypeDefaultDescription
brokerIdleTimeoutMs integer 1000 Time in milliseconds to wait for MQTT broker to send us some data after which we will consider that we have "caught up" and end the polling loop.
## `NetworkNamespaceType` Corresponds to podman's `containers.conf::netns` We podman is used inside containers (e.g. podman-in-docker or podman-in-k8s) it usually runs uses host network namespace.
Variants
Private
Host
## `OutboxConfig`
FieldTypeDefaultDescription
awaitingStepSecs integer 1
batchSize integer 20
## `Password` Base type: `string` ## `PasswordPolicyConfig`
FieldTypeDefaultDescription
minNewPasswordLength integer 8
## `PrivateKey` Base type: `string` ## `ProtocolConfig`
FieldTypeDefaultDescription
flightSql FlightSqlConfig 
\{
      "allowAnonymous": true,
      "anonSessionExpirationTimeout": "5m",
      "anonSessionInactivityTimeout": "5s",
      "authedSessionExpirationTimeout": "30m",
      "authedSessionInactivityTimeout": "5s"
      }
 FlightSQL configuration
ipfs IpfsConfig 
\{
      "httpGateway": "http\://localhost:8080/",
      "preResolveDnslink": true
      }
 IPFS configuration
## `QuotaAccountConfig`
FieldTypeDefaultDescription
defaultStorageLimitInBytes integer null
## `QuotaConfig`
FieldTypeDefaultDescription
account QuotaAccountConfig \{}
## `RepoCachingConfig`
FieldTypeDefaultDescription
metadataLocalFsCachePath string null
registryCacheEnabled boolean false
## `RepoConfig`
FieldTypeDefaultDescription
caching RepoCachingConfig 
\{
      "registryCacheEnabled": false
      }
dataBlocksPageSize integer null
repoUrl UrlOrPath null
## `RetryPolicyConfig`
FieldTypeDefaultDescription
backoffType RetryPolicyConfigBackoffType null
maxAttempts integer null
minDelaySecs integer null
## `RetryPolicyConfigBackoffType`
Variants
Fixed
Linear
Exponential
ExponentialWithJitter
## `RuntimeConfig`
FieldTypeDefaultDescription
maxBlockingThreads integer null
threadStackSize integer null
workerThreads integer null
## `SearchConfig`
FieldTypeDefaultDescription
embeddingsChunker EmbeddingsChunkerConfig 
\{
      "kind": "Simple",
      "splitParagraphs": false,
      "splitSections": false
      }
 Embeddings chunker configuration
embeddingsEncoder EmbeddingsEncoderConfig 
\{
      "dimensions": 1536,
      "kind": "OpenAi",
      "modelName": "text-embedding-ada-002"
      }
 Embeddings encoder configuration
indexer SearchIndexerConfig 
\{
      "clearOnStart": false,
      "incrementalIndexing": false
      }
 Indexer configuration
repo SearchRepositoryConfig 
\{
      "kind": "Dummy"
      }
 Search repository configuration
semanticSearchThresholdScore number 0.0
## `SearchIndexerConfig`
FieldTypeDefaultDescription
clearOnStart boolean false
incrementalIndexing boolean false Whether incremental indexing is enabled
## `SearchRepositoryConfig`
Variants
Dummy
Elasticsearch
## `SearchRepositoryConfig::Dummy`
FieldTypeDefaultDescription
kind string
## `SearchRepositoryConfig::Elasticsearch`
FieldTypeDefaultDescription
caCertPemPath string null
embeddingDimensions integer 1536
enableCompression boolean false
indexPrefix string "kamu-node"
kind string
password string null
timeoutSecs integer 30
url string "http\://localhost:9200"
## `SourceConfig`
FieldTypeDefaultDescription
ethereum EthereumSourceConfig 
\{
      "commitAfterBlocksScanned": 1000000,
      "getLogsBlockStride": 100000,
      "rpcEndpoints": \[],
      "useBlockTimestampFallback": false
      }
 Ethereum-specific configuration
mqtt MqttSourceConfig 
\{
      "brokerIdleTimeoutMs": 1000
      }
 MQTT-specific configuration
targetRecordsPerSlice integer 10000 Target number of records after which we will stop consuming from the resumable source and commit data, leaving the rest for the next iteration. This ensures that one data slice doesn't become too big.
## `TaskAgentConfig`
FieldTypeDefaultDescription
taskCheckingIntervalSecs integer 1
## `UploadRepoConfig`
FieldTypeDefaultDescription
maxFileSizeMb integer 50
storage UploadRepoStorageConfig 
\{
      "kind": "Local"
      }
## `UploadRepoStorageConfig`
Variants
S3
Local
## `UploadRepoStorageConfig::S3`
FieldTypeDefaultDescription
bucketS3Url string
kind string
## `UploadRepoStorageConfig::Local`
FieldTypeDefaultDescription
kind string
## `UrlConfig`
FieldTypeDefaultDescription
baseUrlFlightsql UrlOrPath "grpc://localhost:50050"
baseUrlPlatform UrlOrPath "http\://localhost:4200/"
baseUrlRest UrlOrPath "http\://localhost:8080/"
## `UrlOrPath` Base type: `string` ## `WebhooksConfig`
FieldTypeDefaultDescription
deliveryTimeoutSecs integer 10
maxConsecutiveFailures integer 5
secretEncryptionEnabled boolean false
secretEncryptionKey string null Represents the encryption key for the webhooks secret. This field is required if `secret_encryption_enabled` is `true` or `None`. The encryption key must be a 32-character alphanumeric string, which includes both uppercase and lowercase Latin letters (A-Z, a-z) and digits (0-9). # Example let config = WebhooksConfig \{ ... secret\_encryption\_enabled: Some(true), encryption\_key: Some(String::from("aBcDeFgHiJkLmNoPqRsTuVwXyZ012345")) }; \`\`\`