Introduces a sourceState field into AddData event to store the state of the generic data source from which the data is being added, allowing for incremental ingestion.
ODF implementations often need to store the state of the data source from which data is ingested to avoid expensive recomputations.For example when using SetPollingSource source state can be in the form of:
ETag of Last-Modified header for HTTP sources
filename of the last file matched by glob pattern
or a height of the block of indexed blockchain.
Currently, we don’t have anywhere to store such data.
When ingesting data, an ODF implementation will be able to attach opaque state data to the AddData event in order to resume ingestion most efficiently on the next iteration. The source state data will allow differentiating the kind of state that is preserved (similarly to MIME type) and will specify the identity of the source.
New SourceState metadata fragment will be introduced.The schema of AddData metadata event will:
be extended with an optional sourceState field
have outputData field made optional (as it’s possible for ingest to produce no new data but update source state, watermark, or checkpoint)
Two predefined source state kinds will be added:
odf/etag - for state identifiers similar to ETag HTTP header
odf/last-modified - for RFC3338 timestamps with meaning similar to Last-Modified HTTP header
One predefined source ID will be added:
odf/polling - referring to the source specified in the SetPollingSource metadata event
Both fields will be plain strings and not enums allowing different implementations to define their own extensions.It should be safe to ignore the source state that an implementation does not understand.Example SourceState:
The source identity (source field) should allow us to support multiple concurrent ingestion sources if needed
Last modified on October 15, 2026
{"docsConfig":{"theme":"mint","$schema":"https://mintlify.com/docs.json","name":"Kamu Docs","colors":{"primary":"#ec6e2a","light":"#ec6e2a","dark":"#ec6e2a"},"logo":{"light":"https://mintcdn.com/kamu/j1tx5fgzyFHd1DYS/images/kamu-logo-light.svg?fit=max\u0026auto=format\u0026n=j1tx5fgzyFHd1DYS\u0026q=85\u0026s=a09261b617da6e1515dea3d28495f633","dark":"https://mintcdn.com/kamu/j1tx5fgzyFHd1DYS/images/kamu-logo-dark.svg?fit=max\u0026auto=format\u0026n=j1tx5fgzyFHd1DYS\u0026q=85\u0026s=8c6d0d7ffa23ef050859453f64354192"},"favicon":"https://mintcdn.com/kamu/j1tx5fgzyFHd1DYS/images/favicon.svg?fit=max\u0026auto=format\u0026n=j1tx5fgzyFHd1DYS\u0026q=85\u0026s=a7b01f8f352427f5a4afe3aef1f8a9d3","background":{"color":{"dark":"#050C1E"}},"navbar":{"links":[{"href":"https://discord.gg/nU6TXRQNXC","label":"Get Help"}],"primary":{"type":"button","label":"Install","href":"/cli/install/"}},"footer":{"socials":{"github":"https://github.com/kamu-data","discord":"https://discord.gg/nU6TXRQNXC","youtube":"https://www.youtube.com/@kamudata","linkedin":"https://www.linkedin.com/company/kamu-data","medium":"https://medium.com/kamu-data","bluesky":"https://bsky.app/profile/kamu.dev","x":"https://x.com/kamu_data"},"links":[{"header":"Company","items":[{"label":"Blog","href":"https://kamu.dev/blog"},{"label":"Careers","href":"https://kamu.dev/jobs"}]}]},"seo":{"metatags":{"canonical":"https://docs.kamu.dev"}},"styling":{"eyebrows":"breadcrumbs","latex":false},"contextual":{"options":["copy","view","chatgpt","claude","perplexity","mcp","cursor","vscode"],"display":"header"},"metadata":{"timestamp":true},"navigation":{"global":{"anchors":[]},"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"kamu","hasAudioNarration":false,"actualSubdomain":"kamu","customDomains":["docs.kamu.dev"],"gitSource":{"type":"github","owner":"kamu-data","repo":"kamu-docs","deployBranch":"master","contentDirectory":"","isPrivate":true,"mountPath":"","label":""},"chroma":{"liveDeploymentHistoryId":"6ad05633011b02bffcb4463d","collectionId":"0127742b-6059-4668-b046-f443923b1bdd","schemaVersion":"v3","latestBuiltDeploymentHistoryId":"6ad05633011b02bffcb4463d"},"entitlements":{},"buildId":"6ad05633011b02bffcb4463d","clientVersion":"0.0.4044","searchFilterCounts":[{"language":"default","tag":"Guides","count":75},{"language":"default","tag":"Reference","count":42},{"language":"default","tag":"Open Data Fabric","count":39},{"language":"default","tag":"Kamu CLI","count":31},{"language":"default","tag":"RFCs","count":22},{"language":"default","tag":"Kamu Node","count":17},{"language":"default","tag":"Specification","count":15},{"language":"default","tag":"APIs","count":13},{"language":"default","tag":"Get Started","count":12},{"language":"default","tag":"Datasets","count":10}]},"assistantConfig":{"enableAskAiButton":false,"enableStarterQuestions":false,"enableContextualStarterQuestions":false,"enableAskAiSearchBar":true},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"byok-sep-25","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"vite-origin","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-updated-9-15","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"sso-setup-deadline-banner","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"googleFontsInlined":false},"customCss":[{"filePath":"style.css","content":"a.card:is(.dark *),div.card:is(.dark *){background:#09162d}a.card:is(.light *),div.card:is(.light *){background:#fafafa}a:has(svg[style*=\"wrench.svg\"]){pointer-events:none;opacity:.7}button:has(svg[style*=\"wrench.svg\"]){opacity:.7}a:has(svg[style*=\"wrench.svg\"])>div:first-child,button:has(svg[style*=\"wrench.svg\"])>svg[style*=\"wrench.svg\"]{order:2;margin-left:auto}.prose a.glossary-term:is(.dark *, .light *){color:inherit;font-weight:inherit;border-bottom:2px dotted #325495}.prose a.schema-object:is(.dark *, .light *){color:inherit;font-weight:inherit;border-bottom:2px dotted #2c802c}h4.project-title:is(.dark *, .light *){display:block;text-align:center}div.badges a,h4.project-title a{border-bottom:0}a.project-quicklink:is(.dark *, .light *){font-size:15px;font-weight:300;color:#ec6e2a;border-bottom:0}.prose div img:is(\n [src*=\"projects/odf.svg\"],\n [src*=\"projects/kamu-cli.svg\"],\n [src*=\"projects/kamu-node.svg\"],\n [src*=\"projects/kamu-platform.svg\"]\n ){display:block;margin:30px auto 0;height:100px;width:auto}div.badges{display:flex;flex-direction:row;flex-wrap:wrap;justify-content:center;gap:10px;margin:20px}div.badges img{display:inline;margin:0}"}],"page":{"pagePath":"odf/rfcs/009-ingest-source-state","slug":"/odf/rfcs/009-ingest-source-state","pageMetadata":{"title":"RFC-009: Ingestion Source State","description":null,"href":"/odf/rfcs/009-ingest-source-state"},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];const YouTubeList=({id})=>{const src=`https://www.youtube.com/embed/videoseries?list=${id}`;return _jsx(`iframe`,{className:`mint-w-full mint-aspect-video mint-rounded-xl`,src,allow:`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture`,allowFullScreen:true})};const YouTube=({id,width})=>{const src=`https://www.youtube.com/embed/${id}`;return _jsx(`iframe`,{className:`mint-w-full mint-aspect-video mint-rounded-xl`,src,allow:`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture`,allowFullScreen:true,width})};const Schema=({t,id})=>{const _components={a:`a`,..._provideComponents()};const anchor=id?id:t.toLowerCase().replace(/\\s+/g,`-`);const link=`/odf/schemas#${anchor}`;return _jsx(_components.a,{class:`schema-object`,href:link,children:t})};const Term=({t,id})=>{const _components={a:`a`,..._provideComponents()};const anchor=id?id:t.toLowerCase().replace(/\\s+/g,`-`);const link=`/general/glossary#${anchor}`;return _jsx(_components.a,{class:`glossary-term`,href:link,children:t})};const Diagram=({src,alt})=>{return _jsxs(`div`,{style:{display:`flex`,\"flex-direction\":`column`,\"align-items\":`center`},children:[_jsx(`img`,{src,alt,style:{background:`#dddddd`,\"margin-bottom\":0}}),_jsx(`span`,{children:alt})]})};function _createMdxContent(props){const _components={a:`a`,code:`code`,img:`img`,li:`li`,p:`p`,pre:`pre`,span:`span`,strong:`strong`,ul:`ul`,..._provideComponents(),...props.components},{CodeBlock,Heading}=_components;if(!CodeBlock)_missingMdxReference(`CodeBlock`,true);if(!Heading)_missingMdxReference(`Heading`,true);return _jsxs(_Fragment,{children:[_jsxs(_components.p,{children:[_jsx(_components.strong,{children:`Start Date`}),`: 2023-04-28`]}),`\n`,_jsxs(_components.p,{children:[_jsx(_components.strong,{children:`Specs`}),`:`]}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsx(_components.li,{children:_jsx(_components.a,{href:`/odf/spec/datasets/ingestion`,children:`Ingestion`})}),`\n`]}),`\n`,_jsx(_components.p,{children:_jsx(_components.a,{href:`https://github.com/kamu-data/open-data-fabric/issues/50`,children:_jsx(_components.img,{src:`https://img.shields.io/github/issues/detail/state/kamu-data/open-data-fabric/50?label=RFC%20Status`,alt:`RFC`})})}),`\n`,_jsx(_components.p,{children:_jsx(_components.a,{href:`https://github.com/kamu-data/open-data-fabric/pull/51`,children:_jsx(_components.img,{src:`https://img.shields.io/github/pulls/detail/state/kamu-data/open-data-fabric/51?label=Spec%20PR`,alt:`Spec PR`})})}),`\n`,_jsxs(_components.p,{children:[_jsx(_components.strong,{children:`Compatibility`}),`: Backwards-compatible, forwards-incompatible`]}),`\n`,_jsx(Heading,{level:`2`,id:`summary`,children:`Summary`}),`\n`,_jsxs(_components.p,{children:[`Introduces a `,_jsx(_components.code,{children:`sourceState`}),` field into `,_jsx(_components.code,{children:`AddData`}),` event to store the state of the generic data source from which the data is being added, allowing for incremental ingestion.`]}),`\n`,_jsx(Heading,{level:`2`,id:`motivation`,children:`Motivation`}),`\n`,_jsx(_components.p,{children:`ODF implementations often need to store the state of the data source from which data is ingested to avoid expensive recomputations.`}),`\n`,_jsxs(_components.p,{children:[`For example when using `,_jsx(_components.code,{children:`SetPollingSource`}),` source state can be in the form of:`]}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsx(_components.li,{children:`ETag of Last-Modified header for HTTP sources`}),`\n`,_jsx(_components.li,{children:`filename of the last file matched by glob pattern`}),`\n`,_jsx(_components.li,{children:`or a height of the block of indexed blockchain.`}),`\n`]}),`\n`,_jsx(_components.p,{children:`Currently, we don’t have anywhere to store such data.`}),`\n`,_jsx(Heading,{level:`2`,id:`guide-level-explanation`,children:`Guide-level explanation`}),`\n`,_jsxs(_components.p,{children:[`When ingesting data, an ODF implementation will be able to attach opaque state data to the `,_jsx(_components.code,{children:`AddData`}),` event in order to resume ingestion most efficiently on the next iteration. The source state data will allow differentiating the kind of state that is preserved (similarly to MIME type) and will specify the identity of the source.`]}),`\n`,_jsx(Heading,{level:`2`,id:`reference-level-explanation`,children:`Reference-level explanation`}),`\n`,_jsxs(_components.p,{children:[`New `,_jsx(_components.code,{children:`SourceState`}),` metadata fragment will be introduced.`]}),`\n`,_jsxs(_components.p,{children:[`The schema of `,_jsx(_components.code,{children:`AddData`}),` metadata event will:`]}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[`be extended with an optional `,_jsx(_components.code,{children:`sourceState`}),` field`]}),`\n`,_jsxs(_components.li,{children:[`have `,_jsx(_components.code,{children:`outputData`}),` field made optional (as it’s possible for ingest to produce no new data but update source state, watermark, or checkpoint)`]}),`\n`]}),`\n`,_jsx(_components.p,{children:`Two predefined source state kinds will be added:`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.code,{children:`odf/etag`}),` - for state identifiers similar to `,_jsx(_components.code,{children:`ETag`}),` HTTP header`]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.code,{children:`odf/last-modified`}),` - for RFC3338 timestamps with meaning similar to `,_jsx(_components.code,{children:`Last-Modified`}),` HTTP header`]}),`\n`]}),`\n`,_jsx(_components.p,{children:`One predefined source ID will be added:`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.code,{children:`odf/polling`}),` - referring to the source specified in the `,_jsx(_components.code,{children:`SetPollingSource`}),` metadata event`]}),`\n`]}),`\n`,_jsx(_components.p,{children:`Both fields will be plain strings and not enums allowing different implementations to define their own extensions.`}),`\n`,_jsxs(_components.p,{children:[`It should be `,_jsx(_components.strong,{children:`safe to ignore`}),` the source state that an implementation does not understand.`]}),`\n`,_jsxs(_components.p,{children:[`Example `,_jsx(_components.code,{children:`SourceState`}),`:`]}),`\n`,_jsx(CodeBlock,{filename:``,numberOfLines:`4`,language:`yaml`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`yaml`,children:_jsxs(_components.code,{language:`yaml`,numberOfLines:`4`,children:[_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#116329`,\"--shiki-dark\":`#569CD6`},children:`sourceState`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`:`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#116329`,\"--shiki-dark\":`#569CD6`},children:` kind`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`odf/etag`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#116329`,\"--shiki-dark\":`#569CD6`},children:` source`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`odf/polling`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#116329`,\"--shiki-dark\":`#569CD6`},children:` value`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`W/\"95c5fde3918cba7c33eaac7ff9d02b22\"`})]}),`\n`]})})}),`\n`,_jsx(Heading,{level:`2`,id:`compatibility`,children:`Compatibility`}),`\n`,_jsxs(_components.p,{children:[`This change will be fully backwards-compatible. It will be forwards-incompatible due to making of `,_jsx(_components.code,{children:`AddData::outputData`}),` field optional.`]}),`\n`,_jsx(Heading,{level:`2`,id:`drawbacks`,children:`Drawbacks`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsx(_components.li,{children:`More complexity in metadata`}),`\n`]}),`\n`,_jsx(Heading,{level:`2`,id:`alternatives`,children:`Alternatives`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[`Store source state as part of the ingest checkpoint:`,`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[`Would require storing source state alongside engine checkpoints within one file (e.g. by `,_jsx(_components.code,{children:`tar`}),`’ing them together)`]}),`\n`,_jsx(_components.li,{children:`Would complicate and slows down engine checkpoint extraction`}),`\n`,_jsx(_components.li,{children:`Would only work for a single source`}),`\n`]}),`\n`]}),`\n`]}),`\n`,_jsx(Heading,{level:`2`,id:`prior-art`,children:`Prior art`}),`\n`,_jsx(_components.p,{children:`N/A`}),`\n`,_jsx(Heading,{level:`2`,id:`unresolved-questions`,children:`Unresolved questions`}),`\n`,_jsx(_components.p,{children:`N/A`}),`\n`,_jsx(Heading,{level:`2`,id:`future-possibilities`,children:`Future possibilities`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[`The source identity (`,_jsx(_components.code,{children:`source`}),` field) should allow us to support multiple concurrent ingestion sources if needed`]}),`\n`]})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{YouTubeList,YouTube,Schema,Term,Diagram,default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"RFC-009: Ingestion Source State","description":null,"href":"/odf/rfcs/009-ingest-source-state"}}},"mdxSourceWithNoJs":null,"noJsSplices":[[147,1427,")=>null;const YouTube=()=>null;const Schema=()=>null;const Term=()=>null;const Diagram=()=>null;func"]]},"mdxExtracts":{"tableOfContents":[{"title":"Summary","slug":"summary","depth":2,"children":[]},{"title":"Motivation","slug":"motivation","depth":2,"children":[]},{"title":"Guide-level explanation","slug":"guide-level-explanation","depth":2,"children":[]},{"title":"Reference-level explanation","slug":"reference-level-explanation","depth":2,"children":[]},{"title":"Compatibility","slug":"compatibility","depth":2,"children":[]},{"title":"Drawbacks","slug":"drawbacks","depth":2,"children":[]},{"title":"Alternatives","slug":"alternatives","depth":2,"children":[]},{"title":"Prior art","slug":"prior-art","depth":2,"children":[]},{"title":"Unresolved questions","slug":"unresolved-questions","depth":2,"children":[]},{"title":"Future possibilities","slug":"future-possibilities","depth":2,"children":[]}],"codeExamples":{}},"jsLinks":[],"dynamicCss":".mint-aspect-video {\n aspect-ratio: 16 / 9\n}\n.mint-w-full {\n width: 100%\n}\n.mint-rounded-xl {\n border-radius: 0.75rem\n}","subdomain":"kamu","siteName":"Kamu Docs","title":"RFC-009: Ingestion Source State - Kamu Docs","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"lastModified":"2026-10-15T04:13:42.409Z","gitLastModified":"2026-10-15T04:13:20.000Z","originalFileLocation":"odf/rfcs/009-ingest-source-state.md"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"application-name\" content=\"Kamu Docs\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon/android-chrome-192x192.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon/apple-touch-icon.png\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon/favicon-16x16.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon/favicon-32x32.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon/favicon.ico\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon-dark/android-chrome-192x192.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon-dark/favicon-16x16.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon-dark/favicon-32x32.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon-dark/favicon.ico\" media=\"(prefers-color-scheme: dark)\">\u003cmeta data-mint-head property=\"og:title\" content=\"RFC-009: Ingestion Source State - Kamu Docs\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"Kamu Docs\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://kamu.mintlify.app/_mintlify/api/og/v2?division=RFCs\u0026amp;title=RFC-009%3A+Ingestion+Source+State\u0026amp;theme=4425de33767c8dc7c1d6072e\u0026amp;signature=a598cd7ef14d7528eafa02de8592f65db980757837b3b0f212697d75bc4460aa\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"RFC-009: Ingestion Source State - Kamu Docs\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://kamu.mintlify.app/_mintlify/api/og/v2?division=RFCs\u0026amp;title=RFC-009%3A+Ingestion+Source+State\u0026amp;theme=4425de33767c8dc7c1d6072e\u0026amp;signature=a598cd7ef14d7528eafa02de8592f65db980757837b3b0f212697d75bc4460aa\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"/mintlify-assets/_mintlify/favicons/kamu/FxhmdHpAfy0cwcpV/_generated/favicon/browserconfig.xml\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"Kamu Docs\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#ec6e2a\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/odf/rfcs/009-ingest-source-state.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.kamu.dev/#organization\",\"name\":\"Kamu Docs\",\"url\":\"https://docs.kamu.dev\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintcdn.com/kamu/j1tx5fgzyFHd1DYS/images/kamu-logo-light.svg?fit=max\u0026auto=format\u0026n=j1tx5fgzyFHd1DYS\u0026q=85\u0026s=a09261b617da6e1515dea3d28495f633\"}},{\"@type\":\"WebSite\",\"@id\":\"https://docs.kamu.dev#website\",\"name\":\"Kamu Docs\",\"url\":\"https://docs.kamu.dev\",\"publisher\":{\"@id\":\"https://docs.kamu.dev/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state#webpage\",\"url\":\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state\",\"name\":\"RFC-009: Ingestion Source State\",\"dateModified\":\"2026-10-15T04:13:20.000Z\",\"isPartOf\":{\"@id\":\"https://docs.kamu.dev#website\"},\"breadcrumb\":{\"@id\":\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Open Data Fabric\",\"item\":\"https://docs.kamu.dev/odf/about\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"RFCs\",\"item\":\"https://docs.kamu.dev/odf/rfcs/index\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"RFC-009: Ingestion Source State\",\"item\":\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state\"}]},{\"@type\":[\"Article\",\"TechArticle\"],\"@id\":\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state#article\",\"headline\":\"RFC-009: Ingestion Source State\",\"name\":\"RFC-009: Ingestion Source State\",\"url\":\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state\",\"mainEntityOfPage\":{\"@id\":\"https://docs.kamu.dev/odf/rfcs/009-ingest-source-state#webpage\"},\"dateModified\":\"2026-10-15T04:13:20.000Z\",\"publisher\":{\"@id\":\"https://docs.kamu.dev/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.kamu.dev#website\"}}]}\u003c/script>\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"Kamu Docs\",\"creator\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","siteName":"Kamu Docs","nav":{"global":{"anchors":[]},"tabs":[{"tab":"Guides","icon":"person-chalkboard","pages":[{"title":"Guides","href":"/start/welcome"}]},{"tab":"Reference","icon":"book-open-cover","groups":[{"group":"Open Data Fabric","pages":[{"title":null,"href":"/odf/about"},{"group":"Specification","root":{"title":"Overview","href":"/odf/spec/overview"},"pages":[{"group":"Datasets","pages":[{"title":"Logical Structure","href":"/odf/spec/datasets/logical-structure"},{"title":"Identity","href":"/odf/spec/datasets/identity"},{"title":"Data Model","href":"/odf/spec/datasets/data-model"},{"title":"Data Schema","href":"/odf/spec/datasets/data-schema"},{"title":"Metadata Chain","href":"/odf/spec/datasets/metadata-chain"},{"title":"Data \u0026 Checkpoint Hashing","href":"/odf/spec/datasets/data-and-checkpoint-hashing"},{"title":"Storage Layout","href":"/odf/spec/datasets/storage-layout"},{"title":"Ingestion","href":"/odf/spec/datasets/ingestion"}]},{"group":"Transformations","pages":[{"title":"Processing Model","href":"/odf/spec/transformations/processing-model"},{"title":"Engine Contract","href":"/odf/spec/transformations/engine-contract"}]},{"group":"Exchange","pages":[{"title":"Transfer Protocols","href":"/odf/spec/exchange/transfer-protocols"},{"title":"Verifiable Batch Queries","href":"/odf/spec/exchange/verifiable-batch-queries"}]},{"group":"Automation","pages":[{"title":"Resource Framework","href":"/odf/spec/automation/resource-framework"},{"title":"Tasks and Flows","href":"/odf/spec/automation/tasks-and-flows"}]},{"group":"Auth \u0026 Privacy","pages":[{"title":"Authorization","href":"/odf/spec/auth-and-privacy/authorization"}]}]},{"title":"Schemas","href":"/odf/schemas"},{"group":"RFCs","root":{"title":"RFCs","href":"/odf/rfcs/index"},"pages":[{"title":"RFC-001: Record Offsets","href":"/odf/rfcs/001-record-offsets"},{"title":"RFC-002: Logical Data Digests","href":"/odf/rfcs/002-logical-data-hashes"},{"title":"RFC-003: Content Addressability","href":"/odf/rfcs/003-content-addressability"},{"title":"RFC-004: Metadata Extensibility","href":"/odf/rfcs/004-metadata-extensibility"},{"title":"RFC-005: New Annotation Metadata Events","href":"/odf/rfcs/005-dataset-annotations"},{"title":"RFC-006: Store Checkpoints as Files","href":"/odf/rfcs/006-checkpoints-as-files"},{"title":"RFC-007: Simple Transfer Protocol","href":"/odf/rfcs/007-simple-transfer-protocol"},{"title":"RFC-008: Smart Transfer Protocol","description":null,"href":"/odf/rfcs/008-smart-transfer-protocol"},{"title":"RFC-009: Ingestion Source State","href":"/odf/rfcs/009-ingest-source-state"},{"title":"RFC-010: Data Schema in Metadata","description":null,"href":"/odf/rfcs/010-data-schema-in-metadata"},{"title":"RFC-011: Push Ingest Sources","href":"/odf/rfcs/011-push-ingest-sources"},{"title":"RFC-012: Recommend `base16` encoding for textual representation of hashes and DIDs","href":"/odf/rfcs/012-recommend-base16-encoding"},{"title":"RFC-013: Enum representation in YAML encoding","href":"/odf/rfcs/013-yaml-enum-representation"},{"title":"RFC-014: Minimizing scanning for last offset and block","href":"/odf/rfcs/014-minimize-offset-scanning"},{"title":"RFC-015: Unified changelog stream schema","href":"/odf/rfcs/015-unified-changelog-stream-schema"},{"title":"RFC-016: ODF Schema Format","href":"/odf/rfcs/016-odf-schema"},{"title":"RFC-017: Large Files Linking","href":"/odf/rfcs/017-large-files-linking"},{"title":"RFC-018: Infrastructure-as-Code Resource Framework","href":"/odf/rfcs/018-iac-resource-framework"},{"title":"RFC-019: IaC Task and Flow System","href":"/odf/rfcs/019-iac-task-flow-system"},{"title":"RFC-020: Authorization Model","href":"/odf/rfcs/020-auth"},{"title":"RFC-021: ODF Spec Restructuring","href":"/odf/rfcs/021-spec-restructuring"},{"title":"RFC-022: Batch Query Commitments","href":"/odf/rfcs/022-batch-query-commitments"}]}]},{"group":"Kamu CLI","pages":[{"title":"Commands Reference","href":"/cli/commands"},{"title":"Config Reference","href":"/cli/config"}]},{"group":"Kamu Node","pages":[{"title":"Config Reference","href":"/node/config"}]}]},{"tab":"APIs","icon":"code","pages":[{"title":"APIs","href":"/api/overview"}]},{"tab":"Integrations","icon":"network-wired","pages":[{"title":"Integrations","href":"/integrations/overview"}]}]},"basePath":"","title":"RFC-009: Ingestion Source State - Kamu Docs","buildId":"dpl_DeHjVRGfFnhYQwHnfC2pjeXXv8h9","deploymentId":"dpl_DeHjVRGfFnhYQwHnfC2pjeXXv8h9","dynamics":["a74be98ba7"],"chunks":["zoom-image-lightbox-BG6oD64b.js","lightbox-handle-r17bVUz3.js"]}
Compatibility: Backwards-compatible, forwards-incompatible