Polling Sources

---
kind: DatasetSnapshot
version: 1
content:
  name: com.example.city-populations
  kind: Root
  metadata:
    - kind: SetPollingSource
      # Where to fetch the data from.
      # Supports multiple protocols and file layouts
      fetch:
        kind: Url
        url: https://example.com/city_populations_over_time.zip
      # OPTIONAL: How to prepare the binary data
      # Includes decompression, file filtering, format conversions
      prepare:
        - kind: Decompress
          format: Zip
      # How to interpret the data.
      # Includes data format, schema to apply, error handling
      read:
        kind: Csv
        header: true
        schema:
          fields:
            - name: date
              type: Date
            - name: city
              type: String
            - name: population
              type: String
      # OPTIONAL: Pre-processing query that shapes the data.
      # Useful for converting text data into strict types
      preprocess:
        kind: Sql
        # Use one of the supported engines and a query in its dialect
        # See: https://docs.kamu.dev/cli/transform/supported-engines/
        engine: datafusion
        query: |
          select
            date,
            city,
            -- remove commas between thousands
            cast(replace(population, ",", "") as bigint)
          from input
      # How to combine data ingested in the past with the new data.
      merge:
        kind: Ledger
        primaryKey:
          - date
          - city
      # Lets you manipulate names of the system columns to avoid conflicts
      # or use names better suited for your data.
    - kind: SetVocab
      eventTimeColumn: date

kamu pull com.example.city-populations

fetch:
  kind: FilesGlob
  # Ingest all files matching the pattern (in lexicographic order)
  path: /data/database-dump-*.json.gz
  eventTime:
    kind: FromPath
    # Extract timestamp as the first regex group in the pattern
    pattern: database-dump-(\d+-\d+-\d+)\.json\.gz
    # Parse timestamp
    timestampFormat: '%Y-%m-%d'