feat: separate pruner

.github/workflows/informative-indexer-build.yml

@@ -40,7 +40,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22.7'
+          go-version: '1.23.7'
           cache: true
 
       - name: Run Tests
@@ -88,4 +88,16 @@ jobs:
           body: '{ "event_type": "core_informative_indexer", "client_payload": { "network": "initia", "chain_id": "initiation-2", "sha_short": ${{ needs.push.outputs.sha_short }} } }'
           timeout: 10000
           log-response: true
-          retries: 3
+          retries: 3
+
+      - id: dispatch-interwoven-1
+        name: Dispatch to alleslabs/celatone-infrastructure for interwoven-1
+        uses: indiesdev/curl@v1.1
+        with:
+          url: https://api.github.com/repos/alleslabs/celatone-infrastructure/dispatches
+          method: "POST"
+          headers: '{ "Accept": "application/vnd.github.everest-preview+json", "Authorization": "token ${{ steps.token.outputs.github_token }}" }'
+          body: '{ "event_type": "core_informative_indexer", "client_payload": { "network": "initia", "chain_id": "interwoven-1", "sha_short": ${{ needs.push.outputs.sha_short }} } }'
+          timeout: 10000
+          log-response: true
+          retries: 3

.github/workflows/unified-deployment.yml

@@ -0,0 +1,65 @@
+name: Core Indexer Unified Deployment
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "v*"
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  docker:
+    name: build-image
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        service:
+          - { name: core-indexer-api, dockerfile: dockerfiles/Dockerfile.api }
+          - { name: event-indexer, dockerfile: dockerfiles/Dockerfile.event }
+          - { name: generic-indexer, dockerfile: dockerfiles/Dockerfile.generic }
+          - { name: informative-indexer, dockerfile: dockerfiles/Dockerfile.informative }
+          - { name: pruner, dockerfile: dockerfiles/Dockerfile.pruner }
+          - { name: sweeper, dockerfile: dockerfiles/Dockerfile.sweeper }
+          - { name: uploader, dockerfile: dockerfiles/Dockerfile.uploader }
+
+    env:
+      IMAGE_NAME: ${{ matrix.service.name }}
+      DOCKERFILE_NAME: ${{ matrix.service.dockerfile }}
+
+    permissions:
+      contents: read
+      packages: write
+
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ${{ env.DOCKERFILE_NAME }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            COMMIT_SHA=${{ github.sha }}

.gitignore

@@ -56,3 +56,14 @@ __pycache__
 venv
 *.bin
 .env
+
+tmp
+
+sweeper/sweeper
+informative-indexer/informative-indexer
+generic-indexer/generic-indexer
+
+# Go binaries (e.g. from go build in api/)
+api/api
+event-indexer/event-indexer
+tx-response-uploader/tx-response-uploader

README.md

@@ -1,29 +1,117 @@
 # Core Indexer
 
-## Informative Indexer
+## Components
 
-### Components
+### API
+REST API service that provides endpoints for querying indexed blockchain data. Built with Fiber framework and includes comprehensive Swagger documentation.
 
-**Sweeper** - Polls the RPC for new block data and submits it to the message queue.
+**Features:**
+- Account data queries
+- Block information retrieval
+- Module transaction tracking
+- NFT data and transaction history
+- Governance proposal information
+- Transaction details and history
+- Validator information and metrics
+- Health check endpoints
+- CORS support and request logging
 
-**Flusher** - Consumes messages from the queue and processes them into the database.
+### Event Indexer
+Specialized indexer for blockchain events with comprehensive event tracking capabilities. Processes transaction events, block events, and Move smart contract events.
 
-**Prunner** - Periodically prunes the database and stores backups in a cloud storage service.
+**Features:**
+- Transaction event processing and indexing
+- Block finalization event capture
+- Move smart contract event tracking
+- Database migration management
+- Data pruning with cloud storage backup
+- Command-line interface with multiple modes
 
-### Workflow Summary
+**Commands:**
+- `indexer` - Main indexing process
+- `migrate` - Database migration operations
+- `prunner` - Data pruning and archival
 
-Each service runs continuously and must be executed together for the indexer to function properly.
+### Generic Indexer
+General-purpose blockchain data indexer with both continuous and scheduled processing capabilities. Handles comprehensive blockchain state tracking and account management.
+
+**Features:**
+- Continuous block processing
+- Cron-based batch operations
+- Account data indexing
+- Block result processing
+- Validator state tracking
+- Batch insertion optimization
+- Multi-mode operation support
+
+**Commands:**
+- `indexer` - Continuous indexing mode
+- `indexercron` - Scheduled batch processing mode
+
+### Informative Indexer
+Comprehensive blockchain data processor with specialized module processors for different blockchain components. Features advanced state tracking and caching mechanisms.
+
+**Features:**
+- Modular processor architecture (auth, bank, IBC, move, OPinit, gov, staking)
+- State tracking and management
+- Data caching for performance optimization
+- Genesis block processing
+- Event processing utilities
+- Validator uptime tracking
+- Batch state updates
+
+**Commands:**
+- `indexer` - Main processing engine
+- `migrate` - Database schema management
+
+### Sweeper
+High-performance data collection service that polls RPC endpoints for new blockchain data and distributes it via message queues.
+
+**Features:**
+- RPC endpoint polling
+- Block data retrieval
+- Message queue publishing
+- Database migration on startup
+- Error handling and retry logic
+- Configurable polling intervals
+
+### TX Response Uploader
+Message queue consumer that processes transaction response data and uploads it to cloud storage systems with support for large message handling.
+
+**Features:**
+- Kafka message consumption
+- Cloud storage integration (GCS)
+- Claim check pattern support for large messages
+- Dead letter queue error handling
+- Retry mechanisms with exponential backoff
+- Transaction response archival
+- Sentry integration for monitoring
+
+## Workflow Summary
+
+The indexer ecosystem consists of multiple specialized services that work together to provide comprehensive blockchain data indexing and querying capabilities.
+
+**Core Data Flow:**
+
+1. **Sweeper** retrieves new block data from RPC endpoints and publishes it to message queues
+2. **Indexers** (Event, Generic, Informative) consume messages and process blockchain data into the database
+3. **API** serves the indexed data through REST endpoints
+4. **TX Response Uploader** handles transaction response storage in cloud services
+5. **Pruners** manage data lifecycle and storage optimization
 
 **Sweeper**
 
-1. Retrieves the latest indexed block from the database.
-2. Fetches data for the next block using the RPC methods `/block` and `/block_results`.
-3. Publishes the block data as a message to the message queue.
+1. On startup, applies any pending database migrations.
+2. Retrieves the latest indexed block from the database.
+3. Fetches data for the next block using the RPC methods `/block` and `/block_results`.
+4. Publishes the block data as a message to the message queue.
 
-**Flusher**
+**Indexers**
 
-1. Subscribes to and reads messages from the queue.
-2. Processes each message and inserts the data into the database.
+1. Subscribe to and read messages from the queue.
+2. Process each message using specialized processors for different blockchain modules.
+3. Insert processed data into the database with batch operations for efficiency.
+4. Handle state tracking and caching for optimized performance.
 
 **Prunner**
 
@@ -34,6 +122,6 @@ Each service runs continuously and must be executed together for the indexer to
    2. Uploads the data to a cloud storage service.
    3. Deletes the fetched rows from the database.
 
-### Running Locally
+## Running Locally
 
-Follow the guide for running the Informative Indexer with Docker: [dockerfiles/README.md](dockerfiles/README.md).
+To run the Informative Indexer with Docker locally, follow this [guide](local/README.md).

api/.env.example

@@ -0,0 +1,5 @@
+DB_CONNECTION_STRING_RO=postgres://readonly:password@localhost:5432/initiation-2?sslmode=disable
+ENVIRONMENT=local
+CHAIN_ID=initiation-2
+BUCKETS=newer-bucket-name,legacy-bucket-name
+PORT=8080

api/README.md

@@ -0,0 +1,72 @@
+# Core Indexer API
+
+This is the API service for the Core Indexer project, built with Go Fiber.
+
+## Prerequisites
+
+- Go 1.21 or higher
+- Git
+- Air (for hot reloading)
+
+## Setup
+
+1. Install dependencies:
+
+```bash
+go mod download
+```
+
+2. Create a `.env` file in the root directory (optional):
+
+```bash
+PORT=8080
+```
+
+## Running the Application
+
+### Development Mode (with Hot Reload)
+
+To run the application with hot reload (recommended for development):
+
+```bash
+air
+```
+
+The server will automatically restart when you make changes to the code.
+
+### Production Mode
+
+To run the application without hot reload:
+
+```bash
+go run main.go
+```
+
+The server will start on port 3000 by default (or the port specified in your .env file).
+
+## API Endpoints
+
+- `GET /`: Welcome message
+- `GET /indexer/swagger/*`: Swagger documentation
+
+## Project Structure
+
+```
+api/
+├── main.go           # Application entry point
+├── routes/           # Route definitions
+│   └── routes.go
+├── dto/             # Data Transfer Objects
+│   └── nft.go
+├── go.mod           # Go module file
+├── .air.toml        # Air configuration
+└── README.md        # This file
+```
+
+## Development
+
+To add new routes:
+
+1. Create new handler functions in the appropriate package
+2. Add the routes in `routes/routes.go`
+3. Import and use the handlers in your routes

api/apperror/errors.go

@@ -0,0 +1,23 @@
+package apperror
+
+// Common error messages
+const (
+	ErrMsgBadRequest      = "Invalid request"
+	ErrMsgUnauthorized    = "Unauthorized"
+	ErrMsgNotFound        = "Resource not found"
+	ErrMsgInternal        = "Internal server error"
+	ErrMsgProposalId      = "Proposal id is not a valid int32 integer"
+	ErrMsgBlocks          = "Blocks argument is not a valid integer"
+	ErrMsgBlocksZero      = "Blocks must be greater than 0"
+	ErrMsgBlocksRequired  = "Blocks parameter is required"
+	ErrMsgModuleNotFound  = "Module not found"
+	ErrMsgDuplicateStatus = "Duplicate status in query"
+	ErrMsgTxNotFound      = "Transaction not found for hash %s"
+	ErrMsgNoValidTxFiles  = "No valid transaction files found for hash %s"
+	ErrMsgInvalidLimit    = "Limit must be between 1 and 1000"
+	ErrMsgLimitInteger    = "Limit must be in integer format"
+	ErrMsgOffsetInteger   = "Offset must be in integer format"
+	ErrMsgReverse         = "Reverse must be a boolean"
+	ErrMsgCountTotal      = "CountTotal must be a boolean"
+	ErrMsgHeightInteger   = "Height must be in integer format"
+)