Integration Tests #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Run integration tests daily at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| push: | |
| branches: [ master ] | |
| paths: | |
| - 'internal/storage/**' | |
| - 'internal/output/**' | |
| - 'test/integration/**' | |
| env: | |
| GO_VERSION: '1.23' | |
| jobs: | |
| integration-test: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| services: | |
| clickhouse: | |
| image: clickhouse/clickhouse-server:latest | |
| env: | |
| CLICKHOUSE_DB: parsedmarc_test | |
| CLICKHOUSE_USER: parsedmarc | |
| CLICKHOUSE_PASSWORD: test123 | |
| ports: | |
| - 8123:8123 | |
| - 9000:9000 | |
| options: >- | |
| --health-cmd="wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=3 | |
| zookeeper: | |
| image: confluentinc/cp-zookeeper:latest | |
| env: | |
| ZOOKEEPER_CLIENT_PORT: 2181 | |
| ZOOKEEPER_TICK_TIME: 2000 | |
| ports: | |
| - 2181:2181 | |
| kafka: | |
| image: confluentinc/cp-kafka:latest | |
| env: | |
| KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" | |
| ports: | |
| - 9092:9092 | |
| options: >- | |
| --health-cmd="kafka-broker-api-versions --bootstrap-server localhost:9092" | |
| --health-interval=30s | |
| --health-timeout=10s | |
| --health-retries=3 | |
| mailhog: | |
| image: mailhog/mailhog:latest | |
| ports: | |
| - 1025:1025 | |
| - 8025:8025 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Cache Go modules | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-go- | |
| - name: Download dependencies | |
| run: go mod download | |
| - name: Wait for services to be ready | |
| run: | | |
| # Wait for ClickHouse | |
| echo "Waiting for ClickHouse..." | |
| for i in {1..30}; do | |
| if wget --no-verbose --tries=1 --spider http://localhost:8123/ping; then | |
| echo "ClickHouse is ready!" | |
| break | |
| fi | |
| echo "Waiting for ClickHouse... ($i/30)" | |
| sleep 2 | |
| done | |
| # Wait for Kafka | |
| echo "Waiting for Kafka..." | |
| for i in {1..30}; do | |
| if kafka-broker-api-versions --bootstrap-server localhost:9092 > /dev/null 2>&1; then | |
| echo "Kafka is ready!" | |
| break | |
| fi | |
| echo "Waiting for Kafka... ($i/30)" | |
| sleep 2 | |
| done | |
| - name: Initialize test environment | |
| run: | | |
| # Make scripts executable | |
| chmod +x scripts/init-clickhouse.sh | |
| # Initialize ClickHouse | |
| ./scripts/init-clickhouse.sh | |
| - name: Run integration tests | |
| run: make test-integration-quick | |
| env: | |
| PARSEDMARC_TEST_MODE: true | |
| CLICKHOUSE_URL: tcp://localhost:9000 | |
| CLICKHOUSE_USERNAME: parsedmarc | |
| CLICKHOUSE_PASSWORD: test123 | |
| CLICKHOUSE_DATABASE: parsedmarc_test | |
| KAFKA_BROKERS: localhost:9092 | |
| - name: Generate integration test report | |
| if: always() | |
| run: | | |
| echo "## Integration Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "- ✅ ClickHouse integration tested" >> $GITHUB_STEP_SUMMARY | |
| echo "- ✅ Kafka integration tested" >> $GITHUB_STEP_SUMMARY | |
| echo "- ✅ Email output integration tested" >> $GITHUB_STEP_SUMMARY |