compdemocracy · ballPointPenguin · May 22, 2025 · Apr 3, 2025 · May 8, 2025 · May 8, 2025
diff --git a/.github/workflows/cypress-tests.yml b/.github/workflows/cypress-tests.yml
@@ -25,9 +25,34 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      # Configure proper Docker layer caching
+      - name: Set up Docker Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ github.workspace }}/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
       - name: Build and start Docker containers
         run: |
+          # Build with proper cache configuration
+          docker buildx build \
+            --cache-from=type=local,src=${{ github.workspace }}/.buildx-cache \
+            --cache-to=type=local,dest=${{ github.workspace }}/.buildx-cache-new,mode=max \
+            --load \
+            -f server/Dockerfile \
+            ./server
+
+          # Move cache to prevent cache growth
+          rm -rf ${{ github.workspace }}/.buildx-cache
+          mv ${{ github.workspace }}/.buildx-cache-new ${{ github.workspace }}/.buildx-cache
+
+          # Start containers
           docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env --profile postgres up -d --build
+        env:
+          DOCKER_BUILDKIT: 1
+          COMPOSE_DOCKER_CLI_BUILD: 1
 
       - name: Health Check the Server http response
         uses: jtalk/url-health-check-action@v4

diff --git a/.github/workflows/jest-server-test.yml b/.github/workflows/jest-server-test.yml
@@ -18,50 +18,83 @@ on:
 jobs:
   jest-run:
     runs-on: ubuntu-latest
+    env:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: PdwPNS2mDN73Vfbc
+      POSTGRES_DB: polis-test
+      POSTGRES_PORT: 5432
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_DB }}
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d polis-test"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Setup env
-        run: |
-          cp example.env .env
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and start Docker containers
-        run: |
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up postgres -d --build
-
-
-      - uses: actions/checkout@v3
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
+      # Configure proper Docker layer caching
+      - name: Set up Docker Cache
+        uses: actions/cache@v3
         with:
-          node-version: "18"
-          cache: "npm"
-          cache-dependency-path: server/package-lock.json
+          path: ${{ github.workspace }}/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Copy test.env to .env and configure for GHA service
+        run: |
+          cp test.env .env
+          echo "POSTGRES_HOST=172.17.0.1" >> .env
+          echo "DATABASE_URL=postgres://${{ env.POSTGRES_USER }}:${{ env.POSTGRES_PASSWORD }}@172.17.0.1:${{ env.POSTGRES_PORT }}/${{ env.POSTGRES_DB }}" >> .env
 
-      - name: Setup env in server
-        working-directory: server
+      - name: Build and start Docker containers (excluding Postgres)
         run: |
-          cp example.env .env
+          # Build with proper cache configuration
+          docker buildx build \
+            --cache-from=type=local,src=${{ github.workspace }}/.buildx-cache \
+            --cache-to=type=local,dest=${{ github.workspace }}/.buildx-cache-new,mode=max \
+            --load \
+            -f server/Dockerfile \
+            ./server
+
+          # Move cache to prevent cache growth
+          rm -rf ${{ github.workspace }}/.buildx-cache
+          mv ${{ github.workspace }}/.buildx-cache-new ${{ github.workspace }}/.buildx-cache
+
+          # Start containers - remove --profile postgres as GHA service handles postgres
+          docker compose -f docker-compose.yml -f docker-compose.test.yml up -d --build
+        env:
+          DOCKER_BUILDKIT: 1
+          COMPOSE_DOCKER_CLI_BUILD: 1
 
-      - name: Install dependencies
-        working-directory: server
-        run: npm install
+      - name: Run database migrations
+        env:
+          # Construct DATABASE_URL for the migration script
+          # using the same host (172.17.0.1) and credentials as the application
+          DATABASE_URL: postgres://${{ env.POSTGRES_USER }}:${{ env.POSTGRES_PASSWORD }}@172.17.0.1:${{ env.POSTGRES_PORT }}/${{ env.POSTGRES_DB }}
+        run: |
+          echo "Installing postgresql-client..."
+          sudo apt-get update -y
+          sudo apt-get install -y postgresql-client
+          echo "Running migrations..."
+          bash server/bin/run-migrations.sh
 
-      - name: Build & start server
-        working-directory: server
+      - name: Run Jest tests
         run: |
-          npm run build
-          nohup npm run serve & 
-
-      - name: Jest test
-        working-directory: server
-        run: npm run test
+          docker exec polis-test-server-1 npm test
 
       - name: Stop Docker containers
         if: always()
-        run: docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env --profile postgres down
+        run: docker compose -f docker-compose.yml -f docker-compose.test.yml --profile postgres down
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,5 @@ prod.env
 xids.csv
 preprod.env 
 .venv
+
+**/CLAUDE.local.md
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -8,15 +8,42 @@
 # For more information see https://docs.docker.com/compose/extends/
 
 services:
+  server:
+    environment:
+      - GOOGLE_CREDENTIALS_BASE64
+      - NODE_ENV=development
+      - MAILDEV_HOST=maildev
+
   maildev:
     image: docker.io/maildev/maildev:1.1.1
     labels:
       polis_tag: ${TAG}
     networks:
       - polis-net
     ports:
+      # User interface
       - "1080:1080"
+      # SMTP port
+      - "1025:1025"
 
-  server:
+  postgres:
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
     environment:
-      - GOOGLE_CREDENTIALS_BASE64
+      # Ensure these are explicitly set in test environment
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB}
+
+  file-server:
+    build:
+      args:
+        NODE_ENV: development
+    ports:
+      - ${STATIC_FILES_PORT:-8080}:${STATIC_FILES_PORT:-8080}
diff --git a/example.env b/example.env
@@ -29,7 +29,7 @@ POSTGRES_HOST=postgres:${POSTGRES_PORT}
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=oiPorg3Nrz0yqDLE
 # Always required. Replace with your own database URL if not using dockerized postgres.
-DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}
+DATABASE_URL=postgres://postgres:oiPorg3Nrz0yqDLE@postgres:5432/polis-test
 # Makefile will read this to determine if the database is running in a docker container.
 POSTGRES_DOCKER=true
 

diff --git a/server/.dockerignore b/server/.dockerignore
@@ -1,6 +1,9 @@
 .env
 .git
 .google_creds_temp
+.venv/
+coverage/
+dist
 logs/
 node_modules/
-npm-debug.log*
+npm-debug.log*
diff --git a/server/.eslintrc.js b/server/.eslintrc.js
@@ -7,9 +7,10 @@ module.exports = {
   extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
   overrides: [
     {
-      files: ["bin/*.js"],
+      files: ["bin/*.js", "__tests__/**/*.ts"],
       rules: {
         "no-console": "off",
+        "no-restricted-properties": "off"
       }
     }
   ],
@@ -46,5 +47,5 @@ module.exports = {
     "prefer-rest-params": 1,
     "prefer-spread": 1
   },
-  ignorePatterns: ["dist"]
+  ignorePatterns: ["coverage", "dist"]
 };
diff --git a/server/.gitignore b/server/.gitignore
@@ -1,7 +1,8 @@
 .env
 .google_creds_temp
+.venv/
+coverage/
 dist
 logs/
 node_modules/
-npm-debug.log*
-.venv/
+npm-debug.log*
diff --git a/server/.nvmrc b/server/.nvmrc
diff --git a/server/__tests__/README.md b/server/__tests__/README.md
@@ -0,0 +1,129 @@
+# Testing Guide
+
+This directory contains the test suite for the Polis server. The tests are organized by type (unit, integration, e2e) and use Jest as the test runner.
+
+## Getting Started
+
+To run the tests, you'll need:
+
+- A local PostgreSQL database for testing
+- Node.js and npm installed
+
+## Running Tests
+
+### All Tests
+
+```bash
+npm test
+```
+
+### Unit Tests Only
+
+```bash
+npm run test:unit
+```
+
+### Integration Tests Only
+
+```bash
+npm run test:integration
+```
+
+### Feature Tests Only
+
+```bash
+npm run test:feature
+```
+
+### Run Specific Tests
+
+```bash
+# Run tests in a specific file
+npm test -- __tests__/integration/participation.test.js
+
+# Run tests that match a specific name
+npm test -- -t "should do something specific"
+```
+
+## Database Setup for Tests
+
+The tests require a clean database state to run successfully. There are several ways to manage this:
+
+### Option 1: Reset Database Before Running Tests
+
+This will completely reset your database, dropping and recreating it with a fresh schema:
+
+```bash
+# Reset the database immediately
+npm run db:reset
+
+# Run tests with a database reset first
+RESET_DB_BEFORE_TESTS=true npm test
+```
+
+⚠️ **WARNING**: The `db:reset` script will delete ALL data in the database specified by `DATABASE_URL`.
+
+## Mailer Testing
+
+A maildev container is typically running (see `docker-compose.dev.yml`) and will capture emails sent during testing. You can view the emails at `http://localhost:1080` (SMTP port 1025).
+
+The test suite includes helper functions in `__tests__/setup/email-helpers.js` to interact with MailDev:
+
+```javascript
+// Find an email sent to a specific recipient
+const email = await findEmailByRecipient('[email protected]');
+
+// Get all emails currently in MailDev
+const allEmails = await getEmails();
+
+// Clean up emails before/after tests
+await deleteAllEmails();
+
+// Extract password reset URLs from emails
+const { url, token } = getPasswordResetUrl(email);
+```
+
+## Response Format Handling
+
+The test suite includes robust handling for the various response formats from the API:
+
+- **JSON Responses**: Automatically parsed into JavaScript objects
+- **Text Responses**: Preserved as strings
+- **Gzipped Content**: Automatically detected and decompressed, even when incorrectly marked
+- **Mixed Content-Types**: Handles cases where JSON content is served with non-JSON content types
+
+## Test Safety Features
+
+The test environment includes this safety feature:
+
+- **Production Database Prevention**: Tests will not run against production databases (URLs containing 'amazonaws', 'prod', etc.)
+
+## Troubleshooting Common Issues
+
+### Participant Creation Issues
+
+If tests fail with duplicate participant errors, try:
+
+```bash
+npm run db:reset
+```
+
+### Database Connection Errors
+
+Check that:
+
+1. Your PostgreSQL server is running
+2. Your DATABASE_URL environment variable is correct
+3. Database and schema exist (you can use `npm run db:reset` to create them)
+
+### Test Timeouts
+
+If tests timeout, try:
+
+1. Increase the timeout in individual tests:
+
+   ```javascript
+   jest.setTimeout(90000); // Set timeout to 90 seconds
+   ```
+
+2. Check for any blocking async operations that might not be resolving