Bump yauzl from 3.2.0 to 3.2.1

.github/workflows/build-and-deploy.yml

@@ -8,9 +8,50 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
+  build_ipx:
+    name: build ipx
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            suffix=-${{ github.run_id }}-ipx
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: census/website/ipx
+          file: census/website/ipx/Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
   build_census_api:
     name: build api
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     permissions:
       contents: read
@@ -50,7 +91,7 @@ jobs:
 
   build_census_ui:
     name: build ui
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repository
@@ -59,7 +100,6 @@ jobs:
       - uses: pnpm/action-setup@v4
         name: Install pnpm
         with:
-          version: 9
           run_install: false
 
       - name: Install Node.js
@@ -84,11 +124,11 @@ jobs:
 
   deploy:
     name: deploy to production
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs:
       - build_census_ui
       - build_census_api
-
+      - build_ipx
     permissions:
       id-token: write
 
@@ -105,7 +145,6 @@ jobs:
       - uses: pnpm/action-setup@v4
         name: Install pnpm
         with:
-          version: 9
           run_install: false
 
       - name: Install Node.js
@@ -118,35 +157,49 @@ jobs:
         run: pnpm i
 
       - uses: pulumi/auth-actions@v1
+        name: Login to Pulumi
         with:
           organization: alveusgg
           requested-token-type: urn:pulumi:token-type:access_token:personal
           scope: user:changesbyjames
 
-      - name: Azure CLI Login
+      - name: Login to Azure
         uses: azure/login@v2
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}
           subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
 
-      - uses: pulumi/actions@v5
+      - uses: pulumi/actions@v6
+        name: Deploy infrastructure
         id: pulumi
+        continue-on-error: true
         with:
           command: up
           stack-name: alveusgg/prod
           work-dir: infrastructure/pulumi
           config-map: |
             {
               "image": { value: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main-${{ github.run_id }}-census-api", secret: false },
+              "ipx": { value: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main-${{ github.run_id }}-ipx", secret: false }
             }
         env:
           ARM_USE_OIDC: true
           ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
           ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
 
-      - name: az cli copy
+      - name: Build & upload worker
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          workingDirectory: census/sync
+          preCommands: |
+            echo '${{ steps.pulumi.outputs.wranglerConfig }}' > wrangler.generated.jsonc
+          command: |
+            deploy --config wrangler.generated.jsonc --env prod
+
+      - name: Upload UI & purge CDN
         uses: azure/CLI@v2
         with:
           azcliversion: 2.53.0

.gitignore

@@ -45,7 +45,6 @@ jspm_packages/
 
 # Gatsby files
 .cache/
-public/
 
 # Vuepress build output
 .vuepress/dist
@@ -72,3 +71,4 @@ dist/
 
 docker-compose.local.yml
 
+.wrangler/

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "eslint.workingDirectories": ["census/api"]
+}

census/README.md

@@ -1,53 +1,61 @@
-
 # Getting started
 
 ## Prerequisites
 
 1. Clone the repository.
 2. Ensure you have `pnpm` installed and run `pnpm install`.
 3. Ensure you have `docker` installed and running on your machine.
-4. Clone `./api/.env.example` and rename as `./api/.env` and configure it as follows.
+4. Ensure you have [`ffmpeg`](https://www.ffmpeg.org/) installed
+   - MacOS: run `brew install ffmpeg`
+5. Clone `./api/.env.example` and rename as `./api/.env` and configure it as follows.
 
 ### Twitch setup
+
 1. Go to the [Twitch Developer Dashboard](https://dev.twitch.tv/console).
 2. Create a new application and add the following as OAuth Redirect URLs.
-    - `http://localhost:35523/auth/redirect`
-    - `http://localhost:35523/admin/redirect`
+   - `http://localhost:35523/auth/redirect`
+   - `http://localhost:35523/admin/redirect`
 3. Copy the generated Client ID and Client Secret
- 	- Update the `./api/.env` file with `TWITCH_CLIENT_ID` and `TWITCH_CLIENT_SECRET`.
-
-### Azure setup
-
-Currently, we don't support the Azurite emulator for local development, so you'll need to have an Azure storage account and container that you can use.
-
-1. Go to the Azure Portal.
-2. Create a storage account: 
- 	- Under Azure services, select Storage accounts and create a new storage account.
- 	- Update the `./api/.env` file with `STORAGE_ACCOUNT_NAME`.
-3. Obtain the access key:
-    - From your storage account's page, navigate to Security + networking > Access keys.
- 	- Update the `./api/.env` file with `STORAGE_ACCOUNT_KEY` with either `key1` or `key2`.
-4. Create a storage container:
-    - From your storage account's page, go to Data storage > Containers and create a new container.
-    - Update the `./api/.env` file with `CONTAINER_NAME`.
-5. Enable public access for the container
-	- From your storage account's page, go to Configuration and set `Allow Blob anonymous access` to `Enabled`.
-	- From your container's page, select Change access level and set the `anonymous access level` as `Container` or `Blob`.
+   - Update the `./api/.env` file with `TWITCH_CLIENT_ID` and `TWITCH_CLIENT_SECRET`.
 
 ### `JWT_SECRET` setup
 
 This is the secret that the API uses to sign the JWTs. For local development, you can stick to the default value but know that it makes the token insecure. If you want to generate a new secret, you can run `pnpm --filter=@alveusgg/census-api setup:jwt` to generate a new secret.
 
+## Seeding the database
+
+You will need to seed the database with the correct data. To add yourself as an admin, run `pnpm --filter=@alveusgg/census-api setup:api` and follow the prompts.
+
 ## Running the services
 
-1. Start the database.
-	- In the root of the repo, run `docker compose up`.
+1. Start the local services, the database, cache & blob storage.
+   - In the root of the repo, run `pnpm run deps:up`.
 2. Start the API.
-	- `pnpm --filter=@alveusgg/census-api start`.
-	- `pnpm --filter=@alveusgg/census-api dev` to start in watch mode.
+   - `pnpm --filter=@alveusgg/census-api start`.
+   - `pnpm --filter=@alveusgg/census-api dev` to start in watch mode.
 3. Start the UI.
-	- `pnpm --filter=@alveusgg/census-ui start`.
+   - `pnpm --filter=@alveusgg/census-website start`. This is always started in watch mode.
 
-## Seeding the database
+## Optional & additional setup
+
+### Azure setup
+
+If you want to actually host & store on the internet, you'll need to setup an Azure storage account.
+
+1. Go to the Azure Portal.
+2. Create a storage account:
+   - Under Azure services, select Storage accounts and create a new storage account.
+   - Update the `./api/.env` file with `STORAGE_ACCOUNT_NAME`.
+3. Obtain the access key:
+   - From your storage account's page, navigate to Security + networking > Access keys.
+   - Update the `./api/.env` file with `STORAGE_ACCOUNT_KEY` with either `key1` or `key2`.
+4. Create a storage container:
+   - From your storage account's page, go to Data storage > Containers and create a new container.
+   - Update the `./api/.env` file with `CONTAINER_NAME`.
+5. Enable public access for the container
+   - From your storage account's page, go to Configuration and set `Allow Blob anonymous access` to `Enabled`.
+   - From your container's page, select Change access level and set the `anonymous access level` as `Container` or `Blob`.
+
+# Getting started with development
 
-You will need to seed the database with the correct data. To add yourself as an admin, run `pnpm --filter=@alveusgg/census-api setup:api` and follow the prompts.
+See the [development guide](../docs/dev/README.md) for more information.

census/api/.env.example

@@ -1,3 +1,5 @@
+NODE_ENV=development
+
 TWITCH_CLIENT_ID=
 TWITCH_CLIENT_SECRET=
 
@@ -9,11 +11,14 @@ POSTGRES_PASSWORD=mypassword
 POSTGRES_DB=db01
 POSTGRES_HOST=localhost
 
-UI_URL=http://localhost:5173
+REDIS_HOST=localhost
+REDIS_PORT=6379
+
 JWT_SECRET=bed840003446895cfee2372afad1c9adace070b0b54cdfdc640d61b521b5a5aeb3af9519785d5935b8de0f25469b782701b2276a22df329a971c3065bd59a565
 
-STORAGE_ACCOUNT_NAME=
-STORAGE_ACCOUNT_KEY=
-CONTAINER_NAME=
+WORKER_API_TOKEN=EXAMPLE_WORKER_API_TOKEN
+
+STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+CONTAINER_NAME=azurite
 
-NODE_ENV=development
+DEV_FLAG_USE_TWITCH_CLIP_DIRECTLY=true

census/api/Dockerfile

@@ -1,16 +1,17 @@
-FROM node:22-slim AS base
+FROM node:22
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
+RUN npm i -g corepack@latest
 RUN corepack enable
 
+RUN apt-get update && apt-get install -y ffmpeg
 WORKDIR /ws
 COPY . .
 
-FROM base AS build
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
 RUN pnpm --filter=@alveusgg/census-api --prod deploy /prod/api
 
-FROM base
-COPY --from=build /prod/api /prod/api
+
 WORKDIR /prod/api
+
 CMD ["pnpm", "start"]

census/api/drizzle.config.ts

@@ -3,5 +3,12 @@ import { defineConfig } from 'drizzle-kit';
 export default defineConfig({
   dialect: 'postgresql',
   schema: './src/db/schema/index.ts',
-  out: './drizzle'
+  out: './drizzle',
+  dbCredentials: {
+    host: 'localhost',
+    port: 5432,
+    user: 'myuser',
+    password: 'mypassword',
+    database: 'db01'
+  }
 });