Feat/polymarket

docs/examples/evm/15.polymarket.example.ts

@@ -0,0 +1,101 @@
+import { readFileSync } from 'fs'
+
+import { createDefaultLogger, humanBytes } from '@subsquid/pipes'
+import { evmDecoder, evmPortalStream } from '@subsquid/pipes/evm'
+import { metricsServer } from '@subsquid/pipes/metrics/node'
+import { deltaDbTarget } from '@subsquid/pipes/targets/delta-db'
+
+import { exchangeEvents } from './abi/polymarket/abi'
+
+/**
+ * Polymarket order tracking with insider detection.
+ *
+ * Pipeline:
+ *   Portal (Polygon) → evmDecoder → transform → Delta DB → onDelta
+ *
+ * 1. evmPortalSource streams raw blocks from Polygon via the Portal API,
+ *    starting at block 35,873,440.
+ * 2. evmDecoder filters for OrdersMatched events emitted by the Polymarket
+ *    exchange contract (0x4bFb...982E) and decodes them using the ABI.
+ * 3. transform maps each decoded event into an `orders` row — determining
+ *    buy/sell side from takerAssetId and converting BigInt fields to strings.
+ * 4. deltaDbTarget ingests rows into an embedded Delta DB that runs two
+ *    SQL reducers defined in polymarket.sql:
+ *      - market_stats: per-asset volume, trade count, and price moments
+ *      - insider_classifier: flags traders who place >$4k in buys at <0.95
+ *        within a 15-minute window, then tracks all their subsequent trades
+ *    and two materialized views: token_summary, insider_positions.
+ * 5. onDelta receives each flushed delta batch for downstream consumption.
+ */
+
+const SCHEMA = readFileSync(new URL('./schemas/polymarket.sql', import.meta.url), 'utf-8')
+
+const logger = createDefaultLogger()
+
+async function cli() {
+  let totalOrders = 0
+  const startTime = Date.now()
+
+  await evmPortalStream({
+    id: 'polymarket-insiders',
+    logger,
+    portal: {
+      url: 'https://portal.sqd.dev/datasets/polygon-mainnet',
+    },
+    outputs: evmDecoder({
+      range: { from: '75,873,440' },
+      // range: { from: '55,873,440' },
+      contracts: ['0x4bFb41d5B3570DeFd03C39a9A4D8dE6Bd8B8982E'.toLowerCase()],
+      events: {
+        OrdersMatched: exchangeEvents.OrdersMatched,
+      },
+    }),
+
+    metrics: metricsServer(),
+  })
+    .pipe({
+      profiler: { name: 'transform' },
+      transform: (data, ctx) => {
+        totalOrders += data.OrdersMatched.length
+
+        ctx.logger.debug({
+          message: `input batch ${data.OrdersMatched.length}`,
+          blocksCount: ctx.batch.blocksCount,
+          batchSize: `${humanBytes(ctx.batch.bytesSize)}`,
+        })
+
+        const elapsed = (Date.now() - startTime) / 1000
+        ctx.logger.info(`Input orders/sec: ${(totalOrders / elapsed).toFixed(1)} (total: ${totalOrders})`)
+
+        return {
+          orders: data.OrdersMatched.map((order) => {
+            const isBuy = order.event.takerAssetId === 0n
+
+            return {
+              block_number: order.block.number,
+              timestamp: Math.floor(order.timestamp.valueOf() / 1000),
+              trader: order.event.takerOrderMaker,
+              asset_id: isBuy ? order.event.makerAssetId : order.event.takerAssetId,
+              usdc: isBuy ? order.event.takerAmountFilled : order.event.makerAmountFilled,
+              shares: isBuy ? order.event.makerAmountFilled : order.event.takerAmountFilled,
+              side: isBuy ? 'buy' : 'sell',
+            }
+          }),
+        }
+      },
+    })
+    .pipeTo(
+      deltaDbTarget({
+        schema: SCHEMA,
+        // dataDir: ':memory:',
+        dataDir: './polymarket.delta-db',
+        onDelta: ({ batch }) => {
+          // We will save the batch to a downstream store, e.g. Postgres or Clickhouse,
+          // after processing it with reducers and MVs in Delta DB
+          // logger.info(`Delta batch ${batch.sequence} processed`)
+        },
+      }),
+    )
+}
+
+void cli()

docs/examples/evm/abi/polymarket/abi.ts

@@ -0,0 +1,78 @@
+import { event, indexed } from '@subsquid/evm-abi'
+import * as p from '@subsquid/evm-codec'
+
+// --- ABIs ---
+export const exchangeEvents = {
+  OrderFilled: event(
+    '0xd0a08e8c493f9c94f29311604c9de1b4e8c8d4c06bd0c789af57f2d65bfec0f6',
+    'OrderFilled(bytes32,address,address,uint256,uint256,uint256,uint256,uint256)',
+    {
+      orderHash: indexed(p.bytes32),
+      maker: indexed(p.address),
+      taker: indexed(p.address),
+      makerAssetId: p.uint256,
+      takerAssetId: p.uint256,
+      makerAmountFilled: p.uint256,
+      takerAmountFilled: p.uint256,
+      fee: p.uint256,
+    },
+  ),
+  OrdersMatched: event(
+    '0x63bf4d16b7fa898ef4c4b2b6d90fd201e9c56313b65638af6088d149d2ce956c',
+    'OrdersMatched(bytes32,address,uint256,uint256,uint256,uint256)',
+    {
+      takerOrderHash: indexed(p.bytes32),
+      takerOrderMaker: indexed(p.address),
+      makerAssetId: p.uint256,
+      takerAssetId: p.uint256,
+      makerAmountFilled: p.uint256,
+      takerAmountFilled: p.uint256,
+    },
+  ),
+  TokenRegistered: event(
+    '0xbc9a2432e8aeb48327246cddd6e872ef452812b4243c04e6bfb786a2cd8faf0d',
+    'TokenRegistered(uint256,uint256,bytes32)',
+    {
+      token0: indexed(p.uint256),
+      token1: indexed(p.uint256),
+      conditionId: indexed(p.bytes32),
+    },
+  ),
+}
+
+export const ctfEvents = {
+  ConditionPreparation: event(
+    '0xab3760c3bd2bb38b5bcf54dc79802ed67338b4cf29f3054ded67ed24661e4177',
+    'ConditionPreparation(bytes32,address,bytes32,uint256)',
+    {
+      conditionId: indexed(p.bytes32),
+      oracle: indexed(p.address),
+      questionId: indexed(p.bytes32),
+      outcomeSlotCount: p.uint256,
+    },
+  ),
+  PositionSplit: event(
+    '0x2e6bb91f8cbcda0c93623c54d0403a43514fabc40084ec96b6d5379a74786298',
+    'PositionSplit(address,address,bytes32,bytes32,uint256[],uint256)',
+    {
+      stakeholder: indexed(p.address),
+      collateralToken: p.address,
+      parentCollectionId: indexed(p.bytes32),
+      conditionId: indexed(p.bytes32),
+      partition: p.array(p.uint256),
+      amount: p.uint256,
+    },
+  ),
+  PositionsMerge: event(
+    '0x6f13ca62553fcc2bcd2372180a43949c1e4cebba603901ede2f4e14f36b282ca',
+    'PositionsMerge(address,address,bytes32,bytes32,uint256[],uint256)',
+    {
+      stakeholder: indexed(p.address),
+      collateralToken: p.address,
+      parentCollectionId: indexed(p.bytes32),
+      conditionId: indexed(p.bytes32),
+      partition: p.array(p.uint256),
+      amount: p.uint256,
+    },
+  ),
+}

docs/package.json

@@ -23,13 +23,15 @@
     "@subsquid/borsh": "0.3.0",
     "@subsquid/evm-abi": "0.3.1",
     "@subsquid/evm-codec": "0.3.0",
+    "@sqd-pipes/delta-db": "0.0.1-alpha.20",
     "@subsquid/pipes": "workspace:*",
     "@subsquid/solana-stream": "0.3.1",
     "date-fns": "^4.1.0",
     "drizzle-graphql": "^0.8.5",
     "drizzle-orm": "^0.44.7",
     "graphql": "^16.12.0",
     "pg": "^8.16.3",
+    "pino-pretty": "13.1.1",
     "viem": "^2.45.1"
   }
 }

docs/tsconfig.json

@@ -2,7 +2,7 @@
   "extends": "../tsconfig.json",
   "compilerOptions": {
     "baseUrl": ".",
-    "module": "commonjs",
+    "module": "esnext",
     "moduleResolution": "node",
     "declaration": false,
     "outDir": "dist",

package.json

@@ -13,15 +13,18 @@
     "watch": "turbo run watch"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.4.8",
-    "bun-types": "1.3.5",
+    "@biomejs/biome": "2.4.9",
+    "bun-types": "1.3.11",
     "cpy": "12.1.0",
-    "glob": "11.0.3",
+    "glob": "13.0.6",
     "recast": "0.23.11",
-    "tsup": "8.5.0",
-    "tsx": "4.20.6",
-    "turbo": "2.7.3",
+    "tsup": "8.5.1",
+    "tsx": "4.21.0",
+    "turbo": "2.8.21",
     "zx": "8.8.5"
   },
-  "packageManager": "[email protected]"
+  "packageManager": "[email protected]",
+  "dependencies": {
+    "@sqd-pipes/delta-db": "0.0.1-alpha.20"
+  }
 }

packages/pipe-ui/app/components/ui/code.tsx

@@ -9,14 +9,21 @@ import typescript from 'react-syntax-highlighter/dist/esm/languages/hljs/typescr
 import { CopyButton } from '~/components/ui/copy-button'
 
 const theme: Record<string, React.CSSProperties> = {
-  'hljs': { background: 'transparent', color: 'rgba(255,255,255,0.4)' },
-  'hljs-string': { color: '#7ec89e' },
-  'hljs-number': { color: '#6eb3d4' },
-  'hljs-literal': { color: '#6eb3d4' },
-  'hljs-keyword': { color: '#7ec89e' },
-  'hljs-attr': { color: '#9d8abf' },
-  'hljs-punctuation': { color: 'rgba(255,255,255,0.3)' },
-  'hljs-comment': { color: 'rgba(255,255,255,0.2)' },
+  'hljs': { background: 'transparent', color: 'rgba(255,255,255,0.85)' },
+  'hljs-string': { color: '#a5d6a7' },
+  'hljs-number': { color: '#90caf9' },
+  'hljs-literal': { color: '#90caf9' },
+  'hljs-keyword': { color: '#c792ea' },
+  'hljs-built_in': { color: '#82aaff' },
+  'hljs-type': { color: '#ffcb6b' },
+  'hljs-function': { color: '#82aaff' },
+  'hljs-title': { color: '#82aaff' },
+  'hljs-attr': { color: '#89ddff' },
+  'hljs-params': { color: 'rgba(255,255,255,0.85)' },
+  'hljs-punctuation': { color: 'rgba(255,255,255,0.5)' },
+  'hljs-comment': { color: 'rgba(255,255,255,0.35)', fontStyle: 'italic' },
+  'hljs-variable': { color: '#f07178' },
+  'hljs-property': { color: '#89ddff' },
 }
 import { cn } from '~/lib/utils'
 
@@ -44,8 +51,8 @@ export const Code = memo(function Code({
   wrapLines?: boolean
 }) {
   return (
-    <div className={cn('relative border rounded-md p-1 text-xs', className)}>
-      {!hideCopyButton ? <CopyButton className="absolute right-0 top-0.5" content={children} /> : null}
+    <div className={cn('relative border rounded-md p-3 text-xs', className)}>
+      {!hideCopyButton ? <CopyButton className="absolute right-1 top-1" content={children} /> : null}
       <SyntaxHighlighter
         wrapLongLines={wrapLongLines}
         wrapLines={wrapLines}

packages/pipe-ui/app/dashboard/pipeline-disconnected.tsx

@@ -3,7 +3,6 @@
 import { ArrowUpRightIcon, Terminal } from 'lucide-react'
 
 import { Alert, AlertDescription, AlertTitle } from '~/components/ui/alert'
-import { Button } from '~/components/ui/button'
 import { Code } from '~/components/ui/code'
 
 const DOCS_URL = 'https://beta.docs.sqd.dev'
@@ -12,31 +11,21 @@ const example = `import { commonAbis, evmDecoder, evmPortalStream } from '@subsq
 import { metricsServer } from '@subsquid/pipes/metrics/node'
 
 async function cli() {
-  // Create a data stream from the Ethereum mainnet portal
   const stream = evmPortalStream({
     id: 'erc20-transfers',
     portal: 'https://portal.sqd.dev/datasets/ethereum-mainnet',
     outputs: {
-      // Decode ERC-20 Transfer events starting from block 12,000,000
       erc20: evmDecoder({
         range: { from: '12,000,000' },
         events: {
           transfers: commonAbis.erc20.events.Transfer,
         },
       }),
     },
-
-    /*
-     * IMPORTANT!
-     * ============================
-     * Enable the metrics server to connect with the Pipe UI dashboard.
-     * Without it, no metrics will be collected or displayed.
-     * ============================
-     */
+    // Enable the metrics server to connect with the Pipe UI dashboard
     metrics: metricsServer(),
   })
 
-  // Consume the stream and log the number of parsed transfers in each batch
   for await (const { data } of stream) {
     console.log(\`parsed \${data.erc20.transfers.length} transfers\`)
   }
@@ -54,32 +43,34 @@ export function PipelineDisconnected() {
       </Alert>
 
       <div className="mt-10">
-        <h1 className="mt-4 mb-2 font-medium">Get started with Pipes SDK</h1>
+        <h1 className="text-lg font-semibold tracking-tight">Get started with Pipes SDK</h1>
 
-        <div className="mt-4">
-          <h4 className="mb-1">1. Install npm package</h4>
-          <Code language="bash" className="text-xs">
+        <div className="mt-6">
+          <h4 className="mb-2 text-sm text-muted-foreground font-medium">1. Install npm package</h4>
+          <Code language="bash">
             npm install @subsquid/pipes
           </Code>
         </div>
 
-        <div className="mt-4">
-          <h4 className="mb-1">2. Run a simple pipe</h4>
-          <Code language="typescript" className="text-xs">
+        <div className="mt-6">
+          <h4 className="mb-2 text-sm text-muted-foreground font-medium">2. Run a simple pipe</h4>
+          <Code language="typescript">
             {example}
           </Code>
         </div>
 
-        <div className="mt-4">
-          <h4 className="mb-1">3. Explore docs</h4>
-          <div className="text-xs text-muted pt-2 pb-8">
-            <Button size="xl" asChild variant="default">
-              <a href={`${DOCS_URL}/en/sdk/pipes-sdk/quickstart`} target="_blank">
-                Documentation
-                <ArrowUpRightIcon />
-              </a>
-            </Button>
-          </div>
+        <div className="mt-6">
+          <h4 className="text-sm text-muted-foreground font-medium">
+            3. Explore the{' '}
+            <a
+              href={`${DOCS_URL}/en/sdk/pipes-sdk/quickstart`}
+              target="_blank"
+              className="text-blue-400 underline underline-offset-4 hover:text-blue-300"
+            >
+              documentation
+              <ArrowUpRightIcon className="inline h-3.5 w-3.5 ml-0.5 -translate-y-px" />
+            </a>
+          </h4>
         </div>
       </div>
     </div>

packages/pipe-ui/app/dashboard/pipeline.tsx

@@ -7,15 +7,15 @@ import { Terminal } from 'lucide-react'
 // @ts-ignore
 import { Sparklines, SparklinesLine } from 'react-sparklines'
 
-import { PipeStatus, useStats } from '~/hooks/use-metrics'
-import { useServerIndex } from '~/hooks/use-server-context'
 import { Alert, AlertDescription, AlertTitle } from '~/components/ui/alert'
 import { Tabs, TabsContent, TabsList, TabsTrigger } from '~/components/ui/tabs'
 import { humanBytes } from '~/dashboard/formatters'
 import { PipelineDisconnected } from '~/dashboard/pipeline-disconnected'
 import { Profiler } from '~/dashboard/profiler'
 import { QueryExemplar } from '~/dashboard/query-exemplar'
 import { TransformationExemplar } from '~/dashboard/transformation-exemplar'
+import { PipeStatus, useStats } from '~/hooks/use-metrics'
+import { useServerIndex } from '~/hooks/use-server-context'
 
 const sparklineStyle = { fill: '#d0a9e2' }
 const sparklineColor = 'rgb(170, 140, 235)'