Preserve CLI API filter output and page metadata

README.md

@@ -1795,15 +1795,20 @@ polymorfa api /platform/customers --surface platform \
   page bodies and whose `metadata.pages` is the page count.
 - `--slurp` requires `--paginate` and combines the page bodies into one array
   before `--jq` or `--template` runs. Without `--slurp`, filters run on each page.
+  In human output, `--slurp --include` prints an array with each page's body and
+  response metadata. `--json` prints one document with an array of page bodies
+  and the final response metadata plus the page count; `--include` only changes
+  human output.
 - `-q/--jq` runs a jq 1.8 program with an embedded WebAssembly jq, so no `jq`
-  binary is required. Strings print without quotes. The jq runtime has its own
+  binary is required. Strings print without quotes and keep leading and trailing
+  whitespace. The jq runtime has its own
   environment, so `$ENV` does not expose your shell variables or credentials.
 - `-t/--template` renders a Go-style template: `{{.path.to.value}}`,
   `{{range .list}}…{{else}}…{{end}}`, `{{if .value}}…{{else}}…{{end}}`,
   `{{$.root}}` inside a range, `{{json .x}}`, `{{len .x}}`, `{{upper .x}}`,
   `{{lower .x}}`, `{{join ", " .list}}`, string literals such as `{{"\n"}}`,
   comments, and `{{-`/`-}}` whitespace trimming. Missing values render as an
-  empty string.
+  empty string. Quoted strings can contain `}}`.
 - `--jq`, `--template`, and `--json` are mutually exclusive, as are `--include`
   and a filter.
 

src/api/jq.ts

@@ -7,19 +7,28 @@ import {PolymorfaValidationError} from '@polymorfa/sdk'
  * host credentials.
  */
 export async function applyJq(value: unknown, filter: string): Promise<string> {
-  const {raw} = await import('jq-wasm')
+  const {json, JqError} = await import('jq-wasm')
   // jq-wasm parses a string argument as JSON *text*, so a decoded JSON string
   // body would be re-parsed and fail. Serialize first so every value arrives
-  // as the JSON it stands for.
-  const result = await raw(JSON.stringify(value) ?? 'null', filter, ['-r'])
-  if (result.exitCode !== 0) {
-    const detail = result.stderr
+  // as the JSON it stands for. raw() trims the *entire* stdout stream inside
+  // jq-wasm, including spaces and newlines that belong to string values.
+  try {
+    const values = await json<unknown>(JSON.stringify(value) ?? 'null', filter)
+    return (
+      values
+        .map((item) =>
+          typeof item === 'string' ? item : JSON.stringify(item, undefined, 2),
+        )
+        .join('\n') + (values.length === 0 ? '' : '\n')
+    )
+  } catch (error) {
+    if (!(error instanceof JqError)) throw error
+    const detail = error.stderr
       .trim()
       .replace(/^jq: error(?: \(at [^)]*\))?: /, '')
     throw new PolymorfaValidationError(
-      `Invalid --jq filter: ${detail || `jq exited with ${result.exitCode}`}`,
+      `Invalid --jq filter: ${detail || `jq exited with ${error.exitCode}`}`,
       {code: 'invalid_jq'},
     )
   }
-  return result.stdout
 }

src/api/template.ts

@@ -47,7 +47,7 @@ function tokenize(template: string): Token[] {
       start += 1
     }
     if (text.length > 0) tokens.push({kind: 'text', value: text})
-    const close = template.indexOf('}}', start)
+    const close = actionEnd(template, start)
     if (close < 0) throw templateError('unclosed action "{{"')
     let end = close
     let trimRight = false
@@ -65,6 +65,30 @@ function tokenize(template: string): Token[] {
   return tokens
 }
 
+function actionEnd(template: string, start: number): number {
+  let quoted = false
+  let escaped = false
+  let comment = template.startsWith('/*', start)
+  for (let index = start; index < template.length - 1; index++) {
+    const character = template[index]
+    if (comment) {
+      if (character === '*' && template[index + 1] === '/') {
+        comment = false
+        index++
+      }
+    } else if (quoted) {
+      if (escaped) escaped = false
+      else if (character === '\\') escaped = true
+      else if (character === '"') quoted = false
+    } else if (character === '"') {
+      quoted = true
+    } else if (character === '}' && template[index + 1] === '}') {
+      return index
+    }
+  }
+  return -1
+}
+
 function parse(tokens: Token[]): Node[] {
   let position = 0
 

src/commands/api.ts

@@ -54,7 +54,8 @@ export default class Api extends BaseCommand {
     }),
     include: Flags.boolean({
       char: 'i',
-      description: 'Include response metadata in human output.',
+      description:
+        'Include response metadata in human output, including each page with --paginate --slurp.',
       exclusive: ['jq', 'template'],
     }),
     input: Flags.string({
@@ -64,7 +65,7 @@ export default class Api extends BaseCommand {
     jq: Flags.string({
       char: 'q',
       description:
-        'Filter the response with a jq expression (embedded jq 1.8). Strings print without quotes.',
+        'Filter the response with a jq expression (embedded jq 1.8). Strings print without quotes and keep their whitespace.',
       exclusive: ['json', 'template'],
     }),
     'max-pages': Flags.integer({
@@ -90,7 +91,7 @@ export default class Api extends BaseCommand {
     slurp: Flags.boolean({
       dependsOn: ['paginate'],
       description:
-        'With --paginate, combine all pages into one JSON array before --jq or --template.',
+        'With --paginate, collect pages into one array before filtering or printing.',
     }),
     surface: Flags.option({
       description: 'API credential surface.',
@@ -193,7 +194,7 @@ export default class Api extends BaseCommand {
     const last = pages[pages.length - 1] as RawApiResult
     this.writeValue(
       {data: bodies, metadata: {...last.metadata, pages: pages.length}},
-      formatHuman(bodies),
+      formatHuman(flags.include ? pages : bodies),
       flags,
     )
   }

test/api/template.test.ts

@@ -38,6 +38,18 @@ describe('api template', () => {
     )
   })
 
+  test('keeps closing braces inside quoted action strings', () => {
+    expect(renderTemplate('{{"a}}b"}}/{{join "}}" .data.0.tags}}', data)).toBe(
+      'a}}b/x}}y',
+    )
+    expect(renderTemplate('{{"escaped \\"}} still quoted"}}', data)).toBe(
+      'escaped "}} still quoted',
+    )
+    expect(renderTemplate('a{{/* }} stays in the comment */}}b', data)).toBe(
+      'ab',
+    )
+  })
+
   test('rejects malformed templates', () => {
     expect(() => renderTemplate('{{range .data}}', data)).toThrow(
       'missing {{end}}',

test/commands/api-gh.test.ts

@@ -99,6 +99,15 @@ describe('api gh-style options', () => {
     expect(result.stdout).toBe('plain text\n')
   })
 
+  test('preserves leading and trailing whitespace in jq strings', async () => {
+    server = await startHttpServer({body: {data: ['  padded  ', 'line\n']}})
+
+    const result = await run('api /v1/thing --surface platform --jq .data[]')
+
+    expect(result.error, result.stderr).toBeUndefined()
+    expect(result.stdout).toBe('  padded  \nline\n\n')
+  })
+
   test('applies --jq to each page when paginating', async () => {
     server = await startHttpSequenceServer(pages)
 
@@ -155,6 +164,23 @@ describe('api gh-style options', () => {
     expect(result.stdout).toBe('3\n')
   })
 
+  test('retains each page metadata with --paginate --slurp --include', async () => {
+    server = await startHttpSequenceServer([
+      {...pages[0], headers: {'request-id': 'req_1'}},
+      {...pages[1], headers: {'request-id': 'req_2'}},
+    ])
+
+    const result = await run(
+      'api /platform/customers --surface platform --paginate --slurp --include',
+    )
+
+    expect(result.error, result.stderr).toBeUndefined()
+    expect(JSON.parse(result.stdout)).toMatchObject([
+      {data: pages[0]?.body, metadata: {requestId: 'req_1', status: 200}},
+      {data: pages[1]?.body, metadata: {requestId: 'req_2', status: 200}},
+    ])
+  })
+
   test('renders --template against the response', async () => {
     server = await startHttpServer(pages[1] as {body: unknown})