\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_4_1.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_4_1.html
deleted file mode 100644
index 147c242a1..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_4_1.html
+++ /dev/null
@@ -1,151 +0,0 @@
-
-
-
-
-
-
Q1’24
-
-
Q2’24
-
-
Q3’24
-
-
Q4’24
-
-
Q1’25
-
-
Q2’25
-
-
Q3’25
-
-
Q4’25
-
-
Q1’26
-
-
-
Advertising
-
$
-
35,635
-
$
-
38,329
-
$
-
39,885
-
$
-
46,783
-
$
-
41,392
-
$
-
46,563
-
$
-
50,082
-
$
-
58,137
-
$
-
55,024
-
-
-
Other
-
-
380
-
-
389
-
-
434
-
-
519
-
-
510
-
-
583
-
-
690
-
-
801
-
-
885
-
-
-
Family of Apps
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Revenue
-
-
36,015
-
-
38,718
-
-
40,319
-
-
47,302
-
-
41,902
-
-
47,146
-
-
50,772
-
-
58,938
-
-
55,909
-
-
-
Reality Labs Revenue
-
-
440
-
-
353
-
-
270
-
-
1,083
-
-
412
-
-
370
-
-
470
-
-
955
-
-
402
-
-
-
Total Revenue
-
$
-
36,455
-
$
-
39,071
-
$
-
40,589
-
$
-
48,385
-
$
-
42,314
-
$
-
47,516
-
$
-
51,242
-
$
-
59,893
-
$
-
56,311
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_4_2.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_4_2.html
deleted file mode 100644
index 14c5a0117..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_4_2.html
+++ /dev/null
@@ -1,151 +0,0 @@
-
-
-
-
Family of Apps
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Operating Income
-
$
-
17,664 $
-
-
19,335 $
-
-
21,778 $
-
-
28,332 $
-
-
21,765 $
-
-
24,971 $
-
-
24,967 $
-
-
30,766 $
-
-
26,900
-
-
-
Reality Labs
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Operating (Loss)
-
-
(3,846)
-
-
(4,488)
-
-
(4,428)
-
-
(4,967)
-
-
(4,210)
-
-
(4,530)
-
-
(4,432)
-
-
(6,021)
-
-
(4,028)
-
-
-
Total Income from
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Operations
-
$
-
13,818 $
-
-
14,847 $
-
-
17,350 $
-
-
23,365 $
-
-
17,555 $
-
-
20,441 $
-
-
20,535 $
-
-
24,745 $
-
-
22,872
-
-
-
Operating Margin
-
-
38 %
-
-
38 %
-
-
43 %
-
-
48 %
-
-
41 %
-
-
43 %
-
-
40 %
-
-
41 %
-
-
41 %
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_6_1.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_6_1.html
deleted file mode 100644
index a13e1653f..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-presentation/tables/table_page_6_1.html
+++ /dev/null
@@ -1,64 +0,0 @@
-
-
-
-
-
Q1’24
-
Q2’24
-
Q3’24
-
Q4’24
-
Q1’25
-
Q2’25
-
Q3’25 (1)
-
Q4’25
-
Q1’26 (2)
-
-
-
Income before income taxes $
-
14,183 $
-
15,106 $
-
17,822 $
-
23,553 $
-
18,382 $
-
20,534 $
-
21,663 $
-
25,354 $
-
21,752
-
-
-
Provision (benefit) for
-
-
-
-
-
-
-
-
-
-
-
-
income taxes $
-
1,814 $
-
1,641 $
-
2,134 $
-
2,715 $
-
1,738 $
-
2,197 $
-
18,954 $
-
2,586 $
-
(5,021)
-
-
-
Effective Tax Rate
-
13 %
-
11 %
-
12 %
-
12 %
-
9 %
-
11 %
-
87 %
-
10 %
-
(23)%
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/chunks.json b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/chunks.json
deleted file mode 100644
index 98590d9b7..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/chunks.json
+++ /dev/null
@@ -1,498 +0,0 @@
-{
- "chunks": [
- {
- "chunk_id": "asset_0fa8bac4-af93-5a59-ae3b-c524bc4a2928",
- "type": "table",
- "content": "tables/table_page_1_1.html",
- "path": "tables/table_page_1_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 1
- ],
- "file_path": "tables/table_page_1_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_ad8e5439-900b-53da-ab51-64bd2b77a1d6",
- "type": "table",
- "content": "tables/table_page_5_1.html",
- "path": "tables/table_page_5_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 5
- ],
- "file_path": "tables/table_page_5_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_d2aa7ea5-56cb-5a07-b966-9846d1a180d7",
- "type": "table",
- "content": "tables/table_page_6_1.html",
- "path": "tables/table_page_6_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 6
- ],
- "file_path": "tables/table_page_6_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_193aed87-0f6a-5b0a-a4c7-9f84e6671263",
- "type": "table",
- "content": "tables/table_page_7_1.html",
- "path": "tables/table_page_7_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 7
- ],
- "file_path": "tables/table_page_7_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_bb67915b-87ea-5e5d-922e-ca0498422186",
- "type": "table",
- "content": "tables/table_page_8_1.html",
- "path": "tables/table_page_8_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 8
- ],
- "file_path": "tables/table_page_8_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_536926bc-014f-5b0c-8928-f1e105212edc",
- "type": "table",
- "content": "tables/table_page_9_1.html",
- "path": "tables/table_page_9_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 9
- ],
- "file_path": "tables/table_page_9_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "node_33903cf9-9081-5e45-be08-0dcefd82909b",
- "type": "page",
- "content": "Meta Reports First Quarter 2026 Results\nMENLO PARK, Calif. – April 29, 2026 – Meta Platforms, Inc. (Nasdaq: META) today reported financial results for the \nquarter ended March 31, 2026.\n\"We had a milestone quarter with strong momentum across our apps and the release of our first model from Meta \nSuperintelligence Labs,\" said Mark Zuckerberg, Meta founder and CEO. \"We're on track to deliver personal superintelligence \nto billions of people.\"\nFirst Quarter 2026 Financial Highlights\nThree Months Ended March 31,\n% Change\nIn millions, except percentages and per share amounts\n2026\n2025\nRevenue\n$ \n56,311 $ \n42,314 \n 33 %\nCosts and expenses\n \n33,439 \n24,759 \n 35 %\nIncome from operations\n$ \n22,872 $ \n17,555 \n 30 %\nOperating margin\n 41 %\n 41 %\nProvision (benefit) for income taxes (1)\n$ \n(5,021) $ \n1,738 \nNM\nEffective tax rate (1)\n (23) %\n 9 %\nNet income\n$ \n26,773 $ \n16,644 \n 61 %\nDiluted earnings per share (EPS) (1)\n$ \n10.44 $ \n6.43 \n 62 %\n____________________________________\nNM — not meaningful\n(1) Includes an $8.03 billion income tax benefit recognized in the first quarter of 2026, which partially offsets the $15.93 billion non-cash \ntax charge recorded in the third quarter of 2025 upon enactment of the One Big Beautiful Bill Act. This benefit is the result of U.S. \nTreasury Notice 2026-7, which addressed the Corporate Alternative Minimum Tax treatment of previously capitalized U.S. research and \ndevelopment costs. Excluding this tax benefit, our effective tax rate would have been 37 percentage points higher and our diluted earnings \nper share (EPS) would have been $3.13 lower.\nFirst Quarter 2026 Operational and Other Financial Highlights\n•\nFamily daily active people (DAP) – DAP was 3.56 billion on average for March 2026, an increase of 4% year-\nover-year. The slight decline in DAP on a quarter-over-quarter basis was driven by internet disruptions in Iran, as \nwell as a restriction on access to WhatsApp in Russia. \n•\nAd impressions – Ad impressions delivered across our Family of Apps increased by 19% year-over-year. \n•\nAverage price per ad – Average price per ad increased by 12% year-over-year.\n•\nRevenue – Revenue was $56.31 billion, an increase of 33% year-over-year. Revenue on a constant currency basis \nwould have increased by 29% year-over-year.\n•\nCosts and expenses – Total costs and expenses were $33.44 billion, an increase of 35% year-over-year.\n•\nCapital expenditures – Capital expenditures, including principal payments on finance leases, were $19.84 billion. \n•\nCapital return program – Dividend and dividend equivalent payments were $1.35 billion.\n•\nCash, cash equivalents, and marketable securities – Cash, cash equivalents, and marketable securities were \n$81.18 billion as of March 31, 2026. \n•\nCash flow – Cash flow from operating activities was $32.23 billion, and free cash flow was $12.39 billion.(1)\n•\nHeadcount – Headcount was 77,986 as of March 31, 2026, an increase of 1% year-over-year.\n____________________________________\n(1) For more information on our free cash flow non-GAAP financial measure, see the sections entitled \"Non-GAAP Financial Measures\" \nand \"Reconciliation of GAAP to Non-GAAP Results\" in this press release.\n1",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/First Quarter 2026 Financial Highlights",
- "metadata": {
- "length": 3185,
- "summary": "This section presents a table of financial highlights for Meta Platforms, Inc. for the three months ended March 31, 2026, compared to the same period in 2025. Key metrics include Revenue ($56,311 million), Costs and expenses ($33,439 million), Income from operations ($22,872 million), Net income ($26,773 million), and Diluted earnings per share ($10.44). The table also details the effective tax rate and includes a footnote explaining significant tax benefits related to the One Big Beautiful Bill Act.",
- "page_nums": [
- 1
- ],
- "keywords": [
- "Meta Platforms, Inc.",
- "Nasdaq: META"
- ],
- "connect_to": [
- {
- "target": "asset_0fa8bac4-af93-5a59-ae3b-c524bc4a2928",
- "relation": "embeds",
- "ref": "[tables/table_page_1_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 1,
- "artifact_ref": "page_citation_assets/page-1.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_b6ed821a-bb98-5fd2-a21b-db12da14dc49",
- "type": "page",
- "content": "[SAME-AS Meta Q1 2026 Exhibit 99.1.pdf/Root/First Quarter 2026 Financial Highlights p1]",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/First Quarter 2026 Operational and Other Financial Highlights",
- "metadata": {
- "length": 87,
- "summary": "This section details Meta's operational and financial performance for the first quarter of 2026. It reports that Family daily active people (DAP) reached an average of 3.56 billion, a 4% year-over-year increase, though noting declines in Iran and Russia. Ad impressions grew by 19%, while the average price per ad rose 12%. Total revenue was $56.31 billion (up 33%), with costs and expenses totaling $33.44 billion. The company also reported capital expenditures of $19.84 billion, cash flow from operating activities of $32.23 billion, free cash flow of $12.39 billion, and a headcount of 77,986.",
- "page_nums": [
- 1
- ],
- "keywords": [
- "Meta",
- "Iran",
- "Russia"
- ],
- "connect_to": [
- {
- "target": "asset_0fa8bac4-af93-5a59-ae3b-c524bc4a2928",
- "relation": "related",
- "ref": "[tables/table_page_1_1.html]",
- "same_as_owner": "Meta Q1 2026 Exhibit 99.1.pdf/Root/First Quarter 2026 Financial Highlights"
- }
- ],
- "page_assets": [
- {
- "page_num": 1,
- "artifact_ref": "page_citation_assets/page-1.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_493babef-72e2-5f5f-84d1-4c9517267093",
- "type": "page",
- "content": "CFO Outlook Commentary\nWe expect second quarter 2026 total revenue to be in the range of $58-61 billion. Our guidance assumes foreign currency is \nan approximately 2% tailwind to year-over-year total revenue growth, based on current exchange rates.\nWe expect full year 2026 total expenses to be in the range of $162-169 billion, unchanged from our prior outlook.\nWe continue to expect to deliver operating income this year that is above 2025 operating income.\nWe anticipate 2026 capital expenditures, including principal payments on finance leases, to be in the range of $125-145 \nbillion, increased from our prior range of $115-135 billion. This reflects our expectations for higher component pricing this \nyear and, to a lesser extent, additional data center costs to support future year capacity.\nAbsent any changes to our tax landscape, we expect our tax rate for the remaining quarters of 2026 to be between 13-16%.\nLastly, we continue to monitor active legal and regulatory matters, including headwinds in the EU and the U.S. that could \nsignificantly impact our business and financial results. For example, we continue to see scrutiny on youth-related issues and \nhave additional trials scheduled for this year in the U.S., which may ultimately result in a material loss.\n2",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/CFO Outlook Commentary",
- "metadata": {
- "length": 1280,
- "summary": "This page contains a CFO Outlook Commentary providing financial guidance for 2026. It details expected revenue ranges of $58-61 billion for the second quarter and total expenses between $162-169 billion for the full year. The text also outlines anticipated capital expenditures, tax rates, and mentions ongoing legal and regulatory headwinds in the EU and U.S.",
- "page_nums": [
- 2
- ],
- "keywords": [
- "EU",
- "U.S."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 2,
- "artifact_ref": "page_citation_assets/page-2.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_96ec4a54-77ad-59b1-870d-466f366fedd4",
- "type": "page",
- "content": "Webcast and Conference Call Information\nMeta will host a conference call to discuss its results at 2:30 p.m. PT / 5:30 p.m. ET today. The live webcast of the call can be \naccessed at the Meta Investor Relations website at investor.atmeta.com, along with the company's earnings press release, \nfinancial tables, and slide presentation.\nFollowing the call, a replay will be available at the same website. Transcripts of conference calls with publishing equity \nresearch analysts held today will also be posted to the investor.atmeta.com website.\nDisclosure Information\nMeta uses the investor.atmeta.com and meta.com/news websites as well as Mark Zuckerberg's Facebook profile \n(facebook.com/zuck), Instagram account (instagram.com/zuck) and Threads profile (threads.net/zuck) as means of disclosing \nmaterial non-public information and for complying with its disclosure obligations under Regulation FD. \nAbout Meta\nMeta is building the future of human connection, powered by artificial intelligence and immersive technologies. When \nFacebook launched in 2004, it changed the way people connect. Apps like Messenger, Instagram, and WhatsApp further \nempowered billions around the world. Now, Meta is moving beyond 2D screens toward experiences that foster deeper \nconnections and unlock new possibilities.\nContacts\nInvestors: \nKenneth Dorell\ninvestor@meta.com / investor.atmeta.com\nPress: \nMatt Tye\npress@meta.com / meta.com/news\n3",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Webcast and Conference Call Information",
- "metadata": {
- "length": 1428,
- "summary": "This section provides details regarding a conference call hosted by Meta to discuss its results, scheduled for 2:30 p.m. PT / 5:30 p.m. ET. It specifies that the live webcast and subsequent replay will be available on the Meta Investor Relations website (investor.atmeta.com), along with earnings press releases, financial tables, slide presentations, and transcripts of analyst calls.",
- "page_nums": [
- 3
- ],
- "keywords": [
- "Meta",
- "investor.atmeta.com"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_237a0643-c556-52df-ac0c-e2253fd96bbc",
- "type": "page",
- "content": "[SAME-AS Meta Q1 2026 Exhibit 99.1.pdf/Root/Webcast and Conference Call Information p3]",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Disclosure Information",
- "metadata": {
- "length": 87,
- "summary": "This section outlines Meta's channels for disclosing material non-public information and complying with Regulation FD. These channels include the investor.atmeta.com and meta.com/news websites, as well as Mark Zuckerberg's social media profiles on Facebook, Instagram, and Threads.",
- "page_nums": [
- 3
- ],
- "keywords": [
- "Meta",
- "Mark Zuckerberg",
- "Facebook",
- "Instagram",
- "Threads"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_17037ae6-fb46-56dc-9663-e194754b5716",
- "type": "page",
- "content": "[SAME-AS Meta Q1 2026 Exhibit 99.1.pdf/Root/Webcast and Conference Call Information p3]",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/About Meta",
- "metadata": {
- "length": 87,
- "summary": "This section describes Meta's mission to build the future of human connection using artificial intelligence and immersive technologies. It highlights the company's history starting with Facebook in 2004, mentions its apps like Messenger, Instagram, and WhatsApp, and states that Meta is moving beyond 2D screens to foster deeper connections.",
- "page_nums": [
- 3
- ],
- "keywords": [
- "Meta",
- "Facebook",
- "Messenger",
- "Instagram",
- "WhatsApp"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_9c53103d-67e2-5a04-8177-69a8a2aab456",
- "type": "page",
- "content": "[SAME-AS Meta Q1 2026 Exhibit 99.1.pdf/Root/Webcast and Conference Call Information p3]",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Contacts",
- "metadata": {
- "length": 87,
- "summary": "The 'Contacts' section lists the designated points of contact for Meta, divided into two categories: Investors and Press. For investors, it provides the name Kenneth Dorell along with his email address (investor@meta.com) and website (investor.atmeta.com). For press inquiries, it lists Matt Tye with his email address (press@meta.com) and website (meta.com/news).",
- "page_nums": [
- 3
- ],
- "keywords": [
- "Kenneth Dorell",
- "Matt Tye"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_3cb1e052-e576-5744-83ec-71e18d42b442",
- "type": "page",
- "content": "Forward-Looking Statements\nThis press release contains forward-looking statements regarding our future business plans and expectations. These forward-\nlooking statements are only predictions and may differ materially from actual results due to a variety of factors including: the \nimpact of macroeconomic conditions on our business and financial results, including as a result of geopolitical events; our \nability to retain or increase users and engagement levels; our reliance on advertising revenue; our dependency on data signals \nand mobile operating systems, networks, and standards that we do not control; changes to the content or application of third-\nparty policies that impact our advertising practices; risks associated with new products and changes to existing products as \nwell as other new business initiatives, including our artificial intelligence initiatives and Reality Labs efforts; our emphasis on \ncommunity growth and engagement and the user experience over short-term financial results; maintaining and enhancing our \nbrand and reputation; our ongoing privacy, safety, security, and content and advertising review and enforcement efforts; \ncompetition; risks associated with government actions that could restrict access to our products or impair our ability to sell \nadvertising in certain countries; litigation and government inquiries; privacy, legislative, and regulatory concerns or \ndevelopments; risks associated with acquisitions; security breaches; our ability to manage our scale and geographically-\ndispersed operations; and market conditions or other factors affecting capital return to stockholders. These and other potential \nrisks and uncertainties that could cause actual results to differ from the results predicted are more fully detailed under the \ncaption \"Risk Factors\" in our Annual Report on Form 10-K filed with the SEC on January 29, 2026, which is available on our \nInvestor Relations website at investor.atmeta.com and on the SEC website at www.sec.gov. Additional information will also \nbe set forth in our Quarterly Report on Form 10-Q for the quarter ended March 31, 2026. In addition, please note that the date \nof this press release is April 29, 2026, and any forward-looking statements contained herein are based on assumptions that we \nbelieve to be reasonable as of this date. We undertake no obligation to update these statements as a result of new information \nor future events.\nFor a discussion of limitations in the measurement of certain of our community metrics, see the section entitled \"Limitations \nof Key Metrics and Other Data\" in our most recent quarterly or annual report filed with the SEC.\nNon-GAAP Financial Measures \nTo supplement our condensed consolidated financial statements, which are prepared and presented in accordance with \ngenerally accepted accounting principles in the United States (GAAP), we use the following non-GAAP financial measures: \nrevenue excluding foreign exchange effect, advertising revenue excluding foreign exchange effect, and free cash flow. The \npresentation of these financial measures is not intended to be considered in isolation or as a substitute for, or superior to, \nfinancial information prepared and presented in accordance with GAAP. Investors are cautioned that there are material \nlimitations associated with the use of non-GAAP financial measures as an analytical tool. In addition, these measures may be \ndifferent from non-GAAP financial measures used by other companies, limiting their usefulness for comparison purposes. \nWe compensate for these limitations by providing specific information regarding the GAAP amounts excluded from these \nnon-GAAP financial measures.\nWe believe these non-GAAP financial measures provide investors with useful supplemental information about the financial \nperformance of our business, enable comparison of financial results between periods where certain items may vary \nindependent of business performance, and allow for greater transparency with respect to key metrics used by management in \noperating our business. \nOur non-GAAP financial measures are adjusted for the following items: \nForeign exchange effect on revenue. To calculate revenue on a constant currency basis, we translate current period revenue \nusing the prior year's monthly exchange rates for our settlement or billing currencies other than the U.S. dollar, which we \nbelieve is a useful metric that facilitates comparison to our historical performance.\nPurchases of property and equipment; Principal payments on finance leases. We subtract both purchases of property and \nequipment, and principal payments on finance leases in our calculation of free cash flow because we believe that these two \nitems collectively represent the amount of property and equipment we need to procure to support our business, regardless of \nwhether we procure such property or equipment with a finance lease. We believe that this methodology can provide useful \nsupplemental information to help investors better understand underlying trends in our business. Free cash flow is not intended \nto represent our residual cash flow available for discretionary expenditures. \nFor more information on our non-GAAP financial measures and a reconciliation of GAAP to non-GAAP measures, see the \n\"Reconciliation of GAAP to Non-GAAP Results\" table in this press release.\n4",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Forward-Looking Statements",
- "metadata": {
- "length": 5368,
- "summary": "The section contains a standard legal disclaimer regarding forward-looking statements. It warns that the press release includes predictions about future business plans and expectations which may differ from actual results due to various factors such as macroeconomic conditions, geopolitical events, user retention, advertising revenue reliance, data signal dependency, product changes, AI initiatives, community growth focus, brand reputation, privacy/security efforts, competition, government actions, litigation, acquisitions, security breaches, operational scale, and market conditions. The text references specific filings (Form 10-K filed January 29, 2026, and Form 10-Q for the quarter ended March 31, 2026) available on the SEC website and the company's Investor Relations site (investor.atmeta.com). It notes the press release date is April 29, 2026, and states the company undertakes no obligation to update these statements.",
- "page_nums": [
- 4
- ],
- "keywords": [
- "SEC",
- "Form 10-K",
- "January 29, 2026",
- "Investor Relations website",
- "investor.atmeta.com",
- "www.sec.gov",
- "Quarterly Report on Form 10-Q",
- "March 31, 2026",
- "April 29, 2026"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 4,
- "artifact_ref": "page_citation_assets/page-4.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_1801f1f2-b208-5a57-96f0-b7dc62f52ac0",
- "type": "page",
- "content": "[SAME-AS Meta Q1 2026 Exhibit 99.1.pdf/Root/Forward-Looking Statements p4]\n\nMETA PLATFORMS, INC.\nCONDENSED CONSOLIDATED STATEMENTS OF INCOME\n(In millions, except per share amounts)\n(Unaudited)\nThree Months Ended March 31,\n2026\n2025\nRevenue\n$ \n56,311 $ \n42,314 \nCosts and expenses:\nCost of revenue\n \n10,218 \n7,572 \nResearch and development\n \n17,699 \n12,150 \nMarketing and sales\n \n2,908 \n2,757 \nGeneral and administrative\n \n2,614 \n2,280 \nTotal costs and expenses\n \n33,439 \n24,759 \nIncome from operations\n \n22,872 \n17,555 \nInterest and other income (expense), net\n \n(1,120) \n827 \nIncome before income taxes\n \n21,752 \n18,382 \nProvision (benefit) for income taxes\n \n(5,021) \n1,738 \nNet income\n$ \n26,773 $ \n16,644 \nEarnings per share:\nBasic\n$ \n10.57 $ \n6.59 \nDiluted\n$ \n10.44 $ \n6.43 \nWeighted-average shares used to compute earnings per share:\nBasic\n \n2,534 \n2,527 \nDiluted\n \n2,564 \n2,590 \n5\n\nMETA PLATFORMS, INC.\nCONDENSED CONSOLIDATED BALANCE SHEETS\n(In millions)\n(Unaudited)\nMarch 31, 2026\nDecember 31, 2025\nAssets\nCurrent assets:\nCash and cash equivalents\n$ \n23,426 $ \n35,873 \nMarketable securities\n \n57,754 \n45,719 \nAccounts receivable, net\n \n17,470 \n19,769 \nPrepaid expenses and other current assets\n \n11,115 \n7,361 \nTotal current assets\n \n109,765 \n108,722 \nNon-marketable equity investments\n \n28,410 \n27,524 \nProperty and equipment, net\n \n194,776 \n176,400 \nOperating lease right-of-use assets\n \n23,268 \n20,404 \nGoodwill\n \n24,748 \n24,534 \nOther assets\n \n14,283 \n8,437 \nTotal assets\n$ \n395,250 $ \n366,021 \nLiabilities and stockholders' equity\nCurrent liabilities:\nAccounts payable\n$ \n13,326 $ \n8,894 \nOperating lease liabilities, current\n \n2,414 \n2,213 \nAccrued expenses and other current liabilities\n \n31,013 \n30,729 \nTotal current liabilities\n \n46,753 \n41,836 \nOperating lease liabilities, non-current\n \n25,607 \n22,940 \nLong-term debt\n \n58,748 \n58,744 \nLong-term income taxes\n \n16,849 \n21,005 \nOther liabilities\n \n3,612 \n4,253 \nTotal liabilities\n \n151,569 \n148,778 \nCommitments and contingencies\nStockholders' equity:\nCommon stock and additional paid-in capital\n \n99,337 \n95,793 \nAccumulated other comprehensive income (loss)\n \n(303) \n271 \nRetained earnings\n \n144,647 \n121,179 \nTotal stockholders' equity\n \n243,681 \n217,243 \nTotal liabilities and stockholders' equity\n$ \n395,250 $ \n366,021 \n6\n\nMETA PLATFORMS, INC.\nCONDENSED CONSOLIDATED STATEMENTS OF CASH FLOWS\n(In millions)\n(Unaudited)\nThree Months Ended March 31,\n2026\n2025\nCash flows from operating activities\nNet income\n$ \n26,773 $ \n16,644 \nAdjustments to reconcile net income to net cash provided by operating activities:\nDepreciation and amortization\n \n5,999 \n3,900 \nShare-based compensation\n \n6,032 \n4,147 \nDeferred income taxes\n \n123 \n(993) \nUnrealized (gain) loss on equity investments\n \n1,075 \n(135) \nOther\n \n(17) \n(96) \nChanges in assets and liabilities:\nAccounts receivable\n \n2,128 \n2,804 \nPrepaid expenses and other current assets\n \n(2,424) \n360 \nOther assets\n \n(1,082) \n(52) \nAccounts payable\n \n(937) \n(1,034) \nAccrued expenses and other current liabilities\n \n(271) \n(2,231) \nOther liabilities\n \n(5,173) \n712 \nNet cash provided by operating activities\n \n32,226 \n24,026 \nCash flows from investing activities\nPurchases of property and equipment\n \n(18,997) \n(12,941) \nPurchases of marketable securities\n \n(32,978) \n(11,763) \nSales and maturities of marketable securities\n \n19,176 \n4,784 \nPurchases of non-marketable equity investments\n \n(544) \n(100) \nPayments for held-for-sale assets\n \n(118) \n— \nAcquisitions of businesses and intangible assets\n \n(372) \n(1) \nOther investing activities\n \n155 \n11 \nNet cash used in investing activities\n \n(33,678) \n(20,010) \nCash flows from financing activities\nTaxes paid related to net share settlement of equity awards\n \n(4,423) \n(4,883) \nRepurchases of Class A common stock\n \n— \n(12,754) \nPayments for dividends and dividend equivalents\n \n(1,346) \n(1,329) \nPrincipal payments on finance leases\n \n(843) \n(751) \nOther financing activities\n \n59 \n222 \nNet cash used in financing activities\n \n(6,553) \n(19,495) \nEffect of exchange rate changes on cash, cash equivalents, restricted cash, and \nrestricted cash equivalents\n \n7 \n112 \nNet decrease in cash, cash equivalents, restricted cash, and restricted cash \nequivalents\n \n(7,998) \n(15,367) \nCash, cash equivalents, restricted cash, and restricted cash equivalents at beginning \nof the period\n \n39,100 \n45,438 \nCash, cash equivalents, restricted cash, and restricted cash equivalents at end \nof the period\n$ \n31,102 $ \n30,071 \nReconciliation of cash, cash equivalents, restricted cash, and restricted cash \nequivalents to the condensed consolidated balance sheets\nCash and cash equivalents\n$ \n23,426 $ \n28,750 \nRestricted cash and restricted cash equivalents, included in prepaid expenses and \nother current assets\n \n340 \n71 \nRestricted cash and restricted cash equivalents, included in other assets\n \n7,336 \n1,250 \nTotal cash, cash equivalents, restricted cash, and restricted cash equivalents\n$ \n31,102 $ \n30,071 \nSupplemental cash flow data\nCash paid for income taxes, net\n$ \n541 $ \n448 \n7",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Non-GAAP Financial Measures",
- "metadata": {
- "length": 5102,
- "summary": "This section explains the company's use of non-GAAP financial measures to supplement its GAAP financial statements. It details three specific measures: revenue excluding foreign exchange effect, advertising revenue excluding foreign exchange effect, and free cash flow. The text clarifies that these measures are intended to provide useful supplemental information for investors by enabling comparison of results between periods and offering greater transparency into key metrics used by management. It also describes the adjustments made to calculate these measures, such as translating revenue using prior year's exchange rates and subtracting purchases of property and equipment and principal payments on finance leases from free cash flow.",
- "page_nums": [
- 4,
- 5,
- 6,
- 7
- ],
- "keywords": [
- "SEC",
- "Form 10-K",
- "Form 10-Q"
- ],
- "connect_to": [
- {
- "target": "asset_ad8e5439-900b-53da-ab51-64bd2b77a1d6",
- "relation": "embeds",
- "ref": "[tables/table_page_5_1.html]"
- },
- {
- "target": "asset_d2aa7ea5-56cb-5a07-b966-9846d1a180d7",
- "relation": "embeds",
- "ref": "[tables/table_page_6_1.html]"
- },
- {
- "target": "asset_193aed87-0f6a-5b0a-a4c7-9f84e6671263",
- "relation": "embeds",
- "ref": "[tables/table_page_7_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 4,
- "artifact_ref": "page_citation_assets/page-4.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 5,
- "artifact_ref": "page_citation_assets/page-5.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 6,
- "artifact_ref": "page_citation_assets/page-6.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 7,
- "artifact_ref": "page_citation_assets/page-7.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_5afeb565-fa1f-5df8-8f2c-434aa72a8b03",
- "type": "page",
- "content": "Segment Results\nWe report our financial results for our two reportable segments: Family of Apps (FoA) and Reality Labs (RL). FoA includes \nFacebook, Instagram, Messenger, WhatsApp, and other services. RL includes our virtual and augmented reality related \nconsumer hardware, software, and content.\nThe following table sets forth our segment information of revenue and income (loss) from operations:\nSegment Information\n(In millions)\n(Unaudited)\nThree Months Ended March 31,\n2026\n2025\nRevenue:\nAdvertising\n$ \n55,024 $ \n41,392 \nOther revenue\n \n885 \n510 \nFamily of Apps\n \n55,909 \n41,902 \nReality Labs\n \n402 \n412 \nTotal revenue\n$ \n56,311 $ \n42,314 \nIncome (loss) from operations:\nFamily of Apps\n$ \n26,900 $ \n21,765 \nReality Labs\n \n(4,028) \n(4,210) \nTotal income from operations\n$ \n22,872 $ \n17,555 \n8",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Segment Results",
- "metadata": {
- "length": 800,
- "summary": "This page presents the segment results for a company, detailing financial performance across two reportable segments: Family of Apps (FoA) and Reality Labs (RL). It includes a table titled 'Segment Information' that lists revenue and income from operations for the three months ended March 31, 2026, and 2025. The data is presented in millions and covers specific categories such as advertising, other revenue, and total revenue.",
- "page_nums": [
- 8
- ],
- "keywords": [
- "Family of Apps",
- "Reality Labs",
- "Facebook",
- "Instagram",
- "Messenger",
- "WhatsApp"
- ],
- "connect_to": [
- {
- "target": "asset_bb67915b-87ea-5e5d-922e-ca0498422186",
- "relation": "embeds",
- "ref": "[tables/table_page_8_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_9590d259-b517-513b-aa74-c17104bf0548",
- "type": "page",
- "content": "Reconciliation of GAAP to Non-GAAP Results\n(In millions, except percentages)\n(Unaudited)\nThree Months Ended March 31,\n2026\n2025\nGAAP revenue\n$ \n56,311 $ \n42,314 \nForeign exchange effect on 2026 revenue using 2025 rates\n \n(1,749) \nRevenue excluding foreign exchange effect\n$ \n54,562 \nGAAP revenue year-over-year change %\n 33 %\nRevenue excluding foreign exchange effect year-over-year change %\n 29 %\nGAAP advertising revenue\n$ \n55,024 $ \n41,392 \nForeign exchange effect on 2026 advertising revenue using 2025 rates\n \n(1,734) \nAdvertising revenue excluding foreign exchange effect\n$ \n53,290 \nGAAP advertising revenue year-over-year change %\n 33 %\nAdvertising revenue excluding foreign exchange effect year-over-year change %\n 29 %\nNet cash provided by operating activities\n$ \n32,226 $ \n24,026 \nPurchases of property and equipment\n \n(18,997) \n(12,941) \nPrincipal payments on finance leases\n \n(843) \n(751) \nFree cash flow\n$ \n12,386 $ \n10,334 \n9",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Reconciliation of GAAP to Non-GAAP Results",
- "metadata": {
- "length": 941,
- "summary": "The page displays a financial table titled 'Reconciliation of GAAP to Non-GAAP Results' for the three months ended March 31, comparing data from 2026 and 2025. The table details revenue figures (total and advertising), year-over-year percentage changes, and cash flow metrics including net cash provided by operating activities and free cash flow.",
- "page_nums": [
- 9
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_536926bc-014f-5b0c-8928-f1e105212edc",
- "relation": "embeds",
- "ref": "[tables/table_page_9_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- }
- ]
-}
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/doc_nav.json b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/doc_nav.json
deleted file mode 100644
index 2cff38d51..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/doc_nav.json
+++ /dev/null
@@ -1,140 +0,0 @@
-{
- "version": "1.0",
- "file_name": "Meta Q1 2026 Exhibit 99.1.pdf",
- "stats": {
- "total_chunks": 17,
- "text_chunks": 0,
- "image_chunks": 0,
- "table_chunks": 6,
- "page_chunks": 11,
- "max_depth": 2
- },
- "sections": [
- {
- "title": "Root",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root",
- "level": 1,
- "summary": "",
- "chunk_count": 14,
- "children": [
- {
- "title": "First Quarter 2026 Financial Highlights",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/First Quarter 2026 Financial Highlights",
- "level": 2,
- "summary": "This section presents a table of financial highlights for Meta Platforms, Inc. for the three months ended March 31, 2026, compared to the same period in 2025. Key metrics include Revenue ($56,311 million), Costs and expenses ($33,439 million), Income from operations ($22,872 million), Net income ($26,773 million), and Diluted earnings per share ($10.44). The table also details the effective tax rate and includes a footnote explaining significant tax benefits related to the One Big Beautiful Bill Act.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "First Quarter 2026 Operational and Other Financial Highlights",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/First Quarter 2026 Operational and Other Financial Highlights",
- "level": 2,
- "summary": "This section details Meta's operational and financial performance for the first quarter of 2026. It reports that Family daily active people (DAP) reached an average of 3.56 billion, a 4% year-over-year increase, though noting declines in Iran and Russia. Ad impressions grew by 19%, while the average price per ad rose 12%. Total revenue was $56.31 billion (up 33%), with costs and expenses totaling $33.44 billion. The company also reported capital expenditures of $19.84 billion, cash flow from operating activities of $32.23 billion, free cash flow of $12.39 billion, and a headcount of 77,986.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "CFO Outlook Commentary",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/CFO Outlook Commentary",
- "level": 2,
- "summary": "This page contains a CFO Outlook Commentary providing financial guidance for 2026. It details expected revenue ranges of $58-61 billion for the second quarter and total expenses between $162-169 billion for the full year. The text also outlines anticipated capital expenditures, tax rates, and mentions ongoing legal and regulatory headwinds in the EU and U.S.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Webcast and Conference Call Information",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Webcast and Conference Call Information",
- "level": 2,
- "summary": "This section provides details regarding a conference call hosted by Meta to discuss its results, scheduled for 2:30 p.m. PT / 5:30 p.m. ET. It specifies that the live webcast and subsequent replay will be available on the Meta Investor Relations website (investor.atmeta.com), along with earnings press releases, financial tables, slide presentations, and transcripts of analyst calls.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Disclosure Information",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Disclosure Information",
- "level": 2,
- "summary": "This section outlines Meta's channels for disclosing material non-public information and complying with Regulation FD. These channels include the investor.atmeta.com and meta.com/news websites, as well as Mark Zuckerberg's social media profiles on Facebook, Instagram, and Threads.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "About Meta",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/About Meta",
- "level": 2,
- "summary": "This section describes Meta's mission to build the future of human connection using artificial intelligence and immersive technologies. It highlights the company's history starting with Facebook in 2004, mentions its apps like Messenger, Instagram, and WhatsApp, and states that Meta is moving beyond 2D screens to foster deeper connections.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Contacts",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Contacts",
- "level": 2,
- "summary": "The 'Contacts' section lists the designated points of contact for Meta, divided into two categories: Investors and Press. For investors, it provides the name Kenneth Dorell along with his email address (investor@meta.com) and website (investor.atmeta.com). For press inquiries, it lists Matt Tye with his email address (press@meta.com) and website (meta.com/news).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Forward-Looking Statements",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Forward-Looking Statements",
- "level": 2,
- "summary": "The section contains a standard legal disclaimer regarding forward-looking statements. It warns that the press release includes predictions about future business plans and expectations which may differ from actual results due to various factors such as macroeconomic conditions, geopolitical events, user retention, advertising revenue reliance, data signal dependency, product changes, AI initiatives, community growth focus, brand reputation, privacy/security efforts, competition, government actions, litigation, acquisitions, security breaches, operational scale, and market conditions. The text references specific filings (Form 10-K filed January 29, 2026, and Form 10-Q for the quarter ended March 31, 2026) available on the SEC website and the company's Investor Relations site (investor.atmeta.com). It notes the press release date is April 29, 2026, and states the company undertakes no obligation to update these statements.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Non-GAAP Financial Measures",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Non-GAAP Financial Measures",
- "level": 2,
- "summary": "This section explains the company's use of non-GAAP financial measures to supplement its GAAP financial statements. It details three specific measures: revenue excluding foreign exchange effect, advertising revenue excluding foreign exchange effect, and free cash flow. The text clarifies that these measures are intended to provide useful supplemental information for investors by enabling comparison of results between periods and offering greater transparency into key metrics used by management. It also describes the adjustments made to calculate these measures, such as translating revenue using prior year's exchange rates and subtracting purchases of property and equipment and principal payments on finance leases from free cash flow.",
- "chunk_count": 4,
- "children": []
- },
- {
- "title": "Segment Results",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Segment Results",
- "level": 2,
- "summary": "This page presents the segment results for a company, detailing financial performance across two reportable segments: Family of Apps (FoA) and Reality Labs (RL). It includes a table titled 'Segment Information' that lists revenue and income from operations for the three months ended March 31, 2026, and 2025. The data is presented in millions and covers specific categories such as advertising, other revenue, and total revenue.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Reconciliation of GAAP to Non-GAAP Results",
- "path": "Meta Q1 2026 Exhibit 99.1.pdf/Root/Reconciliation of GAAP to Non-GAAP Results",
- "level": 2,
- "summary": "The page displays a financial table titled 'Reconciliation of GAAP to Non-GAAP Results' for the three months ended March 31, comparing data from 2026 and 2025. The table details revenue figures (total and advertising), year-over-year percentage changes, and cash flow metrics including net cash provided by operating activities and free cash flow.",
- "chunk_count": 1,
- "children": []
- }
- ]
- }
- ],
- "resources": {
- "images": [],
- "tables": [
- {
- "path": "tables/table_page_1_1.html",
- "summary": "tables/table_page_1_1.html"
- },
- {
- "path": "tables/table_page_5_1.html",
- "summary": "tables/table_page_5_1.html"
- },
- {
- "path": "tables/table_page_6_1.html",
- "summary": "tables/table_page_6_1.html"
- },
- {
- "path": "tables/table_page_7_1.html",
- "summary": "tables/table_page_7_1.html"
- },
- {
- "path": "tables/table_page_8_1.html",
- "summary": "tables/table_page_8_1.html"
- },
- {
- "path": "tables/table_page_9_1.html",
- "summary": "tables/table_page_9_1.html"
- }
- ]
- }
-}
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/manifest.json b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/manifest.json
deleted file mode 100644
index 8b0204020..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/manifest.json
+++ /dev/null
@@ -1,260 +0,0 @@
-{
- "version": "2.0",
- "job_id": "job_e28e0306a02a",
- "data_id": "demo-financial-meta-q1-2026-results",
- "source_file_name": "Meta Q1 2026 Exhibit 99.1.pdf",
- "processing_date": "2026-08-18T17:53:47.605062Z",
- "processing": {
- "page_count": 9,
- "billing_status": "charged",
- "cost": {
- "micro_dollars": 135000,
- "credits": 0.135
- },
- "cost_estimate": {
- "currency": "USD",
- "pricing_version": "2026-08-18:874ab8b6c9e4",
- "total_input_cost": 0.00418058,
- "total_output_cost": 0.0008251,
- "total_cost": 0.00500568,
- "by_model": {
- "qwen3.5-flash": {
- "input_tokens": 16633,
- "output_tokens": 224,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- },
- "deepseek-chat": {
- "input_tokens": 997,
- "output_tokens": 145,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.00013958,
- "output_cost": 4.06e-05,
- "total_cost": 0.00018018,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- },
- "qwen3.6-flash": {
- "input_tokens": 16164,
- "output_tokens": 523,
- "calls": 6,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.004041,
- "output_cost": 0.0007845,
- "total_cost": 0.0048255,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 16633,
- "output_tokens": 224,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "page_memory.hierarchy": {
- "input_cost": 0.00013958,
- "output_cost": 4.06e-05,
- "total_cost": 0.00018018,
- "models": {
- "deepseek-chat": {
- "input_tokens": 997,
- "output_tokens": 145,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.00013958,
- "output_cost": 4.06e-05,
- "total_cost": 0.00018018,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- },
- "page_memory.asset_detect": {
- "input_cost": 0.004041,
- "output_cost": 0.0007845,
- "total_cost": 0.0048255,
- "models": {
- "qwen3.6-flash": {
- "input_tokens": 16164,
- "output_tokens": 523,
- "calls": 6,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.004041,
- "output_cost": 0.0007845,
- "total_cost": 0.0048255,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "timing": {
- "started_at": "2026-08-18T17:51:51.996560+00:00",
- "completed_at": "2026-08-18T17:53:47.602103+00:00",
- "duration_ms": 115605
- },
- "stages": {
- "timing_ms": {
- "page_memory.normalize_to_pdf": 0,
- "page_memory.profile": 29563,
- "page_memory.skeleton": 0,
- "page_memory.title_render": 16989,
- "page_memory.title_detection": 8642,
- "page_memory.fine_hierarchy": 1447,
- "page_memory.render": 12343,
- "page_memory.tag": 7671,
- "page_memory.assets": 28018,
- "page_memory.node_summary": 10628,
- "page_memory.node_rows": 30,
- "page_memory.node_assembly": 10662,
- "worker.parse.document": 115374
- },
- "token_usage": {
- "prompt_tokens": 33794,
- "completion_tokens": 892,
- "total_tokens": 34686,
- "calls": 8,
- "by_model": {
- "qwen3.5-flash": {
- "prompt_tokens": 16633,
- "completion_tokens": 224,
- "total_tokens": 16857,
- "calls": 1
- },
- "deepseek-chat": {
- "prompt_tokens": 997,
- "completion_tokens": 145,
- "total_tokens": 1142,
- "calls": 1
- },
- "qwen3.6-flash": {
- "prompt_tokens": 16164,
- "completion_tokens": 523,
- "total_tokens": 16687,
- "calls": 6
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "prompt_tokens": 16633,
- "completion_tokens": 224,
- "total_tokens": 16857,
- "calls": 1,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 16633,
- "completion_tokens": 224,
- "total_tokens": 16857,
- "calls": 1
- }
- }
- },
- "page_memory.hierarchy": {
- "prompt_tokens": 997,
- "completion_tokens": 145,
- "total_tokens": 1142,
- "calls": 1,
- "models": {
- "deepseek-chat": {
- "prompt_tokens": 997,
- "completion_tokens": 145,
- "total_tokens": 1142,
- "calls": 1
- }
- }
- },
- "page_memory.asset_detect": {
- "prompt_tokens": 16164,
- "completion_tokens": 523,
- "total_tokens": 16687,
- "calls": 6,
- "models": {
- "qwen3.6-flash": {
- "prompt_tokens": 16164,
- "completion_tokens": 523,
- "total_tokens": 16687,
- "calls": 6
- }
- }
- }
- }
- }
- }
- },
- "statistics": {
- "total_chunks": 17,
- "text_chunks": 0,
- "image_chunks": 0,
- "table_chunks": 6,
- "page_chunks": 11,
- "total_pages": null
- },
- "HIERARCHY": {
- "Root": {
- "First Quarter 2026 Financial Highlights": {},
- "First Quarter 2026 Operational and Other Financial Highlights": {},
- "CFO Outlook Commentary": {},
- "Webcast and Conference Call Information": {},
- "Disclosure Information": {},
- "About Meta": {},
- "Contacts": {},
- "Forward-Looking Statements": {},
- "Non-GAAP Financial Measures": {},
- "Segment Results": {},
- "Reconciliation of GAAP to Non-GAAP Results": {}
- }
- }
-}
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-1.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-1.png
deleted file mode 100644
index d4d6c6ac6..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-2.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-2.png
deleted file mode 100644
index b1fdf66ae..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-3.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-3.png
deleted file mode 100644
index 87d1a3bb8..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-3.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-4.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-4.png
deleted file mode 100644
index ba6638388..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-4.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-5.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-5.png
deleted file mode 100644
index 535130b97..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-5.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-6.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-6.png
deleted file mode 100644
index f5dad7d8e..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-6.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-7.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-7.png
deleted file mode 100644
index 018896564..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-7.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-8.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-8.png
deleted file mode 100644
index 8b7515cab..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-8.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-9.png b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-9.png
deleted file mode 100644
index 11938ca96..000000000
Binary files a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/page_citation_assets/page-9.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_1_1.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_1_1.html
deleted file mode 100644
index b260bc73b..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_1_1.html
+++ /dev/null
@@ -1,64 +0,0 @@
-
-
-
-
-
-
Three Months Ended March 31,
-
-
-
-
In millions, except percentages and per share amounts
-
-
2026 2025
-
% Change
-
-
-
Revenue
-
$
-
56,311 $ 42,314
-
33 %
-
-
-
Costs and expenses
-
-
33,439 24,759
-
35 %
-
-
-
Income from operations
-
$
-
22,872 $ 17,555
-
30 %
-
-
-
Operating margin
-
-
41 % 41 %
-
-
-
-
Provision (benefit) for income taxes (1)
-
$
-
(5,021) $ 1,738
-
NM
-
-
-
Effective tax rate (1)
-
-
(23) % 9 %
-
-
-
-
Net income
-
$
-
26,773 $ 16,644
-
61 %
-
-
-
Diluted earnings per share (EPS) (1)
-
$
-
10.44 $ 6.43
-
62 %
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_5_1.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_5_1.html
deleted file mode 100644
index 167a3d718..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_5_1.html
+++ /dev/null
@@ -1,104 +0,0 @@
-
-
-
-
-
-
Three Months Ended March 31,
-
-
-
-
-
2026 2025
-
-
-
Revenue
-
$
-
56,311 $ 42,314
-
-
-
Costs and expenses:
-
-
-
-
-
Cost of revenue
-
-
10,218 7,572
-
-
-
Research and development
-
-
17,699 12,150
-
-
-
Marketing and sales
-
-
2,908 2,757
-
-
-
General and administrative
-
-
2,614 2,280
-
-
-
Total costs and expenses
-
-
33,439 24,759
-
-
-
Income from operations
-
-
22,872 17,555
-
-
-
Interest and other income (expense), net
-
-
(1,120) 827
-
-
-
Income before income taxes
-
-
21,752 18,382
-
-
-
Provision (benefit) for income taxes
-
-
(5,021) 1,738
-
-
-
Net income
-
$
-
26,773 $ 16,644
-
-
-
Earnings per share:
-
-
-
-
-
Basic
-
$
-
10.57 $ 6.59
-
-
-
Diluted
-
$
-
10.44 $ 6.43
-
-
-
Weighted-average shares used to compute earnings per share:
-
-
-
-
-
Basic
-
-
2,534 2,527
-
-
-
Diluted
-
-
2,564 2,590
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_6_1.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_6_1.html
deleted file mode 100644
index db530b695..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_6_1.html
+++ /dev/null
@@ -1,235 +0,0 @@
-
-
-
-
-
(Unaudited)
-
-
-
-
-
-
-
-
-
March 31, 2026
-
December 31, 2025
-
-
-
Assets
-
-
-
-
-
-
-
Current assets:
-
-
-
-
-
-
-
Cash and cash equivalents
-
-
$
-
23,426
-
$ 35,873
-
-
-
Marketable securities
-
-
-
57,754
-
45,719
-
-
-
Accounts receivable, net
-
-
-
17,470
-
19,769
-
-
-
Prepaid expenses and other current assets
-
-
-
11,115
-
7,361
-
-
-
Total current assets
-
-
-
109,765
-
108,722
-
-
-
Non-marketable equity investments
-
-
-
28,410
-
27,524
-
-
-
Property and equipment, net
-
-
-
194,776
-
176,400
-
-
-
Operating lease right-of-use assets
-
-
-
23,268
-
20,404
-
-
-
Goodwill
-
-
-
24,748
-
24,534
-
-
-
Other assets
-
-
-
14,283
-
8,437
-
-
-
Total assets
-
-
$
-
395,250
-
$ 366,021
-
-
-
Liabilities and stockholders' equity
-
-
-
-
-
-
-
Current liabilities:
-
-
-
-
-
-
-
Accounts payable
-
-
$
-
13,326
-
$ 8,894
-
-
-
Operating lease liabilities, current
-
-
-
2,414
-
2,213
-
-
-
Accrued expenses and other current liabilities
-
-
-
31,013
-
30,729
-
-
-
Total current liabilities
-
-
-
46,753
-
41,836
-
-
-
Operating lease liabilities, non-current
-
-
-
25,607
-
22,940
-
-
-
Long-term debt
-
-
-
58,748
-
58,744
-
-
-
Long-term income taxes
-
-
-
16,849
-
21,005
-
-
-
Other liabilities
-
-
-
3,612
-
4,253
-
-
-
Total liabilities
-
-
-
151,569
-
148,778
-
-
-
Commitments and contingencies
-
-
-
-
-
-
-
Stockholders' equity:
-
-
-
-
-
-
-
Common stock and additional paid-in capital
-
-
-
99,337
-
95,793
-
-
-
Accumulated other comprehensive income (loss)
-
-
-
(303)
-
271
-
-
-
Retained earnings
-
-
-
144,647
-
121,179
-
-
-
Total stockholders' equity
-
-
-
243,681
-
217,243
-
-
-
Total liabilities and stockholders' equity
-
-
$
-
395,250
-
$ 366,021
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_7_1.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_7_1.html
deleted file mode 100644
index ed5d254e1..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_7_1.html
+++ /dev/null
@@ -1,259 +0,0 @@
-
-
-
-
-
-
Three Months Ended March 31,
-
-
-
-
-
2026 2025
-
-
-
Cash flows from operating activities
-
-
-
-
-
Net income
-
$
-
26,773 $ 16,644
-
-
-
Adjustments to reconcile net income to net cash provided by operating activities:
-
-
-
-
-
Depreciation and amortization
-
-
5,999 3,900
-
-
-
Share-based compensation
-
-
6,032 4,147
-
-
-
Deferred income taxes
-
-
123 (993)
-
-
-
Unrealized (gain) loss on equity investments
-
-
1,075 (135)
-
-
-
Other
-
-
(17) (96)
-
-
-
Changes in assets and liabilities:
-
-
-
-
-
Accounts receivable
-
-
2,128 2,804
-
-
-
Prepaid expenses and other current assets
-
-
(2,424) 360
-
-
-
Other assets
-
-
(1,082) (52)
-
-
-
Accounts payable
-
-
(937) (1,034)
-
-
-
Accrued expenses and other current liabilities
-
-
(271) (2,231)
-
-
-
Other liabilities
-
-
(5,173) 712
-
-
-
Net cash provided by operating activities
-
-
32,226 24,026
-
-
-
Cash flows from investing activities
-
-
-
-
-
Purchases of property and equipment
-
-
(18,997) (12,941)
-
-
-
Purchases of marketable securities
-
-
(32,978) (11,763)
-
-
-
Sales and maturities of marketable securities
-
-
19,176 4,784
-
-
-
Purchases of non-marketable equity investments
-
-
(544) (100)
-
-
-
Payments for held-for-sale assets
-
-
(118) —
-
-
-
Acquisitions of businesses and intangible assets
-
-
(372) (1)
-
-
-
Other investing activities
-
-
155 11
-
-
-
Net cash used in investing activities
-
-
(33,678) (20,010)
-
-
-
Cash flows from financing activities
-
-
-
-
-
Taxes paid related to net share settlement of equity awards
-
-
(4,423) (4,883)
-
-
-
Repurchases of Class A common stock
-
-
— (12,754)
-
-
-
Payments for dividends and dividend equivalents
-
-
(1,346) (1,329)
-
-
-
Principal payments on finance leases
-
-
(843) (751)
-
-
-
Other financing activities
-
-
59 222
-
-
-
Net cash used in financing activities
-
-
(6,553) (19,495)
-
-
-
Effect of exchange rate changes on cash, cash equivalents, restricted cash, and
-
-
-
-
-
restricted cash equivalents
-
-
7 112
-
-
-
Net decrease in cash, cash equivalents, restricted cash, and restricted cash
-
-
-
-
-
equivalents
-
-
(7,998) (15,367)
-
-
-
Cash, cash equivalents, restricted cash, and restricted cash equivalents at beginning
-
-
-
-
-
of the period
-
-
39,100 45,438
-
-
-
Cash, cash equivalents, restricted cash, and restricted cash equivalents at end
-
-
-
-
-
of the period
-
$
-
31,102 $ 30,071
-
-
-
Reconciliation of cash, cash equivalents, restricted cash, and restricted cash
-
-
-
-
-
equivalents to the condensed consolidated balance sheets
-
-
-
-
-
Cash and cash equivalents
-
$
-
23,426 $ 28,750
-
-
-
Restricted cash and restricted cash equivalents, included in prepaid expenses and
-
-
-
-
-
other current assets
-
-
340 71
-
-
-
Restricted cash and restricted cash equivalents, included in other assets
-
-
7,336 1,250
-
-
-
Total cash, cash equivalents, restricted cash, and restricted cash equivalents
-
$
-
31,102 $ 30,071
-
-
-
Supplemental cash flow data
-
-
-
-
-
Cash paid for income taxes, net
-
$
-
541 $ 448
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_8_1.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_8_1.html
deleted file mode 100644
index 6c67b4b24..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_8_1.html
+++ /dev/null
@@ -1,94 +0,0 @@
-
-
-
-
-
Segment Information
-
-
-
-
-
-
(In millions)
-
-
-
-
-
-
(Unaudited)
-
-
-
-
-
-
-
-
Three Months Ended March 31,
-
-
-
-
-
-
2026 2025
-
-
-
Revenue:
-
-
-
-
-
-
Advertising
-
-
$
-
55,024 $ 41,392
-
-
-
Other revenue
-
-
-
885 510
-
-
-
Family of Apps
-
-
-
55,909 41,902
-
-
-
Reality Labs
-
-
-
402 412
-
-
-
Total revenue
-
-
$
-
56,311 $ 42,314
-
-
-
Income (loss) from operations:
-
-
-
-
-
-
Family of Apps
-
-
$
-
26,900 $ 21,765
-
-
-
Reality Labs
-
-
-
(4,028) (4,210
-
-
-
Total income from operations
-
-
$
-
22,872 $ 17,555
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_9_1.html b/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_9_1.html
deleted file mode 100644
index ed21bb6ed..000000000
--- a/apps/api/app/data/demo_documents/financial-meta-q1-2026-results/tables/table_page_9_1.html
+++ /dev/null
@@ -1,84 +0,0 @@
-
-
-
-
-
-
Three Months Ended March 31,
-
-
-
-
-
2026 2025
-
-
-
GAAP revenue
-
$
-
56,311 $ 42,314
-
-
-
Foreign exchange effect on 2026 revenue using 2025 rates
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/chunks.json b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/chunks.json
deleted file mode 100644
index 18f0ec0c5..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/chunks.json
+++ /dev/null
@@ -1,4210 +0,0 @@
-{
- "chunks": [
- {
- "chunk_id": "d55ac241-0983-5356-98e6-5388e5b473d0",
- "type": "image",
- "content": "\nThe cover of the Microsoft Annual Report for 2025, featuring a photo of two colleagues collaborating on a laptop.\n[images/image-1 Annual Report 2025.png]\n",
- "path": "images/image-1 Annual Report 2025.png",
- "metadata": {
- "length": 155,
- "summary": "image-1\nThe cover of the Microsoft Annual Report for 2025, featuring a photo of two colleagues collaborating on a laptop.",
- "page_nums": [],
- "file_path": "images/image-1 Annual Report 2025.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "9e0d738a-c578-5816-afc1-93e714a7ab5d",
- "type": "image",
- "content": "\nA handwritten signature appearing to read 'Sarga N.'\n[images/image-2 Microsoft has an imme.jpeg]\n",
- "path": "images/image-2 Microsoft has an imme.jpeg",
- "metadata": {
- "length": 99,
- "summary": "image-2\nA handwritten signature appearing to read 'Sarga N.'",
- "page_nums": [],
- "file_path": "images/image-2 Microsoft has an imme.jpeg",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "f0fa5122-814d-55dd-aac9-1a40c56c0a8c",
- "type": "table",
- "content": "tables/table-1 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html",
- "path": "tables/table-1 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html",
- "metadata": {
- "length": 93,
- "summary": "table-1\nThe table shows quarterly share repurchases for fiscal years 2025, 2024, and 2023. In 2025, shares repurchased were 7, 8, 8, and 8 per quarter, totaling 31 shares for $13,000 million. In 2024, quarterly shares were 11, 7, 7, and 7, totaling 32 shares for $11,960 million. In 2023, shares were 17, 20, 18, and 14, totaling 69 shares for $18,400 million. The highest quarterly amount was $4,600 million in 2023 quarters, and the lowest was $2,800 million in 2024 quarters.",
- "page_nums": [],
- "file_path": "tables/table-1 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "0e66fea7-c5b7-53f8-8471-b407613a66b3",
- "type": "table",
- "content": "tables/table-2 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html",
- "path": "tables/table-2 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html",
- "metadata": {
- "length": 86,
- "summary": "table-2\nThe table lists quarterly dividends for fiscal years 2025 and 2024, with per-share dividends of $0.83 for each of the four 2025 payments (totaling $3.32 and $24,678 million) and $0.75 for each of the four 2024 payments (totaling $3.00 and $22,295 million), with declaration, record, and payment dates specified for each quarter.",
- "page_nums": [],
- "file_path": "tables/table-2 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "77c3bd2d-5b47-5b5a-929b-e8988bc51db5",
- "type": "image",
- "content": "\nLine chart tracking the performance of Microsoft Corporation, S&P 500, and NASDAQ Computer from June 20 to June 25. All indices start at $100, dip on June 22, then rise sharply, with Microsoft reaching approximately $250 by June 25.\n[images/image-3 and the NASDAQ Comput.jpeg]\n",
- "path": "images/image-3 and the NASDAQ Comput.jpeg",
- "metadata": {
- "length": 279,
- "summary": "image-3\nLine chart tracking the performance of Microsoft Corporation, S&P 500, and NASDAQ Computer from June 20 to June 25. All indices start at $100, dip on June 22, then rise sharply, with Microsoft reaching approximately $250 by June 25.",
- "page_nums": [],
- "file_path": "images/image-3 and the NASDAQ Comput.jpeg",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "b1e3c716-1628-5ef8-9628-f29f51a19862",
- "type": "table",
- "content": "tables/table-3 Microsoft Corporation vs S&P 500 vs NASDAQ Computer Index Values June 20-25.html",
- "path": "tables/table-3 Microsoft Corporation vs S&P 500 vs NASDAQ Computer Index Values June 20-25.html",
- "metadata": {
- "length": 95,
- "summary": "table-3\nMicrosoft Corporation rose from 100.00 on 6/20 to 255.13 on 6/25, S&P 500 from 100.00 to 215.89, and NASDAQ Computer from 100.00 to 254.97, with peaks on 6/24 for all three (227.51, 187.47, 223.97) and a dip on 6/22.",
- "page_nums": [],
- "file_path": "tables/table-3 Microsoft Corporation vs S&P 500 vs NASDAQ Computer Index Values June 20-25.html",
- "keywords": [
- "Microsoft Corporation",
- "S&P 500",
- "NASDAQ Computer"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "421aaa5d-a19e-55f8-b19f-9f0d269aa779",
- "type": "table",
- "content": "tables/table-4 Microsoft Cloud Key Metrics Definitions.html",
- "path": "tables/table-4 Microsoft Cloud Key Metrics Definitions.html",
- "metadata": {
- "length": 59,
- "summary": "table-4\nThis table defines three key Microsoft Cloud metrics: commercial remaining performance obligation (unearned revenue and future invoiced revenue), Microsoft Cloud revenue and revenue growth (covering Microsoft 365 Commercial cloud, Azure, other cloud services, commercial LinkedIn, and Dynamics 365), and Microsoft Cloud gross margin percentage.",
- "page_nums": [],
- "file_path": "tables/table-4 Microsoft Cloud Key Metrics Definitions.html",
- "keywords": [
- "Microsoft",
- "Microsoft 365",
- "Azure",
- "LinkedIn",
- "Dynamics 365"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "ca7bdf17-9611-509f-9921-ed11c2b257a0",
- "type": "table",
- "content": "tables/table-5 Microsoft 365 and Azure Key Performance Indicators Definitions.html",
- "path": "tables/table-5 Microsoft 365 and Azure Key Performance Indicators Definitions.html",
- "metadata": {
- "length": 82,
- "summary": "table-5\nThis table defines key performance indicators for Microsoft's commercial and consumer cloud services, including revenue growth metrics for Microsoft 365 Commercial, Consumer, LinkedIn, Dynamics 365, and Azure, along with seat and subscriber counts, with no numerical values provided.",
- "page_nums": [],
- "file_path": "tables/table-5 Microsoft 365 and Azure Key Performance Indicators Definitions.html",
- "keywords": [
- "Microsoft",
- "LinkedIn",
- "Dynamics 365",
- "Azure",
- "Microsoft 365 Commercial",
- "Microsoft 365 Consumer",
- "Enterprise Mobility + Security",
- "Windows Commercial",
- "Power BI",
- "Exchange",
- "SharePoint",
- "Microsoft Teams",
- "Microsoft 365 Security and Compliance",
- "Microsoft 365 Copilot",
- "GitHub",
- "Nuance Healthcare"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "c70fc2de-5ee0-5069-8ae4-4f9152c8737b",
- "type": "table",
- "content": "tables/table-6 Windows OEM and Devices revenue growth and Xbox content and services revenue growth and Search and news advertising revenue ex TAC growth.html",
- "path": "tables/table-6 Windows OEM and Devices revenue growth and Xbox content and services revenue growth and Search and news advertising revenue ex TAC growth.html",
- "metadata": {
- "length": 157,
- "summary": "table-6\nThe table defines three revenue growth metrics: Windows OEM and Devices revenue (from Windows Pro/non-Pro OEM licenses and first-party devices like Surface), Xbox content and services revenue (including games, Game Pass, cloud gaming, and advertising), and Search and news advertising revenue excluding traffic acquisition costs. No numerical data is provided.",
- "page_nums": [],
- "file_path": "tables/table-6 Windows OEM and Devices revenue growth and Xbox content and services revenue growth and Search and news advertising revenue ex TAC growth.html",
- "keywords": [
- "Microsoft",
- "Xbox",
- "Bing",
- "Surface"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "21d686c9-1f19-5e9b-99f4-7b4fdbc5cd0b",
- "type": "table",
- "content": "tables/table-7 Revenue and Profit Metrics for 2025 vs 2024.html",
- "path": "tables/table-7 Revenue and Profit Metrics for 2025 vs 2024.html",
- "metadata": {
- "length": 63,
- "summary": "table-7\nRevenue rose 15% to $281,724 million in 2025 from $245,122 million in 2024; gross margin increased 13% to $193,893 million, operating income up 17% to $128,528 million, net income up 16% to $101,832 million, and diluted EPS up 16% to $13.64 from $11.80.",
- "page_nums": [],
- "file_path": "tables/table-7 Revenue and Profit Metrics for 2025 vs 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "e4a917a1-1b21-5bf4-8860-fdc09eb4b30e",
- "type": "table",
- "content": "tables/table-8 Microsoft Segment Revenue and Operating Income FY2025 vs FY2024.html",
- "path": "tables/table-8 Microsoft Segment Revenue and Operating Income FY2025 vs FY2024.html",
- "metadata": {
- "length": 83,
- "summary": "table-8\nMicrosoft's total revenue rose 15% to $281,724 million in 2025 from $245,122 million in 2024, with operating income up 17% to $128,528 million; Productivity and Business Processes revenue grew 13% to $120,810 million, Intelligent Cloud revenue grew 21% to $106,265 million, and More Personal Computing revenue grew 7% to $54,649 million.",
- "page_nums": [],
- "file_path": "tables/table-8 Microsoft Segment Revenue and Operating Income FY2025 vs FY2024.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "65e313e3-db40-55ee-a0b9-a74657e07419",
- "type": "table",
- "content": "tables/table-9 Research and development expenses for fiscal years 2025 and 2024.html",
- "path": "tables/table-9 Research and development expenses for fiscal years 2025 and 2024.html",
- "metadata": {
- "length": 84,
- "summary": "table-9\nResearch and development expenses were $32,488 million in 2025, up 10% from $29,510 million in 2024, with R&D as a percent of revenue unchanged at 12% in both years (0 percentage point change).",
- "page_nums": [],
- "file_path": "tables/table-9 Research and development expenses for fiscal years 2025 and 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "301de650-4cd3-562e-a6a4-72b0cfb244d4",
- "type": "table",
- "content": "tables/table-10 Sales and marketing expenses for 2025 and 2024.html",
- "path": "tables/table-10 Sales and marketing expenses for 2025 and 2024.html",
- "metadata": {
- "length": 67,
- "summary": "table-10\nSales and marketing expenses were $25,654 million in 2025, up 5% from $24,456 million in 2024, with the expense as a percent of revenue decreasing from 10% to 9% (a 1 percentage point decline).",
- "page_nums": [],
- "file_path": "tables/table-10 Sales and marketing expenses for 2025 and 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "4058dfb8-d7a1-5283-af19-829f40330f3a",
- "type": "table",
- "content": "tables/table-11 General and Administrative Expenses 2025 vs 2024.html",
- "path": "tables/table-11 General and Administrative Expenses 2025 vs 2024.html",
- "metadata": {
- "length": 69,
- "summary": "table-11\nGeneral and administrative expenses decreased from $7,609 million in 2024 to $7,223 million in 2025, a 5% decline, while remaining steady at 3% of revenue in both years.",
- "page_nums": [],
- "file_path": "tables/table-11 General and Administrative Expenses 2025 vs 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "c0a4e50c-e185-55ca-9c4a-a71bbaf7ce12",
- "type": "table",
- "content": "tables/table-12 Year Ended June 30, 2025 and 2024 Financial Data.html",
- "path": "tables/table-12 Year Ended June 30, 2025 and 2024 Financial Data.html",
- "metadata": {
- "length": 69,
- "summary": "table-12\nFor the year ended June 30, 2025, total net losses were $4,901 million, compared to $1,646 million in 2024. Key components include interest and dividends income of $2,647 million (2025) vs $3,157 million (2024), interest expense of $2,385 million vs $2,935 million, and other net losses of $4,725 million vs $1,319 million.",
- "page_nums": [],
- "file_path": "tables/table-12 Year Ended June 30, 2025 and 2024 Financial Data.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "82944171-cfed-507c-a902-82c89f467531",
- "type": "table",
- "content": "tables/table-13 Three Months Ending September 30 2025 to Thereafter Financial Obligations Table.html",
- "path": "tables/table-13 Three Months Ending September 30 2025 to Thereafter Financial Obligations Table.html",
- "metadata": {
- "length": 100,
- "summary": "table-13\nThe table shows financial obligations totaling $67,265 million, with the largest amount of $25,191 million due by September 30, 2025, followed by $19,733 million by December 31, 2025, $13,742 million by March 31, 2026, $5,889 million by June 30, 2026, and $2,710 million thereafter.",
- "page_nums": [],
- "file_path": "tables/table-13 Three Months Ending September 30 2025 to Thereafter Financial Obligations Table.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "8d1714c5-e877-501e-83fa-d4750007bb69",
- "type": "table",
- "content": "tables/table-14 Long-term Debt and Other Commitments (In Millions).html",
- "path": "tables/table-14 Long-term Debt and Other Commitments (In Millions).html",
- "metadata": {
- "length": 71,
- "summary": "table-14\nThe table shows contractual obligations for 2026 and thereafter, with principal payments of $3,000 and $46,206, interest payments of $1,509 and $25,527, construction commitments of $26,859 and $5,290, leases of $12,798 and $165,903, and purchase commitments of $103,940 and $6,013, totaling $397,045.",
- "page_nums": [],
- "file_path": "tables/table-14 Long-term Debt and Other Commitments (In Millions).html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "7d1314f0-065b-54b3-84f1-55e25d235575",
- "type": "table",
- "content": "tables/table-15 Risk Categories and Hypothetical Market Change Impacts on Financial Metrics.html",
- "path": "tables/table-15 Risk Categories and Hypothetical Market Change Impacts on Financial Metrics.html",
- "metadata": {
- "length": 96,
- "summary": "table-15\nThe table shows hypothetical impacts on financial metrics: a 10% decrease in foreign exchange rates would reduce revenue by $11,596 million (earnings) and investments by $17 million (fair value); a 100 basis point increase in U.S. treasury rates reduces fair value by $1,415 million; a 100 basis point credit spread increase reduces fair value by $436 million; a 10% equity market price decrease reduces earnings by $1,213 million.",
- "page_nums": [],
- "file_path": "tables/table-15 Risk Categories and Hypothetical Market Change Impacts on Financial Metrics.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "172af7d7-ea9b-5403-8809-b2545b8ea94b",
- "type": "table",
- "content": "tables/table-16 Microsoft Cloud Revenue and Income Statement FY2023-2025.html",
- "path": "tables/table-16 Microsoft Cloud Revenue and Income Statement FY2023-2025.html",
- "metadata": {
- "length": 77,
- "summary": "table-16\nMicrosoft reported total revenue of $281,724 million in FY2025, up from $245,122 million in FY2024 and $211,915 million in FY2023; net income rose to $101,832 million in FY2025 from $88,136 million in FY2024 and $72,361 million in FY2023, with diluted EPS increasing from $9.68 to $13.64.",
- "page_nums": [],
- "file_path": "tables/table-16 Microsoft Cloud Revenue and Income Statement FY2023-2025.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "f996e041-a3b3-5bc1-b947-010fa5e45999",
- "type": "table",
- "content": "tables/table-17 Consolidated Statements of Comprehensive Income.html",
- "path": "tables/table-17 Consolidated Statements of Comprehensive Income.html",
- "metadata": {
- "length": 68,
- "summary": "table-17\nMicrosoft reported net income of $101,832 million in 2025, $88,136 million in 2024, and $72,361 million in 2023. Other comprehensive income was $2,243 million, $753 million, and $(1,665) million respectively, yielding comprehensive income of $104,075 million, $88,889 million, and $70,696 million.",
- "page_nums": [],
- "file_path": "tables/table-17 Consolidated Statements of Comprehensive Income.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "8fec4155-9cc3-5780-a857-96e83308701e",
- "type": "table",
- "content": "tables/table-18 Microsoft Corporation Consolidated Balance Sheets (Unaudited).html",
- "path": "tables/table-18 Microsoft Corporation Consolidated Balance Sheets (Unaudited).html",
- "metadata": {
- "length": 82,
- "summary": "table-18\nMicrosoft Corporation's consolidated balance sheet as of June 30, 2025, shows total assets of $619,003 million, up from $512,163 million in 2024, with total liabilities of $275,524 million and stockholders' equity of $343,479 million.",
- "page_nums": [],
- "file_path": "tables/table-18 Microsoft Corporation Consolidated Balance Sheets (Unaudited).html",
- "keywords": [
- "Microsoft Corporation"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "aaa89319-cae0-5959-a6fa-3b1a400b0a60",
- "type": "table",
- "content": "tables/table-19 Microsoft Cash Flow Statement for Fiscal Years 2023-2025.html",
- "path": "tables/table-19 Microsoft Cash Flow Statement for Fiscal Years 2023-2025.html",
- "metadata": {
- "length": 77,
- "summary": "table-19\nNet cash from operations rose from $87,582M in 2023 to $118,548M in 2024 and $136,162M in 2025; net income grew from $72,361M to $101,832M; cash and equivalents ended at $30,242M in 2025, down from $34,704M in 2023; investing used $72,599M in 2025, financing used $51,699M.",
- "page_nums": [],
- "file_path": "tables/table-19 Microsoft Cash Flow Statement for Fiscal Years 2023-2025.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "75b59a24-798a-519f-bcef-06088c97f96d",
- "type": "table",
- "content": "tables/table-20 Microsoft Stockholders' Equity Statement.html",
- "path": "tables/table-20 Microsoft Stockholders' Equity Statement.html",
- "metadata": {
- "length": 61,
- "summary": "table-20\nThe statement shows stockholders' equity components for fiscal years 2025, 2024, and 2023, with total equity rising from $206,223 million to $343,479 million, driven by net income of $101,832 million in 2025, common stock repurchases, and dividends per share increasing from $2.72 to $3.32.",
- "page_nums": [],
- "file_path": "tables/table-20 Microsoft Stockholders' Equity Statement.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "d7c618ea-146e-534a-ba94-d4a1ee5df5b6",
- "type": "table",
- "content": "tables/table-21 Microsoft Earnings Per Share Table.html",
- "path": "tables/table-21 Microsoft Earnings Per Share Table.html",
- "metadata": {
- "length": 55,
- "summary": "table-21\nThe table shows Microsoft's earnings per share for fiscal years 2025, 2024, and 2023. Net income available for common shareholders was $101,832 million in 2025, $88,136 million in 2024, and $72,361 million in 2023. Weighted average shares outstanding were 7,433, 7,431, and 7,446 million respectively. Basic EPS was $13.70, $11.86, and $9.72; diluted EPS was $13.64, $11.80, and $9.68 for the three years.",
- "page_nums": [],
- "file_path": "tables/table-21 Microsoft Earnings Per Share Table.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "8f0ab224-deaf-565e-89b5-1642ad95e14e",
- "type": "table",
- "content": "tables/table-22 Year Ended June 30, 2025, 2024, and 2023 Financial Data.html",
- "path": "tables/table-22 Year Ended June 30, 2025, 2024, and 2023 Financial Data.html",
- "metadata": {
- "length": 76,
- "summary": "table-22\nThe table shows total net losses of $4,901 million in 2025, $1,646 million in 2024, and net gains of $788 million in 2023, with interest and dividends income declining from $2,994 million to $2,647 million, and other net losses increasing sharply to $4,725 million in 2025.",
- "page_nums": [],
- "file_path": "tables/table-22 Year Ended June 30, 2025, 2024, and 2023 Financial Data.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "577e18e7-d331-50fc-9972-61a95fd1d6bc",
- "type": "table",
- "content": "tables/table-23 Line 1_ Realized Gains and Losses from Available-for-Sale Securities and Impairments.html",
- "path": "tables/table-23 Line 1_ Realized Gains and Losses from Available-for-Sale Securities and Impairments.html",
- "metadata": {
- "length": 105,
- "summary": "table-23\nLine 2: For fiscal years 2025, 2024, and 2023, realized gains were $40M, $22M, and $36M; realized losses were $(65)M, $(98)M, and $(124)M; impairments/allowances were $8M, $23M, and $(10)M. Totals: $(17)M, $(53)M, and $(98)M.",
- "page_nums": [],
- "file_path": "tables/table-23 Line 1_ Realized Gains and Losses from Available-for-Sale Securities and Impairments.html",
- "keywords": [
- "Line 3:"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "56ca9fcb-4e77-5b94-af9d-0e6f3a01dde4",
- "type": "table",
- "content": "tables/table-24 Year Ended June 30 2025 2024 2023 Investment Gains and Impairments.html",
- "path": "tables/table-24 Year Ended June 30 2025 2024 2023 Investment Gains and Impairments.html",
- "metadata": {
- "length": 87,
- "summary": "table-24\nNet realized gains on investments sold were $83 million in 2025, $18 million in 2024, and $75 million in 2023; net unrealized gains on investments still held were $536 million, $146 million, and $303 million; impairments were $(951) million, $(229) million, and $(20) million, resulting in totals of $(332) million, $(65) million, and $358 million.",
- "page_nums": [],
- "file_path": "tables/table-24 Year Ended June 30 2025 2024 2023 Investment Gains and Impairments.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "09f4dfcd-8f5a-502c-a2ce-c81f1c7502b6",
- "type": "table",
- "content": "tables/table-25 Investments Measured at Fair Value on Recurring Basis as of June 30, 2025.html",
- "path": "tables/table-25 Investments Measured at Fair Value on Recurring Basis as of June 30, 2025.html",
- "metadata": {
- "length": 94,
- "summary": "table-25\nTotal investments of $109,970 million include debt securities with adjusted cost basis of $85,589 million, unrealized gains of $295 million, losses of $1,625 million, and recorded basis of $84,259 million; equity investments total $13,718 million; cash is $11,711 million; derivatives net $282 million.",
- "page_nums": [],
- "file_path": "tables/table-25 Investments Measured at Fair Value on Recurring Basis as of June 30, 2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "4a3c2966-3788-501d-a4bb-63808be3ee18",
- "type": "table",
- "content": "tables/table-26 Investments Measured at Fair Value on Recurring Basis as of June 30, 2024.html",
- "path": "tables/table-26 Investments Measured at Fair Value on Recurring Basis as of June 30, 2024.html",
- "metadata": {
- "length": 94,
- "summary": "table-26\nAs of June 30, 2024, total investments measured at fair value were $90,143 million, including $64,899 million in debt investments (with $68,209 million adjusted cost basis and $3,348 million unrealized losses) and $13,661 million in equity investments; cash and cash equivalents totaled $18,315 million, short-term investments $57,228 million, and equity and other investments $14,600 million.",
- "page_nums": [],
- "file_path": "tables/table-26 Investments Measured at Fair Value on Recurring Basis as of June 30, 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "1539d95f-5366-5c5a-98e1-23db91787a71",
- "type": "table",
- "content": "tables/table-27 Investments with Unrealized Losses as of June 30, 2025 and 2024.html",
- "path": "tables/table-27 Investments with Unrealized Losses as of June 30, 2025 and 2024.html",
- "metadata": {
- "length": 84,
- "summary": "table-27\nAs of June 30, 2025, total fair value of securities with unrealized losses was $42,736 million with total unrealized losses of $1,625 million, compared to $54,589 million fair value and $3,348 million unrealized losses at June 30, 2024; the largest category was U.S. government and agency securities ($37,177 million fair value, $1,462 million losses in 2025; $46,350 million, $2,948 million in 2024).",
- "page_nums": [],
- "file_path": "tables/table-27 Investments with Unrealized Losses as of June 30, 2025 and 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "eba625c6-80bb-5830-8fb7-4881edfb68a6",
- "type": "table",
- "content": "tables/table-28 Debt Securities by Maturity Date.html",
- "path": "tables/table-28 Debt Securities by Maturity Date.html",
- "metadata": {
- "length": 53,
- "summary": "table-28\nThe table shows debt securities by maturity at June 30, 2025, with adjusted cost basis totaling $85,589 million and estimated fair value totaling $84,259 million. Due in one year or less: cost $35,108, fair value $34,952; after one through five years: cost $42,460, fair value $41,481; after five through ten years: cost $6,530, fair value $6,424; after ten years: cost $1,491, fair value $1,402.",
- "page_nums": [],
- "file_path": "tables/table-28 Debt Securities by Maturity Date.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "8474c325-a28e-5fe3-9244-2d9d66a680c8",
- "type": "table",
- "content": "tables/table-29 Derivative Assets and Liabilities at Fair Value.html",
- "path": "tables/table-29 Derivative Assets and Liabilities at Fair Value.html",
- "metadata": {
- "length": 68,
- "summary": "table-29\nThe table shows derivative notional amounts as of June 30, 2025 and 2024. Designated hedging instruments include foreign exchange contracts purchased ($1,492 million both years) and interest rate contracts purchased ($1,150 million in 2025, $1,100 million in 2024). Non-designated instruments include foreign exchange contracts purchased ($15,214 million in 2025, $7,167 million in 2024), sold ($43,307 million vs. $31,793 million), equity contracts purchased ($5,434 million vs. $4,016 million), sold ($2,189 million vs. $2,165 million), other contracts purchased ($2,769 million vs. $2,113 million), and sold ($1,242 million vs. $811 million).",
- "page_nums": [],
- "file_path": "tables/table-29 Derivative Assets and Liabilities at Fair Value.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "d7f38500-41ab-5488-951e-0a6105cf64e1",
- "type": "table",
- "content": "tables/table-30 Derivative Assets and Liabilities as of June 30, 2025 and 2024.html",
- "path": "tables/table-30 Derivative Assets and Liabilities as of June 30, 2025 and 2024.html",
- "metadata": {
- "length": 83,
- "summary": "table-30\nThe table shows derivative assets and liabilities, gross and net, for June 30, 2025 and 2024. Gross assets were $758 million (2025) and $331 million (2024); gross liabilities were $(1,837) million and $(800) million. Net assets were $500 million and $180 million; net liabilities were $(1,676) million and $(752) million. Offsets and cash collateral reduced gross amounts.",
- "page_nums": [],
- "file_path": "tables/table-30 Derivative Assets and Liabilities as of June 30, 2025 and 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "9b401ce0-62b5-57f2-81a9-f1ae471ae44e",
- "type": "table",
- "content": "tables/table-31 Derivative Assets and Liabilities by Fair Value Hierarchy.html",
- "path": "tables/table-31 Derivative Assets and Liabilities by Fair Value Hierarchy.html",
- "metadata": {
- "length": 78,
- "summary": "table-31\nDerivative assets totaled $758 million at June 30, 2025, with $1 million in Level 1, $474 million in Level 2, and $283 million in Level 3; derivative liabilities were $(1,837) million, including $(1,832) million in Level 2 and $(5) million in Level 3. At June 30, 2024, assets were $331 million (Level 1: $0, Level 2: $327, Level 3: $4) and liabilities were $(800) million (Level 1: $(1), Level 2: $(799), Level 3: $0).",
- "page_nums": [],
- "file_path": "tables/table-31 Derivative Assets and Liabilities by Fair Value Hierarchy.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "774362b2-4d34-5748-b999-a72bfbf1e1ee",
- "type": "table",
- "content": "tables/table-32 Line 1_ Derivative Instruments and Hedging Activities Fair Value and Cash Flow Hedges.html",
- "path": "tables/table-32 Line 1_ Derivative Instruments and Hedging Activities Fair Value and Cash Flow Hedges.html",
- "metadata": {
- "length": 106,
- "summary": "table-32\nLine 2: For fiscal years 2025, 2024, and 2023, fair value hedge interest rate contracts show derivative gains/losses of $5, $(23), $(65) and hedged items of $(45), $(25), $38. Cash flow hedge FX reclassifications: $103, $(48), $61. Non-designated FX: $(938), $367, $(73); equity: $(266), $(177), $(420); other: $21, $(15), $(41).",
- "page_nums": [],
- "file_path": "tables/table-32 Line 1_ Derivative Instruments and Hedging Activities Fair Value and Cash Flow Hedges.html",
- "keywords": [
- "Line 3:"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "4911001d-4060-58e9-815b-96f22f51545b",
- "type": "table",
- "content": "tables/table-33 Cash Flow Hedging Instruments - Foreign Exchange Contracts.html",
- "path": "tables/table-33 Cash Flow Hedging Instruments - Foreign Exchange Contracts.html",
- "metadata": {
- "length": 79,
- "summary": "table-33\nForeign exchange contracts designated as cash flow hedging instruments show gains of $77 million in 2025, losses of $(14) million in 2024, and gains of $34 million in 2023, all included in effectiveness assessment.",
- "page_nums": [],
- "file_path": "tables/table-33 Cash Flow Hedging Instruments - Foreign Exchange Contracts.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "67202e30-5eeb-5d05-a101-99d342e69724",
- "type": "table",
- "content": "tables/table-34 Property, Plant and Equipment.html",
- "path": "tables/table-34 Property, Plant and Equipment.html",
- "metadata": {
- "length": 50,
- "summary": "table-34\nThe table shows property, plant and equipment at cost and net of accumulated depreciation as of June 30, 2025 and 2024. Total at cost increased from $212,012 million to $298,619 million, with buildings and improvements rising from $93,943 to $137,921 million. Accumulated depreciation grew from $76,421 to $93,653 million, resulting in net totals of $204,966 million (2025) and $135,591 million (2024).",
- "page_nums": [],
- "file_path": "tables/table-34 Property, Plant and Equipment.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "9a5bfc83-30b1-5018-8ff7-41a738c3dc90",
- "type": "table",
- "content": "tables/table-35 Purchase Price Allocation Table.html",
- "path": "tables/table-35 Purchase Price Allocation Table.html",
- "metadata": {
- "length": 52,
- "summary": "table-35\nThe table shows the allocation of a total purchase price of $75,408 million, with major components including cash and cash equivalents of $12,976 million, goodwill of $51,001 million, intangible assets of $21,969 million, and other assets of $2,503 million, offset by liabilities such as long-term debt of $2,799 million, long-term income taxes of $1,946 million, deferred income taxes of $4,676 million, and other liabilities of $3,620 million.",
- "page_nums": [],
- "file_path": "tables/table-35 Purchase Price Allocation Table.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "48034024-9c74-53e3-9b34-c6cf7c84825d",
- "type": "table",
- "content": "tables/table-36 Fair value of intangible assets acquired.html",
- "path": "tables/table-36 Fair value of intangible assets acquired.html",
- "metadata": {
- "length": 61,
- "summary": "table-36\nThe table shows acquired intangible assets with a total fair value of $21,969 million and a weighted average life of 15 years. Marketing-related intangibles are the largest at $11,619 million (24 years), followed by technology-based at $9,689 million (4 years) and customer-related at $661 million (4 years).",
- "page_nums": [],
- "file_path": "tables/table-36 Fair value of intangible assets acquired.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "6c9eaeba-45ae-5065-9440-b0d0155eab20",
- "type": "table",
- "content": "tables/table-37 Microsoft Fiscal Year 2024 and 2023 Financial Highlights.html",
- "path": "tables/table-37 Microsoft Fiscal Year 2024 and 2023 Financial Highlights.html",
- "metadata": {
- "length": 77,
- "summary": "table-37\nMicrosoft reported revenue of $247,442 million for fiscal year 2024, up from $219,790 million in 2023, with net income rising to $88,308 million from $71,383 million, and diluted earnings per share increasing to $11.82 from $9.55.",
- "page_nums": [],
- "file_path": "tables/table-37 Microsoft Fiscal Year 2024 and 2023 Financial Highlights.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "7bd19c3e-a5c9-5270-a340-a4edc82d6206",
- "type": "table",
- "content": "tables/table-38 Microsoft Segment Goodwill Rollforward.html",
- "path": "tables/table-38 Microsoft Segment Goodwill Rollforward.html",
- "metadata": {
- "length": 59,
- "summary": "table-38\nMicrosoft's goodwill by segment increased from $67,886 million at June 30, 2023 to $119,220 million at June 30, 2024, driven by $51,235 million in acquisitions (mainly More Personal Computing), then to $119,509 million by June 30, 2025, with $289 million in other changes; Productivity and Business Processes grew from $31,359 to $31,457, Intelligent Cloud from $25,676 to $25,689, and More Personal Computing from $10,851 to $62,363.",
- "page_nums": [],
- "file_path": "tables/table-38 Microsoft Segment Goodwill Rollforward.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "232454cc-54aa-5ec2-a0c2-982930a29730",
- "type": "table",
- "content": "tables/table-39 Intangible Assets - June 30, 2025 and 2024.html",
- "path": "tables/table-39 Intangible Assets - June 30, 2025 and 2024.html",
- "metadata": {
- "length": 63,
- "summary": "table-39\nAs of June 30, 2025, total gross carrying amount of intangible assets was $43,557 million, with accumulated amortization of $20,953 million, resulting in net carrying amount of $22,604 million. This compares to June 30, 2024, when gross was $44,509 million, accumulated amortization was $16,912 million, and net was $27,597 million. Marketing-related net decreased from $13,399 to $12,601; technology-based net fell from $11,172 to $7,601; customer-related net dropped from $2,987 to $2,228; contract-based net rose from $39 to $174.",
- "page_nums": [],
- "file_path": "tables/table-39 Intangible Assets - June 30, 2025 and 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "94c04c68-141f-5982-90ac-5f46acfb78bc",
- "type": "table",
- "content": "tables/table-40 Intangible Assets by Category and Weighted Average Life for Fiscal Years 2025 and 2024.html",
- "path": "tables/table-40 Intangible Assets by Category and Weighted Average Life for Fiscal Years 2025 and 2024.html",
- "metadata": {
- "length": 107,
- "summary": "table-40\nThe table shows intangible asset amounts and weighted average lives for fiscal years 2025 and 2024. For 2025, marketing-related assets are $13 million (10 years), technology-based $912 million (9 years), customer-related $0 (0 years), contract-based $171 million (5 years), totaling $1,096 million (9 years). For 2024, marketing-related $11,619 million (24 years), technology-based $10,947 million (4 years), customer-related $660 million (4 years), contract-based $38 million (4 years), totaling $23,264 million (14 years).",
- "page_nums": [],
- "file_path": "tables/table-40 Intangible Assets by Category and Weighted Average Life for Fiscal Years 2025 and 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "9ff99e1e-0c88-5acf-bc6c-4cdc1c1ab453",
- "type": "table",
- "content": "tables/table-41 Year Ending June 30, 2026 to 2030 and Thereafter Lease Payments.html",
- "path": "tables/table-41 Year Ending June 30, 2026 to 2030 and Thereafter Lease Payments.html",
- "metadata": {
- "length": 84,
- "summary": "table-41\nThe table shows future minimum lease payments by fiscal year: 2026 at $4,594 million, 2027 at $2,901 million, 2028 at $2,034 million, 2029 at $1,851 million, 2030 at $1,382 million, and $9,842 million thereafter, totaling $22,604 million.",
- "page_nums": [],
- "file_path": "tables/table-41 Year Ending June 30, 2026 to 2030 and Thereafter Lease Payments.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "203d64ab-9b51-5ee5-830b-f7042569057c",
- "type": "table",
- "content": "tables/table-42 Long-term Debt Issuances and Maturities as of June 30, 2025 and 2024.html",
- "path": "tables/table-42 Long-term Debt Issuances and Maturities as of June 30, 2025 and 2024.html",
- "metadata": {
- "length": 89,
- "summary": "table-42\nTotal face value of debt is $49,206 million at June 30, 2025, down from $51,221 million at June 30, 2024; after adjustments for discounts, hedge fair value, and premium on debt exchange, total debt is $43,151 million (2025) and $44,937 million (2024), with long-term debt of $40,152 million and $42,688 million, respectively; largest issuance is 2015's $23.8 billion with maturities 2025–2055.",
- "page_nums": [],
- "file_path": "tables/table-42 Long-term Debt Issuances and Maturities as of June 30, 2025 and 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "2a31a2e6-10ff-5f17-b76c-be44e548759b",
- "type": "table",
- "content": "tables/table-43 Line 1_ Future Minimum Lease Payments Table.html",
- "path": "tables/table-43 Line 1_ Future Minimum Lease Payments Table.html",
- "metadata": {
- "length": 64,
- "summary": "table-43\nLine 2: The table shows future minimum lease payments by fiscal year, with $3,000 million in 2026, $9,250 million in 2027, $0 in 2028, $2,054 million in 2029, $0 in 2030, and $34,902 million thereafter, totaling $49,206 million.",
- "page_nums": [],
- "file_path": "tables/table-43 Line 1_ Future Minimum Lease Payments Table.html",
- "keywords": [
- "Line 3:"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "a38735ed-f1e5-59f3-b5b1-bb9c3195605b",
- "type": "table",
- "content": "tables/table-44 Provision for Income Taxes by Category for Fiscal Years Ended June 30, 2025, 2024, and 2023.html",
- "path": "tables/table-44 Provision for Income Taxes by Category for Fiscal Years Ended June 30, 2025, 2024, and 2023.html",
- "metadata": {
- "length": 112,
- "summary": "table-44\nThe table shows current and deferred tax provisions for U.S. federal, state and local, and foreign taxes. Total current taxes rose from $23,009 million in 2023 to $24,389 million in 2024 and $28,851 million in 2025. Deferred taxes were net benefits of $6,059 million (2023), $4,738 million (2024), and $7,056 million (2025). Total provision for income taxes increased from $16,950 million (2023) to $19,651 million (2024) and $21,795 million (2025). Foreign current taxes jumped from $6,678 million to $11,423 million over the period.",
- "page_nums": [],
- "file_path": "tables/table-44 Provision for Income Taxes by Category for Fiscal Years Ended June 30, 2025, 2024, and 2023.html",
- "keywords": [
- "U.S. federal",
- "U.S. state and local",
- "Foreign"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "35db7d9d-f52f-5807-9862-a55597bd303b",
- "type": "table",
- "content": "tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html",
- "path": "tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html",
- "metadata": {
- "length": 95,
- "summary": "table-45\nU.S. income before taxes rose from $52,917 million in 2023 to $62,886 million in 2024 and $69,212 million in 2025; foreign income increased from $36,394 million to $44,901 million and $54,415 million; total income before taxes grew from $89,311 million to $107,787 million and $123,627 million.",
- "page_nums": [],
- "file_path": "tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html",
- "keywords": [
- "U.S."
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "efea2194-29ab-5e8e-9e8a-b1ade3dc4796",
- "type": "table",
- "content": "tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html",
- "path": "tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html",
- "metadata": {
- "length": 81,
- "summary": "table-46\nThe effective tax rate decreased from 19.0% in 2023 to 18.2% in 2024 and 17.6% in 2025, with the federal statutory rate at 21.0% each year. Key adjustments include foreign earnings taxed at lower rates (-1.5% in 2025), FDII deduction (-1.0%), state taxes (+1.5%), R&D credit (-1.1%), stock-based compensation benefits (-0.9%), interest (+1.0%), and other items (-1.4%).",
- "page_nums": [],
- "file_path": "tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "37e9fc8e-3678-5a29-9e1e-7562a9ca8023",
- "type": "table",
- "content": "tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html",
- "path": "tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html",
- "metadata": {
- "length": 92,
- "summary": "table-47\nNet deferred income tax assets increased from $19,652 million in 2024 to $26,273 million in 2025, with total deferred tax assets of $47,112 million (net of $1,169 million valuation allowance) and deferred tax liabilities of $19,670 million; largest asset components include capitalized R&D ($16,891 million) and leasing liabilities ($12,874 million).",
- "page_nums": [],
- "file_path": "tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "84b6f224-48d2-548e-bb74-41134065a57a",
- "type": "table",
- "content": "tables/table-48 Unrecognized Tax Benefits Rollforward.html",
- "path": "tables/table-48 Unrecognized Tax Benefits Rollforward.html",
- "metadata": {
- "length": 58,
- "summary": "table-48\nThe unrecognized tax benefits increased from $17,120 million in 2023 to $22,760 million in 2024, then to $24,729 million in 2025. Key changes include current-year increases of $2,066 (2025) and $1,903 (2024), prior-year increases of $468 (2025) and $4,289 (2024), and settlements decreasing by $240 (2025) and $76 (2024).",
- "page_nums": [],
- "file_path": "tables/table-48 Unrecognized Tax Benefits Rollforward.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "db4a750b-c288-58dc-99c2-ff1f5efc436f",
- "type": "table",
- "content": "tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html",
- "path": "tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html",
- "metadata": {
- "length": 78,
- "summary": "table-49\nProductivity and Business Processes revenue increased from $43,599 million in 2024 to $50,567 million in 2025; Intelligent Cloud rose from $13,683 million to $14,022 million; More Personal Computing fell from $2,902 million to $2,676 million; total revenue grew from $60,184 million to $67,265 million.",
- "page_nums": [],
- "file_path": "tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "087287ba-e4d1-5ad9-b7be-1fdffdd0a05f",
- "type": "table",
- "content": "tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html",
- "path": "tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html",
- "metadata": {
- "length": 78,
- "summary": "table-50\nThe table shows unearned revenue beginning at $60,184 million, with $186,957 million deferred and $179,876 million recognized, resulting in an ending balance of $67,265 million for the year ended June 30, 2025.",
- "page_nums": [],
- "file_path": "tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "4452390e-74dd-5620-b691-6e58c8dc77de",
- "type": "table",
- "content": "tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html",
- "path": "tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html",
- "metadata": {
- "length": 69,
- "summary": "table-51\nOperating lease cost increased from $2,875 million in 2023 to $3,555 million in 2024 and $5,524 million in 2025. Total finance lease cost rose from $1,853 million in 2023 to $2,534 million in 2024 and $4,825 million in 2025, including amortization and interest components.",
- "page_nums": [],
- "file_path": "tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "28d25b09-4699-514f-8b98-01fc0bd50366",
- "type": "table",
- "content": "tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html",
- "path": "tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html",
- "metadata": {
- "length": 102,
- "summary": "table-52\nCash paid for operating leases was $4,931M in 2025, $3,550M in 2024, and $2,706M in 2023; finance lease operating cash flows were $1,372M, $734M, and $501M; financing cash flows from finance leases were $2,283M, $1,286M, and $1,056M. Right-of-use assets obtained: operating leases $7,826M, $6,703M, $3,514M; finance leases $20,511M, $11,633M, $3,128M.",
- "page_nums": [],
- "file_path": "tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "da108065-e4c5-5301-95cb-bfa49a12d84f",
- "type": "table",
- "content": "tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html",
- "path": "tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html",
- "metadata": {
- "length": 87,
- "summary": "table-53\nThe table shows operating lease right-of-use assets of $24,823 million and total operating lease liabilities of $22,861 million as of June 30, 2025, compared to $18,961 million and $19,077 million in 2024. Finance lease property and equipment net is $44,015 million (2025) versus $25,862 million (2024), with total finance lease liabilities of $46,172 million (2025) and $27,145 million (2024). Weighted average remaining lease terms are 6 years (operating) and 13 years (finance) in 2025; discount rates are 3.5% and 4.2%, respectively.",
- "page_nums": [],
- "file_path": "tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "e267be8c-3565-5cd5-9a8a-d0d1596d29f2",
- "type": "table",
- "content": "tables/table-54 Lease Maturities for Operating and Finance Leases.html",
- "path": "tables/table-54 Lease Maturities for Operating and Finance Leases.html",
- "metadata": {
- "length": 70,
- "summary": "table-54\nThe table shows future minimum lease payments for operating and finance leases as of year ending June 30, with annual amounts from 2026 to 2030 and thereafter. Operating lease payments total $25,481 million, with imputed interest of $2,620 million, resulting in a total of $22,861 million. Finance lease payments total $60,506 million, with imputed interest of $14,334 million, resulting in a total of $46,172 million. The largest single-year amounts are $6,111 million (operating, 2026) and $36,251 million (finance, thereafter).",
- "page_nums": [],
- "file_path": "tables/table-54 Lease Maturities for Operating and Finance Leases.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "36c1f271-f9e2-5fb3-b351-1f862f2b95fe",
- "type": "table",
- "content": "tables/table-55 Common Stock Outstanding Rollforward.html",
- "path": "tables/table-55 Common Stock Outstanding Rollforward.html",
- "metadata": {
- "length": 57,
- "summary": "table-55\nThe table shows common stock outstanding for fiscal years 2025, 2024, and 2023, with beginning balances of 7,434, 7,432, and 7,464 million shares, respectively. Issuances were 31, 34, and 37 million, while repurchases were 31, 32, and 69 million, resulting in ending balances of 7,434, 7,434, and 7,432 million shares.",
- "page_nums": [],
- "file_path": "tables/table-55 Common Stock Outstanding Rollforward.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "657a22eb-9366-5d37-b1fa-074b1683ae81",
- "type": "table",
- "content": "tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html",
- "path": "tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html",
- "metadata": {
- "length": 94,
- "summary": "table-56\nThe table shows quarterly share repurchases for fiscal years 2025, 2024, and 2023. In 2025, shares repurchased were 7, 8, 8, and 8 per quarter, totaling 31 shares for $13,000 million. In 2024, quarterly shares were 11, 7, 7, and 7, totaling 32 shares for $11,960 million. In 2023, shares were 17, 20, 18, and 14, totaling 69 shares for $18,400 million. The highest quarterly amount was $4,600 million in 2023 quarters.",
- "page_nums": [],
- "file_path": "tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "0a3c08c8-285f-5234-ba6e-dcd0afd088ac",
- "type": "table",
- "content": "tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html",
- "path": "tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html",
- "metadata": {
- "length": 87,
- "summary": "table-57\nThe table lists quarterly dividends for fiscal years 2025 and 2024, with per-share dividends of $0.83 for FY2025 (total $3.32, amount $24,678 million) and $0.75 for FY2024 (total $3.00, amount $22,295 million), each paid across four dates.",
- "page_nums": [],
- "file_path": "tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "61be9b16-9678-54c0-89a8-bc758632e3a0",
- "type": "table",
- "content": "tables/table-58 Accumulated Other Comprehensive Loss by Component.html",
- "path": "tables/table-58 Accumulated Other Comprehensive Loss by Component.html",
- "metadata": {
- "length": 70,
- "summary": "table-58\nThe table shows accumulated other comprehensive loss components for fiscal years 2025, 2024, and 2023. Derivatives balance changed from $(3) to $(8) in 2025, investments from $(2,625) to $(1,051), translation adjustments from $(2,962) to $(2,288). Total end-of-period loss: $(3,347) in 2025, $(5,590) in 2024, $(6,343) in 2023.",
- "page_nums": [],
- "file_path": "tables/table-58 Accumulated Other Comprehensive Loss by Component.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "94b78184-d619-5a1d-b89d-8bb7afb772ab",
- "type": "table",
- "content": "tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html",
- "path": "tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html",
- "metadata": {
- "length": 112,
- "summary": "table-59\nStock-based compensation expense rose from $9,611 million in 2023 to $10,734 million in 2024 and $11,974 million in 2025; income tax benefits increased from $1,651 million to $1,826 million and $2,027 million over the same periods.",
- "page_nums": [],
- "file_path": "tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "309e74c6-6705-520b-a28b-959fb9661644",
- "type": "table",
- "content": "tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html",
- "path": "tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html",
- "metadata": {
- "length": 86,
- "summary": "table-60\nDividends per share ranged from $0.62 to $0.83 quarterly across fiscal years 2023-2025, with 2025 at $0.75–$0.83, 2024 at $0.68–$0.75, and 2023 at $0.62–$0.68. Interest rates ranged from 2.0% to 5.6%, with 2025 at 3.4%–5.5%, 2024 at 3.8%–5.6%, and 2023 at 2.0%–5.4%.",
- "page_nums": [],
- "file_path": "tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "792d3955-3e4a-56a4-8154-2e6d38d6218b",
- "type": "table",
- "content": "tables/table-61 Nonvested Stock Awards Activity.html",
- "path": "tables/table-61 Nonvested Stock Awards Activity.html",
- "metadata": {
- "length": 52,
- "summary": "table-61\nThe table shows nonvested stock awards activity: beginning balance of 88 million shares at $292.28 weighted average grant-date fair value, 39 million granted at $413.90, 38 million vested at $293.25, 7 million forfeited at $317.23, ending balance of 82 million shares at $347.44.",
- "page_nums": [],
- "file_path": "tables/table-61 Nonvested Stock Awards Activity.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "4e74811f-7ba5-517d-8687-fc6523746a50",
- "type": "table",
- "content": "tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html",
- "path": "tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html",
- "metadata": {
- "length": 73,
- "summary": "table-62\nThe company repurchased 6 million shares in both fiscal 2025 and 2024, and 7 million in 2023, with average prices per share of $385.10, $339.46, and $245.59 respectively, showing rising buyback costs over the three years.",
- "page_nums": [],
- "file_path": "tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "0579405f-2c41-52d7-aca9-c24a01e2a411",
- "type": "table",
- "content": "tables/table-63 Microsoft Segment Operating Income Table.html",
- "path": "tables/table-63 Microsoft Segment Operating Income Table.html",
- "metadata": {
- "length": 61,
- "summary": "table-63\nMicrosoft's segment operating income for fiscal years 2025, 2024, and 2023 shows Productivity and Business Processes leading with $69,773M (2025), followed by Intelligent Cloud at $44,589M, and More Personal Computing at $14,166M; total operating income rose from $88,523M (2023) to $128,528M (2025), with total revenue increasing from $211,915M to $281,724M.",
- "page_nums": [],
- "file_path": "tables/table-63 Microsoft Segment Operating Income Table.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "e3d92df6-6a6e-53d0-abb3-1136d50f5ae9",
- "type": "table",
- "content": "tables/table-64 United States and Other Countries Revenue by Fiscal Year.html",
- "path": "tables/table-64 United States and Other Countries Revenue by Fiscal Year.html",
- "metadata": {
- "length": 77,
- "summary": "table-64\nFor fiscal years 2025, 2024, and 2023, total revenue was $281,724 million, $245,122 million, and $211,915 million, respectively; United States revenue grew from $106,744 million to $144,546 million, and other countries revenue grew from $105,171 million to $137,178 million.",
- "page_nums": [],
- "file_path": "tables/table-64 United States and Other Countries Revenue by Fiscal Year.html",
- "keywords": [
- "United States"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "89bdd525-3b5c-53e1-947e-df3aefb5bdb9",
- "type": "table",
- "content": "tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html",
- "path": "tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html",
- "metadata": {
- "length": 76,
- "summary": "table-65\nMicrosoft's total revenue grew from $211,915 million in fiscal 2023 to $245,122 million in 2024 and $281,724 million in 2025. Server products and cloud services was the largest segment, rising from $65,007M to $79,828M to $98,435M. Microsoft 365 Commercial grew from $66,949M to $76,969M to $87,767M. Gaming increased from $15,466M to $21,503M to $23,455M. Other segments included LinkedIn, Windows and Devices, Search and news advertising, Dynamics, Enterprise and partner services, and Microsoft 365 Consumer.",
- "page_nums": [],
- "file_path": "tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html",
- "keywords": [
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "f84caf3c-647d-582d-a0b5-ddbab86fdb05",
- "type": "table",
- "content": "tables/table-66 United States and Other Countries Revenue by Fiscal Year.html",
- "path": "tables/table-66 United States and Other Countries Revenue by Fiscal Year.html",
- "metadata": {
- "length": 77,
- "summary": "table-66\nThe table shows total revenue of $371,902 million for fiscal year 2025, up from $301,369 million in 2024 and $187,239 million in 2023. United States revenue grew from $114,380 million (2023) to $186,106 million (2024) and $230,069 million (2025); Other countries revenue rose from $72,859 million to $115,263 million and $141,833 million over the same periods.",
- "page_nums": [],
- "file_path": "tables/table-66 United States and Other Countries Revenue by Fiscal Year.html",
- "keywords": [
- "United States"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "6656b43d-55a4-5cfd-9025-359ff3b95182",
- "type": "table",
- "content": "tables/table-67 Deloitte & Touche LLP signature block.html",
- "path": "tables/table-67 Deloitte & Touche LLP signature block.html",
- "metadata": {
- "length": 58,
- "summary": "table-67\nSignature block for Deloitte & Touche LLP, dated July 30, 2025, in Seattle, Washington.",
- "page_nums": [],
- "file_path": "tables/table-67 Deloitte & Touche LLP signature block.html",
- "keywords": [
- "Deloitte & Touche LLP",
- "Seattle",
- "Washington"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "6b48ae77-7813-5b62-87f3-ff41c622c790",
- "type": "table",
- "content": "tables/table-68 Microsoft Board of Directors List.html",
- "path": "tables/table-68 Microsoft Board of Directors List.html",
- "metadata": {
- "length": 54,
- "summary": "table-68\nThe table lists twelve members of Microsoft's Board of Directors, including Satya Nadella (Chairman and CEO), Sandra E. Peterson (Lead Independent Director), and other executives from companies like Citigroup, Wells Fargo, Disney, and GSK, with committee assignments noted.",
- "page_nums": [],
- "file_path": "tables/table-68 Microsoft Board of Directors List.html",
- "keywords": [
- "Satya Nadella",
- "Microsoft Corporation",
- "Catherine MacGregor",
- "Engie S.A.",
- "Carlos Rodriguez",
- "Automatic Data Processing, Inc.",
- "Reid G. Hoffman",
- "Greylock Partners",
- "Mark A. L. Mason",
- "Citigroup Inc.",
- "Charles W. Scharf",
- "Wells Fargo & Company",
- "Hugh F. Johnston",
- "The Walt Disney Company",
- "Sandra E. Peterson",
- "Clayton, Dubilier & Rice, LLC",
- "John W. Stanton",
- "Trilogy Partnerships",
- "Teri L. List",
- "The Gap, Inc.",
- "Penny S. Pritzker",
- "PSP Partners, LLC",
- "Emma N. Walmsley",
- "GSK plc"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "fc7f99a2-f90b-53db-9944-c6216c05ea6c",
- "type": "table",
- "content": "tables/table-69 Microsoft Leadership Table.html",
- "path": "tables/table-69 Microsoft Leadership Table.html",
- "metadata": {
- "length": 47,
- "summary": "table-69\nThis table lists Microsoft executives and their titles, including the Chairman and CEO, CFO, and other senior leaders such as the CEO of Microsoft Commercial, Chief Marketing Officer, Chief Human Resources Officer, Vice Chair and President, and Executive Vice President of Strategy and Transformation.",
- "page_nums": [],
- "file_path": "tables/table-69 Microsoft Leadership Table.html",
- "keywords": [
- "Satya Nadella",
- "Amy E. Hood",
- "Judson B. Althoff",
- "Takeshi Numoto",
- "Amy L. Coleman",
- "Bradford L. Smith",
- "Kathleen T. Hogan",
- "Microsoft"
- ],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "edf99fdb-63b6-532d-a46a-4829e552912c",
- "type": "image",
- "content": "\nAn FSC MIX certification label indicating paper supporting responsible forestry, featuring the organization's logo and website.\n[images/image-4 FSC Certification Labe.jpeg]\n",
- "path": "images/image-4 FSC Certification Labe.jpeg",
- "metadata": {
- "length": 174,
- "summary": "image-4\nAn FSC MIX certification label indicating paper supporting responsible forestry, featuring the organization's logo and website.",
- "page_nums": [],
- "file_path": "images/image-4 FSC Certification Labe.jpeg",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "473dd805-f6b7-57f9-ae47-fbbf9a67daad",
- "type": "text",
- "content": "U.S. income before taxes rose from $52,917 million in 2023 to $62,886 million in 2024 and $69,212 million in 2025; foreign income increased from $36,394 million to $44,901 million and $54,415 million; total income before taxes grew from $89,311 million to $107,787 million and $123,627 million.\n[tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html]\n\nEffective Tax Rate\nThe items accounting for the difference between income taxes computed at the U.S. federal statutory rate and our effective rate were as follows:\n\nThe effective tax rate decreased from 19.0% in 2023 to 18.2% in 2024 and 17.6% in 2025, with the federal statutory rate at 21.0% each year. Key adjustments include foreign earnings taxed at lower rates (-1.5% in 2025), FDII deduction (-1.0%), state taxes (+1.5%), R&D credit (-1.1%), stock-based compensation benefits (-0.9%), interest (+1.0%), and other items (-1.4%).\n[tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html]\n\nThe decrease from the federal statutory rate in fiscal years 2025, 2024, and 2023 is primarily due to earnings taxed at lower rates in foreign jurisdictions resulting from producing and distributing our products and services through our foreign regional operations center in Ireland. In fiscal years 2025, 2024, and 2023, our foreign regional operating center in Ireland, which is taxed at a rate lower than the U.S. rate, generated 81%, 83%, and 81% of our foreign income before tax. Other reconciling items, net consists primarily of tax credits and the U.S. global intangible low-taxed income tax, and in fiscal year 2024, includes tax benefits from tax law changes. In fiscal year 2024, tax benefits from tax law changes primarily relate to the delay of the effective date of final foreign tax credit regulations. In fiscal years 2025, 2024, and 2023, there were no individually significant other reconciling items.\nThe decrease in our effective tax rate for fiscal year 2025 compared to fiscal year 2024 was due to changes in the mix of our earnings and tax expenses between the U.S. and foreign countries. The decrease in our effective tax rate for fiscal year 2024 compared to fiscal year 2023 was primarily due to tax benefits from tax law changes, including the delay of the effective date of final foreign tax credit regulations.\nThe components of the deferred income tax assets and liabilities were as follows:\n\nNet deferred income tax assets increased from $19,652 million in 2024 to $26,273 million in 2025, with total deferred tax assets of $47,112 million (net of $1,169 million valuation allowance) and deferred tax liabilities of $19,670 million; largest asset components include capitalized R&D ($16,891 million) and leasing liabilities ($12,874 million).\n[tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html]\n\nDeferred income tax balances reflect the effects of temporary differences between the carrying amounts of assets and liabilities and their tax bases and are stated at enacted tax rates expected to be in effect when the taxes are paid or recovered.\nAs of June 30, 2025, we had federal, state, and foreign net operating loss carryforwards of $390 million, $836 million, and $2.6 billion, respectively. The federal and state net operating loss carryforwards have varying expiration dates ranging from fiscal year 2026 to 2045 or indefinite carryforward periods, if not utilized. The majority of our foreign net operating loss carryforwards do not expire. Certain acquired net operating loss carryforwards are subject to an annual limitation but are expected to be realized with the exception of those which have a valuation allowance. As of June 30, 2025, we had $816 million federal capital loss carryforwards for U.S. tax purposes. The federal capital loss carryforwards will expire in fiscal year 2030 if not utilized.\nThe valuation allowance disclosed in the table above relates to the foreign net operating loss carryforwards, federal capital loss carryforwards, and other net deferred tax assets that may not be realized.\nIncome taxes paid, net of refunds, were $28.7 billion, $23.4 billion, and $23.1 billion in fiscal years 2025, 2024, and 2023, respectively.\nUncertain Tax Positions",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 1",
- "metadata": {
- "length": 4272,
- "summary": "The text discusses U.S. and foreign income before taxes for fiscal years 2023-2025, showing increases in both. U.S. income rose from $52,917 million in 2023 to $62,886 million in 2024 and $69,212 million in 2025; foreign income increased from $36,394 million to $44,901 million and $54,415 million; total income before taxes grew from $89,311 million to $107,787 million and $123,627 million. The effective tax rate decreased from 19.0% in 2023 to 18.2% in 2024 and 17.6% in 2025, with the federal statutory rate at 21.0% each year. Key adjustments include foreign earnings taxed at lower rates (-1.5% in 2025), FDII deduction (-1.0%), state taxes (+1.5%), R&D credit (-1.1%), stock-based compensation benefits (-0.9%), interest (+1.0%), and other items (-1.4%). The decrease is primarily due to earnings taxed at lower rates in foreign jurisdictions, especially through the Ireland regional operations center, which generated 81%, 83%, and 81% of foreign income before tax in 2025, 2024, and 2023, respectively. Net deferred income tax assets increased from $19,652 million in 2024 to $26,273 million in 2025, with total deferred tax assets of $47,112 million (net of $1,169 million valuation allowance) and deferred tax liabilities of $19,670 million; largest asset components include capitalized R&D ($16,891 million) and leasing liabilities ($12,874 million). As of June 30, 2025, federal, state, and foreign net operating loss carryforwards were $390 million, $836 million, and $2.6 billion, respectively. Federal capital loss carryforwards were $816 million, expiring in fiscal year 2030. Income taxes paid, net of refunds, were $28.7 billion, $23.4 billion, and $23.1 billion in fiscal years 2025, 2024, and 2023, respectively.",
- "page_nums": [],
- "tokens": [
- "income",
- "taxes",
- "rose",
- "52",
- "917",
- "million",
- "2023",
- "62",
- "886",
- "2024",
- "69",
- "212",
- "2025",
- "foreign",
- "increased",
- "36",
- "394",
- "44",
- "901",
- "54",
- "415",
- "total",
- "grew",
- "89",
- "311",
- "107",
- "787",
- "123",
- "627",
- "Effective",
- "Tax",
- "Rate",
- "The",
- "items",
- "accounting",
- "difference",
- "computed",
- "federal",
- "statutory",
- "rate",
- "effective",
- "tax",
- "decreased",
- "19.0%",
- "18.2%",
- "17.6%",
- "21.0%",
- "year",
- "Key",
- "adjustments",
- "include",
- "earnings",
- "taxed",
- "lower",
- "rates",
- "1.5%",
- "FDII",
- "deduction",
- "1.0%",
- "state",
- "credit",
- "1.1%",
- "stock",
- "based",
- "compensation",
- "benefits",
- "0.9%",
- "interest",
- "1.4%",
- "decrease",
- "fiscal",
- "years",
- "primarily",
- "due",
- "jurisdictions",
- "resulting",
- "producing",
- "distributing",
- "products",
- "services",
- "regional",
- "operations",
- "center",
- "Ireland",
- "In",
- "operating",
- "generated",
- "81%",
- "83%",
- "Other",
- "reconciling",
- "net",
- "consists",
- "credits",
- "global",
- "intangible",
- "low",
- "includes",
- "law",
- "relate",
- "delay",
- "date",
- "final",
- "regulations",
- "individually",
- "significant",
- "compared",
- "mix",
- "expenses",
- "countries",
- "including",
- "components",
- "deferred",
- "assets",
- "liabilities",
- "Net",
- "19",
- "652",
- "26",
- "273",
- "47",
- "112",
- "169",
- "valuation",
- "allowance",
- "670",
- "largest",
- "asset",
- "capitalized",
- "16",
- "891",
- "leasing",
- "12",
- "874",
- "Deferred",
- "balances",
- "reflect",
- "effects",
- "temporary",
- "differences",
- "carrying",
- "amounts",
- "bases",
- "stated",
- "enacted",
- "expected",
- "effect",
- "paid",
- "recovered",
- "As",
- "June",
- "30",
- "loss",
- "carryforwards",
- "390",
- "836",
- "2.6",
- "billion",
- "varying",
- "expiration",
- "dates",
- "ranging",
- "2026",
- "2045",
- "indefinite",
- "carryforward",
- "periods",
- "utilized",
- "majority",
- "expire",
- "Certain",
- "acquired",
- "subject",
- "annual",
- "limitation",
- "realized",
- "exception",
- "816",
- "capital",
- "purposes",
- "2030",
- "disclosed",
- "table",
- "relates",
- "Income",
- "refunds",
- "28.7",
- "23.4",
- "23.1",
- "Uncertain",
- "Positions"
- ],
- "keywords": [
- "U.S.",
- "Ireland"
- ],
- "connect_to": [
- {
- "target": "35db7d9d-f52f-5807-9862-a55597bd303b",
- "relation": "embeds",
- "ref": "[tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html]",
- "position": {
- "start": 295,
- "end": 392
- }
- },
- {
- "target": "efea2194-29ab-5e8e-9e8a-b1ade3dc4796",
- "relation": "embeds",
- "ref": "[tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html]",
- "position": {
- "start": 929,
- "end": 1012
- }
- },
- {
- "target": "37e9fc8e-3678-5a29-9e1e-7562a9ca8023",
- "relation": "embeds",
- "ref": "[tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html]",
- "position": {
- "start": 2788,
- "end": 2882
- }
- }
- ]
- }
- },
- {
- "chunk_id": "0cc322b3-c5cc-50dc-b9cb-c08210685d21",
- "type": "text",
- "content": "Gross unrecognized tax benefits related to uncertain tax positions as of June 30, 2025, 2024, and 2023, were $24.7 billion, $22.8 billion, and $17.1 billion, respectively, which were primarily included in long-term income taxes in our consolidated balance sheets. If recognized, the resulting tax benefit would affect our effective tax rates for fiscal years 2025, 2024, and 2023 by $21.2 billion, $19.6 billion, and $14.4 billion, respectively.\nAs of June 30, 2025, 2024, and 2023, we had accrued interest expense related to uncertain tax positions of $8.2 billion, $6.8 billion, and $5.2 billion, respectively, net of income tax benefits. The provision for income taxes for fiscal years 2025, 2024, and 2023 included interest expense related to uncertain tax positions of $1.3 billion, $1.5 billion, and $918 million, respectively, net of income tax benefits.\nThe aggregate changes in the gross unrecognized tax benefits related to uncertain tax positions were as follows:\n\nThe unrecognized tax benefits increased from $17,120 million in 2023 to $22,760 million in 2024, then to $24,729 million in 2025. Key changes include current-year increases of $2,066 (2025) and $1,903 (2024), prior-year increases of $468 (2025) and $4,289 (2024), and settlements decreasing by $240 (2025) and $76 (2024).\n[tables/table-48 Unrecognized Tax Benefits Rollforward.html]\n\nWe remain under audit by the IRS for tax years 2014 to 2017. With respect to the audit for tax years 2004 to 2013, on September 26, 2023, we received Notices of Proposed Adjustment (“NOPAs”) from the IRS. The primary issues in the NOPAs relate to intercompany transfer pricing. In the NOPAs, the IRS is seeking an additional tax payment of $28.9 billion plus penalties and interest. As of June 30, 2025, we believe our allowances for income tax contingencies are adequate. We disagree with the proposed adjustments and will vigorously contest the NOPAs through the IRS’s administrative appeals office and, if necessary, judicial proceedings. We do not expect a final resolution of these issues in the next 12 months. Based on the information currently available, we do not anticipate a significant increase or decrease to our income tax contingencies for these issues within the next 12 months.\nWe are subject to income tax in many jurisdictions outside the U.S., some of which are currently under audit by local tax authorities. The resolution of these audits is not expected to be material to our consolidated financial statements. Our operations in Ireland remain subject to examination for tax years 2020 and thereafter.\nNOTE 12 — UNEARNED REVENUE\nUnearned revenue by segment was as follows:\n\nProductivity and Business Processes revenue increased from $43,599 million in 2024 to $50,567 million in 2025; Intelligent Cloud rose from $13,683 million to $14,022 million; More Personal Computing fell from $2,902 million to $2,676 million; total revenue grew from $60,184 million to $67,265 million.\n[tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html]\n\nWe have recast certain prior period amounts to conform to the way we internally manage and monitor our business. Refer to Note 1 – Accounting Policies for further information.\nChanges in unearned revenue were as follows:\n\nThe table shows unearned revenue beginning at $60,184 million, with $186,957 million deferred and $179,876 million recognized, resulting in an ending balance of $67,265 million for the year ended June 30, 2025.\n[tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html]\n\nRevenue allocated to remaining performance obligations, which includes unearned revenue and amounts that will be invoiced and recognized as revenue in future periods, was $375 billion as of June 30, 2025, of which $368 billion is related to the commercial portion of revenue. We expect to recognize approximately 40% of our total company remaining performance obligation revenue over the next 12 months and the remainder thereafter.\nNOTE 13 — LEASES\nWe have operating and finance leases for datacenters, corporate offices, research and development facilities, Microsoft Experience Centers, and certain equipment. Our leases have remaining lease terms of less than 1 year to 20 years, some of which include options to extend the leases for up to 5 years, and some of which include options to terminate the leases within 1 year.\nThe components of lease expense were as follows:",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 2",
- "metadata": {
- "length": 4432,
- "summary": "The text discusses Microsoft's unrecognized tax benefits, which were $24.7 billion, $22.8 billion, and $17.1 billion as of June 30, 2025, 2024, and 2023, respectively. If recognized, these would affect effective tax rates by $21.2 billion, $19.6 billion, and $14.4 billion for those years. Accrued interest expense related to uncertain tax positions was $8.2 billion, $6.8 billion, and $5.2 billion for the same periods, with provisions for income taxes including interest expense of $1.3 billion, $1.5 billion, and $918 million. The gross unrecognized tax benefits increased from $17,120 million in 2023 to $22,760 million in 2024, then to $24,729 million in 2025, with current-year increases of $2,066 (2025) and $1,903 (2024), prior-year increases of $468 (2025) and $4,289 (2024), and settlements decreasing by $240 (2025) and $76 (2024). Microsoft is under IRS audit for tax years 2014-2017 and received Notices of Proposed Adjustment for 2004-2013 seeking an additional $28.9 billion plus penalties and interest, which they plan to contest. Unearned revenue by segment increased from $60,184 million in 2024 to $67,265 million in 2025, with Productivity and Business Processes rising from $43,599 million to $50,567 million, Intelligent Cloud from $13,683 million to $14,022 million, and More Personal Computing falling from $2,902 million to $2,676 million. The changes in unearned revenue show a beginning balance of $60,184 million, $186,957 million deferred, $179,876 million recognized, and an ending balance of $67,265 million. Revenue allocated to remaining performance obligations was $375 billion as of June 30, 2025, with $368 billion related to commercial revenue, and about 40% expected to be recognized over the next 12 months. The text also mentions lease terms and expenses.",
- "page_nums": [],
- "tokens": [
- "Gross",
- "unrecognized",
- "tax",
- "benefits",
- "related",
- "uncertain",
- "positions",
- "June",
- "30",
- "2025",
- "2024",
- "2023",
- "24.7",
- "billion",
- "22.8",
- "17.1",
- "primarily",
- "included",
- "long",
- "term",
- "income",
- "taxes",
- "consolidated",
- "balance",
- "sheets",
- "If",
- "recognized",
- "resulting",
- "benefit",
- "affect",
- "effective",
- "rates",
- "fiscal",
- "years",
- "21.2",
- "19.6",
- "14.4",
- "As",
- "accrued",
- "interest",
- "expense",
- "8.2",
- "6.8",
- "5.2",
- "net",
- "The",
- "provision",
- "1.3",
- "1.5",
- "918",
- "million",
- "aggregate",
- "gross",
- "increased",
- "17",
- "120",
- "22",
- "760",
- "24",
- "729",
- "Key",
- "include",
- "current",
- "year",
- "increases",
- "066",
- "903",
- "prior",
- "468",
- "289",
- "settlements",
- "decreasing",
- "240",
- "76",
- "We",
- "remain",
- "audit",
- "IRS",
- "2014",
- "2017",
- "With",
- "respect",
- "2004",
- "2013",
- "September",
- "26",
- "received",
- "Notices",
- "Proposed",
- "Adjustment",
- "NOPAs",
- "primary",
- "issues",
- "relate",
- "intercompany",
- "transfer",
- "pricing",
- "In",
- "seeking",
- "additional",
- "payment",
- "28.9",
- "penalties",
- "allowances",
- "contingencies",
- "adequate",
- "disagree",
- "proposed",
- "adjustments",
- "vigorously",
- "contest",
- "administrative",
- "appeals",
- "office",
- "judicial",
- "proceedings",
- "expect",
- "final",
- "resolution",
- "12",
- "months",
- "Based",
- "information",
- "anticipate",
- "significant",
- "increase",
- "decrease",
- "subject",
- "jurisdictions",
- "local",
- "authorities",
- "audits",
- "expected",
- "material",
- "financial",
- "statements",
- "Our",
- "operations",
- "Ireland",
- "examination",
- "2020",
- "NOTE",
- "UNEARNED",
- "REVENUE",
- "Unearned",
- "revenue",
- "segment",
- "Productivity",
- "Business",
- "Processes",
- "43",
- "599",
- "50",
- "567",
- "Intelligent",
- "Cloud",
- "rose",
- "13",
- "683",
- "14",
- "022",
- "More",
- "Personal",
- "Computing",
- "fell",
- "902",
- "676",
- "total",
- "grew",
- "60",
- "184",
- "67",
- "265",
- "recast",
- "period",
- "amounts",
- "conform",
- "internally",
- "manage",
- "monitor",
- "business",
- "Refer",
- "Note",
- "Accounting",
- "Policies",
- "Changes",
- "unearned",
- "table",
- "shows",
- "beginning",
- "186",
- "957",
- "deferred",
- "179",
- "876",
- "ending",
- "ended",
- "Revenue",
- "allocated",
- "remaining",
- "performance",
- "obligations",
- "includes",
- "invoiced",
- "future",
- "periods",
- "375",
- "368",
- "commercial",
- "portion",
- "recognize",
- "approximately",
- "40%",
- "company",
- "obligation",
- "remainder",
- "LEASES",
- "operating",
- "finance",
- "leases",
- "datacenters",
- "corporate",
- "offices",
- "research",
- "development",
- "facilities",
- "Microsoft",
- "Experience",
- "Centers",
- "equipment",
- "lease",
- "terms",
- "20",
- "options",
- "extend",
- "terminate",
- "components"
- ],
- "keywords": [
- "IRS",
- "Microsoft"
- ],
- "connect_to": [
- {
- "target": "84b6f224-48d2-548e-bb74-41134065a57a",
- "relation": "embeds",
- "ref": "[tables/table-48 Unrecognized Tax Benefits Rollforward.html]",
- "position": {
- "start": 1298,
- "end": 1358
- }
- },
- {
- "target": "db4a750b-c288-58dc-99c2-ff1f5efc436f",
- "relation": "embeds",
- "ref": "[tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html]",
- "position": {
- "start": 2960,
- "end": 3040
- }
- },
- {
- "target": "087287ba-e4d1-5ad9-b7be-1fdffdd0a05f",
- "relation": "embeds",
- "ref": "[tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html]",
- "position": {
- "start": 3475,
- "end": 3555
- }
- }
- ]
- }
- },
- {
- "chunk_id": "756413a1-3144-5269-b59b-1ce3345b7b11",
- "type": "text",
- "content": "Operating lease cost increased from $2,875 million in 2023 to $3,555 million in 2024 and $5,524 million in 2025. Total finance lease cost rose from $1,853 million in 2023 to $2,534 million in 2024 and $4,825 million in 2025, including amortization and interest components.\n[tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html]\n\nSupplemental cash flow information related to leases was as follows:\n\nCash paid for operating leases was $4,931M in 2025, $3,550M in 2024, and $2,706M in 2023; finance lease operating cash flows were $1,372M, $734M, and $501M; financing cash flows from finance leases were $2,283M, $1,286M, and $1,056M. Right-of-use assets obtained: operating leases $7,826M, $6,703M, $3,514M; finance leases $20,511M, $11,633M, $3,128M.\n[tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html]\n\nSupplemental balance sheet information related to leases was as follows:\n\nThe table shows operating lease right-of-use assets of $24,823 million and total operating lease liabilities of $22,861 million as of June 30, 2025, compared to $18,961 million and $19,077 million in 2024. Finance lease property and equipment net is $44,015 million (2025) versus $25,862 million (2024), with total finance lease liabilities of $46,172 million (2025) and $27,145 million (2024). Weighted average remaining lease terms are 6 years (operating) and 13 years (finance) in 2025; discount rates are 3.5% and 4.2%, respectively.\n[tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html]\n\nThe following table outlines maturities of our lease liabilities as of June 30, 2025:\n\nThe table shows future minimum lease payments for operating and finance leases as of year ending June 30, with annual amounts from 2026 to 2030 and thereafter. Operating lease payments total $25,481 million, with imputed interest of $2,620 million, resulting in a total of $22,861 million. Finance lease payments total $60,506 million, with imputed interest of $14,334 million, resulting in a total of $46,172 million. The largest single-year amounts are $6,111 million (operating, 2026) and $36,251 million (finance, thereafter).\n[tables/table-54 Lease Maturities for Operating and Finance Leases.html]\n\nAs of June 30, 2025, we had additional leases, primarily for datacenters, that had not yet commenced of $92.7 billion. These leases will commence between fiscal year 2026 and fiscal year 2031 with lease terms of 1 year to 20 years.\nNOTE 14 — CONTINGENCIES\nIrish Data Protection Commission Matter\nIn 2018, the Irish Data Protection Commission (“IDPC”) began investigating a complaint against LinkedIn as to whether LinkedIn’s targeted advertising practices violated the recently implemented European Union General Data Protection Regulation (“GDPR”). Microsoft cooperated throughout the period of inquiry. In October 2024, the IDPC provided LinkedIn with a final decision alleging GDPR violations and assessing a fine. In November 2024, LinkedIn appealed the final decision to the Irish courts, and the next hearing is scheduled for December 2025.\nOther Contingencies\nWe also are subject to a variety of other claims and suits that arise from time to time in the ordinary course of our business. Although management currently believes that resolving claims against us, individually or in aggregate, will not have a material adverse impact in our consolidated financial statements, these matters are subject to inherent uncertainties and management’s view of these matters may change in the future.\nAs of June 30, 2025, we accrued aggregate legal liabilities of $541 million. While we intend to defend these matters vigorously, adverse outcomes that we estimate could reach approximately $600 million in aggregate beyond recorded amounts are reasonably possible. Were unfavorable final outcomes to occur, there exists the possibility of a material adverse impact in our consolidated financial statements for the period in which the effects become reasonably estimable.\nNOTE 15 — STOCKHOLDERS’ EQUITY\nShares Outstanding\nShares of common stock outstanding were as follows:",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 3",
- "metadata": {
- "length": 4137,
- "summary": "The text discusses lease costs, supplemental cash flow and balance sheet information, and future minimum lease payments for operating and finance leases. Operating lease cost increased from $2,875 million in 2023 to $3,555 million in 2024 and $5,524 million in 2025. Total finance lease cost rose from $1,853 million in 2023 to $2,534 million in 2024 and $4,825 million in 2025. Cash paid for operating leases was $4,931M in 2025, $3,550M in 2024, and $2,706M in 2023. Right-of-use assets obtained for operating leases were $7,826M, $6,703M, $3,514M; for finance leases $20,511M, $11,633M, $3,128M. As of June 30, 2025, operating lease right-of-use assets were $24,823 million and total operating lease liabilities $22,861 million; finance lease property and equipment net was $44,015 million with total finance lease liabilities of $46,172 million. Weighted average remaining lease terms are 6 years (operating) and 13 years (finance); discount rates are 3.5% and 4.2%. Future minimum lease payments total $25,481 million for operating leases (imputed interest $2,620 million) and $60,506 million for finance leases (imputed interest $14,334 million). Additional leases not yet commenced total $92.7 billion. The text also covers contingencies: the Irish Data Protection Commission matter involving LinkedIn, and other legal contingencies with accrued liabilities of $541 million and reasonably possible losses up to $600 million beyond recorded amounts.",
- "page_nums": [],
- "tokens": [
- "Operating",
- "lease",
- "cost",
- "increased",
- "875",
- "million",
- "2023",
- "555",
- "2024",
- "524",
- "2025",
- "Total",
- "finance",
- "rose",
- "853",
- "534",
- "825",
- "including",
- "amortization",
- "interest",
- "components",
- "Supplemental",
- "cash",
- "flow",
- "information",
- "related",
- "leases",
- "Cash",
- "paid",
- "operating",
- "931M",
- "550M",
- "706M",
- "flows",
- "372M",
- "734M",
- "501M",
- "financing",
- "283M",
- "286M",
- "056M",
- "Right",
- "assets",
- "obtained",
- "826M",
- "703M",
- "514M",
- "20",
- "511M",
- "11",
- "633M",
- "128M",
- "balance",
- "sheet",
- "The",
- "table",
- "shows",
- "24",
- "823",
- "total",
- "liabilities",
- "22",
- "861",
- "June",
- "30",
- "compared",
- "18",
- "961",
- "19",
- "077",
- "Finance",
- "property",
- "equipment",
- "net",
- "44",
- "015",
- "versus",
- "25",
- "862",
- "46",
- "172",
- "27",
- "145",
- "Weighted",
- "average",
- "remaining",
- "terms",
- "years",
- "13",
- "discount",
- "rates",
- "3.5%",
- "4.2%",
- "outlines",
- "maturities",
- "future",
- "minimum",
- "payments",
- "year",
- "ending",
- "annual",
- "amounts",
- "2026",
- "2030",
- "481",
- "imputed",
- "620",
- "resulting",
- "60",
- "506",
- "14",
- "334",
- "largest",
- "single",
- "111",
- "36",
- "251",
- "As",
- "additional",
- "primarily",
- "datacenters",
- "commenced",
- "92.7",
- "billion",
- "These",
- "commence",
- "fiscal",
- "2031",
- "NOTE",
- "CONTINGENCIES",
- "Irish",
- "Data",
- "Protection",
- "Commission",
- "Matter",
- "In",
- "2018",
- "IDPC",
- "began",
- "investigating",
- "complaint",
- "LinkedIn",
- "targeted",
- "advertising",
- "practices",
- "violated",
- "recently",
- "implemented",
- "European",
- "Union",
- "General",
- "Regulation",
- "GDPR",
- "Microsoft",
- "cooperated",
- "period",
- "inquiry",
- "October",
- "provided",
- "final",
- "decision",
- "alleging",
- "violations",
- "assessing",
- "fine",
- "November",
- "appealed",
- "courts",
- "hearing",
- "scheduled",
- "December",
- "Other",
- "Contingencies",
- "We",
- "subject",
- "variety",
- "claims",
- "suits",
- "arise",
- "time",
- "ordinary",
- "business",
- "Although",
- "management",
- "believes",
- "resolving",
- "individually",
- "aggregate",
- "material",
- "adverse",
- "impact",
- "consolidated",
- "financial",
- "statements",
- "matters",
- "inherent",
- "uncertainties",
- "view",
- "change",
- "accrued",
- "legal",
- "541",
- "While",
- "intend",
- "defend",
- "vigorously",
- "outcomes",
- "estimate",
- "reach",
- "approximately",
- "600",
- "recorded",
- "Were",
- "unfavorable",
- "occur",
- "exists",
- "possibility",
- "effects",
- "estimable",
- "15",
- "STOCKHOLDERS",
- "EQUITY",
- "Shares",
- "Outstanding",
- "common",
- "stock",
- "outstanding"
- ],
- "keywords": [
- "LinkedIn",
- "Irish Data Protection Commission",
- "Microsoft"
- ],
- "connect_to": [
- {
- "target": "4452390e-74dd-5620-b691-6e58c8dc77de",
- "relation": "embeds",
- "ref": "[tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html]",
- "position": {
- "start": 273,
- "end": 344
- }
- },
- {
- "target": "28d25b09-4699-514f-8b98-01fc0bd50366",
- "relation": "embeds",
- "ref": "[tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html]",
- "position": {
- "start": 768,
- "end": 872
- }
- },
- {
- "target": "da108065-e4c5-5301-95cb-bfa49a12d84f",
- "relation": "embeds",
- "ref": "[tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html]",
- "position": {
- "start": 1486,
- "end": 1575
- }
- },
- {
- "target": "e267be8c-3565-5cd5-9a8a-d0d1596d29f2",
- "relation": "embeds",
- "ref": "[tables/table-54 Lease Maturities for Operating and Finance Leases.html]",
- "position": {
- "start": 2195,
- "end": 2267
- }
- }
- ]
- }
- },
- {
- "chunk_id": "9bae5843-8789-5ca7-8b7b-4e36c2048ff1",
- "type": "text",
- "content": "The table shows common stock outstanding for fiscal years 2025, 2024, and 2023, with beginning balances of 7,434, 7,432, and 7,464 million shares, respectively. Issuances were 31, 34, and 37 million, while repurchases were 31, 32, and 69 million, resulting in ending balances of 7,434, 7,434, and 7,432 million shares.\n[tables/table-55 Common Stock Outstanding Rollforward.html]\n\nShare Repurchases\nOn September 14, 2021, our Board of Directors approved a share repurchase program authorizing up to $60.0 billion in share repurchases. This share repurchase program commenced in November 2021 and was completed in April 2025.\nOn September 16, 2024, our Board of Directors approved a share repurchase program authorizing up to $60.0 billion in share repurchases. This share repurchase program commenced in April 2025, following completion of the program approved on September 14, 2021, has no expiration date, and may be terminated at any time. As of June 30, 2025, $57.3 billion remained of this $60.0 billion share repurchase program.\nWe repurchased the following shares of common stock under the share repurchase programs:\n\nThe table shows quarterly share repurchases for fiscal years 2025, 2024, and 2023. In 2025, shares repurchased were 7, 8, 8, and 8 per quarter, totaling 31 shares for $13,000 million. In 2024, quarterly shares were 11, 7, 7, and 7, totaling 32 shares for $11,960 million. In 2023, shares were 17, 20, 18, and 14, totaling 69 shares for $18,400 million. The highest quarterly amount was $4,600 million in 2023 quarters.\n[tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html]\n\nAll repurchases were made using cash resources. Shares repurchased during the fourth quarter of fiscal year 2025 were under the share repurchase programs approved on September 14, 2021 and September 16, 2024. All other shares repurchased were under the share repurchase program approved on September 14, 2021. The above table excludes shares repurchased to settle employee tax withholding related to the vesting of stock awards of $5.4 billion, $5.3 billion, and $3.8 billion for fiscal years 2025, 2024, and 2023, respectively.\nDividends\nOur Board of Directors declared the following dividends:\n\nThe table lists quarterly dividends for fiscal years 2025 and 2024, with per-share dividends of $0.83 for FY2025 (total $3.32, amount $24,678 million) and $0.75 for FY2024 (total $3.00, amount $22,295 million), each paid across four dates.\n[tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html]\n\nThe dividend declared on June 10, 2025 was included in other current liabilities as of June 30, 2025.\nNOTE 16 — ACCUMULATED OTHER COMPREHENSIVE INCOME (LOSS)\nThe following table summarizes the changes in accumulated other comprehensive income (loss) by component:\n\nThe table shows accumulated other comprehensive loss components for fiscal years 2025, 2024, and 2023. Derivatives balance changed from $(3) to $(8) in 2025, investments from $(2,625) to $(1,051), translation adjustments from $(2,962) to $(2,288). Total end-of-period loss: $(3,347) in 2025, $(5,590) in 2024, $(6,343) in 2023.\n[tables/table-58 Accumulated Other Comprehensive Loss by Component.html]\n\nNOTE 17 — EMPLOYEE STOCK AND SAVINGS PLANS\nWe grant stock-based compensation to employees and directors. Awards that expire or are canceled without delivery of shares generally become available for issuance under the plans. We issue new shares of Microsoft common stock to satisfy vesting of awards granted under our stock plans. We also have an ESPP for all eligible employees.\nStock-based compensation expense and related income tax benefits were as follows:\n\nStock-based compensation expense rose from $9,611 million in 2023 to $10,734 million in 2024 and $11,974 million in 2025; income tax benefits increased from $1,651 million to $1,826 million and $2,027 million over the same periods.\n[tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html]\n\nStock Plans\nStock awards entitle the holder to receive shares of Microsoft common stock as the award vests. Stock awards generally vest over a service period of four years or five years.\nExecutive Incentive Plan",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 4",
- "metadata": {
- "length": 4257,
- "summary": "The document reports Microsoft's common stock outstanding for fiscal years 2025, 2024, and 2023, with beginning balances of 7,434, 7,432, and 7,464 million shares, respectively. Issuances were 31, 34, and 37 million, while repurchases were 31, 32, and 69 million, resulting in ending balances of 7,434, 7,434, and 7,432 million shares. It details share repurchase programs: a $60.0 billion program approved on September 14, 2021, completed in April 2025, and another $60.0 billion program approved on September 16, 2024, with $57.3 billion remaining as of June 30, 2025. Quarterly repurchases for fiscal years 2025, 2024, and 2023 totaled 31, 32, and 69 million shares, respectively, with total costs of $13,000 million, $11,960 million, and $18,400 million. The highest quarterly repurchase amount was $4,600 million in 2023. Repurchases were made with cash resources, excluding shares repurchased to settle employee tax withholding of $5.4 billion, $5.3 billion, and $3.8 billion for fiscal years 2025, 2024, and 2023. Dividends declared were $0.83 per share for FY2025 (total $3.32, amount $24,678 million) and $0.75 per share for FY2024 (total $3.00, amount $22,295 million). Accumulated other comprehensive loss components changed: derivatives from $(3) to $(8) in 2025, investments from $(2,625) to $(1,051), translation adjustments from $(2,962) to $(2,288), with total end-of-period losses of $(3,347) in 2025, $(5,590) in 2024, and $(6,343) in 2023. Stock-based compensation expense rose from $9,611 million in 2023 to $10,734 million in 2024 and $11,974 million in 2025; income tax benefits increased from $1,651 million to $1,826 million and $2,027 million over the same periods. Stock awards generally vest over four or five years.",
- "page_nums": [],
- "tokens": [
- "The",
- "table",
- "shows",
- "common",
- "stock",
- "outstanding",
- "fiscal",
- "years",
- "2025",
- "2024",
- "2023",
- "beginning",
- "balances",
- "434",
- "432",
- "464",
- "million",
- "shares",
- "Issuances",
- "31",
- "34",
- "37",
- "repurchases",
- "32",
- "69",
- "resulting",
- "ending",
- "Share",
- "Repurchases",
- "On",
- "September",
- "14",
- "2021",
- "Board",
- "Directors",
- "approved",
- "share",
- "repurchase",
- "program",
- "authorizing",
- "60.0",
- "billion",
- "This",
- "commenced",
- "November",
- "completed",
- "April",
- "16",
- "completion",
- "expiration",
- "date",
- "terminated",
- "time",
- "As",
- "June",
- "30",
- "57.3",
- "remained",
- "We",
- "repurchased",
- "programs",
- "quarterly",
- "In",
- "quarter",
- "totaling",
- "13",
- "000",
- "11",
- "960",
- "17",
- "20",
- "18",
- "400",
- "highest",
- "amount",
- "600",
- "quarters",
- "All",
- "made",
- "cash",
- "resources",
- "Shares",
- "fourth",
- "year",
- "excludes",
- "settle",
- "employee",
- "tax",
- "withholding",
- "related",
- "vesting",
- "awards",
- "5.4",
- "5.3",
- "3.8",
- "Dividends",
- "Our",
- "declared",
- "dividends",
- "lists",
- "0.83",
- "FY2025",
- "total",
- "3.32",
- "24",
- "678",
- "0.75",
- "FY2024",
- "3.00",
- "22",
- "295",
- "paid",
- "dates",
- "dividend",
- "10",
- "included",
- "current",
- "liabilities",
- "NOTE",
- "ACCUMULATED",
- "OTHER",
- "COMPREHENSIVE",
- "INCOME",
- "LOSS",
- "summarizes",
- "accumulated",
- "comprehensive",
- "income",
- "loss",
- "component",
- "components",
- "Derivatives",
- "balance",
- "changed",
- "investments",
- "625",
- "051",
- "translation",
- "adjustments",
- "962",
- "288",
- "Total",
- "end",
- "period",
- "347",
- "590",
- "343",
- "EMPLOYEE",
- "STOCK",
- "AND",
- "SAVINGS",
- "PLANS",
- "grant",
- "based",
- "compensation",
- "employees",
- "directors",
- "Awards",
- "expire",
- "canceled",
- "delivery",
- "generally",
- "issuance",
- "plans",
- "issue",
- "Microsoft",
- "satisfy",
- "granted",
- "ESPP",
- "eligible",
- "Stock",
- "expense",
- "benefits",
- "rose",
- "611",
- "734",
- "974",
- "increased",
- "651",
- "826",
- "027",
- "periods",
- "Plans",
- "entitle",
- "holder",
- "receive",
- "award",
- "vests",
- "vest",
- "service",
- "Executive",
- "Incentive",
- "Plan"
- ],
- "keywords": [
- "Microsoft"
- ],
- "connect_to": [
- {
- "target": "36c1f271-f9e2-5fb3-b351-1f862f2b95fe",
- "relation": "embeds",
- "ref": "[tables/table-55 Common Stock Outstanding Rollforward.html]",
- "position": {
- "start": 319,
- "end": 378
- }
- },
- {
- "target": "657a22eb-9366-5d37-b1fa-074b1683ae81",
- "relation": "embeds",
- "ref": "[tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html]",
- "position": {
- "start": 1543,
- "end": 1639
- }
- },
- {
- "target": "0a3c08c8-285f-5234-ba6e-dcd0afd088ac",
- "relation": "embeds",
- "ref": "[tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html]",
- "position": {
- "start": 2478,
- "end": 2567
- }
- },
- {
- "target": "61be9b16-9678-54c0-89a8-bc758632e3a0",
- "relation": "embeds",
- "ref": "[tables/table-58 Accumulated Other Comprehensive Loss by Component.html]",
- "position": {
- "start": 3162,
- "end": 3234
- }
- },
- {
- "target": "94b78184-d619-5a1d-b89d-8bb7afb772ab",
- "relation": "embeds",
- "ref": "[tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html]",
- "position": {
- "start": 3930,
- "end": 4044
- }
- }
- ]
- }
- },
- {
- "chunk_id": "d1b32a12-c27d-5f18-9237-a0b0742435b1",
- "type": "text",
- "content": "Under the Executive Incentive Plan, the Compensation Committee approves stock awards to executive officers and certain senior executives. RSUs generally vest ratably over a service period of four years. PSUs generally vest over a performance period of three years. The number of shares the PSU holder receives is based on the extent to which the corresponding performance goals have been achieved.\nActivity for All Stock Plans\nThe fair value of stock awards was estimated on the date of grant using the following assumptions:\n\nDividends per share ranged from $0.62 to $0.83 quarterly across fiscal years 2023-2025, with 2025 at $0.75–$0.83, 2024 at $0.68–$0.75, and 2023 at $0.62–$0.68. Interest rates ranged from 2.0% to 5.6%, with 2025 at 3.4%–5.5%, 2024 at 3.8%–5.6%, and 2023 at 2.0%–5.4%.\n[tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html]\n\nDuring fiscal year 2025, the following activity occurred under our stock plans:\n\nThe table shows nonvested stock awards activity: beginning balance of 88 million shares at $292.28 weighted average grant-date fair value, 39 million granted at $413.90, 38 million vested at $293.25, 7 million forfeited at $317.23, ending balance of 82 million shares at $347.44.\n[tables/table-61 Nonvested Stock Awards Activity.html]\n\n(a) Includes 1 million of PSUs granted at target and performance adjustments above target levels for each of the fiscal years 2025, 2024, and 2023.\nAs of June 30, 2025, total unrecognized compensation costs related to stock awards were $21.6 billion. These costs are expected to be recognized over a weighted average period of three years. The weighted average grant-date fair value of stock awards granted was $413.90, $339.46, and $252.59 for fiscal years 2025, 2024, and 2023, respectively. The fair value of stock awards vested was $16.2 billion, $16.0 billion, and $11.9 billion, for fiscal years 2025, 2024, and 2023, respectively. As of June 30, 2025, an aggregate of 98 million shares were authorized for future grant under our stock plans.\nEmployee Stock Purchase Plan\nWe have an ESPP for all eligible employees. Shares of our common stock may be purchased by employees at three-month intervals at 90% of the fair market value on the last trading day of each three-month period. Employees may purchase shares having a value not exceeding 15% of their gross compensation during an offering period.\nEmployees purchased the following shares during the periods presented:\n\nThe company repurchased 6 million shares in both fiscal 2025 and 2024, and 7 million in 2023, with average prices per share of $385.10, $339.46, and $245.59 respectively, showing rising buyback costs over the three years.\n[tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html]\n\nAs of June 30, 2025, 62 million shares of our common stock were reserved for future issuance through the ESPP.\nSavings Plans\nWe have savings plans in the U.S. that qualify under Section 401(k) of the Internal Revenue Code, and a number of savings plans in international locations. Eligible U.S. employees may contribute a portion of their salary into the savings plans, subject to certain limitations. We match a portion of each dollar a participant contributes into the plans. Employer-funded retirement benefits for all plans were $1.8 billion, $1.7 billion, and $1.6 billion in fiscal years 2025, 2024, and 2023, respectively, and were expensed as contributed.\nNOTE 18 — SEGMENT INFORMATION AND GEOGRAPHIC DATA\nIn its operation of the business, management, including our chief operating decision maker (“CODM”), who is also our Chief Executive Officer, reviews certain financial information, including segmented internal profit and loss statements. The primary profitability measure used by the CODM to review segment operating results is operating income. The CODM uses operating income to allocate resources during our annual planning process and throughout the year, as well as to assess the performance of our segments, primarily by monitoring actual results compared to prior periods and expected results. During the periods presented, we reported our financial performance based on the following segments: Productivity and Business Processes, Intelligent Cloud, and More Personal Computing.\nWe have recast certain prior period amounts to conform to the way we internally manage and monitor our business. Refer to Note 1 – Accounting Policies for further information.\nOur reportable segments are described below.",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 5",
- "metadata": {
- "length": 4498,
- "summary": "The text describes Microsoft's stock-based compensation plans, including the Executive Incentive Plan, Employee Stock Purchase Plan (ESPP), and savings plans. It details the vesting of RSUs and PSUs, with RSUs vesting over four years and PSUs over three years. The fair value of stock awards was estimated using assumptions: dividends per share ranged from $0.62 to $0.83 quarterly across fiscal years 2023-2025, with 2025 at $0.75–$0.83, 2024 at $0.68–$0.75, and 2023 at $0.62–$0.68. Interest rates ranged from 2.0% to 5.6%, with 2025 at 3.4%–5.5%, 2024 at 3.8%–5.6%, and 2023 at 2.0%–5.4%. In fiscal 2025, nonvested stock awards activity: beginning balance 88 million shares at $292.28 weighted average grant-date fair value, 39 million granted at $413.90, 38 million vested at $293.25, 7 million forfeited at $317.23, ending balance 82 million shares at $347.44. Unrecognized compensation costs were $21.6 billion as of June 30, 2025, expected to be recognized over three years. Weighted average grant-date fair value of awards granted was $413.90, $339.46, and $252.59 for fiscal years 2025, 2024, and 2023, respectively. Fair value of vested awards was $16.2 billion, $16.0 billion, and $11.9 billion for the same years. 98 million shares were authorized for future grant. The ESPP allows employees to purchase shares at 90% of fair market value, with purchases limited to 15% of gross compensation. Employees purchased 6 million shares in both fiscal 2025 and 2024, and 7 million in 2023, with average prices of $385.10, $339.46, and $245.59 respectively. 62 million shares were reserved for future issuance through ESPP. Savings plans match employee contributions; employer-funded retirement benefits were $1.8 billion, $1.7 billion, and $1.6 billion in fiscal years 2025, 2024, and 2023. The company's segments are Productivity and Business Processes, Intelligent Cloud, and More Personal Computing, with operating income as the primary profitability measure.",
- "page_nums": [],
- "tokens": [
- "Under",
- "Executive",
- "Incentive",
- "Plan",
- "Compensation",
- "Committee",
- "approves",
- "stock",
- "awards",
- "executive",
- "officers",
- "senior",
- "executives",
- "RSUs",
- "generally",
- "vest",
- "ratably",
- "service",
- "period",
- "years",
- "PSUs",
- "performance",
- "The",
- "number",
- "shares",
- "PSU",
- "holder",
- "receives",
- "based",
- "extent",
- "goals",
- "achieved",
- "Activity",
- "All",
- "Stock",
- "Plans",
- "fair",
- "estimated",
- "date",
- "grant",
- "assumptions",
- "Dividends",
- "share",
- "ranged",
- "0.62",
- "0.83",
- "quarterly",
- "fiscal",
- "2023",
- "2025",
- "0.75",
- "2024",
- "0.68",
- "Interest",
- "rates",
- "2.0%",
- "5.6%",
- "3.4%",
- "5.5%",
- "3.8%",
- "5.4%",
- "During",
- "year",
- "activity",
- "occurred",
- "plans",
- "table",
- "shows",
- "nonvested",
- "beginning",
- "balance",
- "88",
- "million",
- "292.28",
- "weighted",
- "average",
- "39",
- "granted",
- "413.90",
- "38",
- "vested",
- "293.25",
- "forfeited",
- "317.23",
- "ending",
- "82",
- "347.44",
- "Includes",
- "target",
- "adjustments",
- "levels",
- "As",
- "June",
- "30",
- "total",
- "unrecognized",
- "compensation",
- "costs",
- "related",
- "21.6",
- "billion",
- "These",
- "expected",
- "recognized",
- "339.46",
- "252.59",
- "16.2",
- "16.0",
- "11.9",
- "aggregate",
- "98",
- "authorized",
- "future",
- "Employee",
- "Purchase",
- "We",
- "ESPP",
- "eligible",
- "employees",
- "Shares",
- "common",
- "purchased",
- "month",
- "intervals",
- "90%",
- "market",
- "trading",
- "day",
- "Employees",
- "purchase",
- "exceeding",
- "15%",
- "gross",
- "offering",
- "periods",
- "presented",
- "company",
- "repurchased",
- "prices",
- "385.10",
- "245.59",
- "showing",
- "rising",
- "buyback",
- "62",
- "reserved",
- "issuance",
- "Savings",
- "savings",
- "qualify",
- "Section",
- "401",
- "Internal",
- "Revenue",
- "Code",
- "international",
- "locations",
- "Eligible",
- "contribute",
- "portion",
- "salary",
- "subject",
- "limitations",
- "match",
- "dollar",
- "participant",
- "contributes",
- "Employer",
- "funded",
- "retirement",
- "benefits",
- "1.8",
- "1.7",
- "1.6",
- "expensed",
- "contributed",
- "NOTE",
- "18",
- "SEGMENT",
- "INFORMATION",
- "AND",
- "GEOGRAPHIC",
- "DATA",
- "In",
- "operation",
- "business",
- "management",
- "including",
- "chief",
- "operating",
- "decision",
- "maker",
- "CODM",
- "Chief",
- "Officer",
- "reviews",
- "financial",
- "information",
- "segmented",
- "internal",
- "profit",
- "loss",
- "statements",
- "primary",
- "profitability",
- "measure",
- "review",
- "segment",
- "results",
- "income",
- "allocate",
- "resources",
- "annual",
- "planning",
- "process",
- "assess",
- "segments",
- "primarily",
- "monitoring",
- "actual",
- "compared",
- "prior",
- "reported",
- "Productivity",
- "Business",
- "Processes",
- "Intelligent",
- "Cloud",
- "More",
- "Personal",
- "Computing",
- "recast",
- "amounts",
- "conform",
- "internally",
- "manage",
- "monitor",
- "Refer",
- "Note",
- "Accounting",
- "Policies",
- "Our",
- "reportable"
- ],
- "keywords": [
- "Executive Incentive Plan",
- "Employee Stock Purchase Plan",
- "Microsoft",
- "Productivity and Business Processes",
- "Intelligent Cloud",
- "More Personal Computing"
- ],
- "connect_to": [
- {
- "target": "309e74c6-6705-520b-a28b-959fb9661644",
- "relation": "embeds",
- "ref": "[tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html]",
- "position": {
- "start": 794,
- "end": 882
- }
- },
- {
- "target": "792d3955-3e4a-56a4-8154-2e6d38d6218b",
- "relation": "embeds",
- "ref": "[tables/table-61 Nonvested Stock Awards Activity.html]",
- "position": {
- "start": 1245,
- "end": 1299
- }
- },
- {
- "target": "4e74811f-7ba5-517d-8687-fc6523746a50",
- "relation": "embeds",
- "ref": "[tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html]",
- "position": {
- "start": 2701,
- "end": 2776
- }
- }
- ]
- }
- },
- {
- "chunk_id": "befab8de-0d3f-5c3e-ba1e-76663e7d906e",
- "type": "text",
- "content": "Productivity and Business Processes\nOur Productivity and Business Processes segment consists of products and services in our portfolio of productivity, communication, and information services, spanning a variety of devices and platforms. This segment primarily comprises:\n• Microsoft 365 Commercial products and cloud services, including Microsoft 365 Commercial cloud, comprising Microsoft 365 Commercial, Enterprise Mobility + Security, the cloud portion of Windows Commercial, the per-user portion of Power BI, Exchange, SharePoint, Microsoft Teams, Microsoft 365 Security and Compliance, and Microsoft 365 Copilot; and Microsoft 365 Commercial products, comprising Windows Commercial on-premises and Office licensed on-premises.\n• Microsoft 365 Consumer products and cloud services, including Microsoft 365 Consumer subscriptions, Office licensed on-premises, and other consumer services.\n• LinkedIn, including Talent Solutions, Marketing Solutions, Premium Subscriptions, and Sales Solutions.\n• Dynamics products and cloud services, including Dynamics 365, comprising a set of intelligent, cloud-based applications across ERP, CRM, Power Apps, and Power Automate; and on-premises ERP and CRM applications.\nIntelligent Cloud\nOur Intelligent Cloud segment consists of our public, private, and hybrid server products and cloud services that power modern business and developers. This segment primarily comprises:\n• Server products and cloud services, including Azure and other cloud services, comprising cloud and AI consumption-based services, GitHub cloud services, Nuance Healthcare cloud services, virtual desktop offerings, and other cloud services; and Server products, comprising SQL Server, Windows Server, Visual Studio, System Center, related Client Access Licenses (“CALs”), and other on-premises offerings.\n• Enterprise and partner services, including Enterprise Support Services, Industry Solutions, Nuance professional services, Microsoft Partner Network, and Learning Experience.\nMore Personal Computing\nOur More Personal Computing segment consists of products and services that put customers at the center of the experience with our technology. This segment primarily comprises:\n• Windows and Devices, including Windows OEM licensing (Windows Pro and non-Pro licenses sold through the OEM channel) and Devices, comprising Surface and PC accessories.\n• Gaming, including Xbox hardware and Xbox content and services, comprising first- and third-party content (including games and in-game content), Xbox Game Pass and other subscriptions, Xbox Cloud Gaming, advertising, and other cloud services.\n• Search and news advertising, comprising Bing and Copilot, Microsoft News, Microsoft Edge, and third-party affiliates.\nRevenue and costs are generally directly attributed to our segments. However, due to the integrated structure of our business, certain revenue recognized and costs incurred by one segment may benefit other segments. Revenue from certain contracts is allocated among the segments based on the relative value of the underlying products and services, which can include allocation based on actual prices charged, prices when sold separately, or estimated costs plus a profit margin. Cost of revenue is allocated in certain cases based on a relative revenue methodology. Operating expenses that are allocated primarily include those relating to marketing of products and services from which multiple segments benefit and are generally allocated based on relative gross margin.\nIn addition, certain costs are incurred at a corporate level and allocated to our segments. These allocated costs generally include legal, including settlements and fines, information technology, human resources, finance, excise taxes, field selling, shared facilities services, customer service and support, and severance incurred as part of a corporate program. Each allocation is measured differently based on the specific facts and circumstances of the costs being allocated and is generally based on relative gross margin or relative headcount.\nSegment revenue, cost of revenue, operating expenses, and operating income were as follows during the periods presented:",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 6",
- "metadata": {
- "length": 4174,
- "summary": "The text describes Microsoft's three business segments: Productivity and Business Processes, Intelligent Cloud, and More Personal Computing. It details the products and services in each segment, such as Microsoft 365, LinkedIn, Dynamics 365, Azure, Windows, and Gaming. It also explains revenue and cost allocation methods, including direct attribution and allocation based on relative value, gross margin, or headcount. The text mentions that segment revenue, cost of revenue, operating expenses, and operating income are presented for the periods shown, but the actual financial figures are not included in the provided excerpt.",
- "page_nums": [],
- "tokens": [
- "Productivity",
- "Business",
- "Processes",
- "Our",
- "segment",
- "consists",
- "products",
- "services",
- "portfolio",
- "productivity",
- "communication",
- "information",
- "spanning",
- "variety",
- "devices",
- "platforms",
- "This",
- "primarily",
- "comprises",
- "Microsoft",
- "365",
- "Commercial",
- "cloud",
- "including",
- "comprising",
- "Enterprise",
- "Mobility",
- "Security",
- "portion",
- "Windows",
- "user",
- "Power",
- "BI",
- "Exchange",
- "SharePoint",
- "Teams",
- "Compliance",
- "Copilot",
- "premises",
- "Office",
- "licensed",
- "Consumer",
- "subscriptions",
- "consumer",
- "LinkedIn",
- "Talent",
- "Solutions",
- "Marketing",
- "Premium",
- "Subscriptions",
- "Sales",
- "Dynamics",
- "set",
- "intelligent",
- "based",
- "applications",
- "ERP",
- "CRM",
- "Apps",
- "Automate",
- "Intelligent",
- "Cloud",
- "public",
- "private",
- "hybrid",
- "server",
- "power",
- "modern",
- "business",
- "developers",
- "Server",
- "Azure",
- "AI",
- "consumption",
- "GitHub",
- "Nuance",
- "Healthcare",
- "virtual",
- "desktop",
- "offerings",
- "SQL",
- "Visual",
- "Studio",
- "System",
- "Center",
- "related",
- "Client",
- "Access",
- "Licenses",
- "CALs",
- "partner",
- "Support",
- "Services",
- "Industry",
- "professional",
- "Partner",
- "Network",
- "Learning",
- "Experience",
- "More",
- "Personal",
- "Computing",
- "put",
- "customers",
- "center",
- "experience",
- "technology",
- "Devices",
- "OEM",
- "licensing",
- "Pro",
- "licenses",
- "sold",
- "channel",
- "Surface",
- "PC",
- "accessories",
- "Gaming",
- "Xbox",
- "hardware",
- "content",
- "party",
- "games",
- "game",
- "Game",
- "Pass",
- "advertising",
- "Search",
- "news",
- "Bing",
- "News",
- "Edge",
- "affiliates",
- "Revenue",
- "costs",
- "generally",
- "directly",
- "attributed",
- "segments",
- "However",
- "due",
- "integrated",
- "structure",
- "revenue",
- "recognized",
- "incurred",
- "benefit",
- "contracts",
- "allocated",
- "relative",
- "underlying",
- "include",
- "allocation",
- "actual",
- "prices",
- "charged",
- "separately",
- "estimated",
- "profit",
- "margin",
- "Cost",
- "cases",
- "methodology",
- "Operating",
- "expenses",
- "relating",
- "marketing",
- "multiple",
- "gross",
- "In",
- "addition",
- "corporate",
- "level",
- "These",
- "legal",
- "settlements",
- "fines",
- "human",
- "resources",
- "finance",
- "excise",
- "taxes",
- "field",
- "selling",
- "shared",
- "facilities",
- "customer",
- "service",
- "support",
- "severance",
- "part",
- "program",
- "Each",
- "measured",
- "differently",
- "specific",
- "facts",
- "circumstances",
- "headcount",
- "Segment",
- "cost",
- "operating",
- "income",
- "periods",
- "presented"
- ],
- "keywords": [
- "Microsoft 365",
- "LinkedIn",
- "Dynamics 365",
- "Azure",
- "Windows",
- "Xbox",
- "Bing",
- "Microsoft Edge",
- "Microsoft Partner Network",
- "Nuance"
- ],
- "connect_to": []
- }
- },
- {
- "chunk_id": "be53fc6d-4967-5b88-99fa-d189e6910e51",
- "type": "text",
- "content": "Microsoft's segment operating income for fiscal years 2025, 2024, and 2023 shows Productivity and Business Processes leading with $69,773M (2025), followed by Intelligent Cloud at $44,589M, and More Personal Computing at $14,166M; total operating income rose from $88,523M (2023) to $128,528M (2025), with total revenue increasing from $211,915M to $281,724M.\n[tables/table-63 Microsoft Segment Operating Income Table.html]\n\nNo sales to an individual customer or country other than the United States accounted for more than 10% of revenue for fiscal years 2025, 2024, or 2023. Revenue, classified by the major geographic areas in which our customers were located, was as follows:\n\nFor fiscal years 2025, 2024, and 2023, total revenue was $281,724 million, $245,122 million, and $211,915 million, respectively; United States revenue grew from $106,744 million to $144,546 million, and other countries revenue grew from $105,171 million to $137,178 million.\n[tables/table-64 United States and Other Countries Revenue by Fiscal Year.html]\n\n(a) Includes billings to OEMs and certain multinational organizations because of the nature of these businesses and the impracticability of determining the geographic source of the revenue.\nRevenue, classified by significant product and service offerings, was as follows:\n\nMicrosoft's total revenue grew from $211,915 million in fiscal 2023 to $245,122 million in 2024 and $281,724 million in 2025. Server products and cloud services was the largest segment, rising from $65,007M to $79,828M to $98,435M. Microsoft 365 Commercial grew from $66,949M to $76,969M to $87,767M. Gaming increased from $15,466M to $21,503M to $23,455M. Other segments included LinkedIn, Windows and Devices, Search and news advertising, Dynamics, Enterprise and partner services, and Microsoft 365 Consumer.\n[tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html]\n\nOur Microsoft Cloud revenue, which includes Microsoft 365 Commercial cloud, Azure and other cloud services, the commercial portion of LinkedIn, and Dynamics 365, was $168.9 billion, $137.7 billion, and $111.6 billion in fiscal years 2025, 2024, and 2023, respectively. These amounts are included in Microsoft 365 Commercial products and cloud services, Server products and cloud services, LinkedIn, and Dynamics products and cloud services in the table above.\nAssets are not allocated to segments for internal reporting presentations. A portion of amortization and depreciation is included with various other costs in an overhead allocation to each segment. It is impracticable for us to separately identify the amount of amortization and depreciation by segment that is included in the measure of segment profit or loss.\nLong-lived assets, excluding financial instruments and tax assets, classified by the location of the controlling statutory company and with countries over 10% of the total shown separately, were as follows:\n\nThe table shows total revenue of $371,902 million for fiscal year 2025, up from $301,369 million in 2024 and $187,239 million in 2023. United States revenue grew from $114,380 million (2023) to $186,106 million (2024) and $230,069 million (2025); Other countries revenue rose from $72,859 million to $115,263 million and $141,833 million over the same periods.\n[tables/table-66 United States and Other Countries Revenue by Fiscal Year.html]\n\nREPORT OF INDEPENDENT REGISTERED PUBLIC ACCOUNTING FIRM\nTo the Stockholders and the Board of Directors of Microsoft Corporation\nOpinion on the Financial Statements\nWe have audited the accompanying consolidated balance sheets of Microsoft Corporation and subsidiaries (the “Company”) as of June 30, 2025 and 2024, the related consolidated statements of income, comprehensive income, cash flows, and stockholders’ equity, for each of the three years in the period ended June 30, 2025, and the related notes (collectively referred to as the “financial statements”). In our opinion, the financial statements present fairly, in all material respects, the financial position of the Company as of June 30, 2025 and 2024, and the results of its operations and its cash flows for each of the three years in the period ended June 30, 2025, in conformity with accounting principles generally accepted in the United States of America.",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 7",
- "metadata": {
- "length": 4296,
- "summary": "Microsoft's fiscal year 2025 results show total revenue of $281,724 million, up from $245,122 million in 2024 and $211,915 million in 2023. Operating income rose to $128,528 million in 2025 from $88,523 million in 2023. Segment operating income for 2025: Productivity and Business Processes $69,773M, Intelligent Cloud $44,589M, More Personal Computing $14,166M. Revenue by geography: United States $144,546M (2025), other countries $137,178M (2025). Microsoft Cloud revenue was $168.9B in 2025, $137.7B in 2024, $111.6B in 2023. Long-lived assets total $371,902M in 2025, with United States $230,069M and other countries $141,833M. No single customer or country other than the US accounted for more than 10% of revenue.",
- "page_nums": [],
- "tokens": [
- "Microsoft",
- "segment",
- "operating",
- "income",
- "fiscal",
- "years",
- "2025",
- "2024",
- "2023",
- "shows",
- "Productivity",
- "Business",
- "Processes",
- "leading",
- "69",
- "773M",
- "Intelligent",
- "Cloud",
- "44",
- "589M",
- "More",
- "Personal",
- "Computing",
- "14",
- "166M",
- "total",
- "rose",
- "88",
- "523M",
- "128",
- "528M",
- "revenue",
- "increasing",
- "211",
- "915M",
- "281",
- "724M",
- "No",
- "sales",
- "individual",
- "customer",
- "country",
- "United",
- "States",
- "accounted",
- "10%",
- "Revenue",
- "classified",
- "major",
- "geographic",
- "areas",
- "customers",
- "located",
- "For",
- "724",
- "million",
- "245",
- "122",
- "915",
- "grew",
- "106",
- "744",
- "144",
- "546",
- "countries",
- "105",
- "171",
- "137",
- "178",
- "Includes",
- "billings",
- "OEMs",
- "multinational",
- "organizations",
- "nature",
- "businesses",
- "impracticability",
- "determining",
- "source",
- "significant",
- "product",
- "service",
- "offerings",
- "Server",
- "products",
- "cloud",
- "services",
- "largest",
- "rising",
- "65",
- "007M",
- "79",
- "828M",
- "98",
- "435M",
- "365",
- "Commercial",
- "66",
- "949M",
- "76",
- "969M",
- "87",
- "767M",
- "Gaming",
- "increased",
- "15",
- "466M",
- "21",
- "503M",
- "23",
- "455M",
- "Other",
- "segments",
- "included",
- "LinkedIn",
- "Windows",
- "Devices",
- "Search",
- "news",
- "advertising",
- "Dynamics",
- "Enterprise",
- "partner",
- "Consumer",
- "Our",
- "includes",
- "Azure",
- "commercial",
- "portion",
- "168.9",
- "billion",
- "137.7",
- "111.6",
- "These",
- "amounts",
- "table",
- "Assets",
- "allocated",
- "internal",
- "reporting",
- "presentations",
- "amortization",
- "depreciation",
- "costs",
- "overhead",
- "allocation",
- "It",
- "impracticable",
- "separately",
- "identify",
- "amount",
- "measure",
- "profit",
- "loss",
- "Long",
- "lived",
- "assets",
- "excluding",
- "financial",
- "instruments",
- "tax",
- "location",
- "controlling",
- "statutory",
- "company",
- "shown",
- "The",
- "371",
- "902",
- "year",
- "301",
- "369",
- "187",
- "239",
- "114",
- "380",
- "186",
- "230",
- "069",
- "72",
- "859",
- "115",
- "263",
- "141",
- "833",
- "periods",
- "REPORT",
- "OF",
- "INDEPENDENT",
- "REGISTERED",
- "PUBLIC",
- "ACCOUNTING",
- "FIRM",
- "To",
- "Stockholders",
- "Board",
- "Directors",
- "Corporation",
- "Opinion",
- "Financial",
- "Statements",
- "We",
- "audited",
- "accompanying",
- "consolidated",
- "balance",
- "sheets",
- "subsidiaries",
- "Company",
- "June",
- "30",
- "related",
- "statements",
- "comprehensive",
- "cash",
- "flows",
- "stockholders",
- "equity",
- "period",
- "ended",
- "notes",
- "collectively",
- "referred",
- "In",
- "opinion",
- "present",
- "fairly",
- "material",
- "respects",
- "position",
- "results",
- "operations",
- "conformity",
- "accounting",
- "principles",
- "generally",
- "accepted",
- "America"
- ],
- "keywords": [
- "Microsoft Corporation",
- "United States"
- ],
- "connect_to": [
- {
- "target": "0579405f-2c41-52d7-aca9-c24a01e2a411",
- "relation": "embeds",
- "ref": "[tables/table-63 Microsoft Segment Operating Income Table.html]",
- "position": {
- "start": 360,
- "end": 423
- }
- },
- {
- "target": "e3d92df6-6a6e-53d0-abb3-1136d50f5ae9",
- "relation": "embeds",
- "ref": "[tables/table-64 United States and Other Countries Revenue by Fiscal Year.html]",
- "position": {
- "start": 956,
- "end": 1035
- }
- },
- {
- "target": "89bdd525-3b5c-53e1-947e-df3aefb5bdb9",
- "relation": "embeds",
- "ref": "[tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html]",
- "position": {
- "start": 1822,
- "end": 1900
- }
- },
- {
- "target": "f84caf3c-647d-582d-a0b5-ddbab86fdb05",
- "relation": "embeds",
- "ref": "[tables/table-66 United States and Other Countries Revenue by Fiscal Year.html]",
- "position": {
- "start": 3293,
- "end": 3372
- }
- }
- ]
- }
- },
- {
- "chunk_id": "90310f12-90c1-5cbb-bcda-6984326b963b",
- "type": "text",
- "content": "We have also audited, in accordance with the standards of the Public Company Accounting Oversight Board (United States) (PCAOB), the Company’s internal control over financial reporting as of June 30, 2025, based on criteria established in Internal Control — Integrated Framework (2013) issued by the Committee of Sponsoring Organizations of the Treadway Commission and our report dated July 30, 2025, expressed an unqualified opinion on the Company’s internal control over financial reporting.\nBasis for Opinion\nThese financial statements are the responsibility of the Company’s management. Our responsibility is to express an opinion on the Company’s financial statements based on our audits. We are a public accounting firm registered with the PCAOB and are required to be independent with respect to the Company in accordance with the U.S. federal securities laws and the applicable rules and regulations of the Securities and Exchange Commission and the PCAOB.\nWe conducted our audits in accordance with the standards of the PCAOB. Those standards require that we plan and perform the audit to obtain reasonable assurance about whether the financial statements are free of material misstatement, whether due to error or fraud. Our audits included performing procedures to assess the risks of material misstatement of the financial statements, whether due to error or fraud, and performing procedures that respond to those risks. Such procedures included examining, on a test basis, evidence regarding the amounts and disclosures in the financial statements. Our audits also included evaluating the accounting principles used and significant estimates made by management, as well as evaluating the overall presentation of the financial statements. We believe that our audits provide a reasonable basis for our opinion.\nCritical Audit Matters\nThe critical audit matters communicated below are matters arising from the current-period audit of the financial statements that were communicated or required to be communicated to the audit committee and that (1) relate to accounts or disclosures that are material to the financial statements and (2) involved our especially challenging, subjective, or complex judgments. The communication of critical audit matters does not alter in any way our opinion on the financial statements, taken as a whole, and we are not, by communicating the critical audit matters below, providing separate opinions on the critical audit matters or on the accounts or disclosures to which they relate.\nRevenue Recognition – Refer to Note 1 to the financial statements\nCritical Audit Matter Description\nThe Company recognizes revenue upon transfer of control of promised products or services to customers in an amount that reflects the consideration the Company expects to receive in exchange for those products or services. The Company offers customers the ability to acquire multiple licenses of software products and services, including cloud-based services, in its customer agreements through its volume licensing programs.\nSignificant judgment is exercised by the Company in determining revenue recognition for certain customer agreements, and includes the following:\n• Determination of whether products and services are considered distinct performance obligations that should be accounted for separately versus together, such as software licenses and related services that are sold with cloud-based services.\n• The pattern of delivery (i.e., timing of when revenue is recognized) for each distinct performance obligation.\n• Identification and treatment of contract terms that may impact the timing and amount of revenue recognized (e.g., variable consideration, optional purchases, and free services).\n• Determination of stand-alone selling prices for each distinct performance obligation and for products and services that are not sold separately.\nGiven these factors and due to the volume of transactions, the related audit effort in evaluating management’s judgments in determining revenue recognition for certain customer agreements was extensive and required a high degree of auditor judgment.\nHow the Critical Audit Matter Was Addressed in the Audit\nOur principal audit procedures related to the Company’s revenue recognition for certain customer agreements included the following:",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 8",
- "metadata": {
- "length": 4318,
- "summary": "The text is an excerpt from an audit report. The auditor states that they audited the company's internal control over financial reporting as of June 30, 2025, and expressed an unqualified opinion. The basis for opinion explains that the financial statements are management's responsibility and the auditor's responsibility is to express an opinion. The audit was conducted in accordance with PCAOB standards. The report then discusses a critical audit matter regarding revenue recognition, which involves significant judgment in determining distinct performance obligations, pattern of delivery, contract terms, and stand-alone selling prices. The auditor's procedures included evaluating management's judgments.",
- "page_nums": [],
- "tokens": [
- "We",
- "audited",
- "accordance",
- "standards",
- "Public",
- "Company",
- "Accounting",
- "Oversight",
- "Board",
- "United",
- "States",
- "PCAOB",
- "internal",
- "control",
- "financial",
- "reporting",
- "June",
- "30",
- "2025",
- "based",
- "criteria",
- "established",
- "Internal",
- "Control",
- "Integrated",
- "Framework",
- "2013",
- "issued",
- "Committee",
- "Sponsoring",
- "Organizations",
- "Treadway",
- "Commission",
- "report",
- "dated",
- "July",
- "expressed",
- "unqualified",
- "opinion",
- "Basis",
- "Opinion",
- "These",
- "statements",
- "responsibility",
- "management",
- "Our",
- "express",
- "audits",
- "public",
- "accounting",
- "firm",
- "registered",
- "required",
- "independent",
- "respect",
- "federal",
- "securities",
- "laws",
- "applicable",
- "rules",
- "regulations",
- "Securities",
- "Exchange",
- "conducted",
- "Those",
- "require",
- "plan",
- "perform",
- "audit",
- "obtain",
- "reasonable",
- "assurance",
- "free",
- "material",
- "misstatement",
- "due",
- "error",
- "fraud",
- "included",
- "performing",
- "procedures",
- "assess",
- "risks",
- "respond",
- "Such",
- "examining",
- "test",
- "basis",
- "evidence",
- "amounts",
- "disclosures",
- "evaluating",
- "principles",
- "significant",
- "estimates",
- "made",
- "presentation",
- "provide",
- "Critical",
- "Audit",
- "Matters",
- "The",
- "critical",
- "matters",
- "communicated",
- "arising",
- "current",
- "period",
- "committee",
- "relate",
- "accounts",
- "involved",
- "challenging",
- "subjective",
- "complex",
- "judgments",
- "communication",
- "alter",
- "communicating",
- "providing",
- "separate",
- "opinions",
- "Revenue",
- "Recognition",
- "Refer",
- "Note",
- "Matter",
- "Description",
- "recognizes",
- "revenue",
- "transfer",
- "promised",
- "products",
- "services",
- "customers",
- "amount",
- "reflects",
- "consideration",
- "expects",
- "receive",
- "exchange",
- "offers",
- "ability",
- "acquire",
- "multiple",
- "licenses",
- "software",
- "including",
- "cloud",
- "customer",
- "agreements",
- "volume",
- "licensing",
- "programs",
- "Significant",
- "judgment",
- "exercised",
- "determining",
- "recognition",
- "includes",
- "Determination",
- "considered",
- "distinct",
- "performance",
- "obligations",
- "accounted",
- "separately",
- "versus",
- "related",
- "sold",
- "pattern",
- "delivery",
- "timing",
- "recognized",
- "obligation",
- "Identification",
- "treatment",
- "contract",
- "terms",
- "impact",
- "variable",
- "optional",
- "purchases",
- "stand",
- "selling",
- "prices",
- "Given",
- "factors",
- "transactions",
- "effort",
- "extensive",
- "high",
- "degree",
- "auditor",
- "How",
- "Was",
- "Addressed",
- "principal"
- ],
- "keywords": [
- "Public Company Accounting Oversight Board",
- "Securities and Exchange Commission",
- "Committee of Sponsoring Organizations of the Treadway Commission"
- ],
- "connect_to": []
- }
- },
- {
- "chunk_id": "c3477c6d-6dc2-55a4-8152-29f6e574d464",
- "type": "text",
- "content": "• We tested the effectiveness of controls related to the identification of distinct performance obligations, the determination of the timing of revenue recognition, and the estimation of variable consideration.\n• We evaluated management’s significant accounting policies related to certain customer agreements for reasonableness.\n• We selected a sample of customer agreements and performed the following procedures:\n Obtained and read contract source documents for each selection, including master agreements, and other documents that were part of the agreement.\n Tested management’s identification and treatment of contract terms.\n Assessed the terms in the customer agreement and evaluated the appropriateness of management’s application of their accounting policies, along with their use of estimates, in the determination of revenue recognition conclusions.\n• We evaluated the reasonableness of management’s estimate of stand-alone selling prices for products and services that are not sold separately.\n• We tested the mathematical accuracy of management’s calculations of revenue and the associated timing of revenue recognized in the financial statements.\nIncome Taxes – Uncertain Tax Positions – Refer to Note 11 to the financial statements\nCritical Audit Matter Description\nThe Company’s long-term income taxes liability includes uncertain tax positions related to transfer pricing issues that remain unresolved with the Internal Revenue Service (“IRS”). The Company remains under IRS audit, or subject to IRS audit, for tax years subsequent to 2003. During fiscal year 2024, the Company received Notices of Proposed Adjustments (“NOPAs”) for the tax years 2004 to 2013, primarily related to intercompany transfer pricing. While the Company has settled a portion of the IRS audits, resolution of the remaining matters could have a material impact on the Company’s financial statements.\nConclusions on recognizing and measuring uncertain tax positions involve significant estimates and management judgment and include complex considerations of the Internal Revenue Code, related regulations, tax case laws, and prior-year audit settlements. Given the complexity and the subjective nature of certain transfer pricing issues that remain unresolved with the IRS, evaluating management’s estimates relating to their determination of uncertain tax positions required a high degree of auditor judgment, including involvement of our tax specialists.\nHow the Critical Audit Matter Was Addressed in the Audit\nOur principal audit procedures to evaluate management’s estimates of uncertain tax positions related to unresolved transfer pricing issues included the following:\n• We evaluated management’s methods and assumptions used in the measurement and disclosure of uncertain tax positions, which included testing the effectiveness of the related internal controls.\n• We tested the reasonableness of management’s judgments regarding the future resolution of uncertain tax positions, as follows:\n We evaluated whether management had appropriately considered new information that could significantly change the measurement of the uncertain tax positions.\n We evaluated the reasonableness of management’s estimates by considering how changes in tax law, including statutes, regulations, and recent case law, impacted management’s judgments.\n• We evaluated the appropriateness of the disclosures in relation to the underlying facts, judgments, and conclusions.\n/S/ DELOITTE & TOUCHE LLP\nSeattle, Washington\nJuly 30, 2025\nWe have served as the Company’s auditor since 1983.\nCHANGES IN AND DISAGREEMENTS WITH ACCOUNTANTS ON ACCOUNTING AND FINANCIAL DISCLOSURE\nNot applicable.\nCONTROLS AND PROCEDURES\nUnder the supervision and with the participation of our management, including the Chief Executive Officer and Chief Financial Officer, we have evaluated the effectiveness of our disclosure controls and procedures as required by Exchange Act Rule 13a-15(b) as of the end of the period covered by this report. Based on that evaluation, the Chief Executive Officer and Chief Financial Officer have concluded that these disclosure controls and procedures are effective.\nREPORT OF MANAGEMENT ON INTERNAL CONTROL OVER FINANCIAL REPORTING",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 9",
- "metadata": {
- "length": 4229,
- "summary": "The audit report covers procedures for testing revenue recognition, including identification of performance obligations, timing of revenue recognition, and estimation of variable consideration. It also addresses income taxes, specifically uncertain tax positions related to transfer pricing issues with the IRS for tax years 2004-2013. The company received Notices of Proposed Adjustments (NOPAs) for those years. The report includes management's evaluation of disclosure controls and procedures, and the report of management on internal control over financial reporting. The auditor is Deloitte & Touche LLP, and the report is dated July 30, 2025.",
- "page_nums": [],
- "tokens": [
- "We",
- "tested",
- "effectiveness",
- "controls",
- "related",
- "identification",
- "distinct",
- "performance",
- "obligations",
- "determination",
- "timing",
- "revenue",
- "recognition",
- "estimation",
- "variable",
- "consideration",
- "evaluated",
- "management",
- "significant",
- "accounting",
- "policies",
- "customer",
- "agreements",
- "reasonableness",
- "selected",
- "sample",
- "performed",
- "procedures",
- "Obtained",
- "read",
- "contract",
- "source",
- "documents",
- "selection",
- "including",
- "master",
- "part",
- "agreement",
- "Tested",
- "treatment",
- "terms",
- "Assessed",
- "appropriateness",
- "application",
- "estimates",
- "conclusions",
- "estimate",
- "stand",
- "selling",
- "prices",
- "products",
- "services",
- "sold",
- "separately",
- "mathematical",
- "accuracy",
- "calculations",
- "recognized",
- "financial",
- "statements",
- "Income",
- "Taxes",
- "Uncertain",
- "Tax",
- "Positions",
- "Refer",
- "Note",
- "11",
- "Critical",
- "Audit",
- "Matter",
- "Description",
- "The",
- "Company",
- "long",
- "term",
- "income",
- "taxes",
- "liability",
- "includes",
- "uncertain",
- "tax",
- "positions",
- "transfer",
- "pricing",
- "issues",
- "remain",
- "unresolved",
- "Internal",
- "Revenue",
- "Service",
- "IRS",
- "remains",
- "audit",
- "subject",
- "years",
- "subsequent",
- "2003",
- "During",
- "fiscal",
- "year",
- "2024",
- "received",
- "Notices",
- "Proposed",
- "Adjustments",
- "NOPAs",
- "2004",
- "2013",
- "primarily",
- "intercompany",
- "While",
- "settled",
- "portion",
- "audits",
- "resolution",
- "remaining",
- "matters",
- "material",
- "impact",
- "Conclusions",
- "recognizing",
- "measuring",
- "involve",
- "judgment",
- "include",
- "complex",
- "considerations",
- "Code",
- "regulations",
- "case",
- "laws",
- "prior",
- "settlements",
- "Given",
- "complexity",
- "subjective",
- "nature",
- "evaluating",
- "relating",
- "required",
- "high",
- "degree",
- "auditor",
- "involvement",
- "specialists",
- "How",
- "Was",
- "Addressed",
- "Our",
- "principal",
- "evaluate",
- "included",
- "methods",
- "assumptions",
- "measurement",
- "disclosure",
- "testing",
- "internal",
- "judgments",
- "future",
- "appropriately",
- "considered",
- "information",
- "significantly",
- "change",
- "law",
- "statutes",
- "recent",
- "impacted",
- "disclosures",
- "relation",
- "underlying",
- "facts",
- "DELOITTE",
- "TOUCHE",
- "LLP",
- "Seattle",
- "Washington",
- "July",
- "30",
- "2025",
- "served",
- "1983",
- "CHANGES",
- "IN",
- "AND",
- "DISAGREEMENTS",
- "WITH",
- "ACCOUNTANTS",
- "ON",
- "ACCOUNTING",
- "FINANCIAL",
- "DISCLOSURE",
- "Not",
- "applicable",
- "CONTROLS",
- "PROCEDURES",
- "Under",
- "supervision",
- "participation",
- "Chief",
- "Executive",
- "Officer",
- "Financial",
- "Exchange",
- "Act",
- "Rule",
- "13a",
- "15",
- "end",
- "period",
- "covered",
- "report",
- "Based",
- "evaluation",
- "concluded",
- "effective",
- "REPORT",
- "OF",
- "MANAGEMENT",
- "INTERNAL",
- "CONTROL",
- "OVER",
- "REPORTING"
- ],
- "keywords": [
- "Internal Revenue Service",
- "IRS",
- "Deloitte & Touche LLP",
- "Seattle",
- "Washington"
- ],
- "connect_to": []
- }
- },
- {
- "chunk_id": "ea671ed1-4d5c-5b6f-a998-136f625f9a8c",
- "type": "text",
- "content": "Our management is responsible for establishing and maintaining adequate internal control over financial reporting for the Company. Internal control over financial reporting is a process to provide reasonable assurance regarding the reliability of our financial reporting for external purposes in accordance with accounting principles generally accepted in the United States of America. Internal control over financial reporting includes maintaining records that in reasonable detail accurately and fairly reflect our transactions; providing reasonable assurance that transactions are recorded as necessary for preparation of our consolidated financial statements; providing reasonable assurance that receipts and expenditures of company assets are made in accordance with management authorization; and providing reasonable assurance that unauthorized acquisition, use, or disposition of company assets that could have a material effect on our consolidated financial statements would be prevented or detected on a timely basis. Because of its inherent limitations, internal control over financial reporting is not intended to provide absolute assurance that a misstatement of our consolidated financial statements would be prevented or detected.\nManagement conducted an evaluation of the effectiveness of our internal control over financial reporting based on the framework in Internal Control – Integrated Framework (2013) issued by the Committee of Sponsoring Organizations of the Treadway Commission. Based on this evaluation, management concluded that the Company’s internal control over financial reporting was effective as of June 30, 2025. There were no changes in our internal control over financial reporting during the quarter ended June 30, 2025 that have materially affected, or are reasonably likely to materially affect, our internal control over financial reporting. Deloitte & Touche LLP has audited our internal control over financial reporting as of June 30, 2025; their report follows.\nREPORT OF INDEPENDENT REGISTERED PUBLIC ACCOUNTING FIRM\nTo the Stockholders and the Board of Directors of Microsoft Corporation\nOpinion on Internal Control over Financial Reporting\nWe have audited the internal control over financial reporting of Microsoft Corporation and subsidiaries (the “Company”) as of June 30, 2025, based on criteria established in Internal Control — Integrated Framework (2013) issued by the Committee of Sponsoring Organizations of the Treadway Commission (COSO). In our opinion, the Company maintained, in all material respects, effective internal control over financial reporting as of June 30, 2025, based on criteria established in Internal Control — Integrated Framework (2013) issued by COSO.\nWe have also audited, in accordance with the standards of the Public Company Accounting Oversight Board (United States) (PCAOB), the consolidated financial statements as of and for the year ended June 30, 2025, of the Company and our report dated July 30, 2025, expressed an unqualified opinion on those financial statements.\nBasis for Opinion\nThe Company’s management is responsible for maintaining effective internal control over financial reporting and for its assessment of the effectiveness of internal control over financial reporting, included in the accompanying Report of Management on Internal Control over Financial Reporting. Our responsibility is to express an opinion on the Company’s internal control over financial reporting based on our audit. We are a public accounting firm registered with the PCAOB and are required to be independent with respect to the Company in accordance with the U.S. federal securities laws and the applicable rules and regulations of the Securities and Exchange Commission and the PCAOB.\nWe conducted our audit in accordance with the standards of the PCAOB. Those standards require that we plan and perform the audit to obtain reasonable assurance about whether effective internal control over financial reporting was maintained in all material respects. Our audit included obtaining an understanding of internal control over financial reporting, assessing the risk that a material weakness exists, testing and evaluating the design and operating effectiveness of internal control based on the assessed risk, and performing such other procedures as we considered necessary in the circumstances. We believe that our audit provides a reasonable basis for our opinion.\nDefinition and Limitations of Internal Control over Financial Reporting",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 10",
- "metadata": {
- "length": 4509,
- "summary": "The report covers Microsoft Corporation's internal control over financial reporting as of June 30, 2025. Management is responsible for establishing and maintaining adequate internal control, which provides reasonable assurance on the reliability of financial reporting. Management evaluated the effectiveness based on the COSO framework and concluded it was effective. No changes materially affected internal control during the quarter. Deloitte & Touche LLP audited the internal control and expressed an unqualified opinion, also auditing the consolidated financial statements with an unqualified opinion. The report includes the basis for opinion and definition of internal control.",
- "page_nums": [],
- "tokens": [
- "Our",
- "management",
- "responsible",
- "establishing",
- "maintaining",
- "adequate",
- "internal",
- "control",
- "financial",
- "reporting",
- "Company",
- "Internal",
- "process",
- "provide",
- "reasonable",
- "assurance",
- "reliability",
- "external",
- "purposes",
- "accordance",
- "accounting",
- "principles",
- "generally",
- "accepted",
- "United",
- "States",
- "America",
- "includes",
- "records",
- "detail",
- "accurately",
- "fairly",
- "reflect",
- "transactions",
- "providing",
- "recorded",
- "preparation",
- "consolidated",
- "statements",
- "receipts",
- "expenditures",
- "company",
- "assets",
- "made",
- "authorization",
- "unauthorized",
- "acquisition",
- "disposition",
- "material",
- "effect",
- "prevented",
- "detected",
- "timely",
- "basis",
- "Because",
- "inherent",
- "limitations",
- "intended",
- "absolute",
- "misstatement",
- "Management",
- "conducted",
- "evaluation",
- "effectiveness",
- "based",
- "framework",
- "Control",
- "Integrated",
- "Framework",
- "2013",
- "issued",
- "Committee",
- "Sponsoring",
- "Organizations",
- "Treadway",
- "Commission",
- "Based",
- "concluded",
- "effective",
- "June",
- "30",
- "2025",
- "There",
- "quarter",
- "ended",
- "materially",
- "affected",
- "affect",
- "Deloitte",
- "Touche",
- "LLP",
- "audited",
- "report",
- "REPORT",
- "OF",
- "INDEPENDENT",
- "REGISTERED",
- "PUBLIC",
- "ACCOUNTING",
- "FIRM",
- "To",
- "Stockholders",
- "Board",
- "Directors",
- "Microsoft",
- "Corporation",
- "Opinion",
- "Financial",
- "Reporting",
- "We",
- "subsidiaries",
- "criteria",
- "established",
- "COSO",
- "In",
- "opinion",
- "maintained",
- "respects",
- "standards",
- "Public",
- "Accounting",
- "Oversight",
- "PCAOB",
- "year",
- "dated",
- "July",
- "expressed",
- "unqualified",
- "Basis",
- "The",
- "assessment",
- "included",
- "accompanying",
- "Report",
- "responsibility",
- "express",
- "audit",
- "public",
- "firm",
- "registered",
- "required",
- "independent",
- "respect",
- "federal",
- "securities",
- "laws",
- "applicable",
- "rules",
- "regulations",
- "Securities",
- "Exchange",
- "Those",
- "require",
- "plan",
- "perform",
- "obtain",
- "obtaining",
- "understanding",
- "assessing",
- "risk",
- "weakness",
- "exists",
- "testing",
- "evaluating",
- "design",
- "operating",
- "assessed",
- "performing",
- "procedures",
- "considered",
- "circumstances",
- "Definition",
- "Limitations"
- ],
- "keywords": [
- "Microsoft Corporation",
- "Deloitte & Touche LLP",
- "Committee of Sponsoring Organizations of the Treadway Commission",
- "Public Company Accounting Oversight Board",
- "Securities and Exchange Commission"
- ],
- "connect_to": []
- }
- },
- {
- "chunk_id": "1b0599dd-e932-5fc4-baf0-bb4acb47f17c",
- "type": "text",
- "content": "A company’s internal control over financial reporting is a process designed to provide reasonable assurance regarding the reliability of financial reporting and the preparation of financial statements for external purposes in accordance with generally accepted accounting principles. A company’s internal control over financial reporting includes those policies and procedures that (1) pertain to the maintenance of records that, in reasonable detail, accurately and fairly reflect the transactions and dispositions of the assets of the company; (2) provide reasonable assurance that transactions are recorded as necessary to permit preparation of financial statements in accordance with generally accepted accounting principles, and that receipts and expenditures of the company are being made only in accordance with authorizations of management and directors of the company; and (3) provide reasonable assurance regarding prevention or timely detection of unauthorized acquisition, use, or disposition of the company’s assets that could have a material effect on the financial statements.\nBecause of its inherent limitations, internal control over financial reporting may not prevent or detect misstatements. Also, projections of any evaluation of effectiveness to future periods are subject to the risk that controls may become inadequate because of changes in conditions, or that the degree of compliance with the policies or procedures may deteriorate.\n\nSignature block for Deloitte & Touche LLP, dated July 30, 2025, in Seattle, Washington.\n[tables/table-67 Deloitte & Touche LLP signature block.html]\n\nDIRECTORS AND EXECUTIVE OFFICERS OF MICROSOFT CORPORATION\nDIRECTORS\n\nThe table lists twelve members of Microsoft's Board of Directors, including Satya Nadella (Chairman and CEO), Sandra E. Peterson (Lead Independent Director), and other executives from companies like Citigroup, Wells Fargo, Disney, and GSK, with committee assignments noted.\n[tables/table-68 Microsoft Board of Directors List.html]\n\nBoard Committees\n1. Audit Committee\n2. Compensation Committee\n3. Governance and Nominating Committee\n4. Environmental, Social, and Public Policy Committee\nEXECUTIVE OFFICERS\n\nThis table lists Microsoft executives and their titles, including the Chairman and CEO, CFO, and other senior leaders such as the CEO of Microsoft Commercial, Chief Marketing Officer, Chief Human Resources Officer, Vice Chair and President, and Executive Vice President of Strategy and Transformation.\n[tables/table-69 Microsoft Leadership Table.html]\n\nINVESTOR RELATIONS\nInvestor Relations\nYou can contact Microsoft Investor Relations by calling toll-free at (800) 285-7772 or outside the United States, call (425) 706-4400. We can be contacted between the hours of 9:00 a.m. to 5:00 p.m. Pacific Time to answer investment-oriented questions about Microsoft.\nFor access to additional financial information, visit the Investor Relations website online at:\nwww.microsoft.com/investor\nOur e-mail is msft@microsoft.com\nOur mailing address is:\nInvestor Relations\nMicrosoft Corporation\nOne Microsoft Way\nRedmond, Washington 98052-6399\nAttending the Annual Meeting\nThe 2025 Annual Shareholders Meeting will be held as a virtual-only meeting. Any shareholder can join the Annual Meeting, while shareholders of record as of September 30 2025, will be able to vote and submit questions during the meeting.\nDate: Friday, December 5, 2025\nTime: 8:30 a.m. Pacific Time\nVirtual Shareholder Meeting:\nwww.virtualshareholdermeeting.com/MSFT25\nSubmit Your Question\nWe invite you to submit any questions via the proxy voting site at www.proxyvote.com. We will include as many of your questions as possible during the Q&A session of the meeting and will provide answers to questions on the Microsoft Investor Relations website under the Annual Meeting page.\nRegistered Shareholder Services\nComputershare, our transfer agent, can help you with a variety of shareholder related services including:\n• Change of address\n• Lost stock certificates\n• Transfer of stock to another person\n• Additional administrative services\nComputershare also administers a direct stock purchase plan and a dividend reinvestment program for the company.\nContact Computershare directly to find out more about these services and programs at 800-285-7772, option 1, or visit online at:\nhttps://www.computershare.com/Microsoft\nYou can e-mail the transfer agent at:\nweb.queries@computershare.com\nYou can also send mail to the transfer agent at:\nComputershare\nP.O. Box 505000\nLouisville, KY 40233-5000",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 11",
- "metadata": {
- "length": 4538,
- "summary": "The text discusses Microsoft Corporation's internal control over financial reporting, defining it as a process to ensure reliable financial statements and compliance with GAAP. It notes inherent limitations and risks of control inadequacy over time. The document includes a signature block from Deloitte & Touche LLP dated July 30, 2025, in Seattle, Washington. It lists the Board of Directors (12 members including Satya Nadella as Chairman and CEO, Sandra E. Peterson as Lead Independent Director) and executive officers (including CFO and other senior leaders). It provides investor relations contact details: toll-free (800) 285-7772, outside US (425) 706-4400, email msft@microsoft.com, and address One Microsoft Way, Redmond, WA 98052-6399. The 2025 Annual Shareholders Meeting will be virtual-only on December 5, 2025, at 8:30 a.m. Pacific Time, with voting for shareholders of record as of September 30, 2025. Questions can be submitted via proxyvote.com. Computershare is the transfer agent for shareholder services, including address changes, lost certificates, and stock transfers, with contact at 800-285-7772 option 1, and mailing address P.O. Box 505000, Louisville, KY 40233-5000.",
- "page_nums": [],
- "tokens": [
- "company",
- "internal",
- "control",
- "financial",
- "reporting",
- "process",
- "designed",
- "provide",
- "reasonable",
- "assurance",
- "reliability",
- "preparation",
- "statements",
- "external",
- "purposes",
- "accordance",
- "generally",
- "accepted",
- "accounting",
- "principles",
- "includes",
- "policies",
- "procedures",
- "pertain",
- "maintenance",
- "records",
- "detail",
- "accurately",
- "fairly",
- "reflect",
- "transactions",
- "dispositions",
- "assets",
- "recorded",
- "permit",
- "receipts",
- "expenditures",
- "made",
- "authorizations",
- "management",
- "directors",
- "prevention",
- "timely",
- "detection",
- "unauthorized",
- "acquisition",
- "disposition",
- "material",
- "effect",
- "Because",
- "inherent",
- "limitations",
- "prevent",
- "detect",
- "misstatements",
- "Also",
- "projections",
- "evaluation",
- "effectiveness",
- "future",
- "periods",
- "subject",
- "risk",
- "controls",
- "inadequate",
- "conditions",
- "degree",
- "compliance",
- "deteriorate",
- "Signature",
- "block",
- "Deloitte",
- "Touche",
- "LLP",
- "dated",
- "July",
- "30",
- "2025",
- "Seattle",
- "Washington",
- "DIRECTORS",
- "AND",
- "EXECUTIVE",
- "OFFICERS",
- "OF",
- "MICROSOFT",
- "CORPORATION",
- "The",
- "table",
- "lists",
- "twelve",
- "members",
- "Microsoft",
- "Board",
- "Directors",
- "including",
- "Satya",
- "Nadella",
- "Chairman",
- "CEO",
- "Sandra",
- "Peterson",
- "Lead",
- "Independent",
- "Director",
- "executives",
- "companies",
- "Citigroup",
- "Wells",
- "Fargo",
- "Disney",
- "GSK",
- "committee",
- "assignments",
- "noted",
- "Committees",
- "Audit",
- "Committee",
- "Compensation",
- "Governance",
- "Nominating",
- "Environmental",
- "Social",
- "Public",
- "Policy",
- "This",
- "titles",
- "CFO",
- "senior",
- "leaders",
- "Commercial",
- "Chief",
- "Marketing",
- "Officer",
- "Human",
- "Resources",
- "Vice",
- "Chair",
- "President",
- "Executive",
- "Strategy",
- "Transformation",
- "INVESTOR",
- "RELATIONS",
- "Investor",
- "Relations",
- "You",
- "contact",
- "calling",
- "toll",
- "free",
- "800",
- "285",
- "7772",
- "United",
- "States",
- "call",
- "425",
- "706",
- "4400",
- "We",
- "contacted",
- "hours",
- "00",
- "Pacific",
- "Time",
- "answer",
- "investment",
- "oriented",
- "questions",
- "For",
- "access",
- "additional",
- "information",
- "visit",
- "website",
- "online",
- "www",
- "microsoft",
- "investor",
- "Our",
- "mail",
- "msft",
- "mailing",
- "address",
- "Corporation",
- "One",
- "Way",
- "Redmond",
- "98052",
- "6399",
- "Attending",
- "Annual",
- "Meeting",
- "Shareholders",
- "held",
- "virtual",
- "meeting",
- "Any",
- "shareholder",
- "join",
- "shareholders",
- "record",
- "September",
- "vote",
- "submit",
- "Date",
- "Friday",
- "December",
- "Virtual",
- "Shareholder",
- "virtualshareholdermeeting",
- "MSFT25",
- "Submit",
- "Your",
- "Question",
- "invite",
- "proxy",
- "voting",
- "site",
- "proxyvote",
- "include",
- "session",
- "answers",
- "page",
- "Registered",
- "Services",
- "Computershare",
- "transfer",
- "agent",
- "variety",
- "related",
- "services",
- "Change",
- "Lost",
- "stock",
- "certificates",
- "Transfer",
- "person",
- "Additional",
- "administrative",
- "administers",
- "direct",
- "purchase",
- "plan",
- "dividend",
- "reinvestment",
- "program",
- "Contact",
- "directly",
- "find",
- "programs",
- "option",
- "https",
- "computershare",
- "web",
- "queries",
- "send",
- "Box",
- "505000",
- "Louisville",
- "KY",
- "40233",
- "5000"
- ],
- "keywords": [
- "Deloitte & Touche LLP",
- "Seattle",
- "Washington",
- "Satya Nadella",
- "Sandra E. Peterson",
- "Microsoft Corporation",
- "Citigroup",
- "Wells Fargo",
- "Disney",
- "GSK",
- "Microsoft",
- "Investor Relations",
- "Computershare",
- "Louisville",
- "Kentucky"
- ],
- "connect_to": [
- {
- "target": "6656b43d-55a4-5cfd-9025-359ff3b95182",
- "relation": "embeds",
- "ref": "[tables/table-67 Deloitte & Touche LLP signature block.html]",
- "position": {
- "start": 1548,
- "end": 1608
- }
- },
- {
- "target": "6b48ae77-7813-5b62-87f3-ff41c622c790",
- "relation": "embeds",
- "ref": "[tables/table-68 Microsoft Board of Directors List.html]",
- "position": {
- "start": 1953,
- "end": 2009
- }
- },
- {
- "target": "fc7f99a2-f90b-53db-9944-c6216c05ea6c",
- "relation": "embeds",
- "ref": "[tables/table-69 Microsoft Leadership Table.html]",
- "position": {
- "start": 2488,
- "end": 2537
- }
- }
- ]
- }
- },
- {
- "chunk_id": "5265b9f5-7c50-5767-9c0d-2724861084cd",
- "type": "text",
- "content": "Shareholders can sign up for electronic alerts to access the annual report and proxy statement online. The service gets you the information you need faster and also gives you the power and convenience of online proxy voting. To sign up for this free service, visit the Annual Report site on the Investor Relations website at:\nhttp://www.microsoft.com/investor/AnnualReports/default.aspx\nEnvironmental, Social, and Governance (ESG)/Corporate Social Responsibility\nMany of our shareholders are focused on the importance of the effective engagement and action on environmental, social, and governance topics. To meet the expectations of our stakeholders and to and maintain their trust, we are committed to conducting our business in ways that are principled, transparent, and accountable. From our CEO and Senior Leadership Team and throughout our organization, people at Microsoft are working to conduct our business in principled ways that make a significant positive impact on important global issues. Microsoft’s Board of Directors provides insight, feedback, and oversight across a broad range of environmental and social matters. In particular, among the responsibilities of the Board’s Environmental, Social, and Public Policy Committee is to review and provide guidance to the Board and management about the Company’s policies and programs that relate to corporate social responsibility.\nFor more about Microsoft’s CSR commitments and performance, please visit:\nwww.microsoft.com/transparency.\n\nAn FSC MIX certification label indicating paper supporting responsible forestry, featuring the organization's logo and website.\n[images/image-4 FSC Certification Labe.jpeg]",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 12",
- "metadata": {
- "length": 1673,
- "summary": "The text informs shareholders about electronic alerts for accessing annual reports and proxy statements online, and provides a link to the Investor Relations website. It also discusses Microsoft's commitment to environmental, social, and governance (ESG) topics, highlighting the role of the Board's Environmental, Social, and Public Policy Committee in overseeing corporate social responsibility. A link to Microsoft's transparency page is provided, and an FSC MIX certification label is mentioned.",
- "page_nums": [],
- "tokens": [
- "Shareholders",
- "sign",
- "electronic",
- "alerts",
- "access",
- "annual",
- "report",
- "proxy",
- "statement",
- "online",
- "The",
- "service",
- "information",
- "faster",
- "power",
- "convenience",
- "voting",
- "To",
- "free",
- "visit",
- "Annual",
- "Report",
- "site",
- "Investor",
- "Relations",
- "website",
- "http",
- "www",
- "microsoft",
- "investor",
- "AnnualReports",
- "default",
- "aspx",
- "Environmental",
- "Social",
- "Governance",
- "ESG",
- "Corporate",
- "Responsibility",
- "Many",
- "shareholders",
- "focused",
- "importance",
- "effective",
- "engagement",
- "action",
- "environmental",
- "social",
- "governance",
- "topics",
- "meet",
- "expectations",
- "stakeholders",
- "maintain",
- "trust",
- "committed",
- "conducting",
- "business",
- "ways",
- "principled",
- "transparent",
- "accountable",
- "From",
- "CEO",
- "Senior",
- "Leadership",
- "Team",
- "organization",
- "people",
- "Microsoft",
- "working",
- "conduct",
- "make",
- "significant",
- "positive",
- "impact",
- "important",
- "global",
- "issues",
- "Board",
- "Directors",
- "insight",
- "feedback",
- "oversight",
- "broad",
- "range",
- "matters",
- "In",
- "responsibilities",
- "Public",
- "Policy",
- "Committee",
- "review",
- "provide",
- "guidance",
- "management",
- "Company",
- "policies",
- "programs",
- "relate",
- "corporate",
- "responsibility",
- "For",
- "CSR",
- "commitments",
- "performance",
- "transparency",
- "An",
- "FSC",
- "MIX",
- "certification",
- "label",
- "indicating",
- "paper",
- "supporting",
- "responsible",
- "forestry",
- "featuring",
- "logo"
- ],
- "keywords": [
- "Microsoft",
- "Investor Relations",
- "Board of Directors",
- "Environmental, Social, and Public Policy Committee"
- ],
- "connect_to": [
- {
- "target": "edf99fdb-63b6-532d-a46a-4829e552912c",
- "relation": "embeds",
- "ref": "[images/image-4 FSC Certification Labe.jpeg]",
- "position": {
- "start": 1629,
- "end": 1673
- }
- }
- ]
- }
- }
- ]
-}
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/doc_nav.json b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/doc_nav.json
deleted file mode 100644
index 974c435fc..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/doc_nav.json
+++ /dev/null
@@ -1,417 +0,0 @@
-{
- "version": "1.0",
- "file_name": "Microsoft 2025 Annual Report.docx",
- "stats": {
- "total_chunks": 85,
- "text_chunks": 12,
- "image_chunks": 4,
- "table_chunks": 69,
- "page_chunks": 0,
- "max_depth": 2
- },
- "sections": [
- {
- "title": "U.S. and foreign components of income before income taxes were as follows:",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:",
- "level": 1,
- "summary": "",
- "chunk_count": 12,
- "children": [
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 1",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 1",
- "level": 2,
- "summary": "The text discusses U.S. and foreign income before taxes for fiscal years 2023-2025, showing increases in both. U.S. income rose from $52,917 million in 2023 to $62,886 million in 2024 and $69,212 million in 2025; foreign income increased from $36,394 million to $44,901 million and $54,415 million; total income before taxes grew from $89,311 million to $107,787 million and $123,627 million. The effective tax rate decreased from 19.0% in 2023 to 18.2% in 2024 and 17.6% in 2025, with the federal statutory rate at 21.0% each year. Key adjustments include foreign earnings taxed at lower rates (-1.5% in 2025), FDII deduction (-1.0%), state taxes (+1.5%), R&D credit (-1.1%), stock-based compensation benefits (-0.9%), interest (+1.0%), and other items (-1.4%). The decrease is primarily due to earnings taxed at lower rates in foreign jurisdictions, especially through the Ireland regional operations center, which generated 81%, 83%, and 81% of foreign income before tax in 2025, 2024, and 2023, respectively. Net deferred income tax assets increased from $19,652 million in 2024 to $26,273 million in 2025, with total deferred tax assets of $47,112 million (net of $1,169 million valuation allowance) and deferred tax liabilities of $19,670 million; largest asset components include capitalized R&D ($16,891 million) and leasing liabilities ($12,874 million). As of June 30, 2025, federal, state, and foreign net operating loss carryforwards were $390 million, $836 million, and $2.6 billion, respectively. Federal capital loss carryforwards were $816 million, expiring in fiscal year 2030. Income taxes paid, net of refunds, were $28.7 billion, $23.4 billion, and $23.1 billion in fiscal years 2025, 2024, and 2023, respectively.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 2",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 2",
- "level": 2,
- "summary": "The text discusses Microsoft's unrecognized tax benefits, which were $24.7 billion, $22.8 billion, and $17.1 billion as of June 30, 2025, 2024, and 2023, respectively. If recognized, these would affect effective tax rates by $21.2 billion, $19.6 billion, and $14.4 billion for those years. Accrued interest expense related to uncertain tax positions was $8.2 billion, $6.8 billion, and $5.2 billion for the same periods, with provisions for income taxes including interest expense of $1.3 billion, $1.5 billion, and $918 million. The gross unrecognized tax benefits increased from $17,120 million in 2023 to $22,760 million in 2024, then to $24,729 million in 2025, with current-year increases of $2,066 (2025) and $1,903 (2024), prior-year increases of $468 (2025) and $4,289 (2024), and settlements decreasing by $240 (2025) and $76 (2024). Microsoft is under IRS audit for tax years 2014-2017 and received Notices of Proposed Adjustment for 2004-2013 seeking an additional $28.9 billion plus penalties and interest, which they plan to contest. Unearned revenue by segment increased from $60,184 million in 2024 to $67,265 million in 2025, with Productivity and Business Processes rising from $43,599 million to $50,567 million, Intelligent Cloud from $13,683 million to $14,022 million, and More Personal Computing falling from $2,902 million to $2,676 million. The changes in unearned revenue show a beginning balance of $60,184 million, $186,957 million deferred, $179,876 million recognized, and an ending balance of $67,265 million. Revenue allocated to remaining performance obligations was $375 billion as of June 30, 2025, with $368 billion related to commercial revenue, and about 40% expected to be recognized over the next 12 months. The text also mentions lease terms and expenses.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 3",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 3",
- "level": 2,
- "summary": "The text discusses lease costs, supplemental cash flow and balance sheet information, and future minimum lease payments for operating and finance leases. Operating lease cost increased from $2,875 million in 2023 to $3,555 million in 2024 and $5,524 million in 2025. Total finance lease cost rose from $1,853 million in 2023 to $2,534 million in 2024 and $4,825 million in 2025. Cash paid for operating leases was $4,931M in 2025, $3,550M in 2024, and $2,706M in 2023. Right-of-use assets obtained for operating leases were $7,826M, $6,703M, $3,514M; for finance leases $20,511M, $11,633M, $3,128M. As of June 30, 2025, operating lease right-of-use assets were $24,823 million and total operating lease liabilities $22,861 million; finance lease property and equipment net was $44,015 million with total finance lease liabilities of $46,172 million. Weighted average remaining lease terms are 6 years (operating) and 13 years (finance); discount rates are 3.5% and 4.2%. Future minimum lease payments total $25,481 million for operating leases (imputed interest $2,620 million) and $60,506 million for finance leases (imputed interest $14,334 million). Additional leases not yet commenced total $92.7 billion. The text also covers contingencies: the Irish Data Protection Commission matter involving LinkedIn, and other legal contingencies with accrued liabilities of $541 million and reasonably possible losses up to $600 million beyond recorded amounts.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 4",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 4",
- "level": 2,
- "summary": "The document reports Microsoft's common stock outstanding for fiscal years 2025, 2024, and 2023, with beginning balances of 7,434, 7,432, and 7,464 million shares, respectively. Issuances were 31, 34, and 37 million, while repurchases were 31, 32, and 69 million, resulting in ending balances of 7,434, 7,434, and 7,432 million shares. It details share repurchase programs: a $60.0 billion program approved on September 14, 2021, completed in April 2025, and another $60.0 billion program approved on September 16, 2024, with $57.3 billion remaining as of June 30, 2025. Quarterly repurchases for fiscal years 2025, 2024, and 2023 totaled 31, 32, and 69 million shares, respectively, with total costs of $13,000 million, $11,960 million, and $18,400 million. The highest quarterly repurchase amount was $4,600 million in 2023. Repurchases were made with cash resources, excluding shares repurchased to settle employee tax withholding of $5.4 billion, $5.3 billion, and $3.8 billion for fiscal years 2025, 2024, and 2023. Dividends declared were $0.83 per share for FY2025 (total $3.32, amount $24,678 million) and $0.75 per share for FY2024 (total $3.00, amount $22,295 million). Accumulated other comprehensive loss components changed: derivatives from $(3) to $(8) in 2025, investments from $(2,625) to $(1,051), translation adjustments from $(2,962) to $(2,288), with total end-of-period losses of $(3,347) in 2025, $(5,590) in 2024, and $(6,343) in 2023. Stock-based compensation expense rose from $9,611 million in 2023 to $10,734 million in 2024 and $11,974 million in 2025; income tax benefits increased from $1,651 million to $1,826 million and $2,027 million over the same periods. Stock awards generally vest over four or five years.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 5",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 5",
- "level": 2,
- "summary": "The text describes Microsoft's stock-based compensation plans, including the Executive Incentive Plan, Employee Stock Purchase Plan (ESPP), and savings plans. It details the vesting of RSUs and PSUs, with RSUs vesting over four years and PSUs over three years. The fair value of stock awards was estimated using assumptions: dividends per share ranged from $0.62 to $0.83 quarterly across fiscal years 2023-2025, with 2025 at $0.75–$0.83, 2024 at $0.68–$0.75, and 2023 at $0.62–$0.68. Interest rates ranged from 2.0% to 5.6%, with 2025 at 3.4%–5.5%, 2024 at 3.8%–5.6%, and 2023 at 2.0%–5.4%. In fiscal 2025, nonvested stock awards activity: beginning balance 88 million shares at $292.28 weighted average grant-date fair value, 39 million granted at $413.90, 38 million vested at $293.25, 7 million forfeited at $317.23, ending balance 82 million shares at $347.44. Unrecognized compensation costs were $21.6 billion as of June 30, 2025, expected to be recognized over three years. Weighted average grant-date fair value of awards granted was $413.90, $339.46, and $252.59 for fiscal years 2025, 2024, and 2023, respectively. Fair value of vested awards was $16.2 billion, $16.0 billion, and $11.9 billion for the same years. 98 million shares were authorized for future grant. The ESPP allows employees to purchase shares at 90% of fair market value, with purchases limited to 15% of gross compensation. Employees purchased 6 million shares in both fiscal 2025 and 2024, and 7 million in 2023, with average prices of $385.10, $339.46, and $245.59 respectively. 62 million shares were reserved for future issuance through ESPP. Savings plans match employee contributions; employer-funded retirement benefits were $1.8 billion, $1.7 billion, and $1.6 billion in fiscal years 2025, 2024, and 2023. The company's segments are Productivity and Business Processes, Intelligent Cloud, and More Personal Computing, with operating income as the primary profitability measure.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 6",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 6",
- "level": 2,
- "summary": "The text describes Microsoft's three business segments: Productivity and Business Processes, Intelligent Cloud, and More Personal Computing. It details the products and services in each segment, such as Microsoft 365, LinkedIn, Dynamics 365, Azure, Windows, and Gaming. It also explains revenue and cost allocation methods, including direct attribution and allocation based on relative value, gross margin, or headcount. The text mentions that segment revenue, cost of revenue, operating expenses, and operating income are presented for the periods shown, but the actual financial figures are not included in the provided excerpt.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 7",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 7",
- "level": 2,
- "summary": "Microsoft's fiscal year 2025 results show total revenue of $281,724 million, up from $245,122 million in 2024 and $211,915 million in 2023. Operating income rose to $128,528 million in 2025 from $88,523 million in 2023. Segment operating income for 2025: Productivity and Business Processes $69,773M, Intelligent Cloud $44,589M, More Personal Computing $14,166M. Revenue by geography: United States $144,546M (2025), other countries $137,178M (2025). Microsoft Cloud revenue was $168.9B in 2025, $137.7B in 2024, $111.6B in 2023. Long-lived assets total $371,902M in 2025, with United States $230,069M and other countries $141,833M. No single customer or country other than the US accounted for more than 10% of revenue.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 8",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 8",
- "level": 2,
- "summary": "The text is an excerpt from an audit report. The auditor states that they audited the company's internal control over financial reporting as of June 30, 2025, and expressed an unqualified opinion. The basis for opinion explains that the financial statements are management's responsibility and the auditor's responsibility is to express an opinion. The audit was conducted in accordance with PCAOB standards. The report then discusses a critical audit matter regarding revenue recognition, which involves significant judgment in determining distinct performance obligations, pattern of delivery, contract terms, and stand-alone selling prices. The auditor's procedures included evaluating management's judgments.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 9",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 9",
- "level": 2,
- "summary": "The audit report covers procedures for testing revenue recognition, including identification of performance obligations, timing of revenue recognition, and estimation of variable consideration. It also addresses income taxes, specifically uncertain tax positions related to transfer pricing issues with the IRS for tax years 2004-2013. The company received Notices of Proposed Adjustments (NOPAs) for those years. The report includes management's evaluation of disclosure controls and procedures, and the report of management on internal control over financial reporting. The auditor is Deloitte & Touche LLP, and the report is dated July 30, 2025.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 10",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 10",
- "level": 2,
- "summary": "The report covers Microsoft Corporation's internal control over financial reporting as of June 30, 2025. Management is responsible for establishing and maintaining adequate internal control, which provides reasonable assurance on the reliability of financial reporting. Management evaluated the effectiveness based on the COSO framework and concluded it was effective. No changes materially affected internal control during the quarter. Deloitte & Touche LLP audited the internal control and expressed an unqualified opinion, also auditing the consolidated financial statements with an unqualified opinion. The report includes the basis for opinion and definition of internal control.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 11",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 11",
- "level": 2,
- "summary": "The text discusses Microsoft Corporation's internal control over financial reporting, defining it as a process to ensure reliable financial statements and compliance with GAAP. It notes inherent limitations and risks of control inadequacy over time. The document includes a signature block from Deloitte & Touche LLP dated July 30, 2025, in Seattle, Washington. It lists the Board of Directors (12 members including Satya Nadella as Chairman and CEO, Sandra E. Peterson as Lead Independent Director) and executive officers (including CFO and other senior leaders). It provides investor relations contact details: toll-free (800) 285-7772, outside US (425) 706-4400, email msft@microsoft.com, and address One Microsoft Way, Redmond, WA 98052-6399. The 2025 Annual Shareholders Meeting will be virtual-only on December 5, 2025, at 8:30 a.m. Pacific Time, with voting for shareholders of record as of September 30, 2025. Questions can be submitted via proxyvote.com. Computershare is the transfer agent for shareholder services, including address changes, lost certificates, and stock transfers, with contact at 800-285-7772 option 1, and mailing address P.O. Box 505000, Louisville, KY 40233-5000.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "U.S. and foreign components of income before income taxes were as follows: part 12",
- "path": "Microsoft 2025 Annual Report.docx/U.S. and foreign components of income before income taxes were as follows:/U.S. and foreign components of income before income taxes were as follows: part 12",
- "level": 2,
- "summary": "The text informs shareholders about electronic alerts for accessing annual reports and proxy statements online, and provides a link to the Investor Relations website. It also discusses Microsoft's commitment to environmental, social, and governance (ESG) topics, highlighting the role of the Board's Environmental, Social, and Public Policy Committee in overseeing corporate social responsibility. A link to Microsoft's transparency page is provided, and an FSC MIX certification label is mentioned.",
- "chunk_count": 1,
- "children": []
- }
- ]
- }
- ],
- "resources": {
- "images": [
- {
- "path": "images/image-1 Annual Report 2025.png",
- "summary": "image-1 The cover of the Microsoft Annual Report for 2025, featuring a photo of two colleagues collaborating on a laptop."
- },
- {
- "path": "images/image-2 Microsoft has an imme.jpeg",
- "summary": "image-2 A handwritten signature appearing to read 'Sarga N.'"
- },
- {
- "path": "images/image-3 and the NASDAQ Comput.jpeg",
- "summary": "image-3 Line chart tracking the performance of Microsoft Corporation, S&P 500, and NASDAQ Computer from June 20 to June 25. All indices start at $100, dip on June 22, then rise sharply, with Microsoft reaching approximately $250 by June 25."
- },
- {
- "path": "images/image-4 FSC Certification Labe.jpeg",
- "summary": "image-4 An FSC MIX certification label indicating paper supporting responsible forestry, featuring the organization's logo and website."
- }
- ],
- "tables": [
- {
- "path": "tables/table-1 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html",
- "summary": "table-1 The table shows quarterly share repurchases for fiscal years 2025, 2024, and 2023. In 2025, shares repurchased were 7, 8, 8, and 8 per quarter, totaling 31 shares for $13,000 million. In 2024, quarterly shares were 11, 7, 7, and 7, totaling 32 shares for $11,960 million. In 2023, shares were 17, 20, 18, and 14, totaling 69 shares for $18,400 million. The highest quarterly amount was $4,600 million in 2023 quarters, and the lowest was $2,800 million in 2024 quarters."
- },
- {
- "path": "tables/table-2 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html",
- "summary": "table-2 The table lists quarterly dividends for fiscal years 2025 and 2024, with per-share dividends of $0.83 for each of the four 2025 payments (totaling $3.32 and $24,678 million) and $0.75 for each of the four 2024 payments (totaling $3.00 and $22,295 million), with declaration, record, and payment dates specified for each quarter."
- },
- {
- "path": "tables/table-3 Microsoft Corporation vs S&P 500 vs NASDAQ Computer Index Values June 20-25.html",
- "summary": "table-3 Microsoft Corporation rose from 100.00 on 6/20 to 255.13 on 6/25, S&P 500 from 100.00 to 215.89, and NASDAQ Computer from 100.00 to 254.97, with peaks on 6/24 for all three (227.51, 187.47, 223.97) and a dip on 6/22."
- },
- {
- "path": "tables/table-4 Microsoft Cloud Key Metrics Definitions.html",
- "summary": "table-4 This table defines three key Microsoft Cloud metrics: commercial remaining performance obligation (unearned revenue and future invoiced revenue), Microsoft Cloud revenue and revenue growth (covering Microsoft 365 Commercial cloud, Azure, other cloud services, commercial LinkedIn, and Dynamics 365), and Microsoft Cloud gross margin percentage."
- },
- {
- "path": "tables/table-5 Microsoft 365 and Azure Key Performance Indicators Definitions.html",
- "summary": "table-5 This table defines key performance indicators for Microsoft's commercial and consumer cloud services, including revenue growth metrics for Microsoft 365 Commercial, Consumer, LinkedIn, Dynamics 365, and Azure, along with seat and subscriber counts, with no numerical values provided."
- },
- {
- "path": "tables/table-6 Windows OEM and Devices revenue growth and Xbox content and services revenue growth and Search and news advertising revenue ex TAC growth.html",
- "summary": "table-6 The table defines three revenue growth metrics: Windows OEM and Devices revenue (from Windows Pro/non-Pro OEM licenses and first-party devices like Surface), Xbox content and services revenue (including games, Game Pass, cloud gaming, and advertising), and Search and news advertising revenue excluding traffic acquisition costs. No numerical data is provided."
- },
- {
- "path": "tables/table-7 Revenue and Profit Metrics for 2025 vs 2024.html",
- "summary": "table-7 Revenue rose 15% to $281,724 million in 2025 from $245,122 million in 2024; gross margin increased 13% to $193,893 million, operating income up 17% to $128,528 million, net income up 16% to $101,832 million, and diluted EPS up 16% to $13.64 from $11.80."
- },
- {
- "path": "tables/table-8 Microsoft Segment Revenue and Operating Income FY2025 vs FY2024.html",
- "summary": "table-8 Microsoft's total revenue rose 15% to $281,724 million in 2025 from $245,122 million in 2024, with operating income up 17% to $128,528 million; Productivity and Business Processes revenue grew 13% to $120,810 million, Intelligent Cloud revenue grew 21% to $106,265 million, and More Personal Computing revenue grew 7% to $54,649 million."
- },
- {
- "path": "tables/table-9 Research and development expenses for fiscal years 2025 and 2024.html",
- "summary": "table-9 Research and development expenses were $32,488 million in 2025, up 10% from $29,510 million in 2024, with R&D as a percent of revenue unchanged at 12% in both years (0 percentage point change)."
- },
- {
- "path": "tables/table-10 Sales and marketing expenses for 2025 and 2024.html",
- "summary": "table-10 Sales and marketing expenses were $25,654 million in 2025, up 5% from $24,456 million in 2024, with the expense as a percent of revenue decreasing from 10% to 9% (a 1 percentage point decline)."
- },
- {
- "path": "tables/table-11 General and Administrative Expenses 2025 vs 2024.html",
- "summary": "table-11 General and administrative expenses decreased from $7,609 million in 2024 to $7,223 million in 2025, a 5% decline, while remaining steady at 3% of revenue in both years."
- },
- {
- "path": "tables/table-12 Year Ended June 30, 2025 and 2024 Financial Data.html",
- "summary": "table-12 For the year ended June 30, 2025, total net losses were $4,901 million, compared to $1,646 million in 2024. Key components include interest and dividends income of $2,647 million (2025) vs $3,157 million (2024), interest expense of $2,385 million vs $2,935 million, and other net losses of $4,725 million vs $1,319 million."
- },
- {
- "path": "tables/table-13 Three Months Ending September 30 2025 to Thereafter Financial Obligations Table.html",
- "summary": "table-13 The table shows financial obligations totaling $67,265 million, with the largest amount of $25,191 million due by September 30, 2025, followed by $19,733 million by December 31, 2025, $13,742 million by March 31, 2026, $5,889 million by June 30, 2026, and $2,710 million thereafter."
- },
- {
- "path": "tables/table-14 Long-term Debt and Other Commitments (In Millions).html",
- "summary": "table-14 The table shows contractual obligations for 2026 and thereafter, with principal payments of $3,000 and $46,206, interest payments of $1,509 and $25,527, construction commitments of $26,859 and $5,290, leases of $12,798 and $165,903, and purchase commitments of $103,940 and $6,013, totaling $397,045."
- },
- {
- "path": "tables/table-15 Risk Categories and Hypothetical Market Change Impacts on Financial Metrics.html",
- "summary": "table-15 The table shows hypothetical impacts on financial metrics: a 10% decrease in foreign exchange rates would reduce revenue by $11,596 million (earnings) and investments by $17 million (fair value); a 100 basis point increase in U.S. treasury rates reduces fair value by $1,415 million; a 100 basis point credit spread increase reduces fair value by $436 million; a 10% equity market price decrease reduces earnings by $1,213 million."
- },
- {
- "path": "tables/table-16 Microsoft Cloud Revenue and Income Statement FY2023-2025.html",
- "summary": "table-16 Microsoft reported total revenue of $281,724 million in FY2025, up from $245,122 million in FY2024 and $211,915 million in FY2023; net income rose to $101,832 million in FY2025 from $88,136 million in FY2024 and $72,361 million in FY2023, with diluted EPS increasing from $9.68 to $13.64."
- },
- {
- "path": "tables/table-17 Consolidated Statements of Comprehensive Income.html",
- "summary": "table-17 Microsoft reported net income of $101,832 million in 2025, $88,136 million in 2024, and $72,361 million in 2023. Other comprehensive income was $2,243 million, $753 million, and $(1,665) million respectively, yielding comprehensive income of $104,075 million, $88,889 million, and $70,696 million."
- },
- {
- "path": "tables/table-18 Microsoft Corporation Consolidated Balance Sheets (Unaudited).html",
- "summary": "table-18 Microsoft Corporation's consolidated balance sheet as of June 30, 2025, shows total assets of $619,003 million, up from $512,163 million in 2024, with total liabilities of $275,524 million and stockholders' equity of $343,479 million."
- },
- {
- "path": "tables/table-19 Microsoft Cash Flow Statement for Fiscal Years 2023-2025.html",
- "summary": "table-19 Net cash from operations rose from $87,582M in 2023 to $118,548M in 2024 and $136,162M in 2025; net income grew from $72,361M to $101,832M; cash and equivalents ended at $30,242M in 2025, down from $34,704M in 2023; investing used $72,599M in 2025, financing used $51,699M."
- },
- {
- "path": "tables/table-20 Microsoft Stockholders' Equity Statement.html",
- "summary": "table-20 The statement shows stockholders' equity components for fiscal years 2025, 2024, and 2023, with total equity rising from $206,223 million to $343,479 million, driven by net income of $101,832 million in 2025, common stock repurchases, and dividends per share increasing from $2.72 to $3.32."
- },
- {
- "path": "tables/table-21 Microsoft Earnings Per Share Table.html",
- "summary": "table-21 The table shows Microsoft's earnings per share for fiscal years 2025, 2024, and 2023. Net income available for common shareholders was $101,832 million in 2025, $88,136 million in 2024, and $72,361 million in 2023. Weighted average shares outstanding were 7,433, 7,431, and 7,446 million respectively. Basic EPS was $13.70, $11.86, and $9.72; diluted EPS was $13.64, $11.80, and $9.68 for the three years."
- },
- {
- "path": "tables/table-22 Year Ended June 30, 2025, 2024, and 2023 Financial Data.html",
- "summary": "table-22 The table shows total net losses of $4,901 million in 2025, $1,646 million in 2024, and net gains of $788 million in 2023, with interest and dividends income declining from $2,994 million to $2,647 million, and other net losses increasing sharply to $4,725 million in 2025."
- },
- {
- "path": "tables/table-23 Line 1_ Realized Gains and Losses from Available-for-Sale Securities and Impairments.html",
- "summary": "table-23 Line 2: For fiscal years 2025, 2024, and 2023, realized gains were $40M, $22M, and $36M; realized losses were $(65)M, $(98)M, and $(124)M; impairments/allowances were $8M, $23M, and $(10)M. Totals: $(17)M, $(53)M, and $(98)M."
- },
- {
- "path": "tables/table-24 Year Ended June 30 2025 2024 2023 Investment Gains and Impairments.html",
- "summary": "table-24 Net realized gains on investments sold were $83 million in 2025, $18 million in 2024, and $75 million in 2023; net unrealized gains on investments still held were $536 million, $146 million, and $303 million; impairments were $(951) million, $(229) million, and $(20) million, resulting in totals of $(332) million, $(65) million, and $358 million."
- },
- {
- "path": "tables/table-25 Investments Measured at Fair Value on Recurring Basis as of June 30, 2025.html",
- "summary": "table-25 Total investments of $109,970 million include debt securities with adjusted cost basis of $85,589 million, unrealized gains of $295 million, losses of $1,625 million, and recorded basis of $84,259 million; equity investments total $13,718 million; cash is $11,711 million; derivatives net $282 million."
- },
- {
- "path": "tables/table-26 Investments Measured at Fair Value on Recurring Basis as of June 30, 2024.html",
- "summary": "table-26 As of June 30, 2024, total investments measured at fair value were $90,143 million, including $64,899 million in debt investments (with $68,209 million adjusted cost basis and $3,348 million unrealized losses) and $13,661 million in equity investments; cash and cash equivalents totaled $18,315 million, short-term investments $57,228 million, and equity and other investments $14,600 million."
- },
- {
- "path": "tables/table-27 Investments with Unrealized Losses as of June 30, 2025 and 2024.html",
- "summary": "table-27 As of June 30, 2025, total fair value of securities with unrealized losses was $42,736 million with total unrealized losses of $1,625 million, compared to $54,589 million fair value and $3,348 million unrealized losses at June 30, 2024; the largest category was U.S. government and agency securities ($37,177 million fair value, $1,462 million losses in 2025; $46,350 million, $2,948 million in 2024)."
- },
- {
- "path": "tables/table-28 Debt Securities by Maturity Date.html",
- "summary": "table-28 The table shows debt securities by maturity at June 30, 2025, with adjusted cost basis totaling $85,589 million and estimated fair value totaling $84,259 million. Due in one year or less: cost $35,108, fair value $34,952; after one through five years: cost $42,460, fair value $41,481; after five through ten years: cost $6,530, fair value $6,424; after ten years: cost $1,491, fair value $1,402."
- },
- {
- "path": "tables/table-29 Derivative Assets and Liabilities at Fair Value.html",
- "summary": "table-29 The table shows derivative notional amounts as of June 30, 2025 and 2024. Designated hedging instruments include foreign exchange contracts purchased ($1,492 million both years) and interest rate contracts purchased ($1,150 million in 2025, $1,100 million in 2024). Non-designated instruments include foreign exchange contracts purchased ($15,214 million in 2025, $7,167 million in 2024), sold ($43,307 million vs. $31,793 million), equity contracts purchased ($5,434 million vs. $4,016 million), sold ($2,189 million vs. $2,165 million), other contracts purchased ($2,769 million vs. $2,113 million), and sold ($1,242 million vs. $811 million)."
- },
- {
- "path": "tables/table-30 Derivative Assets and Liabilities as of June 30, 2025 and 2024.html",
- "summary": "table-30 The table shows derivative assets and liabilities, gross and net, for June 30, 2025 and 2024. Gross assets were $758 million (2025) and $331 million (2024); gross liabilities were $(1,837) million and $(800) million. Net assets were $500 million and $180 million; net liabilities were $(1,676) million and $(752) million. Offsets and cash collateral reduced gross amounts."
- },
- {
- "path": "tables/table-31 Derivative Assets and Liabilities by Fair Value Hierarchy.html",
- "summary": "table-31 Derivative assets totaled $758 million at June 30, 2025, with $1 million in Level 1, $474 million in Level 2, and $283 million in Level 3; derivative liabilities were $(1,837) million, including $(1,832) million in Level 2 and $(5) million in Level 3. At June 30, 2024, assets were $331 million (Level 1: $0, Level 2: $327, Level 3: $4) and liabilities were $(800) million (Level 1: $(1), Level 2: $(799), Level 3: $0)."
- },
- {
- "path": "tables/table-32 Line 1_ Derivative Instruments and Hedging Activities Fair Value and Cash Flow Hedges.html",
- "summary": "table-32 Line 2: For fiscal years 2025, 2024, and 2023, fair value hedge interest rate contracts show derivative gains/losses of $5, $(23), $(65) and hedged items of $(45), $(25), $38. Cash flow hedge FX reclassifications: $103, $(48), $61. Non-designated FX: $(938), $367, $(73); equity: $(266), $(177), $(420); other: $21, $(15), $(41)."
- },
- {
- "path": "tables/table-33 Cash Flow Hedging Instruments - Foreign Exchange Contracts.html",
- "summary": "table-33 Foreign exchange contracts designated as cash flow hedging instruments show gains of $77 million in 2025, losses of $(14) million in 2024, and gains of $34 million in 2023, all included in effectiveness assessment."
- },
- {
- "path": "tables/table-34 Property, Plant and Equipment.html",
- "summary": "table-34 The table shows property, plant and equipment at cost and net of accumulated depreciation as of June 30, 2025 and 2024. Total at cost increased from $212,012 million to $298,619 million, with buildings and improvements rising from $93,943 to $137,921 million. Accumulated depreciation grew from $76,421 to $93,653 million, resulting in net totals of $204,966 million (2025) and $135,591 million (2024)."
- },
- {
- "path": "tables/table-35 Purchase Price Allocation Table.html",
- "summary": "table-35 The table shows the allocation of a total purchase price of $75,408 million, with major components including cash and cash equivalents of $12,976 million, goodwill of $51,001 million, intangible assets of $21,969 million, and other assets of $2,503 million, offset by liabilities such as long-term debt of $2,799 million, long-term income taxes of $1,946 million, deferred income taxes of $4,676 million, and other liabilities of $3,620 million."
- },
- {
- "path": "tables/table-36 Fair value of intangible assets acquired.html",
- "summary": "table-36 The table shows acquired intangible assets with a total fair value of $21,969 million and a weighted average life of 15 years. Marketing-related intangibles are the largest at $11,619 million (24 years), followed by technology-based at $9,689 million (4 years) and customer-related at $661 million (4 years)."
- },
- {
- "path": "tables/table-37 Microsoft Fiscal Year 2024 and 2023 Financial Highlights.html",
- "summary": "table-37 Microsoft reported revenue of $247,442 million for fiscal year 2024, up from $219,790 million in 2023, with net income rising to $88,308 million from $71,383 million, and diluted earnings per share increasing to $11.82 from $9.55."
- },
- {
- "path": "tables/table-38 Microsoft Segment Goodwill Rollforward.html",
- "summary": "table-38 Microsoft's goodwill by segment increased from $67,886 million at June 30, 2023 to $119,220 million at June 30, 2024, driven by $51,235 million in acquisitions (mainly More Personal Computing), then to $119,509 million by June 30, 2025, with $289 million in other changes; Productivity and Business Processes grew from $31,359 to $31,457, Intelligent Cloud from $25,676 to $25,689, and More Personal Computing from $10,851 to $62,363."
- },
- {
- "path": "tables/table-39 Intangible Assets - June 30, 2025 and 2024.html",
- "summary": "table-39 As of June 30, 2025, total gross carrying amount of intangible assets was $43,557 million, with accumulated amortization of $20,953 million, resulting in net carrying amount of $22,604 million. This compares to June 30, 2024, when gross was $44,509 million, accumulated amortization was $16,912 million, and net was $27,597 million. Marketing-related net decreased from $13,399 to $12,601; technology-based net fell from $11,172 to $7,601; customer-related net dropped from $2,987 to $2,228; contract-based net rose from $39 to $174."
- },
- {
- "path": "tables/table-40 Intangible Assets by Category and Weighted Average Life for Fiscal Years 2025 and 2024.html",
- "summary": "table-40 The table shows intangible asset amounts and weighted average lives for fiscal years 2025 and 2024. For 2025, marketing-related assets are $13 million (10 years), technology-based $912 million (9 years), customer-related $0 (0 years), contract-based $171 million (5 years), totaling $1,096 million (9 years). For 2024, marketing-related $11,619 million (24 years), technology-based $10,947 million (4 years), customer-related $660 million (4 years), contract-based $38 million (4 years), totaling $23,264 million (14 years)."
- },
- {
- "path": "tables/table-41 Year Ending June 30, 2026 to 2030 and Thereafter Lease Payments.html",
- "summary": "table-41 The table shows future minimum lease payments by fiscal year: 2026 at $4,594 million, 2027 at $2,901 million, 2028 at $2,034 million, 2029 at $1,851 million, 2030 at $1,382 million, and $9,842 million thereafter, totaling $22,604 million."
- },
- {
- "path": "tables/table-42 Long-term Debt Issuances and Maturities as of June 30, 2025 and 2024.html",
- "summary": "table-42 Total face value of debt is $49,206 million at June 30, 2025, down from $51,221 million at June 30, 2024; after adjustments for discounts, hedge fair value, and premium on debt exchange, total debt is $43,151 million (2025) and $44,937 million (2024), with long-term debt of $40,152 million and $42,688 million, respectively; largest issuance is 2015's $23.8 billion with maturities 2025–2055."
- },
- {
- "path": "tables/table-43 Line 1_ Future Minimum Lease Payments Table.html",
- "summary": "table-43 Line 2: The table shows future minimum lease payments by fiscal year, with $3,000 million in 2026, $9,250 million in 2027, $0 in 2028, $2,054 million in 2029, $0 in 2030, and $34,902 million thereafter, totaling $49,206 million."
- },
- {
- "path": "tables/table-44 Provision for Income Taxes by Category for Fiscal Years Ended June 30, 2025, 2024, and 2023.html",
- "summary": "table-44 The table shows current and deferred tax provisions for U.S. federal, state and local, and foreign taxes. Total current taxes rose from $23,009 million in 2023 to $24,389 million in 2024 and $28,851 million in 2025. Deferred taxes were net benefits of $6,059 million (2023), $4,738 million (2024), and $7,056 million (2025). Total provision for income taxes increased from $16,950 million (2023) to $19,651 million (2024) and $21,795 million (2025). Foreign current taxes jumped from $6,678 million to $11,423 million over the period."
- },
- {
- "path": "tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html",
- "summary": "table-45 U.S. income before taxes rose from $52,917 million in 2023 to $62,886 million in 2024 and $69,212 million in 2025; foreign income increased from $36,394 million to $44,901 million and $54,415 million; total income before taxes grew from $89,311 million to $107,787 million and $123,627 million."
- },
- {
- "path": "tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html",
- "summary": "table-46 The effective tax rate decreased from 19.0% in 2023 to 18.2% in 2024 and 17.6% in 2025, with the federal statutory rate at 21.0% each year. Key adjustments include foreign earnings taxed at lower rates (-1.5% in 2025), FDII deduction (-1.0%), state taxes (+1.5%), R&D credit (-1.1%), stock-based compensation benefits (-0.9%), interest (+1.0%), and other items (-1.4%)."
- },
- {
- "path": "tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html",
- "summary": "table-47 Net deferred income tax assets increased from $19,652 million in 2024 to $26,273 million in 2025, with total deferred tax assets of $47,112 million (net of $1,169 million valuation allowance) and deferred tax liabilities of $19,670 million; largest asset components include capitalized R&D ($16,891 million) and leasing liabilities ($12,874 million)."
- },
- {
- "path": "tables/table-48 Unrecognized Tax Benefits Rollforward.html",
- "summary": "table-48 The unrecognized tax benefits increased from $17,120 million in 2023 to $22,760 million in 2024, then to $24,729 million in 2025. Key changes include current-year increases of $2,066 (2025) and $1,903 (2024), prior-year increases of $468 (2025) and $4,289 (2024), and settlements decreasing by $240 (2025) and $76 (2024)."
- },
- {
- "path": "tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html",
- "summary": "table-49 Productivity and Business Processes revenue increased from $43,599 million in 2024 to $50,567 million in 2025; Intelligent Cloud rose from $13,683 million to $14,022 million; More Personal Computing fell from $2,902 million to $2,676 million; total revenue grew from $60,184 million to $67,265 million."
- },
- {
- "path": "tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html",
- "summary": "table-50 The table shows unearned revenue beginning at $60,184 million, with $186,957 million deferred and $179,876 million recognized, resulting in an ending balance of $67,265 million for the year ended June 30, 2025."
- },
- {
- "path": "tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html",
- "summary": "table-51 Operating lease cost increased from $2,875 million in 2023 to $3,555 million in 2024 and $5,524 million in 2025. Total finance lease cost rose from $1,853 million in 2023 to $2,534 million in 2024 and $4,825 million in 2025, including amortization and interest components."
- },
- {
- "path": "tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html",
- "summary": "table-52 Cash paid for operating leases was $4,931M in 2025, $3,550M in 2024, and $2,706M in 2023; finance lease operating cash flows were $1,372M, $734M, and $501M; financing cash flows from finance leases were $2,283M, $1,286M, and $1,056M. Right-of-use assets obtained: operating leases $7,826M, $6,703M, $3,514M; finance leases $20,511M, $11,633M, $3,128M."
- },
- {
- "path": "tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html",
- "summary": "table-53 The table shows operating lease right-of-use assets of $24,823 million and total operating lease liabilities of $22,861 million as of June 30, 2025, compared to $18,961 million and $19,077 million in 2024. Finance lease property and equipment net is $44,015 million (2025) versus $25,862 million (2024), with total finance lease liabilities of $46,172 million (2025) and $27,145 million (2024). Weighted average remaining lease terms are 6 years (operating) and 13 years (finance) in 2025; discount rates are 3.5% and 4.2%, respectively."
- },
- {
- "path": "tables/table-54 Lease Maturities for Operating and Finance Leases.html",
- "summary": "table-54 The table shows future minimum lease payments for operating and finance leases as of year ending June 30, with annual amounts from 2026 to 2030 and thereafter. Operating lease payments total $25,481 million, with imputed interest of $2,620 million, resulting in a total of $22,861 million. Finance lease payments total $60,506 million, with imputed interest of $14,334 million, resulting in a total of $46,172 million. The largest single-year amounts are $6,111 million (operating, 2026) and $36,251 million (finance, thereafter)."
- },
- {
- "path": "tables/table-55 Common Stock Outstanding Rollforward.html",
- "summary": "table-55 The table shows common stock outstanding for fiscal years 2025, 2024, and 2023, with beginning balances of 7,434, 7,432, and 7,464 million shares, respectively. Issuances were 31, 34, and 37 million, while repurchases were 31, 32, and 69 million, resulting in ending balances of 7,434, 7,434, and 7,432 million shares."
- },
- {
- "path": "tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html",
- "summary": "table-56 The table shows quarterly share repurchases for fiscal years 2025, 2024, and 2023. In 2025, shares repurchased were 7, 8, 8, and 8 per quarter, totaling 31 shares for $13,000 million. In 2024, quarterly shares were 11, 7, 7, and 7, totaling 32 shares for $11,960 million. In 2023, shares were 17, 20, 18, and 14, totaling 69 shares for $18,400 million. The highest quarterly amount was $4,600 million in 2023 quarters."
- },
- {
- "path": "tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html",
- "summary": "table-57 The table lists quarterly dividends for fiscal years 2025 and 2024, with per-share dividends of $0.83 for FY2025 (total $3.32, amount $24,678 million) and $0.75 for FY2024 (total $3.00, amount $22,295 million), each paid across four dates."
- },
- {
- "path": "tables/table-58 Accumulated Other Comprehensive Loss by Component.html",
- "summary": "table-58 The table shows accumulated other comprehensive loss components for fiscal years 2025, 2024, and 2023. Derivatives balance changed from $(3) to $(8) in 2025, investments from $(2,625) to $(1,051), translation adjustments from $(2,962) to $(2,288). Total end-of-period loss: $(3,347) in 2025, $(5,590) in 2024, $(6,343) in 2023."
- },
- {
- "path": "tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html",
- "summary": "table-59 Stock-based compensation expense rose from $9,611 million in 2023 to $10,734 million in 2024 and $11,974 million in 2025; income tax benefits increased from $1,651 million to $1,826 million and $2,027 million over the same periods."
- },
- {
- "path": "tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html",
- "summary": "table-60 Dividends per share ranged from $0.62 to $0.83 quarterly across fiscal years 2023-2025, with 2025 at $0.75–$0.83, 2024 at $0.68–$0.75, and 2023 at $0.62–$0.68. Interest rates ranged from 2.0% to 5.6%, with 2025 at 3.4%–5.5%, 2024 at 3.8%–5.6%, and 2023 at 2.0%–5.4%."
- },
- {
- "path": "tables/table-61 Nonvested Stock Awards Activity.html",
- "summary": "table-61 The table shows nonvested stock awards activity: beginning balance of 88 million shares at $292.28 weighted average grant-date fair value, 39 million granted at $413.90, 38 million vested at $293.25, 7 million forfeited at $317.23, ending balance of 82 million shares at $347.44."
- },
- {
- "path": "tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html",
- "summary": "table-62 The company repurchased 6 million shares in both fiscal 2025 and 2024, and 7 million in 2023, with average prices per share of $385.10, $339.46, and $245.59 respectively, showing rising buyback costs over the three years."
- },
- {
- "path": "tables/table-63 Microsoft Segment Operating Income Table.html",
- "summary": "table-63 Microsoft's segment operating income for fiscal years 2025, 2024, and 2023 shows Productivity and Business Processes leading with $69,773M (2025), followed by Intelligent Cloud at $44,589M, and More Personal Computing at $14,166M; total operating income rose from $88,523M (2023) to $128,528M (2025), with total revenue increasing from $211,915M to $281,724M."
- },
- {
- "path": "tables/table-64 United States and Other Countries Revenue by Fiscal Year.html",
- "summary": "table-64 For fiscal years 2025, 2024, and 2023, total revenue was $281,724 million, $245,122 million, and $211,915 million, respectively; United States revenue grew from $106,744 million to $144,546 million, and other countries revenue grew from $105,171 million to $137,178 million."
- },
- {
- "path": "tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html",
- "summary": "table-65 Microsoft's total revenue grew from $211,915 million in fiscal 2023 to $245,122 million in 2024 and $281,724 million in 2025. Server products and cloud services was the largest segment, rising from $65,007M to $79,828M to $98,435M. Microsoft 365 Commercial grew from $66,949M to $76,969M to $87,767M. Gaming increased from $15,466M to $21,503M to $23,455M. Other segments included LinkedIn, Windows and Devices, Search and news advertising, Dynamics, Enterprise and partner services, and Microsoft 365 Consumer."
- },
- {
- "path": "tables/table-66 United States and Other Countries Revenue by Fiscal Year.html",
- "summary": "table-66 The table shows total revenue of $371,902 million for fiscal year 2025, up from $301,369 million in 2024 and $187,239 million in 2023. United States revenue grew from $114,380 million (2023) to $186,106 million (2024) and $230,069 million (2025); Other countries revenue rose from $72,859 million to $115,263 million and $141,833 million over the same periods."
- },
- {
- "path": "tables/table-67 Deloitte & Touche LLP signature block.html",
- "summary": "table-67 Signature block for Deloitte & Touche LLP, dated July 30, 2025, in Seattle, Washington."
- },
- {
- "path": "tables/table-68 Microsoft Board of Directors List.html",
- "summary": "table-68 The table lists twelve members of Microsoft's Board of Directors, including Satya Nadella (Chairman and CEO), Sandra E. Peterson (Lead Independent Director), and other executives from companies like Citigroup, Wells Fargo, Disney, and GSK, with committee assignments noted."
- },
- {
- "path": "tables/table-69 Microsoft Leadership Table.html",
- "summary": "table-69 This table lists Microsoft executives and their titles, including the Chairman and CEO, CFO, and other senior leaders such as the CEO of Microsoft Commercial, Chief Marketing Officer, Chief Human Resources Officer, Vice Chair and President, and Executive Vice President of Strategy and Transformation."
- }
- ]
- }
-}
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-1 Annual Report 2025.png b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-1 Annual Report 2025.png
deleted file mode 100644
index 6f60dce2c..000000000
Binary files a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-1 Annual Report 2025.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-2 Microsoft has an imme.jpeg b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-2 Microsoft has an imme.jpeg
deleted file mode 100644
index 948132a13..000000000
Binary files a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-2 Microsoft has an imme.jpeg and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-3 and the NASDAQ Comput.jpeg b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-3 and the NASDAQ Comput.jpeg
deleted file mode 100644
index 2938d7796..000000000
Binary files a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-3 and the NASDAQ Comput.jpeg and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-4 FSC Certification Labe.jpeg b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-4 FSC Certification Labe.jpeg
deleted file mode 100644
index 3acec9eab..000000000
Binary files a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/images/image-4 FSC Certification Labe.jpeg and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/manifest.json b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/manifest.json
deleted file mode 100644
index 7744209f5..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/manifest.json
+++ /dev/null
@@ -1,180 +0,0 @@
-{
- "version": "2.0",
- "job_id": "job_b9eb8b0f3f50",
- "data_id": "demo-financial-microsoft-2025-annual-report",
- "source_file_name": "Microsoft 2025 Annual Report.docx",
- "processing_date": "2026-08-18T18:00:17.755532Z",
- "processing": {
- "page_count": 72,
- "billing_status": "charged",
- "cost": {
- "micro_dollars": 1080000,
- "credits": 1.08
- },
- "cost_estimate": {
- "currency": "USD",
- "pricing_version": "2026-08-18:874ab8b6c9e4",
- "total_input_cost": 0.01262366,
- "total_output_cost": 0.0021504,
- "total_cost": 0.01477406,
- "by_model": {
- "deepseek-chat": {
- "input_tokens": 90169,
- "output_tokens": 7680,
- "calls": 70,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.01262366,
- "output_cost": 0.0021504,
- "total_cost": 0.01477406,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- }
- },
- "by_task": {
- "parser.heading_hierarchy": {
- "input_cost": 0.00015918,
- "output_cost": 1.876e-05,
- "total_cost": 0.00017794,
- "models": {
- "deepseek-chat": {
- "input_tokens": 1137,
- "output_tokens": 67,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.00015918,
- "output_cost": 1.876e-05,
- "total_cost": 0.00017794,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- },
- "summary.asset": {
- "input_cost": 0.01246448,
- "output_cost": 0.00213164,
- "total_cost": 0.01459612,
- "models": {
- "deepseek-chat": {
- "input_tokens": 89032,
- "output_tokens": 7613,
- "calls": 69,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.01246448,
- "output_cost": 0.00213164,
- "total_cost": 0.01459612,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- }
- },
- "missing_pricing_models": []
- },
- "timing": {
- "started_at": "2026-08-18T17:57:54.877962+00:00",
- "completed_at": "2026-08-18T18:00:17.715945+00:00",
- "duration_ms": 142837
- },
- "stages": {
- "timing_ms": {
- "document.profile": 0,
- "heading.hierarchy_llm_call": 1568,
- "heading.hierarchy_llm": 1586,
- "document.cleanup_unreferenced_images": 2,
- "document.compress_images": 0,
- "worker.parse.document": 142658
- },
- "token_usage": {
- "prompt_tokens": 90169,
- "completion_tokens": 7680,
- "total_tokens": 97849,
- "calls": 70,
- "by_model": {
- "deepseek-chat": {
- "prompt_tokens": 90169,
- "completion_tokens": 7680,
- "total_tokens": 97849,
- "calls": 70
- }
- },
- "by_task": {
- "parser.heading_hierarchy": {
- "prompt_tokens": 1137,
- "completion_tokens": 67,
- "total_tokens": 1204,
- "calls": 1,
- "models": {
- "deepseek-chat": {
- "prompt_tokens": 1137,
- "completion_tokens": 67,
- "total_tokens": 1204,
- "calls": 1
- }
- }
- },
- "summary.asset": {
- "prompt_tokens": 89032,
- "completion_tokens": 7613,
- "total_tokens": 96645,
- "calls": 69,
- "models": {
- "deepseek-chat": {
- "prompt_tokens": 89032,
- "completion_tokens": 7613,
- "total_tokens": 96645,
- "calls": 69
- }
- }
- }
- }
- }
- }
- },
- "statistics": {
- "total_chunks": 85,
- "text_chunks": 12,
- "image_chunks": 4,
- "table_chunks": 69,
- "page_chunks": 0,
- "total_pages": null
- },
- "HIERARCHY": {
- "U.S. and foreign components of income before income taxes were as follows:": {
- "U.S. and foreign components of income before income taxes were as follows: part 1": {},
- "U.S. and foreign components of income before income taxes were as follows: part 2": {},
- "U.S. and foreign components of income before income taxes were as follows: part 3": {},
- "U.S. and foreign components of income before income taxes were as follows: part 4": {},
- "U.S. and foreign components of income before income taxes were as follows: part 5": {},
- "U.S. and foreign components of income before income taxes were as follows: part 6": {},
- "U.S. and foreign components of income before income taxes were as follows: part 7": {},
- "U.S. and foreign components of income before income taxes were as follows: part 8": {},
- "U.S. and foreign components of income before income taxes were as follows: part 9": {},
- "U.S. and foreign components of income before income taxes were as follows: part 10": {},
- "U.S. and foreign components of income before income taxes were as follows: part 11": {},
- "U.S. and foreign components of income before income taxes were as follows: part 12": {}
- }
- }
-}
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-1 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-1 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html
deleted file mode 100644
index e28e005ed..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-1 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Shares
Amount
Shares
Amount
Shares
Amount
Year Ended June 30,
2025
2024
2023
First Quarter
7
$
2,800
11
$
3,560
17
$
4,600
Second Quarter
8
3,500
7
2,800
20
4,600
Third Quarter
8
3,500
7
2,800
18
4,600
Fourth Quarter
8
3,200
7
2,800
14
4,600
Total
31
$
13,000
32
$
11,960
69
$
18,400
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-10 Sales and marketing expenses for 2025 and 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-10 Sales and marketing expenses for 2025 and 2024.html
deleted file mode 100644
index 0e66437fa..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-10 Sales and marketing expenses for 2025 and 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except percentages)
2025
2024
Percentage Change
Sales and marketing
$
25,654
$
24,456
5%
As a percent of revenue
9%
10%
(1)ppt
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-11 General and Administrative Expenses 2025 vs 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-11 General and Administrative Expenses 2025 vs 2024.html
deleted file mode 100644
index bc22dd5d1..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-11 General and Administrative Expenses 2025 vs 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except percentages)
2025
2024
Percentage Change
General and administrative
$
7,223
$
7,609
(5)%
As a percent of revenue
3%
3%
0ppt
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-12 Year Ended June 30, 2025 and 2024 Financial Data.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-12 Year Ended June 30, 2025 and 2024 Financial Data.html
deleted file mode 100644
index aeaf9b293..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-12 Year Ended June 30, 2025 and 2024 Financial Data.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
Interest and dividends income
$
2,647
$
3,157
Interest expense
(2,385
)
(2,935
)
Net recognized losses on investments
(349
)
(118
)
Net losses on derivatives
(260
)
(187
)
Net gains (losses) on foreign currency remeasurements
171
(244
)
Other, net
(4,725
)
(1,319
)
Total
$
(4,901
)
$
(1,646
)
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-13 Three Months Ending September 30 2025 to Thereafter Financial Obligations Table.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-13 Three Months Ending September 30 2025 to Thereafter Financial Obligations Table.html
deleted file mode 100644
index ac444decb..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-13 Three Months Ending September 30 2025 to Thereafter Financial Obligations Table.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Three Months Ending
September 30, 2025
$
25,191
December 31, 2025
19,733
March 31, 2026
13,742
June 30, 2026
5,889
Thereafter
2,710
Total
$
67,265
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-14 Long-term Debt and Other Commitments (In Millions).html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-14 Long-term Debt and Other Commitments (In Millions).html
deleted file mode 100644
index 7e17603d2..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-14 Long-term Debt and Other Commitments (In Millions).html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
2026
Thereafter
Total
Long-term debt: (a)
Principal payments
$
3,000
$
46,206
$
49,206
Interest payments
1,509
25,527
27,036
Construction commitments (b)
26,859
5,290
32,149
Operating and finance leases, including imputed interest (c)
12,798
165,903
178,701
Purchase commitments (d)
103,940
6,013
109,953
Total
$
148,106
$
248,939
$
397,045
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-15 Risk Categories and Hypothetical Market Change Impacts on Financial Metrics.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-15 Risk Categories and Hypothetical Market Change Impacts on Financial Metrics.html
deleted file mode 100644
index 74aeed8b7..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-15 Risk Categories and Hypothetical Market Change Impacts on Financial Metrics.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Risk Categories
Hypothetical Change
June 30, 2025
Impact
Foreign currency – Revenue
10% decrease in foreign exchange rates
$
(11,596
)
Earnings
Foreign currency – Investments
10% decrease in foreign exchange rates
(17
)
Fair Value
Interest rate
100 basis point increase in U.S. treasury interest rates
(1,415
)
Fair Value
Credit
100 basis point increase in credit spreads
(436
)
Fair Value
Equity
10% decrease in equity market prices
(1,213
)
Earnings
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-16 Microsoft Cloud Revenue and Income Statement FY2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-16 Microsoft Cloud Revenue and Income Statement FY2023-2025.html
deleted file mode 100644
index 153a16936..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-16 Microsoft Cloud Revenue and Income Statement FY2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except per share amounts)
Year Ended June 30,
2025
2024
2023
Revenue:
Product
$
63,946
$
64,773
$
64,699
Service and other
217,778
180,349
147,216
Total revenue
281,724
245,122
211,915
Cost of revenue:
Product
13,501
15,272
17,804
Service and other
74,330
58,842
48,059
Total cost of revenue
87,831
74,114
65,863
Gross margin
193,893
171,008
146,052
Research and development
32,488
29,510
27,195
Sales and marketing
25,654
24,456
22,759
General and administrative
7,223
7,609
7,575
Operating income
128,528
109,433
88,523
Other income (expense), net
(4,901
)
(1,646
)
788
Income before income taxes
123,627
107,787
89,311
Provision for income taxes
21,795
19,651
16,950
Net income
$
101,832
$
88,136
$
72,361
Earnings per share:
Basic
$
13.70
$
11.86
$
9.72
Diluted
$
13.64
$
11.80
$
9.68
Weighted average shares outstanding:
Basic
7,433
7,431
7,446
Diluted
7,465
7,469
7,472
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-17 Consolidated Statements of Comprehensive Income.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-17 Consolidated Statements of Comprehensive Income.html
deleted file mode 100644
index 4ba680766..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-17 Consolidated Statements of Comprehensive Income.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Net income
$
101,832
$
88,136
$
72,361
Other comprehensive income (loss), net of tax:
Net change related to derivatives
(5
)
24
(14
)
Net change related to investments
1,574
957
(1,444
)
Translation adjustments and other
674
(228
)
(207
)
Other comprehensive income (loss)
2,243
753
(1,665
)
Comprehensive income
$
104,075
$
88,889
$
70,696
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-18 Microsoft Corporation Consolidated Balance Sheets (Unaudited).html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-18 Microsoft Corporation Consolidated Balance Sheets (Unaudited).html
deleted file mode 100644
index 0f9e9ef74..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-18 Microsoft Corporation Consolidated Balance Sheets (Unaudited).html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
June 30,
2025
2024
Assets
Current assets:
Cash and cash equivalents
$
30,242
$
18,315
Short-term investments
64,323
57,228
Total cash, cash equivalents, and short-term investments
94,565
75,543
Accounts receivable, net of allowance for doubtful accounts of $944 and $830
69,905
56,924
Inventories
938
1,246
Other current assets
25,723
26,021
Total current assets
191,131
159,734
Property and equipment, net of accumulated depreciation of $93,653 and $76,421
204,966
135,591
Operating lease right-of-use assets
24,823
18,961
Equity and other investments
15,405
14,600
Goodwill
119,509
119,220
Intangible assets, net
22,604
27,597
Other long-term assets
40,565
36,460
Total assets
$
619,003
$
512,163
Liabilities and stockholders’ equity
Current liabilities:
Accounts payable
$
27,724
$
21,996
Short-term debt
0
6,693
Current portion of long-term debt
2,999
2,249
Accrued compensation
13,709
12,564
Short-term income taxes
7,211
5,017
Short-term unearned revenue
64,555
57,582
Other current liabilities
25,020
19,185
Total current liabilities
141,218
125,286
Long-term debt
40,152
42,688
Long-term income taxes
25,986
27,931
Long-term unearned revenue
2,710
2,602
Deferred income taxes
2,835
2,618
Operating lease liabilities
17,437
15,497
Other long-term liabilities
45,186
27,064
Total liabilities
275,524
243,686
Commitments and contingencies
Stockholders’ equity:
Common stock and paid-in capital – shares authorized 24,000; outstanding 7,434 and 7,434
109,095
100,923
Retained earnings
237,731
173,144
Accumulated other comprehensive loss
(3,347
)
(5,590
)
Total stockholders’ equity
343,479
268,477
Total liabilities and stockholders’ equity
$
619,003
$
512,163
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-19 Microsoft Cash Flow Statement for Fiscal Years 2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-19 Microsoft Cash Flow Statement for Fiscal Years 2023-2025.html
deleted file mode 100644
index dd8d9296d..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-19 Microsoft Cash Flow Statement for Fiscal Years 2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Operations
Net income
$
101,832
$
88,136
$
72,361
Adjustments to reconcile net income to net cash from operations:
Depreciation, amortization, and other
34,153
22,287
13,861
Stock-based compensation expense
11,974
10,734
9,611
Net recognized losses on investments and derivatives
609
305
196
Deferred income taxes
(7,056
)
(4,738
)
(6,059
)
Changes in operating assets and liabilities:
Accounts receivable
(10,581
)
(7,191
)
(4,087
)
Inventories
309
1,284
1,242
Other current assets
(3,044
)
(1,648
)
(1,991
)
Other long-term assets
(2,950
)
(6,817
)
(2,833
)
Accounts payable
569
3,545
(2,721
)
Unearned revenue
5,438
5,348
5,535
Income taxes
(38
)
1,687
(358
)
Other current liabilities
5,922
4,867
2,272
Other long-term liabilities
(975
)
749
553
Net cash from operations
136,162
118,548
87,582
Financing
Proceeds from issuance (repayments) of debt, maturities of 90 days or less, net
(5,746
)
5,250
0
Proceeds from issuance of debt
0
24,395
0
Repayments of debt
(3,216
)
(29,070
)
(2,750
)
Common stock issued
2,056
2,002
1,866
Common stock repurchased
(18,420
)
(17,254
)
(22,245
)
Common stock cash dividends paid
(24,082
)
(21,771
)
(19,800
)
Other, net
(2,291
)
(1,309
)
(1,006
)
Net cash used in financing
(51,699
)
(37,757
)
(43,935
)
Investing
Additions to property and equipment
(64,551
)
(44,477
)
(28,107
)
Acquisition of companies, net of cash acquired and divestitures, and purchases of intangible and other assets
(5,978
)
(69,132
)
(1,670
)
Purchases of investments
(29,775
)
(17,732
)
(37,651
)
Maturities of investments
16,079
24,775
33,510
Sales of investments
9,309
10,894
14,354
Other, net
2,317
(1,298
)
(3,116
)
Net cash used in investing
(72,599
)
(96,970
)
(22,680
)
Effect of foreign exchange rates on cash and cash equivalents
63
(210
)
(194
)
Net change in cash and cash equivalents
11,927
(16,389
)
20,773
Cash and cash equivalents, beginning of period
18,315
34,704
13,931
Cash and cash equivalents, end of period
$
30,242
$
18,315
$
34,704
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-2 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-2 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html
deleted file mode 100644
index 02b0ba965..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-2 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
Declaration Date
Record Date
Payment Date
Dividend Per Share
Amount
Fiscal Year 2025
(In millions)
September 16, 2024
November 21, 2024
December 12, 2024
$
0.83
$
6,170
December 3, 2024
February 20, 2025
March 13, 2025
0.83
6,169
March 11, 2025
May 15, 2025
June 12, 2025
0.83
6,169
June 10, 2025
August 21, 2025
September 11, 2025
0.83
6,170
Total
$
3.32
$
24,678
Fiscal Year 2024
September 19, 2023
November 16, 2023
December 14, 2023
$
0.75
$
5,574
November 28, 2023
February 15, 2024
March 14, 2024
0.75
5,573
March 12, 2024
May 16, 2024
June 13, 2024
0.75
5,574
June 12, 2024
August 15, 2024
September 12, 2024
0.75
5,574
Total
$
3.00
$
22,295
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-20 Microsoft Stockholders' Equity Statement.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-20 Microsoft Stockholders' Equity Statement.html
deleted file mode 100644
index cf97d7d55..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-20 Microsoft Stockholders' Equity Statement.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except per share amounts)
Year Ended June 30,
2025
2024
2023
Common stock and paid-in capital
Balance, beginning of period
$
100,923
$
93,718
$
86,939
Common stock issued
2,056
2,002
1,866
Common stock repurchased
(5,856
)
(5,712
)
(4,696
)
Stock-based compensation expense
11,974
10,734
9,611
Other, net
(2
)
181
(2
)
Balance, end of period
109,095
100,923
93,718
Retained earnings
Balance, beginning of period
173,144
118,848
84,281
Net income
101,832
88,136
72,361
Common stock cash dividends
(24,677
)
(22,293
)
(20,226
)
Common stock repurchased
(12,568
)
(11,547
)
(17,568
)
Balance, end of period
237,731
173,144
118,848
Accumulated other comprehensive loss
Balance, beginning of period
(5,590
)
(6,343
)
(4,678
)
Other comprehensive income (loss)
2,243
753
(1,665
)
Balance, end of period
(3,347
)
(5,590
)
(6,343
)
Total stockholders’ equity
$
343,479
$
268,477
$
206,223
Cash dividends declared per common share
$
3.32
$
3.00
$
2.72
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-21 Microsoft Earnings Per Share Table.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-21 Microsoft Earnings Per Share Table.html
deleted file mode 100644
index 98e1c072c..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-21 Microsoft Earnings Per Share Table.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except per share amounts)
Year Ended June 30,
2025
2024
2023
Net income available for common shareholders (A)
$
101,832
$
88,136
$
72,361
Weighted average outstanding shares of common stock (B)
7,433
7,431
7,446
Dilutive effect of stock-based awards
32
38
26
Common stock and common stock equivalents (C)
7,465
7,469
7,472
Earnings Per Share
Basic (A/B)
$
13.70
$
11.86
$
9.72
Diluted (A/C)
$
13.64
$
11.80
$
9.68
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-22 Year Ended June 30, 2025, 2024, and 2023 Financial Data.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-22 Year Ended June 30, 2025, 2024, and 2023 Financial Data.html
deleted file mode 100644
index b2d8283e2..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-22 Year Ended June 30, 2025, 2024, and 2023 Financial Data.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Interest and dividends income
$
2,647
$
3,157
$
2,994
Interest expense
(2,385
)
(2,935
)
(1,968
)
Net recognized gains (losses) on investments
(349
)
(118
)
260
Net losses on derivatives
(260
)
(187
)
(456
)
Net gains (losses) on foreign currency remeasurements
171
(244
)
181
Other, net
(4,725
)
(1,319
)
(223
)
Total
$
(4,901
)
$
(1,646
)
$
788
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-23 Line 1_ Realized Gains and Losses from Available-for-Sale Securities and Impairments.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-23 Line 1_ Realized Gains and Losses from Available-for-Sale Securities and Impairments.html
deleted file mode 100644
index 499ea82a1..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-23 Line 1_ Realized Gains and Losses from Available-for-Sale Securities and Impairments.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Realized gains from sales of available-for-sale securities
$
40
$
22
$
36
Realized losses from sales of available-for-sale securities
(65
)
(98
)
(124
)
Impairments and allowance for credit losses
8
23
(10
)
Total
$
(17
)
$
(53
)
$
(98
)
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-24 Year Ended June 30 2025 2024 2023 Investment Gains and Impairments.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-24 Year Ended June 30 2025 2024 2023 Investment Gains and Impairments.html
deleted file mode 100644
index 32cd8ea40..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-24 Year Ended June 30 2025 2024 2023 Investment Gains and Impairments.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Net realized gains on investments sold
$
83
$
18
$
75
Net unrealized gains on investments still held
536
146
303
Impairments of investments
(951
)
(229
)
(20
)
Total
$
(332
)
$
(65
)
$
358
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-25 Investments Measured at Fair Value on Recurring Basis as of June 30, 2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-25 Investments Measured at Fair Value on Recurring Basis as of June 30, 2025.html
deleted file mode 100644
index 54456bdd2..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-25 Investments Measured at Fair Value on Recurring Basis as of June 30, 2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Fair Value Level
Adjusted Cost Basis
Unrealized Gains
Unrealized Losses
Recorded Basis
Cash and Cash Equivalents
Short-term Investments
Equity and Other Investments
June 30, 2025
Changes in Fair Value Recorded in Other Comprehensive Income
Commercial paper
Level 2
$
10,880
$
0
$
0
$
10,880
$
9,939
$
941
$
0
Certificates of deposit
Level 2
2,653
0
0
2,653
2,309
344
0
U.S. government securities
Level 1
52,878
71
(1,462
)
51,487
4,742
46,745
0
U.S. agency securities
Level 2
2,686
0
0
2,686
496
2,190
0
Foreign government bonds
Level 2
349
24
(9
)
364
0
364
0
Mortgage- and asset-backed securities
Level 2
2,558
10
(27
)
2,541
0
2,541
0
Corporate notes and bonds
Level 2
10,763
124
(101
)
10,786
0
10,786
0
Corporate notes and bonds
Level 3
2,511
65
(5
)
2,571
0
111
2,460
Municipal securities
Level 2
207
1
(7
)
201
0
201
0
Municipal securities
Level 3
104
0
(14
)
90
0
90
0
Total debt investments
$
85,589
$
295
$
(1,625
)
$
84,259
$
17,486
$
64,313
$
2,460
Changes in Fair Value Recorded in Net Income
Equity investments
Level 1
$
4,577
$
1,045
$
0
$
3,532
Equity investments
Other
9,141
0
0
9,141
Total equity investments
$
13,718
$
1,045
$
0
$
12,673
Cash
$
11,711
$
11,711
$
0
$
0
Derivatives, net (a)
282
0
10
272
Total
$
109,970
$
30,242
$
64,323
$
15,405
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-26 Investments Measured at Fair Value on Recurring Basis as of June 30, 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-26 Investments Measured at Fair Value on Recurring Basis as of June 30, 2024.html
deleted file mode 100644
index cbf2c2109..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-26 Investments Measured at Fair Value on Recurring Basis as of June 30, 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Fair Value Level
Adjusted Cost Basis
Unrealized Gains
Unrealized Losses
Recorded Basis
Cash and Cash Equivalents
Short-term Investments
Equity and Other Investments
June 30, 2024
Changes in Fair Value Recorded in Other Comprehensive Income
Commercial paper
Level 2
$
4,666
$
0
$
0
$
4,666
$
4,666
$
0
$
0
Certificates of deposit
Level 2
1,547
0
0
1,547
1,503
44
0
U.S. government securities
Level 1
49,603
4
(2,948
)
46,659
14
46,645
0
U.S. agency securities
Level 2
17
0
0
17
0
17
0
Foreign government bonds
Level 2
319
3
(16
)
306
0
306
0
Mortgage- and asset-backed securities
Level 2
944
3
(35
)
912
0
912
0
Corporate notes and bonds
Level 2
9,106
28
(318
)
8,816
0
8,816
0
Corporate notes and bonds
Level 3
1,641
0
(1
)
1,640
0
140
1,500
Municipal securities
Level 2
262
0
(13
)
249
0
249
0
Municipal securities
Level 3
104
0
(17
)
87
0
87
0
Total debt investments
$
68,209
$
38
$
(3,348
)
$
64,899
$
6,183
$
57,216
$
1,500
Changes in Fair Value Recorded in Net Income
Equity investments
Level 1
$
3,547
$
561
$
0
$
2,986
Equity investments
Other
10,114
0
0
10,114
Total equity investments
$
13,661
$
561
$
0
$
13,100
Cash
$
11,571
$
11,571
$
0
$
0
Derivatives, net (a)
12
0
12
0
Total
$
90,143
$
18,315
$
57,228
$
14,600
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-27 Investments with Unrealized Losses as of June 30, 2025 and 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-27 Investments with Unrealized Losses as of June 30, 2025 and 2024.html
deleted file mode 100644
index 67335c6d3..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-27 Investments with Unrealized Losses as of June 30, 2025 and 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
Less than 12 Months
12 Months or Greater
Total Unrealized Losses
Total Unrealized Losses
(In millions)
Fair Value
Unrealized Losses
Fair Value
Unrealized Losses
Total Fair Value
Total Unrealized Losses
June 30, 2025
U.S. government and agency securities
$
2,569
$
(51
)
$
34,608
$
(1,411
)
$
37,177
$
(1,462
)
Foreign government bonds
43
(2
)
106
(7
)
149
(9
)
Mortgage- and asset-backed securities
841
(4
)
189
(23
)
1,030
(27
)
Corporate notes and bonds
1,107
(8
)
3,105
(98
)
4,212
(106
)
Municipal securities
0
0
168
(21
)
168
(21
)
Total
$
4,560
$
(65
)
$
38,176
$
(1,560
)
$
42,736
$
(1,625
)
Less than 12 Months
12 Months or Greater
Total Unrealized Losses
Total Unrealized Losses
(In millions)
Fair Value
Unrealized Losses
Fair Value
Unrealized Losses
Total Fair Value
Total Unrealized Losses
June 30, 2024
U.S. government and agency securities
$
529
$
(12
)
$
45,821
$
(2,936
)
$
46,350
$
(2,948
)
Foreign government bonds
79
(2
)
180
(14
)
259
(16
)
Mortgage- and asset-backed securities
201
(1
)
409
(34
)
610
(35
)
Corporate notes and bonds
1,310
(9
)
5,779
(310
)
7,089
(319
)
Municipal securities
38
(1
)
243
(29
)
281
(30
)
Total
$
2,157
$
(25
)
$
52,432
$
(3,323
)
$
54,589
$
(3,348
)
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-28 Debt Securities by Maturity Date.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-28 Debt Securities by Maturity Date.html
deleted file mode 100644
index 3ada4788e..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-28 Debt Securities by Maturity Date.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Adjusted Cost Basis
Estimated Fair Value
June 30, 2025
Due in one year or less
$
35,108
$
34,952
Due after one year through five years
42,460
41,481
Due after five years through 10 years
6,530
6,424
Due after 10 years
1,491
1,402
Total
$
85,589
$
84,259
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-29 Derivative Assets and Liabilities at Fair Value.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-29 Derivative Assets and Liabilities at Fair Value.html
deleted file mode 100644
index 2b7db3ae1..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-29 Derivative Assets and Liabilities at Fair Value.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
June 30, 2025
June 30, 2024
Designated as Hedging Instruments
Foreign exchange contracts purchased
$
1,492
$
1,492
Interest rate contracts purchased
1,150
1,100
Not Designated as Hedging Instruments
Foreign exchange contracts purchased
15,214
7,167
Foreign exchange contracts sold
43,307
31,793
Equity contracts purchased
5,434
4,016
Equity contracts sold
2,189
2,165
Other contracts purchased
2,769
2,113
Other contracts sold
1,242
811
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-3 Microsoft Corporation vs S&P 500 vs NASDAQ Computer Index Values June 20-25.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-3 Microsoft Corporation vs S&P 500 vs NASDAQ Computer Index Values June 20-25.html
deleted file mode 100644
index 1aaf90de8..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-3 Microsoft Corporation vs S&P 500 vs NASDAQ Computer Index Values June 20-25.html
+++ /dev/null
@@ -1 +0,0 @@
-
6/20
6/21
6/22
6/23
6/24
6/25
Microsoft Corporation
100.00
134.41
128.48
172.01
227.51
255.13
S&P 500
100.00
140.79
125.85
150.51
187.47
215.89
NASDAQ Computer
100.00
150.44
117.59
154.73
223.97
254.97
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-30 Derivative Assets and Liabilities as of June 30, 2025 and 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-30 Derivative Assets and Liabilities as of June 30, 2025 and 2024.html
deleted file mode 100644
index 03a30921f..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-30 Derivative Assets and Liabilities as of June 30, 2025 and 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Derivative Assets
Derivative Liabilities
Derivative Assets
Derivative Liabilities
June 30, 2025
June 30, 2024
Designated as Hedging Instruments
Foreign exchange contracts
$
89
$
(44
)
$
24
$
(76
)
Interest rate contracts
15
0
19
0
Not Designated as Hedging Instruments
Foreign exchange contracts
248
(809
)
213
(230
)
Equity contracts
385
(983
)
63
(491
)
Other contracts
21
(1
)
12
(3
)
Gross amounts of derivatives
758
(1,837
)
331
(800
)
Gross amounts of derivatives offset in the balance sheets
(258
)
260
(151
)
152
Cash collateral received
0
(99
)
0
(104
)
Net amounts of derivatives
$
500
$
(1,676
)
$
180
$
(752
)
Reported as
Short-term investments
$
10
$
0
$
12
$
0
Other current assets
201
0
149
0
Equity and other investments
272
0
0
0
Other long-term assets
17
0
19
0
Other current liabilities
0
(1,639
)
0
(401
)
Other long-term liabilities
0
(37
)
0
(351
)
Total
$
500
$
(1,676
)
$
180
$
(752
)
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-31 Derivative Assets and Liabilities by Fair Value Hierarchy.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-31 Derivative Assets and Liabilities by Fair Value Hierarchy.html
deleted file mode 100644
index 60dab157f..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-31 Derivative Assets and Liabilities by Fair Value Hierarchy.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Level 1
Level 2
Level 3
Total
June 30, 2025
Derivative assets
$
1
$
474
$
283
$
758
Derivative liabilities
0
(1,832
)
(5
)
(1,837
)
June 30, 2024
Derivative assets
0
327
4
331
Derivative liabilities
(1
)
(799
)
0
(800
)
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-32 Line 1_ Derivative Instruments and Hedging Activities Fair Value and Cash Flow Hedges.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-32 Line 1_ Derivative Instruments and Hedging Activities Fair Value and Cash Flow Hedges.html
deleted file mode 100644
index 5a9cd0914..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-32 Line 1_ Derivative Instruments and Hedging Activities Fair Value and Cash Flow Hedges.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Designated as Fair Value Hedging Instruments
Interest rate contracts
Derivatives
$
5
$
(23
)
$
(65
)
Hedged items
(45
)
(25
)
38
Designated as Cash Flow Hedging Instruments
Foreign exchange contracts
Amount reclassified from accumulated other comprehensive loss
103
(48
)
61
Not Designated as Hedging Instruments
Foreign exchange contracts
(938
)
367
(73
)
Equity contracts
(266
)
(177
)
(420
)
Other contracts
21
(15
)
(41
)
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-33 Cash Flow Hedging Instruments - Foreign Exchange Contracts.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-33 Cash Flow Hedging Instruments - Foreign Exchange Contracts.html
deleted file mode 100644
index 1ca1c4547..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-33 Cash Flow Hedging Instruments - Foreign Exchange Contracts.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Designated as Cash Flow Hedging Instruments
Foreign exchange contracts
Included in effectiveness assessment
$
77
$ (14)
$
34
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-34 Property, Plant and Equipment.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-34 Property, Plant and Equipment.html
deleted file mode 100644
index aadbc30e1..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-34 Property, Plant and Equipment.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
June 30,
2025
2024
Land
$
9,338
$
8,163
Buildings and improvements
137,921
93,943
Leasehold improvements
12,117
9,594
Computer equipment and software
132,836
93,780
Furniture and equipment
6,407
6,532
Total, at cost
298,619
212,012
Accumulated depreciation
(93,653
)
(76,421
)
Total, net
$
204,966
$
135,591
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-35 Purchase Price Allocation Table.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-35 Purchase Price Allocation Table.html
deleted file mode 100644
index 0a73589e2..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-35 Purchase Price Allocation Table.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Cash and cash equivalents
$
12,976
Goodwill
51,001
Intangible assets
21,969
Other assets
2,503
Long-term debt
(2,799
)
Long-term income taxes
(1,946
)
Deferred income taxes
(4,676
)
Other liabilities
(3,620
)
Total purchase price
$
75,408
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-36 Fair value of intangible assets acquired.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-36 Fair value of intangible assets acquired.html
deleted file mode 100644
index 3cbe10c02..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-36 Fair value of intangible assets acquired.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except average life)
Amount
Weighted Average Life
Marketing-related
$
11,619
24 years
Technology-based
9,689
4 years
Customer-related
661
4 years
Fair value of intangible assets acquired
$
21,969
15 years
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-37 Microsoft Fiscal Year 2024 and 2023 Financial Highlights.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-37 Microsoft Fiscal Year 2024 and 2023 Financial Highlights.html
deleted file mode 100644
index 5b12eae6d..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-37 Microsoft Fiscal Year 2024 and 2023 Financial Highlights.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except per share amounts)
Year Ended June 30,
2024
2023
Revenue
$
247,442
$
219,790
Net income
88,308
71,383
Diluted earnings per share
11.82
9.55
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-38 Microsoft Segment Goodwill Rollforward.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-38 Microsoft Segment Goodwill Rollforward.html
deleted file mode 100644
index a510eb2d6..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-38 Microsoft Segment Goodwill Rollforward.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
June 30, 2023
Acquisitions
Other
June 30, 2024
Acquisitions
Other
June 30, 2025
Productivity and Business Processes
$
31,359
$
0
$
2
$
31,361
$
0
$
96
$
31,457
Intelligent Cloud
25,676
0
(28
)
25,648
0
41
25,689
More Personal Computing
10,851
51,235
125
62,211
0
152
62,363
Total
$
67,886
$
51,235
$
99
$
119,220
$
0
$
289
$
119,509
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-39 Intangible Assets - June 30, 2025 and 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-39 Intangible Assets - June 30, 2025 and 2024.html
deleted file mode 100644
index 1fad0d5e4..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-39 Intangible Assets - June 30, 2025 and 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Gross Carrying Amount
Accumulated Amortization
Net Carrying Amount
Gross Carrying Amount
Accumulated Amortization
Net Carrying Amount
June 30,
2025
2024
Marketing-related
$
16,502
$
(3,901
)
$
12,601
$
16,500
$
(3,101
)
$
13,399
Technology-based
22,560
(14,959
)
7,601
21,913
(10,741
)
11,172
Customer-related
4,278
(2,050
)
2,228
6,038
(3,051
)
2,987
Contract-based
217
(43
)
174
58
(19
)
39
Total
$
43,557
$
(20,953
)
$
22,604
$
44,509
$
(16,912
)
$
27,597
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-4 Microsoft Cloud Key Metrics Definitions.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-4 Microsoft Cloud Key Metrics Definitions.html
deleted file mode 100644
index 0a8230155..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-4 Microsoft Cloud Key Metrics Definitions.html
+++ /dev/null
@@ -1 +0,0 @@
-
Commercial remaining performance obligation
Commercial portion of revenue allocated to remaining performance obligations, which includes unearned revenue and amounts that will be invoiced and recognized as revenue in future periods
Microsoft Cloud revenue and revenue growth
Revenue from Microsoft 365 Commercial cloud, Azure and other cloud services, the commercial portion of LinkedIn, and Dynamics 365
Microsoft Cloud gross margin percentage
Gross margin percentage for our Microsoft Cloud business
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-40 Intangible Assets by Category and Weighted Average Life for Fiscal Years 2025 and 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-40 Intangible Assets by Category and Weighted Average Life for Fiscal Years 2025 and 2024.html
deleted file mode 100644
index 459a5f11e..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-40 Intangible Assets by Category and Weighted Average Life for Fiscal Years 2025 and 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Amount
Weighted Average Life
Amount
Weighted Average Life
Year Ended June 30,
2025
2024
Marketing-related
$
13
10 years
$
11,619
24 years
Technology-based
912
9 years
10,947
4 years
Customer-related
0
0 years
660
4 years
Contract-based
171
5 years
38
4 years
Total
$
1,096
9 years
$
23,264
14 years
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-41 Year Ending June 30, 2026 to 2030 and Thereafter Lease Payments.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-41 Year Ending June 30, 2026 to 2030 and Thereafter Lease Payments.html
deleted file mode 100644
index c8e710b62..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-41 Year Ending June 30, 2026 to 2030 and Thereafter Lease Payments.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ending June 30,
2026
$
4,594
2027
2,901
2028
2,034
2029
1,851
2030
1,382
Thereafter
9,842
Total
$
22,604
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-42 Long-term Debt Issuances and Maturities as of June 30, 2025 and 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-42 Long-term Debt Issuances and Maturities as of June 30, 2025 and 2024.html
deleted file mode 100644
index 6d7eaae08..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-42 Long-term Debt Issuances and Maturities as of June 30, 2025 and 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, issuance by calendar year)
Maturities (calendar year)
Stated Interest Rate
Effective Interest Rate
June 30, 2025
June 30, 2024
2009 issuance of $3.8 billion
2039
5.20%
5.24%
$
520
$
520
2010 issuance of $4.8 billion
2040
4.50%
4.57%
486
486
2011 issuance of $2.3 billion
2041
5.30%
5.36%
718
718
2012 issuance of $2.3 billion
2042
3.50%
3.57%
454
454
2013 issuance of $5.2 billion
2043
3.75%–4.88%
3.83%–4.92%
314
314
2013 issuance of €4.1 billion
2028–2033
2.63%–3.13%
2.69%–3.22%
2,700
2,465
2015 issuance of $23.8 billion
2025–2055
3.13%–4.75%
3.18%–4.78%
7,555
9,805
2016 issuance of $19.8 billion
2026–2056
2.40%–3.95%
2.46%–4.03%
7,930
7,930
2017 issuance of $17.1 billion
2026–2057
3.30%–4.50%
3.38%–5.49%
6,833
6,833
2020 issuance of $10.1 billion
2030–2060
1.35%–2.68%
2.53%–5.43%
10,111
10,111
2021 issuance of $8.2 billion
2052–2062
2.92%–3.04%
2.92%–3.04%
8,185
8,185
2023 issuance of $0.1 billion
2026–2050
1.35%–4.50%
5.16%–5.49%
56
56
2024 issuance of $3.3 billion
2026–2050
1.35%–4.50%
5.16%–5.49%
3,344
3,344
Total face value
49,206
51,221
Unamortized discount and issuance costs
(1,155
)
(1,227
)
Hedge fair value adjustments (a)
(36
)
(81
)
Premium on debt exchange
(4,864
)
(4,976
)
Total debt
43,151
44,937
Current portion of long-term debt
(2,999
)
(2,249
)
Long-term debt
$
40,152
$
42,688
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-43 Line 1_ Future Minimum Lease Payments Table.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-43 Line 1_ Future Minimum Lease Payments Table.html
deleted file mode 100644
index 2790ea890..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-43 Line 1_ Future Minimum Lease Payments Table.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ending June 30,
2026
$
3,000
2027
9,250
2028
0
2029
2,054
2030
0
Thereafter
34,902
Total
$
49,206
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-44 Provision for Income Taxes by Category for Fiscal Years Ended June 30, 2025, 2024, and 2023.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-44 Provision for Income Taxes by Category for Fiscal Years Ended June 30, 2025, 2024, and 2023.html
deleted file mode 100644
index e59a4c4b2..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-44 Provision for Income Taxes by Category for Fiscal Years Ended June 30, 2025, 2024, and 2023.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Current Taxes
U.S. federal
$
14,086
$
12,165
$
14,009
U.S. state and local
3,342
2,366
2,322
Foreign
11,423
9,858
6,678
Current taxes
$
28,851
$
24,389
$
23,009
Deferred Taxes
U.S. federal
$
(6,250
)
$
(4,791
)
$
(6,146
)
U.S. state and local
(1,087
)
(379
)
(477
)
Foreign
281
432
564
Deferred taxes
$
(7,056
)
$
(4,738
)
$
(6,059
)
Provision for income taxes
$
21,795
$
19,651
$
16,950
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html
deleted file mode 100644
index 757bb5778..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-45 Income before income taxes by geographic region for fiscal years 2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
U.S.
$
69,212
$
62,886
$
52,917
Foreign
54,415
44,901
36,394
Income before income taxes
$
123,627
$
107,787
$
89,311
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html
deleted file mode 100644
index fbf526a2a..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-46 Effective Tax Rate Reconciliation for Fiscal Years 2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
Year Ended June 30,
2025
2024
2023
Federal statutory rate
21.0%
21.0%
21.0%
Effect of:
Foreign earnings taxed at lower rates
(1.5)%
(1.4)%
(1.8)%
Foreign-derived intangible income deduction
(1.0)%
(1.1)%
(1.3)%
State income taxes, net of federal benefit
1.5%
1.5%
1.6%
Research and development credit
(1.1)%
(1.1)%
(1.1)%
Excess tax benefits relating to stock-based compensation
(0.9)%
(1.1)%
(0.7)%
Interest, net
1.0%
1.1%
0.8%
Other reconciling items, net
(1.4)%
(0.7)%
0.5%
Effective rate
17.6%
18.2%
19.0%
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html
deleted file mode 100644
index 2d79477a9..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-47 Deferred Income Tax Assets and Liabilities as of June 30, 2025 and 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
June 30,
2025
2024
Deferred Income Tax Assets
Stock-based compensation expense
$
909
$
765
Accruals, reserves, and other expenses
5,050
4,381
Loss and credit carryforwards
2,114
1,741
Amortization
4,118
4,159
Leasing liabilities
12,874
6,504
Unearned revenue
4,324
3,717
Book/tax basis differences in investments and debt
303
9
Capitalized research and development
16,891
11,442
Other
529
426
Deferred income tax assets
47,112
33,144
Less valuation allowance
(1,169
)
(1,045
)
Deferred income tax assets, net of valuation allowance
$
45,943
$
32,099
Deferred Income Tax Liabilities
Leasing assets
$
(12,696
)
$
(6,503
)
Depreciation
(5,699
)
(3,940
)
Deferred tax on foreign earnings
(1,148
)
(1,837
)
Other
(127
)
(167
)
Deferred income tax liabilities
$
(19,670
)
$
(12,447
)
Net deferred income tax assets
$
26,273
$
19,652
Reported As
Other long-term assets
$
29,108
$
22,270
Long-term deferred income tax liabilities
(2,835
)
(2,618
)
Net deferred income tax assets
$
26,273
$
19,652
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-48 Unrecognized Tax Benefits Rollforward.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-48 Unrecognized Tax Benefits Rollforward.html
deleted file mode 100644
index 353246b5d..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-48 Unrecognized Tax Benefits Rollforward.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Beginning unrecognized tax benefits
$
22,760
$
17,120
$
15,593
Decreases related to settlements
(240
)
(76
)
(329
)
Increases for tax positions related to the current year
2,066
1,903
1,051
Increases for tax positions related to prior years
468
4,289
870
Decreases for tax positions related to prior years
(300
)
(464
)
(60
)
Decreases due to lapsed statutes of limitations
(25
)
(12
)
(5
)
Ending unrecognized tax benefits
$
24,729
$
22,760
$
17,120
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html
deleted file mode 100644
index 61b433ec8..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-49 Microsoft Segment Revenue Comparison June 30 2025 vs 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
June 30,
2025
2024
Productivity and Business Processes
$
50,567
$
43,599
Intelligent Cloud
14,022
13,683
More Personal Computing
2,676
2,902
Total
$
67,265
$
60,184
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-5 Microsoft 365 and Azure Key Performance Indicators Definitions.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-5 Microsoft 365 and Azure Key Performance Indicators Definitions.html
deleted file mode 100644
index e5c6240b4..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-5 Microsoft 365 and Azure Key Performance Indicators Definitions.html
+++ /dev/null
@@ -1 +0,0 @@
-
Microsoft 365 Commercial cloud revenue growth
Revenue from Microsoft 365 Commercial subscriptions, comprising Microsoft 365 Commercial, Enterprise Mobility + Security, the cloud portion of Windows Commercial, the per-user portion of Power BI, Exchange, SharePoint, Microsoft Teams, Microsoft 365 Security and Compliance, and Microsoft 365 Copilot
Microsoft 365 Commercial seat growth
The number of Microsoft 365 Commercial seats at end of period where seats are paid users covered by a Microsoft 365 Commercial subscription
Microsoft 365 Consumer cloud revenue growth
Revenue from Microsoft 365 Consumer subscriptions and other consumer services
Microsoft 365 Consumer subscribers
The number of Microsoft 365 Consumer subscribers at end of period
LinkedIn revenue growth
Revenue from LinkedIn, including Talent Solutions, Marketing Solutions, Premium Subscriptions, and Sales Solutions
Dynamics 365 revenue growth
Revenue from Dynamics 365, including a set of intelligent, cloud-based applications across ERP, CRM, Power Apps, and Power Automate
Azure and other cloud services revenue growth
Revenue from Azure and other cloud services, including cloud and AI consumption-based services, GitHub cloud services, Nuance Healthcare cloud services, virtual desktop offerings, and other cloud services
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html
deleted file mode 100644
index 125c2d5b8..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-50 Unearned Revenue Rollforward for Year Ended June 30, 2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30, 2025
Balance, beginning of period
$
60,184
Deferral of revenue
186,957
Recognition of unearned revenue
(179,876
)
Balance, end of period
$
67,265
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html
deleted file mode 100644
index 5667bfcde..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-51 Lease Cost Components for Fiscal Years 2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Operating lease cost
$
5,524
$
3,555
$
2,875
Finance lease cost:
Amortization of right-of-use assets
$
3,408
$
1,800
$
1,352
Interest on lease liabilities
1,417
734
501
Total finance lease cost
$
4,825
$
2,534
$
1,853
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html
deleted file mode 100644
index fa3f736dc..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-52 Lease Payments and Right-of-Use Assets Obtained in Exchange for Lease Obligations.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Cash paid for amounts included in the measurement of lease liabilities:
Operating cash flows from operating leases
$
4,931
$
3,550
$
2,706
Operating cash flows from finance leases
1,372
734
501
Financing cash flows from finance leases
2,283
1,286
1,056
Right-of-use assets obtained in exchange for lease obligations:
Operating leases
7,826
6,703
3,514
Finance leases
20,511
11,633
3,128
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html
deleted file mode 100644
index ab70e756b..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-53 Lease Balances and Weighted Average Lease Terms and Discount Rates.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except lease term and discount rate)
June 30,
2025
2024
Operating Leases
Operating lease right-of-use assets
$
24,823
$
18,961
Other current liabilities
$
5,424
$
3,580
Operating lease liabilities
17,437
15,497
Total operating lease liabilities
$
22,861
$
19,077
Finance Leases
Property and equipment, at cost
$
53,876
$
32,248
Accumulated depreciation
(9,861
)
(6,386
)
Property and equipment, net
$
44,015
$
25,862
Other current liabilities
$
3,172
$
2,349
Other long-term liabilities
43,000
24,796
Total finance lease liabilities
$
46,172
$
27,145
Weighted Average Remaining Lease Term
Operating leases
6 years
7 years
Finance leases
13 years
12 years
Weighted Average Discount Rate
Operating leases
3.5%
3.3%
Finance leases
4.2%
3.9%
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-54 Lease Maturities for Operating and Finance Leases.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-54 Lease Maturities for Operating and Finance Leases.html
deleted file mode 100644
index 80d103dd9..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-54 Lease Maturities for Operating and Finance Leases.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ending June 30,
Operating Leases
Finance Leases
2026
$
6,111
$
5,008
2027
5,237
5,157
2028
3,495
5,187
2029
2,419
4,521
2030
2,017
4,382
Thereafter
6,202
36,251
Total lease payments
25,481
60,506
Less imputed interest
(2,620
)
(14,334
)
Total
$
22,861
$
46,172
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-55 Common Stock Outstanding Rollforward.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-55 Common Stock Outstanding Rollforward.html
deleted file mode 100644
index f6cccb6c1..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-55 Common Stock Outstanding Rollforward.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Balance, beginning of year
7,434
7,432
7,464
Issued
31
34
37
Repurchased
(31
)
(32
)
(69
)
Balance, end of year
7,434
7,434
7,432
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html
deleted file mode 100644
index f332cf012..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-56 Quarterly Share Repurchase Activity for Fiscal Years 2025, 2024, and 2023.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Shares
Amount
Shares
Amount
Shares
Amount
Year Ended June 30,
2025
2024
2023
First Quarter
7
$
2,800
11
$
3,560
17
$
4,600
Second Quarter
8
3,500
7
2,800
20
4,600
Third Quarter
8
3,500
7
2,800
18
4,600
Fourth Quarter
8
3,200
7
2,800
14
4,600
Total
31
$
13,000
32
$
11,960
69
$
18,400
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html
deleted file mode 100644
index 02b0ba965..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-57 Quarterly Dividend Payment Schedule for Fiscal Years 2024 and 2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
Declaration Date
Record Date
Payment Date
Dividend Per Share
Amount
Fiscal Year 2025
(In millions)
September 16, 2024
November 21, 2024
December 12, 2024
$
0.83
$
6,170
December 3, 2024
February 20, 2025
March 13, 2025
0.83
6,169
March 11, 2025
May 15, 2025
June 12, 2025
0.83
6,169
June 10, 2025
August 21, 2025
September 11, 2025
0.83
6,170
Total
$
3.32
$
24,678
Fiscal Year 2024
September 19, 2023
November 16, 2023
December 14, 2023
$
0.75
$
5,574
November 28, 2023
February 15, 2024
March 14, 2024
0.75
5,573
March 12, 2024
May 16, 2024
June 13, 2024
0.75
5,574
June 12, 2024
August 15, 2024
September 12, 2024
0.75
5,574
Total
$
3.00
$
22,295
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-58 Accumulated Other Comprehensive Loss by Component.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-58 Accumulated Other Comprehensive Loss by Component.html
deleted file mode 100644
index fc748dc40..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-58 Accumulated Other Comprehensive Loss by Component.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Derivatives
Balance, beginning of period
$
(3
)
$
(27
)
$
(13
)
Unrealized gains (losses), net of tax of $20, $(4), and $9
77
(14
)
34
Reclassification adjustments for (gains) losses included in other income (expense), net
(103
)
48
(61
)
Tax expense (benefit) included in provision for income taxes
21
(10
)
13
Amounts reclassified from accumulated other comprehensive loss
(82
)
38
(48
)
Net change related to derivatives, net of tax of $(1), $6, and $(4)
(5
)
24
(14
)
Balance, end of period
$
(8
)
$
(3
)
$
(27
)
Investments
Balance, beginning of period
$
(2,625
)
$
(3,582
)
$
(2,138
)
Unrealized gains (losses), net of tax of $411, $247, and $(393)
1,560
915
(1,523
)
Reclassification adjustments for losses included in other income (expense), net
17
53
99
Tax benefit included in provision for income taxes
(3
)
(11
)
(20
)
Amounts reclassified from accumulated other comprehensive loss
14
42
79
Net change related to investments, net of tax of $414, $258, and $(373)
1,574
957
(1,444
)
Balance, end of period
$
(1,051
)
$
(2,625
)
$
(3,582
)
Translation Adjustments and Other
Balance, beginning of period
$
(2,962
)
$
(2,734
)
$
(2,527
)
Translation adjustments and other, net of tax of $8, $0, and $0
674
(228
)
(207
)
Balance, end of period
$
(2,288
)
$
(2,962
)
$
(2,734
)
Accumulated other comprehensive loss, end of period
$
(3,347
)
$
(5,590
)
$
(6,343
)
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html
deleted file mode 100644
index dcd465207..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-59 Stock-based Compensation Expense and Related Income Tax Benefits for Fiscal Years 2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Stock-based compensation expense
$
11,974
$
10,734
$
9,611
Income tax benefits related to stock-based compensation
2,027
1,826
1,651
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-6 Windows OEM and Devices revenue growth and Xbox content and services revenue growth and Search and news advertising revenue ex TAC growth.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-6 Windows OEM and Devices revenue growth and Xbox content and services revenue growth and Search and news advertising revenue ex TAC growth.html
deleted file mode 100644
index fe8c56c73..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-6 Windows OEM and Devices revenue growth and Xbox content and services revenue growth and Search and news advertising revenue ex TAC growth.html
+++ /dev/null
@@ -1 +0,0 @@
-
Windows OEM and Devices revenue growth
Revenue from sales of Windows Pro and non-Pro licenses sold through the OEM channel and sales of first-party Devices, including Surface and PC accessories
Xbox content and services revenue growth
Revenue from Xbox content and services, comprising first- and third-party content (including games and in-game content), Xbox Game Pass and other subscriptions, Xbox Cloud Gaming, advertising, and other cloud services
Search and news advertising revenue (ex TAC) growth
Revenue from search and news advertising excluding traffic acquisition costs (“TAC”) paid to Bing Ads network publishers and news partners
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html
deleted file mode 100644
index f0a18a757..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-60 Dividends per share and interest rates for fiscal years 2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
Year Ended June 30,
2025
2024
2023
Dividends per share (quarterly amounts)
$
0.75 – 0.83
$
0.68 – 0.75
$
0.62 – 0.68
Interest rates
3.4% – 5.5%
3.8% – 5.6%
2.0% – 5.4%
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-61 Nonvested Stock Awards Activity.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-61 Nonvested Stock Awards Activity.html
deleted file mode 100644
index 11dd03991..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-61 Nonvested Stock Awards Activity.html
+++ /dev/null
@@ -1 +0,0 @@
-
Shares
Weighted Average Grant-Date Fair Value
(In millions)
Stock Awards
Nonvested balance, beginning of year
88
$
292.28
Granted (a)
39
413.90
Vested
(38
)
293.25
Forfeited
(7
)
317.23
Nonvested balance, end of year
82
$
347.44
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html
deleted file mode 100644
index 53691c34e..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-62 Share Repurchase Activity for Fiscal Years 2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(Shares in millions)
Year Ended June 30,
2025
2024
2023
Shares purchased
6
6
7
Average price per share
$
385.10
$
339.46
$
245.59
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-63 Microsoft Segment Operating Income Table.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-63 Microsoft Segment Operating Income Table.html
deleted file mode 100644
index 5e2cbdf5e..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-63 Microsoft Segment Operating Income Table.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Productivity and Business Processes
Revenue
$
120,810
$
106,820
$
94,151
Cost of revenue
22,422
19,611
17,202
Operating expenses
28,615
27,548
26,875
Operating Income
$
69,773
$
59,661
$
50,074
Intelligent Cloud
Revenue
$
106,265
$
87,464
$
72,944
Cost of revenue
40,171
29,611
24,109
Operating expenses
21,505
20,040
20,424
Operating Income
$
44,589
$
37,813
$
28,411
More Personal Computing
Revenue
$
54,649
$
50,838
$
44,820
Cost of revenue
25,238
24,892
24,552
Operating expenses
15,245
13,987
10,230
Operating Income
$
14,166
$
11,959
$
10,038
Total
Revenue
$
281,724
$
245,122
$
211,915
Cost of revenue
87,831
74,114
65,863
Operating expenses
65,365
61,575
57,529
Operating Income
$
128,528
$
109,433
$
88,523
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-64 United States and Other Countries Revenue by Fiscal Year.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-64 United States and Other Countries Revenue by Fiscal Year.html
deleted file mode 100644
index af25a9250..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-64 United States and Other Countries Revenue by Fiscal Year.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
United States (a)
$
144,546
$
124,704
$
106,744
Other countries
137,178
120,418
105,171
Total
$
281,724
$
245,122
$
211,915
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html
deleted file mode 100644
index 3177208a4..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-65 Microsoft Revenue by Segment for Fiscal Years 2023-2025.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
Year Ended June 30,
2025
2024
2023
Server products and cloud services
$
98,435
$
79,828
$
65,007
Microsoft 365 Commercial products and cloud services
87,767
76,969
66,949
Gaming
23,455
21,503
15,466
LinkedIn
17,812
16,372
14,989
Windows and Devices
17,314
17,026
17,147
Search and news advertising
13,878
12,306
12,125
Dynamics products and cloud services
7,827
6,831
5,796
Enterprise and partner services
7,760
7,594
7,900
Microsoft 365 Consumer products and cloud services
7,404
6,648
6,417
Other
72
45
119
Total
$
281,724
$
245,122
$
211,915
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-66 United States and Other Countries Revenue by Fiscal Year.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-66 United States and Other Countries Revenue by Fiscal Year.html
deleted file mode 100644
index 11b53e966..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-66 United States and Other Countries Revenue by Fiscal Year.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions)
June 30,
2025
2024
2023
United States
$
230,069
$
186,106
$
114,380
Other countries
141,833
115,263
72,859
Total
$
371,902
$
301,369
$
187,239
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-67 Deloitte & Touche LLP signature block.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-67 Deloitte & Touche LLP signature block.html
deleted file mode 100644
index 20b07ca86..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-67 Deloitte & Touche LLP signature block.html
+++ /dev/null
@@ -1 +0,0 @@
-
/S/ DELOITTE & TOUCHE LLP
Seattle, Washington
July 30, 2025
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-68 Microsoft Board of Directors List.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-68 Microsoft Board of Directors List.html
deleted file mode 100644
index e62c1b1b8..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-68 Microsoft Board of Directors List.html
+++ /dev/null
@@ -1 +0,0 @@
-
Satya Nadella Chairman and Chief Executive Officer, Microsoft Corporation
Catherine MacGregor 2,4 Group Chief Executive Officer and Director, Engie S.A.
Carlos Rodriguez 1, 2 Director, Automatic Data Processing, Inc.
Reid G. Hoffman 4 Partner, Greylock Partners
Mark A. L. Mason 3 Chief Financial Officer, Citigroup Inc.
Charles W. Scharf 2,3 Chief Executive Officer, President, and Director, Wells Fargo & Company
Hugh F. Johnston 1 Senior Executive Vice President and Chief Financial Officer, The Walt Disney Company
Sandra E. Peterson 2,3 Lead Independent Director, Microsoft Corporation Operating Partner, Clayton, Dubilier & Rice, LLC
John W. Stanton 1,4 Founder and Chairman, Trilogy Partnerships
Teri L. List 1,3 Former Executive Vice President and Chief Financial Officer, The Gap, Inc.
Penny S. Pritzker 4 Founder and Chairman, PSP Partners, LLC
Emma N. Walmsley 2,4 Chief Executive Officer and Director, GSK plc
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-69 Microsoft Leadership Table.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-69 Microsoft Leadership Table.html
deleted file mode 100644
index 263fe36bf..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-69 Microsoft Leadership Table.html
+++ /dev/null
@@ -1 +0,0 @@
-
Satya Nadella Chairman and Chief Executive Officer
Amy E. Hood Executive Vice President and Chief Financial Officer
Judson B. Althoff Executive Vice President and CEO Microsoft Commercial
Takeshi Numoto Executive Vice President and Chief Marketing Officer
Amy L. Coleman Executive Vice President and Chief Human Resources Officer
Bradford L. Smith Vice Chair and President
Kathleen T. Hogan Executive Vice President, Office of Strategy and Transformation
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-7 Revenue and Profit Metrics for 2025 vs 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-7 Revenue and Profit Metrics for 2025 vs 2024.html
deleted file mode 100644
index a7fadd7cf..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-7 Revenue and Profit Metrics for 2025 vs 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except percentages and per share amounts)
2025
2024
Percentage Change
Revenue
$
281,724
$
245,122
15%
Gross margin
193,893
171,008
13%
Operating income
128,528
109,433
17%
Net income
101,832
88,136
16%
Diluted earnings per share
13.64
11.80
16%
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-8 Microsoft Segment Revenue and Operating Income FY2025 vs FY2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-8 Microsoft Segment Revenue and Operating Income FY2025 vs FY2024.html
deleted file mode 100644
index 0b8185c57..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-8 Microsoft Segment Revenue and Operating Income FY2025 vs FY2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
(In millions, except percentages)
2025
2024
Percentage Change
Productivity and Business Processes
Revenue
$
120,810
$
106,820
13%
Cost of revenue
22,422
19,611
14%
Operating expenses
28,615
27,548
4%
Operating Income
$
69,773
$
59,661
17%
Intelligent Cloud
Revenue
$
106,265
$
87,464
21%
Cost of revenue
40,171
29,611
36%
Operating expenses
21,505
20,040
7%
Operating Income
$
44,589
$
37,813
18%
More Personal Computing
Revenue
$
54,649
$
50,838
7%
Cost of revenue
25,238
24,892
1%
Operating expenses
15,245
13,987
9%
Operating Income
$
14,166
$
11,959
18%
Total
Revenue
$
281,724
$
245,122
15%
Cost of revenue
87,831
74,114
19%
Operating expenses
65,365
61,575
6%
Operating Income
$
128,528
$
109,433
17%
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-9 Research and development expenses for fiscal years 2025 and 2024.html b/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-9 Research and development expenses for fiscal years 2025 and 2024.html
deleted file mode 100644
index 2c51f9d5e..000000000
--- a/apps/api/app/data/demo_documents/financial-microsoft-2025-annual-report/tables/table-9 Research and development expenses for fiscal years 2025 and 2024.html
+++ /dev/null
@@ -1 +0,0 @@
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_10_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_10_2.html
deleted file mode 100644
index 0a80c57fa..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_10_2.html
+++ /dev/null
@@ -1,58 +0,0 @@
-
-
-
-
-
-
Future Amortization Expense
-
-
-
-
-
-
(In millions)
-
-
-
-
Fiscal Year:
-
-
-
-
-
-
2027 (excluding the first quarter of fiscal year 2027)
-
$
-
-
689.0
-
-
-
2028
-
-
-
754.0
-
-
-
2029
-
-
-
610.0
-
-
-
2030
-
-
-
516.0
-
-
-
2031
-
-
-
468.0
-
-
-
2032 and thereafter
-
-
-
83.0
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_11_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_11_1.html
deleted file mode 100644
index fd10224a7..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_11_1.html
+++ /dev/null
@@ -1,522 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
Apr 26, 2026
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Reported as
-
-
-
-
-
-
-
-
-
Cost or
-
-
-
-
-
-
-
-
-
Marketable
-
-
-
-
-
-
Pricing
-
-
Amortized
-
-
Unrealized
-
-
Unrealized
-
Estimated
-
-
Cash
-
Marketable
-
Equity
-
-
Other
-
-
-
-
Category
-
-
Cost
-
-
Gain
-
-
Loss
-
Fair Value
-
-
Equivalents
-
Debt Securities
-
Securities
-
-
Assets
-
-
-
-
-
-
-
-
-
-
-
(In millions)
-
-
-
-
-
-
-
-
-
Debt securities issued
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
by the U.S. Treasury
-
Level 2
-
$
-
21,883
-
$
-
43
-
$
-
(8)
-
$ 21,918
-
$
-
470
-
$ 21,448 $
-
—
-
$
-
—
-
-
-
Corporate debt
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
securities
-
Level 2
-
-
15,092
-
-
48
-
-
(8)
-
15,132
-
-
1,533
-
13,599
-
—
-
-
—
-
-
-
Debt securities issued
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
by U.S. government
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
agencies
-
Level 2
-
-
2,009
-
-
2
-
-
(1)
-
2,010
-
-
—
-
2,010
-
—
-
-
—
-
-
-
Certificates of deposit
-
Level 2
-
-
132
-
-
—
-
-
—
-
132
-
-
132
-
—
-
—
-
-
—
-
-
-
Foreign government
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
bonds
-
Level 2
-
-
40
-
-
1
-
-
—
-
41
-
-
—
-
41
-
—
-
-
—
-
-
-
Money market funds
-
Level 1
-
-
10,212
-
-
—
-
-
—
-
10,212
-
-
10,212
-
—
-
—
-
-
—
-
-
-
Publicly-held equity
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
securities (1) (2)
-
Level 1
-
-
-
-
-
-
-
29,887
-
-
—
-
—
-
21,023
-
-
8,864
-
-
-
Publicly-held equity
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
securities (1) (3)
-
Level 2
-
-
-
-
-
-
-
9,214
-
-
—
-
—
-
9,214
-
-
—
-
-
-
Total
-
-
$
-
49,368
-
$
-
94
-
$
-
(17)
-
$ 88,546
-
$
-
12,347
-
$ 37,098 $
-
30,237
-
$
-
8,864
-
-
-
Level 2
-
$
-
21,635 $
-
77
-
$
-
(3)
-
$
-
21,709
-
$
-
—
-
-
$ 21,709 $
-
—
-
$
-
—
-
-
Level 2
-
-
15,410
-
92
-
-
(3)
-
-
15,499
-
-
345
-
-
15,154
-
—
-
-
—
-
-
Level 2
-
-
2,157
-
4
-
-
—
-
-
2,161
-
-
—
-
-
2,161
-
—
-
-
—
-
-
Level 2
-
-
110
-
—
-
-
—
-
-
110
-
-
110
-
-
—
-
—
-
-
—
-
-
Level 2
-
-
40
-
1
-
-
—
-
-
41
-
-
—
-
-
41
-
—
-
-
—
-
-
Level 1
-
-
7,830
-
—
-
-
—
-
-
7,830
-
-
7,830
-
-
—
-
—
-
-
—
-
-
Level 1
-
-
-
-
-
-
-
17,726
-
-
—
-
-
—
-
12,886
-
-
4,840
-
-
-
$
-
47,182 $
-
174
-
$
-
(6)
-
$
-
65,076
-
$
-
8,285
-
-
$ 39,065 $
-
12,886
-
$
-
4,840
-
-
uary 25, 2026 included $10.5 billion of investments that are subject to short-term lock-up restrictions on the ability to sell.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_12_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_12_2.html
deleted file mode 100644
index b29150c58..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_12_2.html
+++ /dev/null
@@ -1,49 +0,0 @@
-
-
-
-
-
Apr 26, 2026
-
Jan 25, 2026
-
-
-
-
Less than 12 months
-
Less than 12 months
-
-
-
-
Estimated Fair Gross Unrealized Estimated Fair
-
Gross Unrealized
-
-
-
-
Value Loss
-
Value Loss
-
-
-
-
(In millions)
-
-
-
-
Debt securities issued by the U.S. Treasury
-
$ 12,238 $ (8) $
-
10,666 $ (3)
-
-
-
Corporate debt securities
-
2,658 (8)
-
1,332 (3)
-
-
-
Debt securities issued by U.S. government agencies
-
1,291 (1)
-
1,134 —
-
-
-
Total
-
$ 16,187 $ (17) $
-
13,132 $ (6)
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_12_3.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_12_3.html
deleted file mode 100644
index c5b52e3e1..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_12_3.html
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-
-
-
-
(In millions)
-
-
-
-
Less than one year
-
$
-
-
24,307
-
-
-
Due in 1 - 5 years
-
-
-
14,926
-
-
-
Total
-
$
-
-
39,233
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_13_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_13_1.html
deleted file mode 100644
index 22c8c6e92..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_13_1.html
+++ /dev/null
@@ -1,64 +0,0 @@
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
-
Apr 26, 2026 Apr 27, 2025
-
-
-
-
-
-
(In millions)
-
-
-
-
Balance at beginning of period
-
$
-
22,251 $
-
3,387
-
-
-
Adjustments related to non-marketable equity securities:
-
-
-
-
-
-
Net additions
-
-
17,899
-
649
-
-
-
Unrealized gains (1)
-
-
2,603
-
63
-
-
-
Reclassification (2)
-
-
(389)
-
(843)
-
-
-
Impairments and unrealized losses
-
-
(28)
-
(16)
-
-
-
Balance at end of period
-
$
-
42,336 $
-
3,240
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_13_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_13_2.html
deleted file mode 100644
index df23f5fe9..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_13_2.html
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-
-
-
-
Apr 26, 2026
-
-
Jan 25, 2026
-
-
-
Inventories:
-
-
(In millions)
-
-
-
-
-
Raw materials
-
$
-
6,647
-
$
-
3,807
-
-
-
Work in process
-
-
9,949
-
-
8,822
-
-
-
Finished goods
-
-
9,201
-
-
8,774
-
-
-
Total inventories (1)
-
$
-
25,797
-
$
-
21,403
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_1.html
deleted file mode 100644
index 9c0ea7a4f..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_1.html
+++ /dev/null
@@ -1,84 +0,0 @@
-
-
-
-
-
-
Apr 26, 2026
-
-
Jan 25, 2026
-
-
-
-
Accrued and Other Current Liabilities:
-
-
(In millions)
-
-
-
-
-
-
Taxes payable
-
$
-
10,638
-
$
-
-
2,669
-
-
-
Customer program accruals
-
-
4,182
-
-
-
5,318
-
-
-
Accrued purchase consideration (1)
-
-
3,957
-
-
-
3,921
-
-
-
Excess inventory purchase obligations (2)
-
-
3,121
-
-
-
2,739
-
-
-
Product warranty
-
-
2,948
-
-
-
2,807
-
-
-
Deferred revenue (3)
-
-
1,714
-
-
-
1,379
-
-
-
Accrued payroll and related expenses
-
-
1,033
-
-
-
1,146
-
-
-
Other
-
-
2,194
-
-
-
1,373
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_2.html
deleted file mode 100644
index 0cebe7363..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_2.html
+++ /dev/null
@@ -1,60 +0,0 @@
-
-
-
-
-
-
Apr 26, 2026
-
-
Jan 25, 2026
-
-
-
-
Other Long-Term Liabilities:
-
-
-
(In millions)
-
-
-
-
-
Income tax payable (1)
-
$
-
-
4,830 $
-
-
3,958
-
-
-
Deferred income tax
-
-
-
1,798
-
-
1,774
-
-
-
Deferred revenue (2)
-
-
-
1,403
-
-
1,193
-
-
-
Other
-
-
-
737
-
-
381
-
-
-
Total other long-term liabilities
-
$
-
-
8,768 $
-
-
7,306
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_3.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_3.html
deleted file mode 100644
index 2e6d7b3fb..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_14_3.html
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-
-
The following table shows the changes in short- and long-term deferred revenue during the first quarter of fiscal years 2027 and 2026:
-
-
-
Three Months Ended
-
-
-
Apr 26, 2026 Apr 27, 2025
-
-
-
(In millions)
-
-
-
Balance at beginning of period $ 2,572 $ 1,813
-
-
-
Deferred revenue additions (1) 2,530 6,493
-
-
-
Revenue recognized (2) (1,985) (6,228)
-
-
-
Balance at end of period $ 3,117 $ 2,078
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_15_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_15_1.html
deleted file mode 100644
index b2aa33654..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_15_1.html
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-
-
-
-
Apr 26, 2026
-
-
Jan 25, 2026
-
-
-
-
-
-
-
(In millions)
-
-
-
-
-
Designated as accounting hedges
-
$
-
-
2,114 $
-
-
1,765
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_16_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_16_1.html
deleted file mode 100644
index d7dacaa66..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_16_1.html
+++ /dev/null
@@ -1,116 +0,0 @@
-
-
-
-
-
Expected
-
-
-
-
-
-
-
-
Remaining Term
-
Effective
-
-
-
-
-
-
-
(years)
-
Interest Rate
-
-
Apr 26, 2026 Jan 25, 2026
-
-
-
-
-
-
-
-
(In millions)
-
-
-
-
3.20% Notes Due 2026
-
0.4
-
3.31%
-
$
-
1,000 $
-
1,000
-
-
-
1.55% Notes Due 2028
-
2.1
-
1.64%
-
-
1,250
-
1,250
-
-
-
2.85% Notes Due 2030
-
3.9
-
2.93%
-
-
1,500
-
1,500
-
-
-
2.00% Notes Due 2031
-
5.1
-
2.09%
-
-
1,250
-
1,250
-
-
-
3.50% Notes Due 2040
-
13.9
-
3.54%
-
-
1,000
-
1,000
-
-
-
3.50% Notes Due 2050
-
23.9
-
3.54%
-
-
2,000
-
2,000
-
-
-
3.70% Notes Due 2060
-
34.0
-
3.73%
-
-
500
-
500
-
-
-
Unamortized debt discount and issuance costs
-
-
-
-
(30)
-
(32
-
-
-
Net carrying amount
-
-
-
$
-
8,470 $
-
8,468
-
-
-
Less short-term portion
-
-
-
-
(1,000)
-
(999
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_17_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_17_1.html
deleted file mode 100644
index 002874472..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_17_1.html
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
-
Apr 26, 2026 Apr 27, 2025
-
-
-
-
-
-
(In millions)
-
-
-
-
Balance at beginning of period
-
$
-
2,807 $
-
1,290
-
-
-
Additions
-
-
330
-
870
-
-
-
Utilization
-
-
(189)
-
(80
-
-
-
Balance at end of period
-
$
-
2,948 $
-
2,080
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_19_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_19_1.html
deleted file mode 100644
index 6e66fa8e1..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_19_1.html
+++ /dev/null
@@ -1,114 +0,0 @@
-
-
-
-
-
-
Compute &
-
-
-
-
-
-
-
-
-
-
Networking
-
-
Graphics
-
-
Total
-
-
-
-
-
-
-
-
(In millions)
-
-
-
-
-
-
Three Months Ended Apr 26, 2026
-
-
-
-
-
-
-
-
-
-
Revenue
-
$
-
74,550
-
$
-
7,065
-
$
-
-
81,615
-
-
-
Other segment items (1)
-
-
21,215
-
-
4,124
-
-
-
25,339
-
-
-
Operating income
-
$
-
53,335
-
$
-
2,941
-
$
-
-
56,276
-
-
-
Three Months Ended Apr 27, 2025
-
-
-
-
-
-
-
-
-
-
Revenue
-
$
-
39,589
-
$
-
4,473
-
$
-
-
44,062
-
-
-
Other segment items (1)
-
-
17,535
-
-
2,833
-
-
-
20,368
-
-
-
Operating income
-
$
-
22,054
-
$
-
1,640
-
$
-
-
23,694
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_20_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_20_1.html
deleted file mode 100644
index 99951bb49..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_20_1.html
+++ /dev/null
@@ -1,59 +0,0 @@
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
Apr 26, 2026 Apr 27, 2025
-
-
-
-
-
(In millions)
-
-
-
Segment operating income
-
$
-
56,276 $ 23,694
-
-
-
Stock-based compensation expense
-
-
(1,928) (1,474)
-
-
-
Unallocated operating expenses
-
-
(565) (419)
-
-
-
Acquisition-related and other costs
-
-
(247) (163)
-
-
-
Interest income
-
-
540 515
-
-
-
Interest expense
-
-
(102) (63)
-
-
-
Other income (expense), net
-
-
15,929 (180)
-
-
-
Consolidated income before income tax
-
$
-
69,903 $ 21,910
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_20_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_20_2.html
deleted file mode 100644
index f28316442..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_20_2.html
+++ /dev/null
@@ -1,73 +0,0 @@
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
Apr 26, 2026 Apr 27, 2025
-
-
-
-
-
(In millions)
-
-
-
Geographic Revenue based upon Customer Headquarters Location:
-
-
-
-
-
United States
-
$
-
63,769 $ 25,685
-
-
-
Taiwan
-
-
12,006 7,648
-
-
-
China (including Hong Kong)
-
-
4,550 9,659
-
-
-
Other
-
-
1,290 1,070
-
-
-
Total revenue
-
$
-
81,615 $ 44,062
-
-
-
Revenue by Market Platform
-
-
-
-
Data Center
-
$
-
75,246 $ 39,112
-
-
Hyperscale
-
-
37,869 17,599
-
-
AI Clouds, Industrial, & Enterprise
-
-
37,377 21,513
-
-
Edge Computing
-
-
6,369 4,950
-
-
Total revenue
-
$
-
81,615 $ 44,062
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_21_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_21_2.html
deleted file mode 100644
index e6f95dc1f..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_21_2.html
+++ /dev/null
@@ -1,60 +0,0 @@
-
-
-
-
-
Operating Lease Obligations
-
-
-
-
(In millions)
-
-
-
Fiscal Year:
-
-
-
-
2027 (excluding the first quarter of fiscal 2027)
-
$ 460
-
-
-
2028
-
626
-
-
-
2029
-
602
-
-
-
2030
-
530
-
-
-
2031
-
462
-
-
-
2032 and thereafter
-
2,924
-
-
-
Total
-
5,604
-
-
-
Less imputed interest
-
1,260
-
-
-
Present value of net future minimum lease payments
-
4,344
-
-
-
Less short-term operating lease liabilities
-
466
-
-
-
Long-term operating lease liabilities
-
$ 3,878
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_21_3.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_21_3.html
deleted file mode 100644
index b3772b9a4..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_21_3.html
+++ /dev/null
@@ -1,40 +0,0 @@
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
-
Apr 26, 2026 Apr 27, 2025
-
-
-
-
-
-
(In millions)
-
-
-
-
Supplemental cash flows information
-
-
-
-
-
-
Operating cash flow used for operating leases
-
$
-
185 $
-
96.0
-
-
-
Operating lease assets obtained in exchange for lease obligations
-
$
-
1,516 $
-
98.0
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_24_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_24_1.html
deleted file mode 100644
index 22711a14e..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_24_1.html
+++ /dev/null
@@ -1,92 +0,0 @@
-
-
-
-
First Quarter of Fiscal Year 2027 Summary
-
-
-
-
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
-
-
-
-
-
Quarter-over-Quarter
-
Year-over-Year
-
-
-
-
Apr 26, 2026
-
Jan 25, 2026 Apr 27, 2025
-
-
Change
-
Change
-
-
-
-
($ in millions, except per share data)
-
-
-
-
-
-
-
Revenue $
-
81,615
-
$ 68,127 $
-
44,062
-
20 %
-
85 %
-
-
-
Gross margin
-
74.9 %
-
75.0 %
-
60.5 %
-
(0.1) pts
-
14.4 pts
-
-
-
Operating expenses $
-
7,621
-
$ 6,794 $
-
5,030
-
12 %
-
52 %
-
-
-
Operating income $
-
53,536
-
$ 44,299 $
-
21,638
-
21 %
-
147 %
-
-
-
Net income $
-
58,321
-
$ 42,960 $
-
18,775
-
36 %
-
211 %
-
-
-
Net income per diluted share $
-
2.39
-
$ 1.76 $
-
0.76
-
36 %
-
214 %
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_25_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_25_1.html
deleted file mode 100644
index 2fd43018b..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_25_1.html
+++ /dev/null
@@ -1,85 +0,0 @@
-
-
-
-
-
-
-
Three Months Ended
-
-
Quarter-over-Quarter
-
Year-over-Year
-
-
-
-
-
Apr 26, 2026
-
Jan 25, 2026
-
Apr 27, 2025
-
Change
-
Change
-
-
-
-
-
-
($ in millions)
-
-
-
-
-
-
Revenue by Market Platform (1)
-
-
-
-
-
-
-
-
-
Data Center
-
$
-
75,246
-
$ 62,314 $
-
39,112
-
21 %
-
92
-
-
-
Hyperscale
-
-
37,869
-
33,814
-
17,599
-
12 %
-
115
-
-
-
AI Clouds, Industrial, & Enterprise
-
-
37,377
-
28,500
-
21,513
-
31 %
-
74
-
-
-
Edge Computing
-
-
6,369
-
5,813
-
4,950
-
10 %
-
29
-
-
-
Total revenue
-
$
-
81,615
-
$ 68,127 $
-
44,062
-
20 %
-
85
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_1.html
deleted file mode 100644
index 2c233ab64..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_1.html
+++ /dev/null
@@ -1,72 +0,0 @@
-
-
-
-
-
Three Months Ended
-
-
-
-
Apr 26, 2026 Apr 27, 2025
-
-
-
Revenue
-
100.0 % 100.0 %
-
-
-
Cost of revenue
-
25.1 39.5
-
-
-
Gross profit
-
74.9 60.5
-
-
-
Operating expenses
-
-
-
-
Research and development
-
7.7 9.1
-
-
-
Sales, general and administrative
-
1.6 2.4
-
-
-
Total operating expenses
-
9.3 11.5
-
-
-
Operating income
-
65.6 49.0
-
-
-
Interest income
-
0.7 1.2
-
-
-
Interest expense
-
(0.1) (0.1)
-
-
-
Other income (expense), net
-
19.5 (0.4)
-
-
-
Total other income, net
-
20.1 0.7
-
-
-
Income before income tax
-
85.7 49.7
-
-
-
Income tax expense
-
14.2 7.1
-
-
-
Net income
-
71.5 % 42.6 %
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_2.html
deleted file mode 100644
index d9eb181bb..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_2.html
+++ /dev/null
@@ -1,64 +0,0 @@
-
-
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
-
-
-
Apr 26, 2026
-
-
Apr 27, 2025 Change $
-
%Change
-
-
-
-
-
-
-
-
-
($ in millions)
-
-
-
-
-
-
Compute & Networking
-
$
-
74,550
-
$
-
39,589 $ 34,961
-
-
-
88 %
-
-
-
Graphics
-
-
7,065
-
-
4,473 2,592
-
-
-
58 %
-
-
-
Total
-
$
-
81,615
-
$
-
44,062 $ 37,553
-
-
-
85 %
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_3.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_3.html
deleted file mode 100644
index b12082ded..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_26_3.html
+++ /dev/null
@@ -1,91 +0,0 @@
-
-
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
-
-
-
Apr 26, 2026
-
-
Apr 27, 2025 Change $
-
%Change
-
-
-
-
-
-
-
-
-
($ in millions)
-
-
-
-
-
-
Compute & Networking
-
$
-
53,335
-
$
-
22,054 $ 31,281
-
-
-
142 %
-
-
-
Graphics
-
-
2,941
-
-
1,640 1,301
-
-
-
79 %
-
-
-
Total
-
$
-
56,276
-
$
-
23,694 $ 32,582
-
-
-
138 %
-
-
-
Research and development
-
$
-
6,321
-
$
-
3,989 $ 2,332
-
-
-
58 %
-
-
Sales, general and administrative
-
-
1,300
-
-
1,041 259
-
-
-
25 %
-
-
Total operating expenses
-
$
-
7,621
-
$
-
5,030 $ 2,591
-
-
-
52 %
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_1.html
deleted file mode 100644
index a214e90b0..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_1.html
+++ /dev/null
@@ -1,67 +0,0 @@
-
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
-
-
-
Apr 26, 2026
-
Apr 27, 2025
-
-
$Change
-
-
-
-
-
-
-
($ in millions)
-
-
-
-
-
-
Interest income
-
$
-
540
-
$ 515
-
$
-
-
25
-
-
-
Interest expense
-
-
(102)
-
(63)
-
-
-
(39)
-
-
-
Other income (expense), net
-
-
15,929
-
(180)
-
-
-
16,109
-
-
-
Total other income, net
-
$
-
16,367
-
$ 272
-
$
-
-
16,095
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_2.html
deleted file mode 100644
index 9a0e49b04..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_2.html
+++ /dev/null
@@ -1,39 +0,0 @@
-
-
-
-
-
-
Apr 26, 2026
-
-
Jan 25, 2026
-
-
-
-
-
(In millions)
-
-
-
-
-
Cash and cash equivalents
-
$
-
13,237
-
$
-
10,605
-
-
-
Marketable debt securities
-
-
37,098
-
-
39,065
-
-
-
Cash, cash equivalents, and marketable debt securities
-
$
-
50,335
-
$
-
49,670
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_3.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_3.html
deleted file mode 100644
index d0d959282..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_28_3.html
+++ /dev/null
@@ -1,39 +0,0 @@
-
-
-
-
-
-
Apr 26, 2026
-
-
Apr 27, 2025
-
-
-
-
-
(In millions)
-
-
-
-
-
Net cash provided by operating activities
-
$
-
50,344
-
$
-
27,414
-
-
-
Net cash used in investing activities
-
$
-
(26,429)
-
$
-
(5,216)
-
-
-
Net cash used in financing activities
-
$
-
(21,283)
-
$
-
(15,553)
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_30_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_30_1.html
deleted file mode 100644
index 275be9d29..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_30_1.html
+++ /dev/null
@@ -1,58 +0,0 @@
-
-
-
-
-
-
Apr 26, 2026
-
-
-
-
-
-
(In millions)
-
-
-
-
Due in one year
-
$
-
-
1,000
-
-
-
Due in one to five years
-
-
-
2,750
-
-
-
Due in five to ten years
-
-
-
1,250
-
-
-
Due in greater than ten years
-
-
-
3,500
-
-
-
Unamortized debt discount and issuance costs
-
-
-
(30)
-
-
-
Net carrying amount
-
$
-
-
8,470
-
-
-
Less short-term portion
-
-
-
(1,000)
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_39_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_39_1.html
deleted file mode 100644
index be98f4e0c..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_39_1.html
+++ /dev/null
@@ -1,67 +0,0 @@
-
-
-
-
-
-
-
Total Number of Shares
-
Approximate Dollar Value
-
-
-
-
Total Number
-
-
Purchased as Part of
-
of Shares that May Yet B
-
-
-
-
of Shares
-
-
Publicly Announced
-
Purchased Under the
-
-
-
-
Purchased
-
Average Price Paid
-
Program
-
Program
-
-
-
Period
-
(In millions)
-
per Share (1)
-
(In millions)
-
(In billions)
-
-
-
January 26, 2026 - February 22, 2026
-
8.2
-
$ 186.35
-
8.2
-
$ 57.0
-
-
-
February 23, 2026 - March 22, 2026
-
9.0
-
$ 183.38
-
9.0
-
$ 55.4
-
-
-
March 23, 2026 - April 26, 2026
-
91.1
-
$ 184.98
-
91.1
-
$ 38.5
-
-
-
Total
-
108.3
-
-
108.3
-
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_39_2.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_39_2.html
deleted file mode 100644
index 7fd784cac..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_39_2.html
+++ /dev/null
@@ -1,60 +0,0 @@
-
-
-
-
Name
-
Title of Director or
-
Action
-
Date
-
Total Shares of
-
Expiration Date
-
-
-
-
Officer
-
-
-
Common Stock to be
-
-
-
-
-
-
-
-
Sold
-
-
-
-
Tench Coxe
-
Director
-
Adoption
-
3/19/2026
-
8,000,000*
-
10/30/2027
-
-
-
Colette M. Kress
-
Executive Vice President
-
Termination
-
4/10/2026
-
500,000**
-
N/A
-
-
-
-
and Chief Financial
-
-
-
-
-
-
-
-
Officer
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_3_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_3_1.html
deleted file mode 100644
index a16c6a9a8..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_3_1.html
+++ /dev/null
@@ -1,166 +0,0 @@
-
-
-
-
-
NVIDIA Corporation and Subsidiaries
-
-
-
-
-
-
Condensed Consolidated Statements of Income
-
-
-
-
-
-
(In millions, except per share data)
-
-
-
-
-
-
(Unaudited)
-
-
-
-
-
-
Three Months Ended
-
-
-
-
-
-
Apr 26, 2026
-
Apr 27, 2025
-
-
-
-
Revenue
-
$
-
81,615 $
-
44,062
-
-
-
Cost of revenue
-
-
20,458
-
17,394
-
-
-
Gross profit
-
-
61,157
-
26,668
-
-
-
Operating expenses
-
-
-
-
-
-
Research and development
-
-
6,321
-
3,989
-
-
-
Sales, general and administrative
-
-
1,300
-
1,041
-
-
-
Total operating expenses
-
-
7,621
-
5,030
-
-
-
Operating income
-
-
53,536
-
21,638
-
-
-
Interest income
-
-
540
-
515
-
-
-
Interest expense
-
-
(102)
-
(63)
-
-
-
Other income (expense), net
-
-
15,929
-
(180)
-
-
-
Total other income, net
-
-
16,367
-
272
-
-
-
Income before income tax
-
-
69,903
-
21,910
-
-
-
Income tax expense
-
-
11,582
-
3,135
-
-
-
Net income
-
$
-
58,321 $
-
18,775
-
-
-
Net income per share:
-
-
-
-
-
-
Basic
-
$
-
2.40 $
-
0.77
-
-
-
Diluted
-
$
-
2.39 $
-
0.76
-
-
-
Weighted average shares used in per share computation:
-
-
-
-
-
-
Basic
-
-
24,286
-
24,441
-
-
-
Diluted
-
-
24,391
-
24,611
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_40_1.html b/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_40_1.html
deleted file mode 100644
index 4ae3e8a5f..000000000
--- a/apps/api/app/data/demo_documents/financial-nvda-q1-fy27-results/tables/table_page_40_1.html
+++ /dev/null
@@ -1,112 +0,0 @@
-
-
-
-
-
-
Incorporated by Reference
-
-
-
-
Exhibit No.
-
Exhibit Description
-
Schedule/Form Exhibit
-
Filing Date
-
-
-
10.1+
-
Variable Compensation Plan - Fiscal Year 2027
-
8-K 10.1
-
3/6/2026
-
-
-
31.1*
-
Certification of Chief Executive Officer as required by Rule 13a-14(a) of the Securities
-
-
-
-
-
-
Exchange Act of 1934
-
-
-
-
-
31.2*
-
Certification of Chief Financial Officer as required by Rule 13a-14(a) of the Securities
-
-
-
-
-
-
Exchange Act of 1934
-
-
-
-
-
32.1#*
-
Certification of Chief Executive Officer as required by Rule 13a-14(b) of the Securities
-
-
-
-
-
-
Exchange Act of 1934
-
-
-
-
-
32.2#*
-
Certification of Chief Financial Officer as required by Rule 13a-14(b) of the Securities
-
-
-
-
-
-
Exchange Act of 1934
-
-
-
-
-
101.INS*
-
Inline XBRL Instance Document - the instance document does not appear in the InteractiveData File because its XBRL tags are embedded within the Inline XBRL document.
Vinyals & Kaiser el al. (2014) [37] semi-supervised 92.1
-
-
-
Transformer (4 layers) semi-supervised 92.7
-
-
-
Luong et al. (2015) [23] multi-task 93.0
-
-
-
Dyer et al. (2016) [8] generative 93.3
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/research-attention-is-all-you-need/tables/table_page_9_1.html b/apps/api/app/data/demo_documents/research-attention-is-all-you-need/tables/table_page_9_1.html
deleted file mode 100644
index 42374713d..000000000
--- a/apps/api/app/data/demo_documents/research-attention-is-all-you-need/tables/table_page_9_1.html
+++ /dev/null
@@ -1,394 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
train
-
PPL
-
BLEU
-
params
-
-
-
-
N
-
dmodel dff
-
h
-
dk
-
dv
-
Pdrop
-
εls
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
steps
-
(dev)
-
(dev)
-
×106
-
-
-
base
-
6
-
512 2048
-
8
-
64
-
64
-
0.1
-
0.1
-
-
100K
-
4.92
-
25.8
-
65
-
-
-
-
-
-
1
-
512
-
512
-
-
-
-
-
5.29
-
24.9
-
-
-
-
-
-
-
4
-
128
-
128
-
-
-
-
-
5.00
-
25.5
-
-
-
-
(A)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
16
-
32
-
32
-
-
-
-
-
4.91
-
25.8
-
-
-
-
-
-
-
32
-
16
-
16
-
-
-
-
-
5.01
-
25.4
-
-
-
-
-
-
-
-
16
-
-
-
-
-
-
5.16
-
25.1
-
58
-
-
-
(B)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
32
-
-
-
-
-
-
5.01
-
25.4
-
60
-
-
-
-
2
-
-
-
-
-
-
-
-
-
6.11
-
23.7
-
36
-
-
-
-
4
-
-
-
-
-
-
-
-
-
5.19
-
25.3
-
50
-
-
-
-
8
-
-
-
-
-
-
-
-
-
4.88
-
25.5
-
80
-
-
-
(C)
-
-
256
-
-
32
-
32
-
-
-
-
-
5.75
-
24.5
-
28
-
-
-
-
-
1024
-
-
128
-
128
-
-
-
-
-
4.66
-
26.0
-
168
-
-
-
-
-
1024
-
-
-
-
-
-
-
-
5.12
-
25.4
-
53
-
-
-
-
-
4096
-
-
-
-
-
-
-
-
4.75
-
26.2
-
90
-
-
-
-
-
-
-
-
-
0.0
-
-
-
-
5.77
-
24.6
-
-
-
-
-
-
-
-
-
-
0.2
-
-
-
-
4.95
-
25.5
-
-
-
-
(D)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
0.0
-
-
-
4.67
-
25.3
-
-
-
-
-
-
-
-
-
-
-
0.2
-
-
-
5.47
-
25.7
-
-
-
-
(E)
-
-
positional embedding instead of sinusoids
-
-
-
-
-
-
-
-
4.92
-
25.7
-
-
-
-
big
-
6
-
1024 4096
-
16
-
-
-
0.3
-
-
-
300K
-
4.33
-
26.4
-
213
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/chunks.json b/apps/api/app/data/demo_documents/research-rag-realized/chunks.json
deleted file mode 100644
index 52f9b4c4e..000000000
--- a/apps/api/app/data/demo_documents/research-rag-realized/chunks.json
+++ /dev/null
@@ -1,1680 +0,0 @@
-{
- "chunks": [
- {
- "chunk_id": "asset_64c4fd25-db45-5e0d-afc0-27b63977ffff",
- "type": "image",
- "content": "[images/image_page_1_figure_1.png]",
- "path": "images/image_page_1_figure_1.png",
- "metadata": {
- "length": 34,
- "summary": "",
- "page_nums": [
- 1
- ],
- "file_path": "images/image_page_1_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_b6c55aab-84c7-5cfb-b518-d8cd7ae5ab58",
- "type": "image",
- "content": "Challenges of LLM-Only Approaches diagram\n[images/image_page_7_figure_1.png]",
- "path": "images/image_page_7_figure_1.png",
- "metadata": {
- "length": 76,
- "summary": "",
- "page_nums": [
- 7
- ],
- "file_path": "images/image_page_7_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_f8cbf310-8e37-5424-8120-d4ab1e2fe789",
- "type": "image",
- "content": "What LLM Hallucination Looks Like example\n[images/image_page_7_figure_2.png]",
- "path": "images/image_page_7_figure_2.png",
- "metadata": {
- "length": 76,
- "summary": "",
- "page_nums": [
- 7
- ],
- "file_path": "images/image_page_7_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_dd6a0f64-1bb6-5e82-b172-48663b64a7f9",
- "type": "image",
- "content": "Why is Retrieval-augmented Generation (RAG) Important for Enterprises? grid\n[images/image_page_7_figure_3.png]",
- "path": "images/image_page_7_figure_3.png",
- "metadata": {
- "length": 110,
- "summary": "",
- "page_nums": [
- 7
- ],
- "file_path": "images/image_page_7_figure_3.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_5ac1ae99-fd71-5ae6-ab11-4f13e040ec4f",
- "type": "image",
- "content": "[images/image_page_8_figure_1.png]",
- "path": "images/image_page_8_figure_1.png",
- "metadata": {
- "length": 34,
- "summary": "",
- "page_nums": [
- 8
- ],
- "file_path": "images/image_page_8_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_7586961b-a15c-5b60-8d3a-c7319ae42586",
- "type": "image",
- "content": "Interesting Question: Does Long Context Window Solve Everything?\n[images/image_page_8_figure_2.png]",
- "path": "images/image_page_8_figure_2.png",
- "metadata": {
- "length": 99,
- "summary": "",
- "page_nums": [
- 8
- ],
- "file_path": "images/image_page_8_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_0e609320-93dc-55d1-a23f-8a29a9b3cd98",
- "type": "image",
- "content": "Challenges of a Naive RAG System\n[images/image_page_9_figure_1.png]",
- "path": "images/image_page_9_figure_1.png",
- "metadata": {
- "length": 67,
- "summary": "",
- "page_nums": [
- 9
- ],
- "file_path": "images/image_page_9_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_6e3d5e2e-c8c9-574b-b54f-0d4486a78553",
- "type": "image",
- "content": "From Naive RAG to Advanced and Modular RAG\n[images/image_page_9_figure_2.png]",
- "path": "images/image_page_9_figure_2.png",
- "metadata": {
- "length": 77,
- "summary": "",
- "page_nums": [
- 9
- ],
- "file_path": "images/image_page_9_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_590337a0-bfca-5ab3-9d17-3209cbda9ee3",
- "type": "image",
- "content": "RAG: Strategic & Technical Insights\n[images/image_page_9_figure_3.png]",
- "path": "images/image_page_9_figure_3.png",
- "metadata": {
- "length": 70,
- "summary": "",
- "page_nums": [
- 9
- ],
- "file_path": "images/image_page_9_figure_3.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_2177dee8-30e8-5ed5-99f0-6ed89f2022de",
- "type": "table",
- "content": "tables/table_page_12_1.html",
- "path": "tables/table_page_12_1.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 12
- ],
- "file_path": "tables/table_page_12_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_82250ba7-f4fc-5fbd-bd53-db2d33df30b4",
- "type": "image",
- "content": "[images/image_page_15_figure_1.png]",
- "path": "images/image_page_15_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 15
- ],
- "file_path": "images/image_page_15_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_a217340b-8aae-5b55-864c-3cd54b679ce4",
- "type": "image",
- "content": "[images/image_page_15_figure_2.png]",
- "path": "images/image_page_15_figure_2.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 15
- ],
- "file_path": "images/image_page_15_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_0071fbf7-bee1-5b71-afea-d1123f49530d",
- "type": "table",
- "content": "tables/table_page_16_1.html",
- "path": "tables/table_page_16_1.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 16
- ],
- "file_path": "tables/table_page_16_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_4b9d9b23-c3d8-5f1d-9399-5d03e6e7319f",
- "type": "image",
- "content": "[images/image_page_16_figure_2.png]",
- "path": "images/image_page_16_figure_2.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 16
- ],
- "file_path": "images/image_page_16_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_0f22389e-2cf0-5900-8636-7e17653021d7",
- "type": "table",
- "content": "tables/table_page_16_3.html",
- "path": "tables/table_page_16_3.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 16
- ],
- "file_path": "tables/table_page_16_3.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_942ce9bd-36e8-5ab1-915e-9a71172518ea",
- "type": "image",
- "content": "[images/image_page_16_figure_4.png]",
- "path": "images/image_page_16_figure_4.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 16
- ],
- "file_path": "images/image_page_16_figure_4.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_36fe7964-1a89-51be-b9c4-71519207bcad",
- "type": "image",
- "content": "Query Engineering, Fusion, & HyDE Process Flowchart\n[images/image_page_17_figure_1.png]",
- "path": "images/image_page_17_figure_1.png",
- "metadata": {
- "length": 87,
- "summary": "",
- "page_nums": [
- 17
- ],
- "file_path": "images/image_page_17_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_fad0bd83-3ee7-5252-95c6-1017279b6565",
- "type": "image",
- "content": "Metadata Filtering & Customized Reranking Logic Diagram\n[images/image_page_17_figure_2.png]",
- "path": "images/image_page_17_figure_2.png",
- "metadata": {
- "length": 91,
- "summary": "",
- "page_nums": [
- 17
- ],
- "file_path": "images/image_page_17_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_ffedfc12-6791-5971-bf1d-9dc33ba19f7b",
- "type": "image",
- "content": "Metadata for Enhancement of Relevance and Consistency\n[images/image_page_18_figure_1.png]",
- "path": "images/image_page_18_figure_1.png",
- "metadata": {
- "length": 89,
- "summary": "",
- "page_nums": [
- 18
- ],
- "file_path": "images/image_page_18_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_21302afc-9155-5cfb-bb21-c13c3ca879bd",
- "type": "image",
- "content": "The RAGAR Approaches: Chain of RAG (CoRAG) & Tree of RAG (ToRAG)\n[images/image_page_20_figure_1.png]",
- "path": "images/image_page_20_figure_1.png",
- "metadata": {
- "length": 100,
- "summary": "",
- "page_nums": [
- 20
- ],
- "file_path": "images/image_page_20_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_f49d6d75-0237-52a9-8167-b74aef5950c8",
- "type": "image",
- "content": "RAG-Augmented Reasoning Technique (Chain of RAG)\n[images/image_page_21_figure_1.png]",
- "path": "images/image_page_21_figure_1.png",
- "metadata": {
- "length": 84,
- "summary": "",
- "page_nums": [
- 21
- ],
- "file_path": "images/image_page_21_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_a8014394-038c-5d4d-b73a-bd6bdfa8abb1",
- "type": "image",
- "content": "RAG-Augmented Reasoning Technique (Tree of RAG)\n[images/image_page_21_figure_2.png]",
- "path": "images/image_page_21_figure_2.png",
- "metadata": {
- "length": 83,
- "summary": "",
- "page_nums": [
- 21
- ],
- "file_path": "images/image_page_21_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_8a5ee5a7-0819-5845-ac96-79088150cc37",
- "type": "image",
- "content": "[images/image_page_23_figure_1.png]",
- "path": "images/image_page_23_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 23
- ],
- "file_path": "images/image_page_23_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_72af184c-53f5-5352-8d28-8736962902fe",
- "type": "image",
- "content": "A Mind Map of RAG Metrics: What do they Measure?\n[images/image_page_25_figure_1.png]",
- "path": "images/image_page_25_figure_1.png",
- "metadata": {
- "length": 84,
- "summary": "",
- "page_nums": [
- 25
- ],
- "file_path": "images/image_page_25_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_664ab58a-7bba-537c-b891-49ea359813e5",
- "type": "image",
- "content": "[images/image_page_30_figure_1.png]",
- "path": "images/image_page_30_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 30
- ],
- "file_path": "images/image_page_30_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_03fc90cc-0291-566d-a56c-af3ff2b08bc0",
- "type": "image",
- "content": "[images/image_page_31_figure_1.png]",
- "path": "images/image_page_31_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 31
- ],
- "file_path": "images/image_page_31_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_f97527e6-ad1b-5b8a-be4d-a8715f5fe19f",
- "type": "image",
- "content": "[images/image_page_31_figure_2.png]",
- "path": "images/image_page_31_figure_2.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 31
- ],
- "file_path": "images/image_page_31_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_d75b0c42-8a92-5a35-abad-106b216c692d",
- "type": "image",
- "content": "Contributors\n[images/image_page_32_figure_1.png]",
- "path": "images/image_page_32_figure_1.png",
- "metadata": {
- "length": 48,
- "summary": "",
- "page_nums": [
- 32
- ],
- "file_path": "images/image_page_32_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_f78e041f-4b77-5495-8b2d-acf02de5188a",
- "type": "image",
- "content": "Contributing Companies\n[images/image_page_32_figure_2.png]",
- "path": "images/image_page_32_figure_2.png",
- "metadata": {
- "length": 58,
- "summary": "",
- "page_nums": [
- 32
- ],
- "file_path": "images/image_page_32_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_685c9285-10f4-5f16-9004-99b1b774bfaa",
- "type": "image",
- "content": "We offer a unique set of offerings to help companies on their way to becoming AI shapers\n[images/image_page_33_figure_1.png]",
- "path": "images/image_page_33_figure_1.png",
- "metadata": {
- "length": 124,
- "summary": "",
- "page_nums": [
- 33
- ],
- "file_path": "images/image_page_33_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "node_899fb3b8-5a27-5581-874a-036e2ae39610",
- "type": "page",
- "content": "June 2024\nRetrieval-augmented \nGeneration Realized: \nStrategic & Technical Insights \nfor Industrial Applications\n\nappliedAI \n2\nContents\nContents \b\n2 \nAbout This White Paper \b\n4\n Key Takeaways \b\n5\n \nSection 1: \nRAG Industrialization - Landscape & Strategy \b\n6\nLet's Start with a Crash Course of RAG \b\n7\nWait! Naive RAG Doesn't Work Well! \b\n9\nSo, What Are Some Techniques to Enhance RAG? \b\n10\nAnd What Tools Can I Use to Develop a RAG System? \b\n12\nAlright, How Can I Strategically Prioritize My Development Focus to \nIndustrialize a RAG Project? \b\n13\nSection 2: \nRAG Recipes for Real-World Challenges \b\n14\nThe Cold Start Recipe: Data-Driven Chunking & Embedding Strategy \nWithout Evaluation Dataset \b\n15\nThe Virtual Havruta Recipe: Optimizing Queries for Multifaceted Contexts \n& Precise Citations \b\n17\nThe Deepset Recipe: Unlocking the Mastery of Metadata for Filtering, \nSearching, and Reranking \b\n18\nThe Jina AI Recipe: Agent-Driven SQL Scoping and Task-Specific \nFinetuning for Patent Search \b\n19\nThe RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-\nchecking \b\n20\n\nRAG: Strategic & Technical Insights\n3\nSection 3: \nA Deep Dive into RAG Evaluation & Metrics \b\n24\nNow, how do we assess RAG systems? \b\n25\nA Further Look into Existing RAG Evaluation Frameworks \b\n26\nUnveiling the Mechanisms Behind These Metrics \b\n28\nReflections and Future Opportunities in RAG Technology \b\n29\nReferences \b\n30\nAuthors \b\n32\nContributors \b\n33\nContributing Companies \b\n33\nAbout appliedAI Initiative GmbH \b\n34\nAcknowledgement \b\n35",
- "path": "Retrieval-Augmented Generation Realized.pdf/Contents",
- "metadata": {
- "length": 1550,
- "summary": "The 'Contents' section outlines the structure of a white paper on Retrieval-Augmented Generation (RAG). It lists three main sections: Section 1 covers RAG industrialization, landscape, and strategy; Section 2 details 'RAG Recipes' for real-world challenges like data-driven chunking and multimodal reasoning; and Section 3 provides a deep dive into RAG evaluation and metrics. The table also includes preliminary sections like 'About This White Paper' and concluding sections such as 'References', 'Authors', and 'Acknowledgements'.",
- "page_nums": [
- 1,
- 2,
- 3
- ],
- "keywords": [
- "appliedAI Initiative GmbH"
- ],
- "connect_to": [
- {
- "target": "asset_64c4fd25-db45-5e0d-afc0-27b63977ffff",
- "relation": "embeds",
- "ref": "[images/image_page_1_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 1,
- "artifact_ref": "page_citation_assets/page-1.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 2,
- "artifact_ref": "page_citation_assets/page-2.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_566cc7cf-312b-5554-9453-09e239e5db9f",
- "type": "page",
- "content": "appliedAI \n4\nManagers, strategists, and \ntechnical leaders may use this \nsection to:\n•\t Quickly grasp fundamental \nRAG concepts.\n•\t Understand the importance of \nRAG for the industry and the \nRAG technical landscape.\n•\t Gain strategic insights on \nprioritizing RAG enhancement \nmethods.\n•\t Navigate through different \nstages of RAG industrialization.\nRAG developers, engineers, \nand practitioners may use this \nsection to:\n•\t Explore enhancement and \noptimization strategies \nthrough five real-world use \ncases.\n•\t Consider engineering recipes \nto address challenges such \nas precise citation based on \nlessons learned in these cases.\nRAG developers, engineers, \nand practitioners may use this \nsection to:\n•\t Understand how various RAG \nevaluation metrics interact \nwith different components of \nRAG.\n•\t Explore existing frameworks \nfor RAG evaluation and their \nfeatures.\n•\t Gain an intuitive understanding \nof how key metrics function.\nAbout This White Paper\nBackground & Purpose\nT\nhis white paper is the culmination of a series of \nstudies and appliedAI RAG roundtable discussions \nconducted with both internal teams and appliedAI \nindustry partners. It delves into the latest developments \nand challenges surrounding Retrieval-Augmented \nGeneration (RAG) in industry, highlighting its emergence \nas a pivotal cost-effective technique in enhancing the \ntrustworthiness and controllability of Large Language \nModel (LLM) applications over the past year.\nIt aims to analyze industry demands, current \nmethodologies, and hurdles concerning the \ndevelopment and evaluation of RAG, facilitating strategy \ndevelopment and knowledge exchange regarding \npractical use cases across diverse industrial sectors.\nDesign Principles\nS\nimplify Complexity: We adopt the philosophy of \n\"less is more\". Wherever possible, we emphasize \nconciseness and attempt to minimize lengthy \ntextual explanations in favor of succinct, cheat-sheet \nstyle messages that convey essential information.\nF\noster Intuitiveness: Wherever possible, we employ \nvisual illustrations to elucidate intricate ideas, \nincluding the overlapping and distinctive features \nof RAG frameworks, the RAG industrialization journey, \nmethodological approaches for addressing challenges, \nand so on.\nSection 1: \nRAG Industrialization - \nLandscape & Strategy\nSection 2: \nRAG Recipes for Real-World \nChallenges\nSection 3: \nA Deep Dive into RAG \nEvaluation & Metrics\nHow to Use This White Paper",
- "path": "Retrieval-Augmented Generation Realized.pdf/About This White Paper",
- "metadata": {
- "length": 2444,
- "summary": "This section outlines the background, purpose, and design principles of the white paper. It explains that the document is based on studies and roundtable discussions regarding Retrieval-Augmented Generation (RAG), aiming to analyze industry demands and facilitate strategy development. The design principles emphasize simplifying complexity and fostering intuitiveness through visual illustrations. Additionally, it details how to use the white paper by describing three main sections: RAG Industrialization Landscape & Strategy, RAG Recipes for Real-World Challenges, and a Deep Dive into RAG Evaluation & Metrics.",
- "page_nums": [
- 4
- ],
- "keywords": [
- "appliedAI"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 4,
- "artifact_ref": "page_citation_assets/page-4.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_63b1fac2-6d61-58ac-bebe-cc27112bfd05",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation Realized.pdf/About This White Paper p4]\n\nRAG: Strategic & Technical Insights\n5\nKey Takeaways\nRAG Industrialization - Landscape & Strategy\nFor sustainable industrial knowledge retrieval and question-answering, RAG \nsolutions are essential due to their trustworthiness, consistency, controllability, \ncost efficiency, etc. Exploring advanced techniques like HyDE and adaptive \nretrieval can enhance quality, though resource constraints must be considered. \nRecognizing challenges early in the RAG industrialization journey is crucial for \neffectively prioritizing development tasks and reducing potential risks related to \nquality, robustness and costs in productionizing RAG solutions.\nRAG Recipes for Real-World Challenges\nWe present five recipes addressing challenges such as limited initial evaluation \ndata for chunking and embedding method selections, adapting to complex \ncontexts and domain-specific conventions, and enhancing relevance through \nmetadata, SQL queries, task-specific finetuning, and multimodal RAG-augmented \nreasoning. Improving retrieval quality is essential for creating reliable and robust \nRAG solutions. This begins with cost-effective strategies such as metadata \nfiltering and hybrid search, and is followed by advanced agentic approaches for \nfurther enhancement.\nA Deep Dive into RAG Evaluation & Metrics\nAssessing RAG systems is complex due to the need to evaluate the interplay \namong questions, contexts, ground truth, and responses using metrics like \ncontext relevance, recall, precision, and answer correctness. Although LLM \nframeworks are emerging to support RAG evaluation, no single framework covers \nall aspects comprehensively. The industry seeks a standardized framework to \nensure consistent quality, reliability, and scalability assessments throughout RAG \ndevelopment and benchmarking.\n1\n2\n3",
- "path": "Retrieval-Augmented Generation Realized.pdf/Key Takeaways",
- "metadata": {
- "length": 1879,
- "summary": "The 'Key Takeaways' section summarizes the white paper's three main parts. It highlights that RAG solutions are essential for sustainable industrial knowledge retrieval due to their trustworthiness and cost efficiency, though resource constraints must be managed. It introduces five recipes for real-world challenges like limited data and complex contexts, emphasizing improved retrieval quality through strategies like metadata filtering and hybrid search. Finally, it notes the complexity of RAG evaluation involving multiple metrics and the industry's need for a standardized framework to ensure consistent quality and scalability.",
- "page_nums": [
- 4,
- 5
- ],
- "keywords": [
- "RAG",
- "HyDE"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 4,
- "artifact_ref": "page_citation_assets/page-4.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 5,
- "artifact_ref": "page_citation_assets/page-5.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_6f822639-69d1-56fb-90e5-3e4f127a0a08",
- "type": "page",
- "content": "appliedAI \n6\nSection 1: \nRAG Industrialization - \nLandscape & Strategy\n\nRAG: Strategic & Technical Insights\n7\nIn the era of Generative AI, Large Language Models (LLMs) are \ntransforming information processing and question answering \nacross industries. A typical LLM-only pipeline looks like this:\nLet's Start with a Crash Course of RAG\nWhat LLM Hallucination Looks Like\nGround Truth Data\nQuestion Answering with LLM\nThree Major Challenges Encountered \nby LLM-Only Approach\nHallucinations\nLLMs can \ngenerate \ninaccurate or \nnonsensical \ninformation. \nKnowledge \nCut-off\nLLMs' knowledge \nare limited to \nthe information \navailable in their \ntraining dataset \nup to their last \nupdate. \nNo Access \nto Private \nEnterprise Data\nLLMs cannot \naccess or \nutilize private, \nproprietary \ndata.\nUser Query\nLLM\nAnswer\nWhy is Retrieval-augmented Generation (RAG) Important for Enterprises?\nTrustworthiness & Reliability\nRAG responses are backed by \nverified and up-to-date documents, \nimproving the reliability and \ntrustworthiness of the generated \ncontent.\nCost Efficiency\nMinimizing the need for expensive and \ntime-consuming model finetuning \nwhile flexibly integrating different \nLLMs and small language models \n(SLMs) into various components to \noptimize cost efficiency.\nConsistency & Robustness\nEnsuring consistent and robust \nanswers by always referencing the \nsame documents, which can be \nreviewed and updated.\nProcess Optimization\nInformation retrieval, database design \nand contents, search modules, and \nother components can be optimized \nwhenever needed, independently of \nthe model.\nAuditability, Explanability, and \nTransparency\nEasier to trace the source of \ninformation provided in responses.\nScalability\nEasier to scale as updates involve \nmodifying the data sources rather than \nretraining the entire model.\nMulti-step Reasoning and Retrieval\nRAG enables integration with \nreasoning capabilities to tackle \ncomplex retrieval and planning tasks \nthat demand multi-layered analysis \nand verification.\nControllability & Configurability\nAllowing enterprises to control and \nupdate the knowledge base and \npipeline components separately from \nthe model.\nIP & Data Secrecy\nFine-grained control over who can \nupdate or access the underlying data \nas well as the range of data that may \nbe retrieved to generate responses.\nChallenges of LLM-Only Approaches\n(cf. Ye et al. [1]) \nIgor Fyodorovich Stravinsky (17 June 1882 – 6 April \n1971) was a Russian composer and conductor \nwith French and American citizenship.\nBéla Viktor János Bartók (25 March 1881 – 26 \nSeptember 1945) was a Hungarian composer, \npianist and ethnomusicologist.\nQ: In which field of music did Igor \nFyodorovich Stravinsky and Béla Viktor \nJános Bartók both have expertise?\n Wrong Answer: Both Igor Fyodorovich \nStravinsky and Béla Viktor János Bartók \nwere famous violinists.\n Right Answer: Both Igor Fyodorovich \nStravinsky and Béla Viktor János Bartók \nwere famous composers.\n Info",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/Let's Start with a Crash Course of RAG",
- "metadata": {
- "length": 2968,
- "summary": "This section provides an overview of the challenges associated with using Large Language Models (LLMs) in isolation, such as hallucinations, knowledge cutoffs, and lack of access to private data. It illustrates these issues with examples involving historical figures like Igor Stravinsky and Béla Bartók. The content then details the importance of Retrieval-Augmented Generation (RAG) for enterprises by listing key benefits including trustworthiness, consistency, controllability, auditability, process optimization, IP security, cost efficiency, scalability, and multi-step reasoning capabilities.",
- "page_nums": [
- 6,
- 7
- ],
- "keywords": [
- "Igor Fyodorovich Stravinsky",
- "Béla Viktor János Bartók"
- ],
- "connect_to": [
- {
- "target": "asset_b6c55aab-84c7-5cfb-b518-d8cd7ae5ab58",
- "relation": "embeds",
- "ref": "[images/image_page_7_figure_1.png]"
- },
- {
- "target": "asset_f8cbf310-8e37-5424-8120-d4ab1e2fe789",
- "relation": "embeds",
- "ref": "[images/image_page_7_figure_2.png]"
- },
- {
- "target": "asset_dd6a0f64-1bb6-5e82-b172-48663b64a7f9",
- "relation": "embeds",
- "ref": "[images/image_page_7_figure_3.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 6,
- "artifact_ref": "page_citation_assets/page-6.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 7,
- "artifact_ref": "page_citation_assets/page-7.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_56c5fe45-287d-50bd-96b8-8c3cd38aab3a",
- "type": "page",
- "content": "appliedAI \n8\n...I am unable to provide \ncomments on future \nevents. Currently, I do not \nhave any information \nregarding the debate \nbetween Elon Musk and \nYann LeCun in May 2024...\nHow would you evaluate the \nrecent debate between Elon \nMusk and Yann LeCun \nregarding what science is \naround the end of May 2024?\nQuestion :\nHow would you evaluate the \nrecent debate ...?\nPlease answer the above \nquestions based on the \nfollowing information :\nChunk 1 :\nChunk 2 :\nChunk 3 :\nChunk 1: \nThe debate started on May 27 \nafter Musk advocated for a \nrigorous pursuit of truth, \ndisregarding popularity or \npolitical correctness, on the \nplatform X.\nChunk 2: \nLeCun criticized Musk for \nspreading conspiracy theories \nand highlighted his own \nresearch contributions, \nchallenging Musk about his \nscientific endeavors.\nChunk 3: \nLeCun argued that scientific \nresearch needs to be published \nto qualify as science, \nemphasizing its role in \ntechnological progress.\nwithout RAG\nThe debate initiated on May \n27, 2024, with Musk \nadvocating for truth pursuit \nwithout popularity or \npolitical correctness on \nplatform X. LeCun \ncountered Musk's stance, \ncriticizing him for spreading \nconspiracy theories and \nstressing the necessity of \npublishing research for \nscientific validity and \ntechnological \nadvancement.\nwith RAG\nQuery\nRelevant Documents\nembeddings\nCombine \nContext and \nPrompts\nAnswer\nLLM\nGeneration\nIndexing\nRetrieval\nInput\nDocuments\nChunks Vectors\nOutput\n3.\nPost-retrieval Stage\n4. \nAugmentation & \nGeneration Stage\nUser\n1.\nPre-retrieval Stage\n2.\nRetrieval Stage\nLet's Start with a Crash Course of RAG\nA Brief Overview of RAG as a Solution\nRetrieval-Augmented Generation (RAG) pairs the capabilities of LLMs with dynamic content from external databases.\nSpecifically, RAG involves four stages of work:\n1.\t Pre-retrieval Stage: Preprocess and index the data, and \nstore them in databases. This may also involve chunking \nthe data and obtain their semantic embeddings as well as \npreprocessing incoming queries on the fly.\n2.\t Retrieval Stage: Retrieve relevant documents based on \nsemantic similarities, BM25, or other methods.\n3.\t Post-retrieval Stage: Post-process the retrieved contents, \nsuch as re-ranking the documents.\n4.\t Augmentation & Generation Stage: Combine the post-\nprocessed documents with prompts and generate final \nresponses using an LLM.\n(cf. Gao et al.[2] )\nInteresting Question: \nDoes Long Context Window Solve Everything?\nBackground: With advanced model design and training, LLMs are \nincreasingly efficient and context windows are expanding.\nQuestion: Will we eventually be able to input entire databases \ninto prompts, thereby eliminating concerns about factual \ncorrectness, etc.?\nConsideration: While feasible, it is doubtful that this will be \noptimal for industry use cases, given concerns about cost \nefficiency, controllability, troubleshooting speed, model lock-in \nrisks, IP and data secrecy, and other factors discussed in this \nwhite paper.\n Info",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/Wait! Naive RAG Doesn't Work Well!",
- "metadata": {
- "length": 2990,
- "summary": "This page provides a crash course on Retrieval-Augmented Generation (RAG), defining it as a method that pairs LLMs with external databases. It details the four stages of RAG: Pre-retrieval, Retrieval, Post-retrieval, and Augmentation & Generation. A central diagram illustrates this workflow using a query about a debate between Elon Musk and Yann LeCun to demonstrate how RAG retrieves relevant context to generate an answer compared to a standard LLM response. The bottom section poses an 'Interesting Question' regarding whether long context windows can solve all data retrieval issues.",
- "page_nums": [
- 8
- ],
- "keywords": [
- "Elon Musk",
- "Yann LeCun",
- "Gao et al.",
- "appliedAI"
- ],
- "connect_to": [
- {
- "target": "asset_5ac1ae99-fd71-5ae6-ab11-4f13e040ec4f",
- "relation": "embeds",
- "ref": "[images/image_page_8_figure_1.png]"
- },
- {
- "target": "asset_7586961b-a15c-5b60-8d3a-c7319ae42586",
- "relation": "embeds",
- "ref": "[images/image_page_8_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_6c8bb125-5909-540e-b7ff-f176994c0d70",
- "type": "page",
- "content": "RAG: Strategic & Technical Insights\n9\nWait! Naive RAG Doesn't Work Well!\nNaive RAG systems face inherent limitations of information retrieval and dependence on LLMs, such as those typical failure points \nreported in Barnett et al. [3], and more.\n\t\n●In response to the challenges of \nnaive RAG, Gao et al. [2] introduced \nthe concepts of advanced RAG \nand modular RAG to describe the \nevolution of RAG paradigms during \ndevelopment.\n\t\n●Advanced RAG: Enhances Naive RAG \nby improving retrieval quality with \npre- and post-retrieval strategies \nand refining indexing, e.g., through \nmetadata filtering.\n\t\n●Modular RAG: Introduces modularity \nfor greater flexibility, featuring \nenhanced functional modules and \nvarious module combination patterns, \nsuch as an additional search module \nfor retrieval (see Gao et al. [2] for \nfurther details).\n\t\n●\nLLM optimization is a growing area \nof interest, involving techniques \nvaring in model adaptation and \nexternal knowledge needed. Prompt \nengineering uses the model’s existing \nabilities, while comprehensive \noptimization often requires both RAG \nand finetuning.\n\t\n●\nThe decision to use RAG or finetuning \nshould align with the specific needs \nof the context, such as subject matter \nknowledge, domain specificity, and the \ngoal of improving accuracy.\nChallenges of a Naive RAG System\nFrom Naive RAG to Advanced and Modular RAG\nRAG\nModules\nPatterns\nRetrieve\nRead\nRetrieve\nRead\nRetrieve\nRead\nRewrite\nRerank\nNaive RAG\nAdvanced RAG\nDSP\n(Khattab et al.,2022)\nITER-RETGEN\n(Shao et al.,2023)\nPredict\nDemonstrate\nSearch\nRetrieve\nRead\nModularRAG\nNaive RAG\nAdvanced RAG\nUser\nQuery\nDocuments\nUser\nQuery\nIndexing\nRouting\nDemonstrate\nDocuments\nIndexing\nOutput\nPre-retrieval\n•Query Routing\n•Query Rewriting\n•Query Expansion\nRetrieval\nRetrieval\nFrozen LLM\nPrompt\nOutput\nFrozen LLM\nPrompt\nPost-retrieval\nRerank\nSummary\nFusion\nRewrite\nRerank\nRead\nRetrieve\nSearch\nMemory\nFusion\nPredict\nExternal \nKnowledge \nRequired\nModel \nAdaptation \nRequired\nHigh\nHigh\nLow\nLow\nFinetuning\nRAG\nPrompt \nEngineering\nAll of the above\nRetriever Finetuning\nStandard Prompt\nFew-shot Prompt\nXoT Prompt\ne.g. CoT, ToT\nNaive RAG\nAdd relevant contextual \nparagraphs\nAdvanced RAG\nIndex/pre-retrieval/ \npost-retrieval optimization\nModular RAG\nOrganic combination of \nmultiple modules\nCollaborative Finetuning\nGenerator Finetuning\nQuery\nRelevant Documents\nembeddings\nFragility\nImprecision\nMisleadingness\nDomain-specific \nranking expectations\nMissed Top-ranked\nNot in Context\nMissing Content\nNot Extracted or \nIncomplete\nInconsistency\nWrong Cut-off Point\nIncorrect \nSpecificity\nWrong Format\nUnfaithfulness\nCombine Context \nand Prompts\nAnswer\nLLM\nGeneration\nIndexing\nRetrieval\nInput\nDocuments\nChunks Vectors\nOutput\n1.\nPre-retrieval Stage\n2.\nRetrieval Stage\nUser\n3.\nPost-retrieval Stage\n4. \nAugmentation & \nGeneration Stage\n(cf. Gao et al. [2], Barnett et al.[3])\n(Adapted from Gao et al. [2])\n(Adapted from Gao et al. [2])\n\nappliedAI \n10\nSo, What Are Some Techniques to Enhance RAG?\nWe identified 12 advanced or modular RAG techniques that \ncan enhance different RAG components and estimated their \noverall complexity in terms of resource consumption, cost, \nlatency, duration of development cycle, and maintenance, \nalong with their projected contribution to overall quality and \ntrustworthiness.\nThis serves as a starting point to assess which techniques to \nprioritize in a project, considering different levels of resource \nconstraints, especially after identifying weak points in a RAG \nsystem.\nAdvanced RAG Techniques and Modules\nMetadata Filtering\nSearch Module & \nHybrid Search\nTask Adapter\nMultiple Indexing \nStructures\nHypothetical \nDocument \nEmbeddings (HyDE)\nInformation \nor Context \nCompression\nExtra Generation \n& Predicting\nQuery Engineering \n& Fusion\nRetrieval\nRetrieval\nRetrieval\nRetrieval\nRetrieval\nAugmentation\nAugmentation\nAugmentation\nAugmentation\nAugmentation\nPre-retrieval Query Processing\nGeneration\nGeneration\nGeneration\nRetrieval-augmented \nReasoning: \nRAT & RAGAR\nAdaptive Retrieval & \nSelf-RAG\nRouting\nMemory\nRetrieval\nPre-retrieval Query Processing\nRetrieval\nRetrieval\nRetrieval\nAugmentation\nAugmentation\nGeneration\nGeneration\nGeneration\nLOW\nLOW\nMEDIUM\nMEDIUM\nHIGH\nHIGH\nDegree of Complexity \n*Associated with resource consumption, cost, latency, duration of develoment cycle, and complexity of maintenance\nProjected Contribution to Overall Quality & Trustworthiness\nPre-retrieval Data Indexing\nRetrieval",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/So, What Are Some Techniques to Enhance RAG?",
- "metadata": {
- "length": 4430,
- "summary": "This section introduces 12 advanced or modular RAG techniques designed to enhance different components of a RAG system. It presents a chart that plots these techniques based on their 'Degree of Complexity' (considering resource consumption, cost, latency, etc.) versus their 'Projected Contribution to Overall Quality & Trustworthiness'. The identified techniques include Metadata Filtering, Hypothetical Document Embeddings (HyDE), Query Engineering & Fusion, Routing, Search Module & Hybrid Search, Task Adapter, Adaptive Retrieval & Self-RAG, Multiple Indexing Structures, Information or Context Compression, Extra Generation & Predicting, Memory, and Retrieval-augmented Reasoning (RAT & RAGAR).",
- "page_nums": [
- 9,
- 10
- ],
- "keywords": [
- "appliedAI"
- ],
- "connect_to": [
- {
- "target": "asset_0e609320-93dc-55d1-a23f-8a29a9b3cd98",
- "relation": "embeds",
- "ref": "[images/image_page_9_figure_1.png]"
- },
- {
- "target": "asset_6e3d5e2e-c8c9-574b-b54f-0d4486a78553",
- "relation": "embeds",
- "ref": "[images/image_page_9_figure_2.png]"
- },
- {
- "target": "asset_590337a0-bfca-5ab3-9d17-3209cbda9ee3",
- "relation": "embeds",
- "ref": "[images/image_page_9_figure_3.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_79297e09-b227-54a9-97bb-dd2a1dfce6d7",
- "type": "page",
- "content": "RAG: Strategic & Technical Insights\n11\nSo, What Are Some Techniques to Enhance RAG?\nMetadata Filtering\nMain Improvement Areas: \nRetrieval\n Augmentation\nWhat & How:\n•\tAutomatically recognizing scope of \nretrieval and filtering the documents \nusing their metadata.\n•\tAttaching metadata like dates, \npurpose, chapter summaries, etc., to \nchunks.\nMemory\nMain Improvement Areas: \nRetrieval\n \nGeneration\nWhat & How:\n•\tIdentifying LLM memories most \nsimilar to the current input.\n•\tUtilizing a retrieval-enhanced \ngenerator for iterative memory \ncreation and self-improvement.\n•\tAligning the output with the target \ndata distribution during this reasoning \nprocess.\nRouting\nMain Improvement Areas: \nPre-retrieval Query Processing\nRetrieval\n \nWhat & How:\n•\tFlexibly alternating between sources \ndiverse in domain, language, and \nformat based on situation.\n•\tDetermining subsequent action to \nuser queries, including summarization, \nspecific database searches (vector, \ngraph, or relational databases, or \na hierarchy of indices), or merging \ndifferent paths.\nSearch Module & Hybrid Search\nMain Improvement Areas: \nRetrieval\n \nWhat & How:\n•\tImplementing direct searches on \nadditional data sources, such as \nsearch engines, SQL/No-SQL/graph \ndatabases, or user-specified texts or \ntables.\n•\tIntegrating results with those based \non semantic search from vector \ndatabases.\nQuery Engineering & Fusion\nMain Improvement Areas: \nPre-retrieval Query Processing\nRetrieval\n Augmentation\nGeneration\nWhat & How:\n•\t Editing, rewriting, or spliting user queries \nto remove errors and biases or to retrieve \nspecific information.\n•\t Expanding user queries into multiple \ndiverse perspectives and running parallel \nvector searches (multi-query approach).\n•\t Ranking/merging multiple responses \nand aligning the final response with both \nexplicit information and implicit user \nintentions.\nInformation or \nContext Compression\nMain Improvement Areas: \n \n \nAugmentation\n \nGeneration\nWhat & How:\n•\tCondensing and compressing a \nvast amount of relevant information \nextracted from extensive knowledge \nbases.\n•\tFiltering the contents and keeping \nonly the most relevant points before \npassing to LLM.\nMultiple Indexing Structures\nMain Improvement Areas: \nPre-retrieval Data Indexing\nRetrieval\n \nWhat & How:\n•\t Introducing graph structures to enhance \nretrieval by leveraging nodes and their \nrelationships.\n•\t Creating multi-index paths to increase \nefficiency.\nAdaptive Retrieval & Self-RAG\nMain Improvement Areas: \nRetrieval\n Augmentation\nGeneration\nWhat & How:\n•\t Enabling LLMs to determine when to \nsearch for necessary information, similar \nto how an agent uses tools.\n•\t Evaluating relevance and level of support \nof retrieved contexts.\n•\t Critiquing and assessing quality of final \noutput.\nTask Adapter\nMain Improvement Areas: \nRetrieval\n \nWhat & How:\n•\tAutomating retrieval of prompts for \nzero-shot task inputs from a pre-\nconstructed data pool, enhancing \nuniversality across tasks and models.\n•\tUtilizing LLM as a few-shot query \ngenerator and creates task-specific \nretrievers based on generated data.\n•\tLeveraging LLM's generalization \ncapability to develop task-specific \nretrievers with minimal examples.\nExtra Generation & Predicting\nMain Improvement Areas: \nAugmentation\n \nGeneration\nWhat & How:\n•\tUtilizing LLM to generate necessary \ncontext instead of direct retrieval.\n•\tAddressing redundancy and noise in \nretrieved content.\nHypothetical Document \nEmbeddings (HyDE)\nMain Improvement Areas: \nRetrieval\n Augmentation\nWhat & How:\n•\tCreating a hypothetical document \n(answer) to a query and retrieve \ncontexts similar to the hypothetical \ndocument (answer) based on its \nembeddings (HyDE).\n•\tEmphasizing the similarities \nbetween embeddings of potential \ndocuments (answers) and those of \nreal answers.\nRetrieval-augmented Reasoning: \nRAT & RAGAR\nMain Improvement Areas: \nRetrieval\n Augmentation\nGeneration\nWhat & How:\n•\t Merging the concepts of RAG with \nChain of Thought, enabling the system \nto logically reason in a certain direction \nand retrieve relevant contexts (RAT, \nRetrieval-augmented Thought [4]).\n•\t Incorporating both Chain of RAG \nand Tree of RAG alongside Chain of \nVerification steps against the most \ncurrent external web resources for \nmultimodal fact-checking (RAGAR, RAG-\naugmented reasoning [5]).",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/And What Tools Can I Use to Develop a RAG System?",
- "metadata": {
- "length": 4322,
- "summary": "This page, titled 'So, What Are Some Techniques to Enhance RAG?', outlines twelve distinct strategies for improving Retrieval-Augmented Generation systems. The techniques are presented in a grid format and include Metadata Filtering, Query Engineering & Fusion, Task Adapter, Memory, Information or Context Compression, Extra Generation & Predicting, Routing, Multiple Indexing Structures, Hypothetical Document Embeddings (HyDE), Search Module & Hybrid Search, Adaptive Retrieval & Self-RAG, and Retrieval-augmented Reasoning (RAT & RAGAR). Each section details the main improvement areas (such as Retrieval, Augmentation, or Generation) and explains the specific methods used.",
- "page_nums": [
- 11
- ],
- "keywords": [
- "LLM",
- "SQL/No-SQL/graph databases",
- "HyDE",
- "RAT",
- "RAGAR"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_b08c2510-f595-59f4-b60c-20b0e9a0acb8",
- "type": "page",
- "content": "appliedAI \n12\nAnd What Tools Can I Use to Develop a RAG System? \nFrequently Used Tools for RAG Solutions: A Quick Glance at Common and Distinct Features\nUnstructured Data Formats: Plain Text (HTML, MARKDOWN, TXT), Image (JPEG, PNG), Document (CSV, PDF)\nDense Retrieval: Top-k Listing, Similarity Threshold Listing\nGenerators: OpenAI, Cohere, AWS, Hugging Face Models\nCross-encoder Reranker: Hugging Face Sentence Transformer\nUnstructured Data Formats: JSON\nChunking Strategies: Header-based Chunking, Semantic Chunking\nTime-weighted Search\nGenerator: Anthropic\nCross-encoder Reranker: Flashrank\nSelf-querying/Auto-retrieval\nGenerator: Google\nGenerator: Google\nGenerator: Ollama\nGenerator: Azure\nMulti-querying\nGenerator: Ollama\nContextual Compression\nGenerator: Azure\nParent Document\nDiversity Ranker\nMetaFieldRanker\nRecursive Retrieval\nGenerator: Mistral\nGenerator: AlephAlpha\nGuardrail: NVIDIA NeMo\nGuardrail: Guardrails.ai\nGenerator: Mistral\nAuto-merging\nEmbedding Models: AI21, Aleph Alpha, Baidu, Google, Azure, Cohere, \nFastembed, Gradient, Jina, Mistral, Voyage...\nVector Databases: AstraDB, Chroma, Marqo, Neo4j, Pgvector, Azure, \nBaudu, Apache Cassandra, MyScale...\nChunking Strategies: Recursive \nChunking\nChunking Strategies: Hierachical \nChunking\nCross-encoder Reranker: ColBert \nRerank\nCross-encoder Reranker: Jina \nRerank\nCross-encoder Reranker: LLM \nRerank\nCross-encoder Reranker: Jina \nRanker\nUnstructured Data Formats: JSON, \nEPUB, HWP, IPYNB, DOCX, PPT, \nPPTX\nUnstructured Data Formats: TIFF, \nBMP, JSON, DOCX\nEmbedding Models: Anthropic, \nvLLM\nUnstructured Data Formats: TIFF, \nBMP, DOCX, PPTX, XLSX\nEmbedding Models: Azure, \nFastembed, Gradient, Jina, Mistral, \nOllama...\nVector Databases: AstraDB, Chroma, \nMarqo, Neo4j, Pgvector\nChunking Strategies: Fixed-size Chunking\nSparse Retrieval: Keyword-based Listing\nPrompt Templates (query, retrieved documents, instructions, examples, tone, style, output format)\nCross-encoder Reranker: Cohere Rerank\nEmbedding Models: OpenAI, Hugging Face, Cohere, AWS Models\nEmsemble Retrieval: Hybrid Search, Multi-embedding Search\nOutput Parsing (e.g., structured output, syntactically valid output, semantic validation)\nLong-context Reordering (Reordering most similar documents to context beginning/end to avoid Lost-in-the-Middle issue)\nVector Databases: Elasticsearch, FAISS, Milvus, OpenSearch, Pinecone, Qdrant, Weaviate\nMetadata Filtering\nPre-\nretrieval \nStage\nCommon \nFeatures\nCommon \nFeatures\nCommon \nFeatures\nCommon \nFeatures\nDistinct Features\nDistinct Features\nDistinct Features\nDistinct Features\nLangchain \n(v0.01.13)\nLlamaIndex \n(v0.10)\nHaystack \n(v1.25)\nHaystack \n(v2.0)\nRetrieval \nStage\nPost-\nretrieval \nStage\nAugmentation & \nGeneration Stage\n\nRAG: Strategic & Technical Insights\n13\nAlright, How Can I Strategically Prioritize My \nDevelopment Focus to Industrialize a RAG Project?\nThe RAG Industrialization Journey\nUse case identification & pre-\nevaluation (User requirments, \nbudget, infrastructure, critical \nlevel definition, etc.)\nChoosing the right technologies \naccording to company \nrequirements and existing \ninfrastructure\nAbility for fast technique \nassessment\nPairwise statistics\nData & metadata preparation\nSketching the overall scaffold to \nenhance scalable development\nProficiency in designing scalable, \nextensible, and maintainable \nsystem architectures\nHuman feedback (e.g., binary \nlikes/dislikes or A/B tests)\nUse case data exploration\nDetermining whether finetuning \nwill be needed and securing \nresources\nProficiency in data synthesis, \naugmentation, or bootstrapping \ntechniques\nData structure design (e.g., \nadditional keyword indices or \ntraditional search modules)\nMaximize long-term re-usability \nof data/metadata structure and \ncontents\nAbility to identify, prioritize, and \nmitigate technical debt early in \nthe development process\nCustom LLM-based or RAGAR \napproaches\nChunking strategy pre-\nselection/pre-evaluation\nCold-start problem: Little or \nno evaluation dataset in the \nbeginning\nVector DB establishment (data \npreprocessor, dataloader, text \nembedder, indexer)\nManaging technical debt \nto enhance scalability and \nmaintanability in the end.\nEmbedding model pre-\nselection/pre-evaluation & \ndecision on finetuning\nRisk identification: Discover \nmajor risks regarding resources, \nmarket demand\nInitial prompt development\nEvaluation given a limited \namount of data\nVector DB pre-selection/pre-\nevaluation\nManaging stakeholder \nexpectations\nInitial retrieval pipeline \ndevelopment\nLLM tech stack pre-selection/\npre-evaluation\nInitial augmentation \n& generation pipeline \ndevelopment\nLLM pre-selection/pre-\nevaluation\nInitial dIalogue flow and \norchestration development\nInitial Chat UI\nInitial evaluation mechanism\nContinually collecting expert \nfeedback and guarantee value\nSystem architecture \ndevelopment\nAttaining precise retrieval given \na wide spectrum of queries and \nscenarios\nProficiency in identifying key \nmodules for improvement\nRegular RAG evaluation \nmetrics (context relevancy, \nanswer relevancy, response \ncompleteness, faithfulness, \ncontext utilization, factual \naccuracy, context precision, \ncontext recall, answer \ncorrectness, answer similarity)\nLong-term infrastructure \ndevelopment (optimizing \ncomputing & data storage \nresources)\nEnsuring scalability & \ninteroperability across different \nunits/regions during fan-out\nAbility to envision the long-term \ngoals and requirements of the \nsystem\nQuality metrics (regular RAG \nmetrics, RAGAR etc.)\nDeploying model updates and \nsystem patches and upgrades\nEnsuring continuous uptime and \navailability of the system\nExpertise in implementing fault-\ntolerant, scalable architectures\nSystem uptime, health, and \nfailure metrics\nPre-retrieval optimization \n(Reconsideration of chunking/\nembedding strategies, data \nstructure, expansion of data)\nPrioritizing optimization modules \nand recipes\nExpertise in compliance and \nsecurity\nCustom LLM-based or RAGAR \napproaches.\nHuman feedback (e.g., binary \nlikes/dislikes or A/B tests)\nLong-term deployment \nAutomation\nEnsuring system robustness & \nreliability\nProficiency in optimizing \nresource utilization\nPerformance metrics (latency, \ncost, token counts etc)\nResource utilization metrics \n(computing, storage)\nContinual retrieval/response \nquality evaluation and \nmonitoring\nScaling the system dynamically \nto handle fluctuating workloads \nand demand spikes\nProficiency in implementing CI/\nCD pipelines\nOutage recovery metrics\nResource utilization metrics \n(computing, storage)\nPerformance metrics (latency, \ncost, token counts etc)\nQuality metrics (regular RAG \nmetrics, RAGAR etc.)\nQuery optimization (e.g., \ncorrection, elaboration, \nquotation, situational info, etc)\nHandling edge cases, \nunexpected inputs, errors.\nEnhancing scalability & \ninteroperability\nOptimizing resource utilization \nto maximize efficiency and \nminimize costs.\nSystem performance, health, \nand failure monitoring\nManaging updates, patches, \nupgrades without disrupting \noperations\nRetrieval optimization (e.g., \ndifferent retrieval methods, \nadditional hybrid search or \nmetadata filtering modules)\nHandling noises and \ninconsistencies in the data\nEnhancing system robustness \n& reliability\nCollaborating with cross-\nfunctional teams, including \noperations, security, compliance \netc, to drive improvement\nResource & cost monitoring, \navoiding over-provisioning.\nResponse quality optimization \n(e.g., iterative response \ngeneration)\nAddressing security, privacy, \nethical, & legal concerns\nEnhancing data privacy \nprotection\nContinual quality, performance, \n& cost optimization\nDIalogue flow and orchestration \noptimization\nDeveloping long-term \nsustainable evaluation metrics \nand automatic dataset updating \nmechanisms\nOptimizing cost & system \nperformance\nConducting regular backups, \ndisaster recovery drills, data \nconsistency checks to ensure \nbusiness continuity\nAnti-attack & ethical/legal \ncompliance mechanism \ndevelopment\nChange Management Process \nestablishment & continual \nbenchmarking (e.g., after \nmodel/system/data updates)\nResponding to incidents, \noutages, and emergencies \npromptly and effectively to \nminimize impact on users and \nbusiness operations.\nUI development\nLong-term monitoring \nand alerting mechanism \ndevelopment\nContinually collecting expert \nfeedback and guarantee value\nPersonal data and log \nmanagement\nMaintence SOP development & \noperational Documentation\nData updating pipeline \ndevelopment\nTraining and knowledge transfer\nDevOps cycle establishment\nContinually collecting expert \nfeedback and guarantee value\nContinually collecting expert \nfeedback and guarantee value\nEvaluation mechanism and \nmetrics development (e.g., \nhuman feedback, automated \nevaluation etc.) -retrieval/\nresponse/system performance.\nBackup, failover, & recovery \nmechanism development\nCore Development Focus\nKey Challenges\nKey Capabilities\nKey Metrics\nIdeation \nIdeation \nIdeation \nIdeation \nPrototyping \nPrototyping \nPrototyping \nPrototyping \nProof of Concept (PoC)\nProof of Concept (PoC)\nProof of Concept (PoC)\nProof of Concept (PoC)\nBest-known Methods (BKM) \nBest-known Methods (BKM) \nBest-known Methods (BKM) \nBest-known Methods (BKM) \nOperation\nOperation \nOperation \nOperation",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/Alright, How Can I Strategically Prioritize My Development Focus to Industrialize a RAG Project?",
- "metadata": {
- "length": 9193,
- "summary": "This section outlines the strategic framework for industrializing a RAG project, structured as 'The RAG Industrialization Journey'. It details four key dimensions—Core Development Focus, Key Challenges, Key Capabilities, and Key Metrics—across five distinct phases: Ideation, Prototyping, Proof of Concept (PoC), Best-known Methods (BKM), and Operation. The content provides specific tasks, potential obstacles, required skills, and performance indicators relevant to each phase of the development lifecycle.",
- "page_nums": [
- 12,
- 13
- ],
- "keywords": [
- "RAG",
- "appliedAI"
- ],
- "connect_to": [
- {
- "target": "asset_2177dee8-30e8-5ed5-99f0-6ed89f2022de",
- "relation": "embeds",
- "ref": "[tables/table_page_12_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 12,
- "artifact_ref": "page_citation_assets/page-12.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 13,
- "artifact_ref": "page_citation_assets/page-13.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_6df0cc41-e6e4-5e9b-b829-c3da83cda97f",
- "type": "page",
- "content": "appliedAI \n14\nSection 2: \nRAG Recipes for \nReal-World Challenges\n\nRAG: Strategic & Technical Insights\n15\nRAG Cold Start Analytics: Exploiting the Potential of Pairwise Cosine Similarities\nThe Underlying Idea\n\t\n●\nNguyen et al. [6] assessed the quality of embeddings \nfrom a general LLM (GPT-3) and a finetuned LLM \nspecific to astronomy (AstroLLaMA) by examining the \ndistribution of pairwise cosine similarity scores (see \nright, divided into 10 equal bins based on similarity levels \nfrom GPT-3).\n\t\n●\nGiven this set of domain-specific documents, \nembeddings by GPT-3 are overly generic with similarities \nclustering around 0.7–0.9, suggesting a lack of \ndiscriminative power.\n\t\n●\nEmbeddings generated by the finetuned model exhibits \na much higher variance within each decile, pointing to a \nhigher proficiency at capturing the semantic diversity in \nthis domain.\n\t\n●\nThis hints at the possibility that quality of embeddings \nmay be reflected in the distribution of pairwise cosine \nsimilarity scores.\n\t\n●\nOn the basis of thorough sanity checks, an even \ndistribution of embeddings may be indicative of a \nmore granular semantic representation, contributing to \nimproved document retrieval.\nThe Cold Start Recipe: \nData-Driven Chunking & Embedding Strategy Without Evaluation Dataset\nBackground & Goals of this Recipe\n\t\n●\nAcross diverse domains, a crucial objective in the initial \nphase of RAG development is to establish an effective \nstrategy for chunking and embedding the data to enable \nefficient and relevant retrieval.\n\t\n●\nThe decisions concerning chunking and embedding at \nthe outset have a significant impact on system design \nand output quality. These choices not only involve the \ncosts of computing resources but are also difficult to \nchange once determined.\nChallenges Addressed\n\t\n●\nScant Availability of an Evaluation Dataset: At project \noutset, a common bottleneck is the absence of a well-\nconstructed dataset developed by domain experts.\n\t\n●\nLack of Time and Resource: In the early stage, developers \noften face time and resource constraints that limit \nextensive validation of their approach. However, they \nrequire a tool to assist in making quick design decisions \ndue to the urgency of showcasing a working prototype to \nstakeholders.\nApplied to the appliedAI AI Act White Paper\n\t\n●\nData: As a prelimiary experiment, we applied this idea \nof looking into pairwise cosine similarities for initial \nchunking/embedding assessment to the appliedAI \nwhite paper: AI Act: Risk Classification of AI Systems \nfrom a Practical Perspective (7511 words).\n\t\n●\nGoal: Gain a quick understanding of the quality of \nembeddings from text-embedding-ada-002 and albert-\nsmall-v2.\n\t\n●\nExperimental Configurations: The Langchain \nCharacterTextSplitter was employed to segment \nthe data, with a chunk size of 1000 and zero overlap \nbetween chunks. Additionally, the Langchain \nQAGenerationChain was utilized to automatically \ngenerate question-answer pairs from these chunks for \na basic RAGAS evaluation using a naive RAG scenario \n(Top-k=1, GPT-3.5 Turbo as the response language \nmodel).",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The Cold Start Recipe: Data-Driven Chunking & Embedding Strategy Without Evaluation Dataset",
- "metadata": {
- "length": 3100,
- "summary": "This section outlines a 'Cold Start Recipe' for establishing data-driven chunking and embedding strategies in Retrieval-Augmented Generation (RAG) systems when an evaluation dataset is unavailable. It addresses challenges such as the scarcity of expert-constructed datasets and resource constraints. The core methodology involves analyzing pairwise cosine similarity scores to assess embedding quality, drawing on research by Nguyen et al. that compares general LLMs (GPT-3) with domain-specific models (AstroLLaMA). A higher variance in similarity scores indicates better semantic representation. The section concludes with a practical application of this analytics approach to the 'appliedAI AI Act White Paper', utilizing Langchain tools like CharacterTextSplitter and QAGenerationChain to generate synthetic question-answer pairs for initial system assessment.",
- "page_nums": [
- 14,
- 15
- ],
- "keywords": [
- "Nguyen et al.",
- "GPT-3",
- "AstroLLaMA",
- "appliedAI",
- "Langchain",
- "RAGAS",
- "GPT-3.5 Turbo"
- ],
- "connect_to": [
- {
- "target": "asset_82250ba7-f4fc-5fbd-bd53-db2d33df30b4",
- "relation": "embeds",
- "ref": "[images/image_page_15_figure_1.png]"
- },
- {
- "target": "asset_a217340b-8aae-5b55-864c-3cd54b679ce4",
- "relation": "embeds",
- "ref": "[images/image_page_15_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 14,
- "artifact_ref": "page_citation_assets/page-14.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 15,
- "artifact_ref": "page_citation_assets/page-15.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_2656378a-05bd-5a23-aa48-66fcdd9d3176",
- "type": "page",
- "content": "appliedAI \n16\n\t\n●\nBased on this idea, appliedAI Initiative developed \na prototype RAG cold start analytic app that \nallows:\n\t\nȃ\nGrid search over different embedding \nmodels, chunking methods, chunk sizes, \nchunk overlaps, retriever search types, and \nnumber of top-K chunks.\n\t\nȃ\nAutomatic QA generation and evaluation \nusing RAGAS metrics.\n\t\nȃ\nExploratory data analysis (EDA) on pair-wise \ncosine similarity score distributions the such \nas descriptive univariate statistics.\n\t\nȃ\nComparative analysis of pair-wise cosine \nsimilarity score distributions by decile \nbetween any two grid points.\n\t\n●\nCompared to other approaches that project \nembeddings into 2D spaces for visual inspection \nof embedding quality, this approach offers a \nconcrete, quantitative measure.\n\t\n●\nBasic sanity checks, e.g., ensuring that similar \ndocuments receive higher pairwise cosine \nsimilarity scores, should be conducted to ensure \nthat the models did not yield noises.\n\t\n●\nResults:\n\t\nȃ The range of pairwise cosine similarity scores from either \ntext-embedding-ada-002 or albert-small-v2 clustered \nabove 0.6, which is not ideal for threshold-based retrieval. \nThis suggests that neither model may be optimal for this \nparticular dataset.\n\t\nȃ However, an initial exploratory data analysis revealed \nthat albert-small-v2 yielded a distribution that was \nslightly more peaked, indicating potentially lower quality \ncompared to text-embedding-ada-002, as also reflected \nin the RAGAS scores.\nQuestions\nGround Truth\nResponse \ntext-embedding-\nada-002\nResponse \nalbert-small-v2\nAccording to \nthe study, what \npercentage of \nthe AI systems \nexamined were in \nthe high-risk class?\n 18%\nAccording to the \nstudy, 18% of the AI \nsystems examined \nwere in the high-risk \nclass.\nThe study does not \nprovide information \non the percentage of \nAI systems examined \nthat were in the high-\nrisk class.\nWhat are the three \nrisk classes outlined \nin the AI Regulation, \nand what are the \nmain requirements \nfor each?\nThe three risk classes \noutlined in the AI \nRegulation are high-risk, \nlow-risk, and unclear. The \nmain requirements for \neach are not specified in \nthe provided context.\nProhibited AI \nsystems, High-risk \nAI systems, and \nLow-risk AI systems.\nI'm sorry, but the \ncontext provided \ndoes not mention \nthe three risk classes \noutlined in the AI \nRegulation or their \nmain requirements.\nContext \nPrecision\nContext \nRecall\nContext \nRelevance\nFaithfulness\nAnswer \nCorrectness\nAnswer \nRelevance\ntext-embedding-\nada-002\n0.85\n0.58\n0.10\n0.67\n0.53\n0.89\nalbert-small-v2\n0.65\n0.37\n0.05\n0.46\n0.29\n0.47\nThe Cold Start Recipe: \nData-Driven Chunking & Embedding Strategy Without Evaluation Dataset\n Prototyping a RAG Cold Start Analytic App",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The Virtual Havruta Recipe: Optimizing Queries for Multifaceted Contexts & Precise Citations",
- "metadata": {
- "length": 2708,
- "summary": "This page details the results of a 'Cold Start Recipe' for data-driven chunking and embedding strategies. It compares two models, text-embedding-ada-002 and albert-small-v2, using tables to show pairwise cosine similarity distributions, RAGAS metrics (like Context Precision and Answer Relevance), and specific QA examples. The bottom section introduces a prototype RAG Cold Start Analytic App developed by appliedAI Initiative, which features grid search experiments, automatic evaluation, and exploratory data analysis visualizations.",
- "page_nums": [
- 16
- ],
- "keywords": [
- "The Cold Start Recipe",
- "text-embedding-ada-002",
- "albert-small-v2",
- "RAGAS",
- "appliedAI Initiative",
- "appliedAI"
- ],
- "connect_to": [
- {
- "target": "asset_0071fbf7-bee1-5b71-afea-d1123f49530d",
- "relation": "embeds",
- "ref": "[tables/table_page_16_1.html]"
- },
- {
- "target": "asset_4b9d9b23-c3d8-5f1d-9399-5d03e6e7319f",
- "relation": "embeds",
- "ref": "[images/image_page_16_figure_2.png]"
- },
- {
- "target": "asset_0f22389e-2cf0-5900-8636-7e17653021d7",
- "relation": "embeds",
- "ref": "[tables/table_page_16_3.html]"
- },
- {
- "target": "asset_942ce9bd-36e8-5ab1-915e-9a71172518ea",
- "relation": "embeds",
- "ref": "[images/image_page_16_figure_4.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 16,
- "artifact_ref": "page_citation_assets/page-16.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_1bcc2537-de4d-559b-953b-52f4e31a95cb",
- "type": "page",
- "content": "RAG: Strategic & Technical Insights\n17\nChallenges Addressed\n\t\n●\nIncomplete Queries: User \nqueries don’t always contain \ncritical information for \nsemantic search.\n\t\n●\nImplicit Assumptions: User \nqueries may be short or \nfragile but carry rich implicit \nbackground knowledge and \nassumptions.\n\t\n●\nNon-Aligned Reranking: \nSimilarity scores or reranking \nmay not always align with \ndomain-specific conventions, \nuser habits, concepts, or \nexpectations about the best \nreferences, e.g., when primary \nsources (e.g., Tanakh, Talmud) \nare preferred over secondary \nsources (e.g., liturgy, \ncommentary).\n\t\n●\nPrecision Demands: Highly \nprecise citations among \nsimilar, neighboring chunks \nmay be required.\nMetadata Filtering & Customized \nReranking\n\t\n●\nMetadata of retrieved references are used to \nlimit the scope on primary sources (e.g., Tanakh, \nTalmud) or secondary sources (e.g., liturgy, \ncommentary).\n\t\n●\nThe results are then ranked based on a weighted \ncombination of cosine similarity scores, \ncustomized LLM-based suitability ratings, and/or \nauthoritative scores specific to Sefaria data.\n\t\n●\nThe authoratative scores indicate the degree of \nimportance of certain references in this context \nand are particular suitable to highlight primary \nsources.\nThe Virtual Havruta Recipe: \nOptimizing Queries for Multifaceted Contexts & Precise Citations\nBackground & Goals of this Recipe\n\t\n●\nVirtual Havruta is a Judaism study companion that uses RAG techniques to generate research-oriented explanations based on \nreliable scriptural references.\n\t\n●\nThe project is an ongoing open-source endeavor, resulting from the collaborative efforts of TUM Venture Labs, Sefaria, \nand the appliedAI Initiative GmbH. It leverages the substantial array of resources and digitalization work that Sefaria has \ncontributed to the realm of Jewish scriptures.\n\t\n●\nTypically, answering a specific question requires rich and multifaceted contextual knowledge, and users expect highly \naccurate responses.\nQuery Engineering, Fusion, & HyDE\n\t\n●\nUser queries are firstly screened, edited, and adapted into \nerror-free, unbiased, serious, and pertinent research-oriented \nquestions.\n\t\n●\nSeveral further query bricks are then evolved, including the \nEnglish translation, key words and concepts associated with the \nquery, an elaborated version of the query with more contexts, \n(hypothetical) scriptural quotations related to the query, critiques \non the query, and potential directions to answer the question.\n\t\n●\nThese query bricks are then recombined into primary source \nquery and secondary source query for retrieval.\nReferences\n1. Reference: Sanhedrin 23b:9. Version Title: \nWilliam Davidson Edition - English. Document \nCategory: Talmud. URL: https://www.sefaria.org/\nSanhedrin.23b.9.\n2. Reference: Kiddushin 76b:13, Version Title: \nWilliam Davidson Edition - English. Document \nCategory: Talmud. URL: https://www.sefaria.org/\nKiddushin.76b.13.\n3. Reference: Mishneh Torah, Forbidden Intercourse \n20:4. Version Title: Mishneh Torah, trans. by Eliyahu \nTouger. Jerusalem, Moznaim Pub. c1986-c2007. \nDocument Category: Halakhah. URL:\nhttps://www.sefaria.org/Mishneh_Torah,_\nForbidden_Intercourse.20.4.l\nReference: Mishneh Torah, Forbidden Intercourse \n20:4. Version Title: Sefaria Community Translation. \nDocument Category: Halakhah. URL:\nhttps://www.sefaria.org/Mishneh_Torah,_\nForbidden_Intercourse.20.4.\nwhat are the cases in the Talmud pertaining to \nlaws regarding a pwrsons ,ייחוסwhere a testimony \nof an individual is believed only immediately, but \nas some very short time elapses, this individual's \ntestimony is not to be believed\n“Should have retrieved this one [Kiddushin.73b.8]”\n“Marcus Jastrow \nDictionary”\n“Encyclopedia \nJudaica”\n“Maccabees 1 ?” \nReferences\n1. Reference: Rabbeinu Bahya, Shemot 15:25:1. Version Title: \nTorah Commentary by Rabbi Bachya ben Asher, trans. \nEliyahu Munk, 1998. Document Category: Commentary URL: \nhttps://www.sefaria.org/Rabbeinu_Bahya,_Shemot.15.25.1.\n2. Reference: Sforno on Deuteronomy 4:8:1. Version Title: \nEliyahu Munk, HaChut Hameshulash. Document Category: \nCommentary. URL: https://www.sefaria.org/Sforno_on_\nDeuteronomy.4.8.1.\n3. Reference: Rabbeinu Bahya, Bamidbar 19:2:4. Version \nTitle: Torah Commentary by Rabbi Bachya ben Asher, trans. \nEliyahu Munk, 1998.. Document Category: Commentary \nURL: https://www.sefaria.org/Rabbeinu_Bahya,_\nBamidbar.19.2.4.\n“Can you explain the \nconcept of ”דרכי שלום\nPrimary sources (e.g., Tanakh, Talmud), not commentaries, \nshould be the top-ranking sources.\nPrimary Source\nMetadata Filtering\nSimilarity\nLLM Rating\nAuthoritative Score\n:\n&\n*\n*\nSecondary Source\nMetadata Filtering\nSimilarity\nLLM Rating\n:\n&\n*\n*\nUser Query\nScreening LLM\nTranslation\nSecondary Source Query \n(Liturgy, Commentary )\nPrimary Source Query \n(Tanakh, Talmud )\nKey Concepts\nElaboration\nQuotation\nChallenge\nDirections\nReformed Query\nEditing\nAdapting\nMulti-querying and query-brick \nrecombination for multifaceted \nsemantic search",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The Deepset Recipe: Unlocking the Mastery of Metadata for Filtering, Searching, and Reranking",
- "metadata": {
- "length": 4971,
- "summary": "This document page outlines the 'Virtual Havruta Recipe,' a project designed to optimize queries for multifaceted contexts and precise citations in Jewish scripture study. It details the background, goals, and challenges addressed by this RAG-based system, such as incomplete queries and non-aligned reranking. The page further explains the query engineering process, including screening, editing, and adapting user inputs into primary and secondary source queries, alongside metadata filtering strategies.",
- "page_nums": [
- 17
- ],
- "keywords": [
- "TUM Venture Labs",
- "Sefaria",
- "appliedAI Initiative GmbH"
- ],
- "connect_to": [
- {
- "target": "asset_36fe7964-1a89-51be-b9c4-71519207bcad",
- "relation": "embeds",
- "ref": "[images/image_page_17_figure_1.png]"
- },
- {
- "target": "asset_fad0bd83-3ee7-5252-95c6-1017279b6565",
- "relation": "embeds",
- "ref": "[images/image_page_17_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_79ce7558-0622-5206-97cc-68534eb96e56",
- "type": "page",
- "content": "appliedAI \n18\nThe Deepset Recipe: \nUnlocking the Mastery of Metadata for Filtering, Searching, and Reranking\nBackground & Goals of this Recipe\n\t\n●\nThis recipe aims at enhancing document retrieval and \nanswer quality by leveraging metadata to preserve \ncontextual understanding across the pipeline of a RAG-\nbased question answering system.\n\t\n●\nThis recipe has been applied in several use cases, \nexpanding the functionality of RAG-based systems \nwithin legal, academic, and media domains, particularly \nin developing RAG-enabled chatbots for efficiently \nanswering queries within large legal and newspaper \ncorpora.\nMetadata for Hybrid Search\n\t\n●\nMetadata can be utilized as strict filters that refine the \nscope of search results.\n\t\n●\nMetadata can be used to guide keyword-based search \nalgorithms (e.g. BM25) by including specific metadata \nfields, enhancing the relevancy of results.\n\t\n●\nMetadata can be used also for embedding retrieval (vector \nsearch) through vectorization of both the document's \ntextual content and selected metadata fields to improve \nsearch precision.\nMetadata for Enhancement of Relevance and Consistensy\nChallenges Addressed\n\t\n●\nData Variability: Handling missing or inconsistent metadata \nacross different documents is a significant challenge.\n\t\n●\nMetadata Relevance: Identifying which metadata fields are \nrelevant for the types of questions that end-users will ask \ncan be challenging.\nMetadata for Reranking\n\t\n●\nSemantic reranking models, such as Cross Encoders, can \nincorporate metadata by prepending its values to the \ndocument text, ensuring that this additional context is \nfactored into the reranking process.\n\t\n●\nTemporal or recentness reranking models prioritize \ndocuments not just by relevance but also by recency, \nutilizing date-based metadata like the Issue Date to \nbalance the importance of content freshness with topical \npertinence.\nHow much money is the contract with \nPear LLC from 2022?\nThe Pear LLC contract is worth 1 \nmillion euros. The project will …\nThe Pear LLC contract is worth 1 \nmillion euros. The project will …\nQuestion\nThe question cannot be answered \nconclusively. There are two contracts \nfor two different amounts.\nAnswer\nPrompt: \nYou are a helpful assistant …\nFile 1:\nThe Pear LLC contract is 1 million \neuros. The project will …\nFile 2:\nThe Pear LLC contract is 20 million \neuros. The project will … …\nGenerator\nFiles\n{\n Issue Date: 2024\n}\nMetadata\nHybrid Search\n + Reranking\nHow much money is the contract with \nPear LLC from 2022?\nOptional: \nAutomatic Filter Extraction\nE.g. Issue Date: 2022\nIssue Date: 2024\nThe Pear LLC contract is 1 million \neuros. The project will …\nQuestion\nThe Pear LLC contract from 2022 is \nworth 20 million euros. \nAnswer\nPrompt: \nYou are a helpful assistant …\nFile 1:\nIssue Date: 2024\nThe Pear LLC contract is 1 million \neuros. The project will …\nFile 2:\nIssue Date: 2022\nThe Pear LLC contract is 20 million \neuros. The project will …\nGenerator w/ Metadata\nHybrid Search\n + Reranking\nExternal Data\nNo Metadata\nWith Metadata\n\nRAG: Strategic & Technical Insights\n19\nAgent-Driven SQL Scoping\n\t\n●\nLLM-agents are implemented to actively identify \nappropriate response scopes as well as document filtering \ncriteria.\n\t\n●\nThe agents then construct corresponding SQL queries \nfrom natural language inputs, incorporating the suitable \nfilters.\n\t\n●\nThe generated SQL queries are enhanced using LLM \nthrough multiple iterations, leading to improved retrieval \nresult quality.\nTask-specific Finetuning\n\t\n●\nTask-specific finetuning focuses on learning private \ncompany knowledge, while domain-specific finetuning is \nbroader, optimizing the model with public data within a \nprofessional field.\n\t\n●\nTriplets of [query, true answer, hard negative answer], \nwere utilized for finetuning [7].\n\t\n●\nOverall performance was dominated by the quality of the \nhard negative answers. \n\t\n●\nIn this case, auto-tuning, i.e., using a LLM to automatically \ngenerate high-quality training data, especially hard \nnegative answers, appears a promising approach.\n\t\n●\nTask-specific finetuning eventually outperforms domain-\nspecific finetuning on NDCG@10.\nThe Jina AI Recipe: \nAgent-Driven SQL Scoping and Task-Specific Finetuning for Patent Search\n Background & Goals of this Recipe\n\t\n●\nWhen assessing a new technology for patenting, legal \nexperts must take much time to thoroughly comprehend \nthe innovation, identify keywords for patent searches, and \ndetermine key elements of the patent application.\n\t\n●\nThis recipe creates a co-pilot to assist patent \nprofessionals in interactively analyzing new patents, \nmitigating IP risks, and identifying innovation scope \nthrough optimized semantic search and patent retrieval \ncapabilities.\nChallenges Addressed\n\t\n●\nInadequate Precision: Classic keyword matching, vector \nsearch, and reranking fall short for intricate, high-\nprecision semantic matching.\n\t\n●\nScoping for SQL Search: Creating dynamic scoping filters \nand crafting reliable SQL queries for structured database \nsearches presents significant complexities.\n\t\n●\nDomain-Specific Embeddings: Tailoring an embedding \nmodel for a specific domain demands a carefully \ndesigned finetuning approach to optimize its \ndiscriminative power for domain-specific texts.\nBicycle Equipped with \nVersatile Dual Chain Drive\n…\ntwo cranksets \nsymmetrically\nmounted\n…\nQuery\nTandem bicycle\n…\nRetrieved Contexts\nStep 1: Establish Baseline\n[Key Techniques] \nEmbeddings + Vector DB\n[Observations]\n�� Decent high-level semantic matching and retrieval.\n�� Detailed-level semantic matching and retrieval didn’t \nwork.\nQuery\n Dual pedaling \nmechanism bicycle\nRetrieved Contexts\nStep 2: Implement Out-of-the-Box Improvements\n[Key Techniques] \nEmbeddings + Vector DB + Reranking (testing different \ncombinations of embeddings and rerankers)\n[Observations]\n�� Improved high-level semantic matching and retrieval.\n�� improved detailed-level semantic matching and retrieval.\nBicycle Equipped with \nVersatile Dual Chain Drive\n…\ntwo cranksets \nsymmetrically\nmounted\n…\nQuery\nTwo wheel drive bicycle \nwith a dual \nchain-and-sprocket \ndrive mechanism driven \nby a two-sided \npaddle-sprocket \nsystem\n...\nRetrieved Contexts\nStep 3a: Apply Additional Optimization \nTechniques without Finetuning\n[Key Techniques] \n Embeddings + Vector DB + Reranking + Chunking + \nAgent-Driven Scoping & SQL Queries\n[Observations]\n�� Detailed-level semantic matching and retrieval was \nsignificantly improved.\nBicycle Equipped with \nVersatile Dual Chain Drive\n…\ntwo cranksets \nsymmetrically\nmounted\n…\nQuery\nTwo wheel drive bicycle \nwith a dual \nchain-and-sprocket \ndrive mechanism driven \nby a two-sided \npaddle-sprocket \nsystem\n...\nRetrieved Contexts\nStep 3b: Task-specific Finetuning of Embedding \nModels\n[Key Techniques] \nLoRA finetuning\n[Observations]\n�� There is a potential for task-specific, “automated” \noptimization.\n�� Detailed-level semantic matching and retrieval was \nsignificantly improved.\n�� Finetuning demands the time-consuming and \nresource-intensive preparation of training data, which \ncan be a costly endeavor.\nBicycle Equipped with \nVersatile Dual Chain Drive\n…\ntwo cranksets \nsymmetrically\nmounted\n…",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The Jina AI Recipe: Agent-Driven SQL Scoping and Task-Specific Finetuning for Patent Search",
- "metadata": {
- "length": 7145,
- "summary": "This section outlines 'The Jina AI Recipe' for patent search, which utilizes agent-driven SQL scoping and task-specific finetuning. It addresses challenges like inadequate precision in classic search methods by implementing LLM agents to construct dynamic SQL queries for filtering. The process involves establishing a baseline, implementing out-of-the-box improvements, and applying task-specific finetuning (e.g., LoRA) to optimize embedding models for domain-specific texts, ultimately improving retrieval quality.",
- "page_nums": [
- 18,
- 19
- ],
- "keywords": [
- "Jina AI",
- "LLM-agents",
- "LoRA"
- ],
- "connect_to": [
- {
- "target": "asset_ffedfc12-6791-5971-bf1d-9dc33ba19f7b",
- "relation": "embeds",
- "ref": "[images/image_page_18_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 18,
- "artifact_ref": "page_citation_assets/page-18.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 19,
- "artifact_ref": "page_citation_assets/page-19.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c1d20713-1136-505f-a061-1a1e94f9b4c4",
- "type": "page",
- "content": "appliedAI \n20\nThe RAGAR Recipe: \nMultimodal RAG-augmented Reasoning for Fact-checking\n Background & Goals of this Recipe\n\t\n●\nThe increasing prevalence of misinformation, particularly \nin the context of political discourse, necessitates \nadvanced solutions for fact-checking.\n\t\n●\nThis recipe introduces RAG-augmented reasoning \n(RAGAR) approaches, in particular Chain of RAG (CoRAG) \nand Tree of RAG (ToRAG) in combination with Chain of \nVerification to achieve evidence-based verification of \nopen-ended multimodal claims [5] .\n\t\n●\nOriginally designed for political fact-checking, RAGAR \ncan be repurposed for various industry applications and \nis ideal for situations requiring thorough verification, \ncomplex reasoning, and accurate fact-checking.\nChallenges Addressed\n\t\n●\nContextualized Multimodal Claim Generation: Generating \nclaims from textual input, while selectively highlighting \nrelevant contextual information and filtering out irrelevant \ndetails from the image, presents its own challenge.\n\t\n●\nOpen-ended Multimodal Evidence Retrieval: Another \nchallenge involves retrieving text and image evidence \nfrom the internet while adhering to the correct time \nframe, filtering out inappropriate sources, and effectively \nutilizing image metadata.\n\t\n●\nDynamic, Agentic Reasoning Actions: Dynamically \ndetermining the most logical reasoning path to follow, \nand deciding when to stop based on sufficient evidence \naccumulation while applying chain of thought and tree \nof thought necessitates careful management of prompts \nand workflow.\nThe RAGAR Approaches: Chain of RAG (CoRAG) & Tree of RAG (ToRAG)\nQuestion\nQA Pair\nRAG\nQuestion\nQA Pair\nRAG\nQuestion\nQA Pair\nRAG\nQuestion\nQA Pair\nRAG\nQuestion\nQA Pair\nRAG\nCollected QA Pairs\nVeracity Prediction\nLabel\nExplanation\nMultimodal Claim Generation\nClaim + Image Context\nQuestion\nQA Pair 1\nRAG\nQuestion\nQA Pair 2\nRAG\nQuestion\nQA Pair 3\nRAG\nQA Elimination\nQA Pair 3\nQuestion\nQA Pair 1\nRAG\nQuestion\nQA Pair 2\nRAG\nQuestion\nQA Pair 3\nRAG\nQA Elimination\nQuestion\nQA Pair 1\nRAG\nQuestion\nQA Pair 2\nRAG\nQuestion\nQA Pair 3\nRAG\nQA Elimination\nQA Pair 1\nQA Pair 2\nTree of RAG\nChain of RAG\nCollected QA Pairs\nVeracity Prediction\nLabel\nExplanation\nClaim\nWestern \nInterconnection\nEastern \nInterconnection\nERCOT \nInterconnection",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Background & Goals of this Recipe",
- "metadata": {
- "length": 2267,
- "summary": "This section outlines the background and goals of the RAGAR recipe, which addresses the increasing prevalence of misinformation in political discourse. It introduces RAG-augmented reasoning (RAGAR) approaches, specifically Chain of RAG (CoRAG) and Tree of RAG (ToRAG), combined with Chain of Verification for evidence-based fact-checking. The text notes that while originally designed for political fact-checking, the method is repurposable for various industry applications requiring thorough verification.",
- "page_nums": [
- 20
- ],
- "keywords": [
- "RAGAR",
- "Chain of RAG",
- "CoRAG",
- "Tree of RAG",
- "ToRAG"
- ],
- "connect_to": [
- {
- "target": "asset_21302afc-9155-5cfb-bb21-c13c3ca879bd",
- "relation": "embeds",
- "ref": "[images/image_page_20_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 20,
- "artifact_ref": "page_citation_assets/page-20.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c66f841f-028e-54cc-917f-fa7dff351345",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Background & Goals of this Recipe p20]",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Challenges Addressed",
- "metadata": {
- "length": 211,
- "summary": "The section titled 'Challenges Addressed' outlines three primary difficulties in multimodal fact-checking. These include Contextualized Multimodal Claim Generation, which involves generating claims from text while filtering relevant image details; Open-ended Multimodal Evidence Retrieval, focusing on retrieving text and image evidence with correct timeframes and source filtering; and Dynamic, Agentic Reasoning Actions, which requires managing prompts to determine logical reasoning paths and when to stop based on evidence accumulation.",
- "page_nums": [
- 20
- ],
- "keywords": [
- "RAGAR",
- "Chain of RAG (CoRAG)",
- "Tree of RAG (ToRAG)"
- ],
- "connect_to": [
- {
- "target": "asset_21302afc-9155-5cfb-bb21-c13c3ca879bd",
- "relation": "related",
- "ref": "[images/image_page_20_figure_1.png]",
- "same_as_owner": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Background & Goals of this Recipe"
- }
- ],
- "page_assets": [
- {
- "page_num": 20,
- "artifact_ref": "page_citation_assets/page-20.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_2e168076-e2ea-560f-a4b1-1fae45fc72f1",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Background & Goals of this Recipe p20]",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Approaches: Chain of RAG (CoRAG) & Tree of RAG (ToRAG)",
- "metadata": {
- "length": 211,
- "summary": "This section illustrates the RAGAR approaches, specifically comparing Chain of RAG (CoRAG) and Tree of RAG (ToRAG). It details a workflow starting with a 'Claim' and 'Multimodal Claim Generation' to produce context. The CoRAG approach is depicted as a linear sequence where questions are generated one by one, each followed by a RAG process to create QA pairs. In contrast, the ToRAG approach shows a branching structure where multiple questions are generated in parallel, processed through RAG, and then undergo 'QA Elimination' steps to filter results before proceeding to the next level. Both methods ultimately collect QA pairs for 'Veracity Prediction', resulting in a label and explanation.",
- "page_nums": [
- 20
- ],
- "keywords": [
- "RAGAR",
- "Chain of RAG",
- "CoRAG",
- "Tree of RAG",
- "ToRAG",
- "Western Interconnection",
- "Eastern Interconnection",
- "ERCOT Interconnection",
- "appliedAI"
- ],
- "connect_to": [
- {
- "target": "asset_21302afc-9155-5cfb-bb21-c13c3ca879bd",
- "relation": "related",
- "ref": "[images/image_page_20_figure_1.png]",
- "same_as_owner": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Background & Goals of this Recipe"
- }
- ],
- "page_assets": [
- {
- "page_num": 20,
- "artifact_ref": "page_citation_assets/page-20.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_498f4fad-220b-5d38-9c14-7934cc66618d",
- "type": "page",
- "content": "RAG: Strategic & Technical Insights\n21\nThe RAGAR Recipe: \nMultimodal RAG-augmented Reasoning for Fact-checking\nClaim:\nThe man next to Mike Pompeo in a November \n2020 photo “is the guy the Trump \nadministration helped get out of jail in 2018 and \nwho is now the ‘president’ of Afghanistan”\nImage context:\nThe image shows Mike Pompeo standing next to a man wearing traditional Afghan attire and a face mask. The \nsetting appears to be a formal meeting room with a painting on the wall, two gold-colored chairs, and a table with \nwater bottles. Both individuals are standing apart, likely adhering to social distancing guidelines prevalent during the \nCOVID-19 pandemic. The background and attire suggest the meeting is official in nature and likely related to \ndiplomatic or governmental affairs. Claim stated in 2021.\nVeracity Label and Explanation\nFollow-up \nRequired?\nQA Pair x\nQA Elimination\nGoogle Reverse\nImage Search \nAPI\nQuestion n\nImage \nQuestion?\nYES\nNO\nQA Pair 2\nQA Pair 3\nQA Pair 1\nQuestion/Followup \nBranch\nEvidence\nRetrieval\n(DuckDuckGo \nSearch)\nGoogle Reverse\nImage Search \nAPI\nQuestion n\nImage \nQuestion?\nEvidence\nRetrieval\n(DuckDuckGo \nSearch)\nQuestion n\nImage \nQuestion?\nEvidence\nRetrieval\n(DuckDuckGo \nSearch)\nGoogle Reverse\nImage Search \nAPI\nYES\nNO\nYES\nNO\nYES\nNO\nRAG-Augmented Reasoning Technique (Chain of RAG)\nRAG-Augmented Reasoning Technique (Tree of RAG)\nClaim:\nThe man next to Mike Pompeo in a \nNovember 2020 photo “is the guy \nthe Trump administration helped \nget out of jail in 2018 and who is \nnow the ‘president’ of Afghanistan”\nImage context:\nThe image shows Mike Pompeo standing next to a man wearing traditional Afghan attire \nand a face mask. The setting appears to be a formal meeting room with a painting on \nthe wall, two gold-colored chairs, and a table with water bottles. Both individuals are \nstanding apart, likely adhering to social distancing guidelines prevalent during the \nCOVID-19 pandemic. The background and attire suggest the meeting is official in nature \nand likely related to diplomatic or governmental affairs. Claim stated in 2021.\nVeracity Label and Explanation\nQuestion n\nImage Question?\nFollow-up \nRequired?\nGoogle Reverse\nImage Search API\nEvidence\nRetrieval\n(DuckDuckGo Search)\nRelevant Evidence\nIf image is relevant, use \nreverse image search to \nextract image captions from \nsources where the image \nappears\nYES\nYES\nNO\nNO",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-Augmented Reasoning Technique (Chain of RAG)",
- "metadata": {
- "length": 2381,
- "summary": "The section titled 'RAG-Augmented Reasoning Technique (Chain of RAG)' outlines a linear workflow for fact-checking using multimodal data. It begins with a specific claim and image context regarding Mike Pompeo. The process involves a decision loop where the system determines if an image question is needed; if yes, it uses Google Reverse Image Search API to find relevant evidence, otherwise it performs standard Evidence Retrieval via DuckDuckGo Search. A follow-up check determines if further questioning is required before generating a final Veracity Label and Explanation.",
- "page_nums": [
- 21
- ],
- "keywords": [
- "Mike Pompeo",
- "Trump administration",
- "Google Reverse Image Search API",
- "DuckDuckGo Search"
- ],
- "connect_to": [
- {
- "target": "asset_f49d6d75-0237-52a9-8167-b74aef5950c8",
- "relation": "embeds",
- "ref": "[images/image_page_21_figure_1.png]"
- },
- {
- "target": "asset_a8014394-038c-5d4d-b73a-bd6bdfa8abb1",
- "relation": "embeds",
- "ref": "[images/image_page_21_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 21,
- "artifact_ref": "page_citation_assets/page-21.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_3e62a695-ffe1-5274-9ca0-f1230f788be7",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-Augmented Reasoning Technique (Chain of RAG) p21]",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-Augmented Reasoning Technique (Tree of RAG)",
- "metadata": {
- "length": 297,
- "summary": "This section details the 'RAG-Augmented Reasoning Technique (Tree of RAG)', a multimodal fact-checking workflow. It begins with a specific claim and image context, then branches into parallel processing paths ('Question/Followup Branch'). Each path independently evaluates an image using either Google Reverse Image Search or DuckDuckGo Evidence Retrieval to generate multiple QA pairs. These pairs undergo elimination before a final decision on whether follow-up is required, ultimately leading to a veracity label.",
- "page_nums": [
- 21
- ],
- "keywords": [
- "Google Reverse Image Search API",
- "DuckDuckGo"
- ],
- "connect_to": [
- {
- "target": "asset_f49d6d75-0237-52a9-8167-b74aef5950c8",
- "relation": "related",
- "ref": "[images/image_page_21_figure_1.png]",
- "same_as_owner": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-Augmented Reasoning Technique (Chain of RAG)"
- },
- {
- "target": "asset_a8014394-038c-5d4d-b73a-bd6bdfa8abb1",
- "relation": "related",
- "ref": "[images/image_page_21_figure_2.png]",
- "same_as_owner": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-Augmented Reasoning Technique (Chain of RAG)"
- }
- ],
- "page_assets": [
- {
- "page_num": 21,
- "artifact_ref": "page_citation_assets/page-21.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_2c0cbb71-94b0-54de-a6e6-1f73f8abf372",
- "type": "page",
- "content": "appliedAI \n22\nChain of RAG & Tree of RAG\n\t\n●\nChain of RAG (CoRAG):\n\t\nȃ\nUses sequential follow-up questions augmented from \nthe RAG response to retrieve further evidence.\n\t\nȃ\nAn early termination check step takes as input the \ngenerated claim and question-answer pair(s) and \nchecks whether enough information to answer the \nclaim has been gathered. \n\t\n●\nTree of RAG (ToRAG):\n\t\nȃ\nCreates question branches at each step of the \nreasoning.\n\t\nȃ\nIn each step, the question-answer pairs are eliminated \nand only the best question-answer branch is chosen \nas the candidate evidence, based on the criteria of \nrelevance, detail, additional information, and answer \nconfidence.\nRAG-enhanced Agents for Enterprises\n\t\n●\nUnfolding developments of RAG-enhanced agents signal \na transformative potential in the realm of enterprise \nsolutions through the harmonization of GenAI and \nadvanced information retrieval techniques and the \nanalytical, reasoning, and agentic prowess of LLMs.\n\t\n●\nThis confluence empowers the advent of intelligent \nagents tailored for complex enterprise applications—from \ndecision-making to strategic planning—heralding an era \nof RAG-enhanced agents equipped to navigate the \nnuanced demands of strategic enterprise contexts.\nThe RAGAR Recipe: \nMultimodal RAG-augmented Reasoning for Fact-checking",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Chain of RAG & Tree of RAG",
- "metadata": {
- "length": 1315,
- "summary": "This section outlines two methodologies for RAG-augmented reasoning: Chain of RAG (CoRAG) and Tree of RAG (ToRAG). CoRAG utilizes sequential follow-up questions to retrieve further evidence and includes an early termination check to determine if sufficient information has been gathered. ToRAG creates question branches at each step of the reasoning process, eliminating pairs until only the best branch is chosen based on relevance, detail, additional information, and answer confidence.",
- "page_nums": [
- 22
- ],
- "keywords": [
- "Chain of RAG",
- "Tree of RAG",
- "CoRAG",
- "ToRAG"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 22,
- "artifact_ref": "page_citation_assets/page-22.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_0f143033-d938-58a5-9d63-15999fe222ce",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Chain of RAG & Tree of RAG p22]\n\nRAG: Strategic & Technical Insights\n23\n“The AI shouldn’t just \nanswer; it should \ndo research first to \ndetermine which of the \nanswers are the best.”\n \n Jenson Huang \nCEO, NVIDIA",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-enhanced Agents for Enterprises",
- "metadata": {
- "length": 385,
- "summary": "This section discusses the transformative potential of RAG-enhanced agents in enterprise solutions. It highlights how the combination of Generative AI, advanced information retrieval, and Large Language Models (LLMs) enables intelligent agents to handle complex applications such as decision-making and strategic planning.",
- "page_nums": [
- 22,
- 23
- ],
- "keywords": [
- "GenAI",
- "LLMs"
- ],
- "connect_to": [
- {
- "target": "asset_8a5ee5a7-0819-5845-ac96-79088150cc37",
- "relation": "embeds",
- "ref": "[images/image_page_23_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 22,
- "artifact_ref": "page_citation_assets/page-22.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 23,
- "artifact_ref": "page_citation_assets/page-23.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_0dd11f91-8f33-57a8-83de-9012469021a3",
- "type": "page",
- "content": "appliedAI \n24\nSection 3: \nA Deep Dive into RAG \nEvaluation & Metrics",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics/Now, how do we assess RAG systems?",
- "metadata": {
- "length": 72,
- "summary": "The page is a section header titled 'Section 3: A Deep Dive into RAG Evaluation & Metrics'. It introduces the third part of a document, which focuses on evaluating and measuring performance metrics for Retrieval-Augmented Generation (RAG) systems.",
- "page_nums": [
- 24
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 24,
- "artifact_ref": "page_citation_assets/page-24.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_eb8b56e9-6ac4-56c3-96d5-978aeff33fb0",
- "type": "page",
- "content": "RAG: Strategic & Technical Insights\n25\nNow, how do we assess RAG systems?\nA Mind Map of RAG Metrics: What do they Measure?\n\t\n●\nOverall, assessing the quality of a RAG system output \nrequires consideration of 4 types of contents: questions \n(user queries), contexts (retrieved relevant references), \nground truth (golden answers), and responses (final \nsystem output).\n\t\n●\nIn the contexts of RAG, besides assessment of a single \ntype of contents, it is particularly important to consider \nthe relationship (relevance, coherence, etc.) between (1) \nquestions and contexts; (2) contexts and ground truth; (3) \ncontexts and responses; (4) responses and ground truth; \n(5) questions and responses.\n\t\n●\nBelow is a mind map summarizing different metrics \nassociated with various aspects of RAG contents, \nshowing what these metrics assess.\n\t\n●\nThis compilation reflects existing RAG metrics \nimplemented in major RAG frameworks, as outlined in \nthe detailed table comparing major RAG evaluation \nframeworks on the next two pages.\n\t\n●\nAn intuitive explanation of key RAG metrics, including their \ncalculation methods, is provided on page 28.\nContexts-\nResponses\nResponses\nContexts\nQuestions-\nContexts\n•\tContext relevance\n•\tRetrieval precision\n•\tString evaluation\n•\tSub-query \ncompleteness\n•\tMulti-query accuracy\n•\tJailbreak detection\n•\tPrompt injection \ndetection\n•\tBias & Toxicity\n•\tGround truth agreement\n [LLM-based Metrics]\n•\tLLM rubric/criteria checking\n•\tGuideline adherence\n•\tAspect critique (Harmfulness, \nmaliciousness, etc.)\n•\tBias & Toxicity\n•\tCoherence, conciseness, \nhelpfulness, consistency\n•\tAnswer relevance\n•\tLLM rubrics/criteria checking\n•\tGuideline adherence\n•\tAnswer completeness\n [Classifier-based Metrics]\n•\tSentiment\n•\tTone & emotion\n•\tBias & Toxicity\n•\tHarassment\n•\tHelpfulness\n•\tEvidence Type\n•\tPersonal Identifiable \nInformation (PII)\n [Deterministic metrics]\n•\tcontains, equals, perplexity, \nregex, starts-with, etc.\n [Conversation Metrics]\n•\tKnowledge Retention\n•\tUser satisfaction\n[Deterministic Metrics]\n•\tLevenshtein distance\n•\tRouge-N\n•\tequals, contains, regex...\n[LLM-based Metrics]\n•\tAnswer Correctness\n•\tAnswer Semantic Similarity\n•\tLLM-rubric/criteria checking\n•\tFactuality\n•\tComprehensiveness\nContexts- \nGround Truth\nQuestions-\nResponses\nGround Truth\n•\tFaithfulness/Groundedness/Utilization\n•\tAugmentation precision\n•\tAugmentation accuracy\n•\tAnswer consistency\n•\tString evaluation\n•\tRelevancy\n•\tContext Precision\n•\tContext Recall\n•\tContext Entities Recall\n•\tMean Reciprocal Rank (MRR)\n•\tMean Average Precision \n(mAP)\n•\tString evaluation (equals, \ncontains, regex...)\nResponses-\nGround Truth\nQuestions\n\nRAG: Strategic & Technical Insights\nappliedAI \n26\n27\nFrameworks\nVersion\nLicense\nIntegration: Compatibile \nFrameworks\nIntegration: Models\nMetrics: Questions-Contexts\nMetrics: Questions-Responses \n (given contexts)\nMetrics: Contexts-Responses \n(given questions)\nMetrics: Contexts-\nGround Truth (given \nquestions and/or \nresponses)\nMetrics: Responses-Ground \nTruth (given questions)\nMetrics: Responses (given \nquestions and/or contexts)\nMetrics: \nGroundTruth\nMetrics: Questions\nMetrics: \nConversation\nCustomized Metrics\nPerformance \nMetrics\nHuman Feedback\nA/B Tests\nLLM Vulnerability Scan\nML Vulnerability \nScan\nModel/Pipeline \nComparison\nPrompt Playground \nUI\nManual Test Set \nCreation UI\nAutomatic Test Set \nCreation\nEvaluation UI\nLLM Monitoring \nDashboard\nEasy Deployment\nTeam Management/\nCollaboration\nGiskard\n2.10.0\nApache-2.0\nGithub, MLflow, Weights & \nBiases, DagsHub, HuggingFace, \nAVID, Pytest\nCatboost, Hugging Face models, \nLangchain models, Pytorch models, \nSklearn models, Tensorflow models\n•\tragas_answer_relevancy\n•\tragas_faithfulness\n•\tragas_context_precision\n•\tragas_context_recall\n•\tCorrectness (CoT)\n•\tLLM-as-a-judge\n•\tGround Truth Similarity\n•\t Hallucination and Misinformation (Plausibility \n& coherency)\n•\t Harmful Content Generation\n•\t Prompt Injection (character level, jailbreaking)\n•\tRobustness (LLM output coherency)\n•\tOutput Formatting\n•\tInformation Disclosure\n•\t Stereotypes and Discrimination\n•\tPerformance Bias\n•\tUnrobustness\n•\tOverconfidence\n•\tUnderconfidence\n•\tUnethical behaviour\n•\tData Leakage\n•\tStochasticity\n•\tSpurious correlation\npromptfoo\n0.50.1\nMIT license\nGitHub Actions, GitLab CI, \nJenkins, Jest, Mocha/Chai\nOpenAI, Anthropic, Azure OpenAI \nmodels, Llama.cpp, Ollama, Google \nVertex models, Google AI Studio, \nGeneric webhook, Custom API \nProvider, Custom scripts, HuggingFace \nmodels, LocalAI models, Replicate \nmodels, Amazon Bedrock models, \nCohere, Groq, Mistral AI models, \nOpenLLM, OpenRouter, Perplexity, \ntext-generation-webui, Together \nAI, vllm\n[Model-graded metrics]\n•\tcontext-relevance\n[Model-graded metrics]\n•\tllm-rubric, answer\n•\trelevance- Similarity\n[Model-graded metrics]\n•\tcontext-faithfulness\n[Model-graded metrics] \n•\tcontext-recall\n[Deterministic metrics]\n•\tcontains, equals, Levenshtein \ndistance, perplexity, regex, \nrouge-n, starts-with\n[Model-graded metrics] \n•\tllm-rubric, model-graded-\nclosedqa, factuality- Similarity\n•\tModel-graded metrics: llm-\nrubric, model-graded-closedqa, \nclassifier grading \n•\tSentiment, Tone and emotion, \nHelpfulness, Grounding, \nfactuality, and evidence-type\n•\tLatency\n•\tCost\n•\tModel-graded metrics: classifier grading - \nToxicity\nRagas\n0.1.6\nApache-2.0\nLlamaIndex, Langchain, \nLangsmith, Arize-Phoenix, \nLangfuse, Athina, Zeno, \nTonicValidate, Haystack\nOpenAI models, Azure OpenAI models, \nAmazon Bedrock models, Google \nVertex AI models\n•\tContext Relevancy\n•\tAnswer Relevance\n•\tFaithfulness\n•\tContext Precision\n•\tContext Recall\n•\tContext entities recall\n•\tAnswer semantic similarity\n•\tAnswer correctness\n•\tAspect Critique (harmfulness, \nmaliciousness)\n•\tAspect Critique\nDeepEval\n0.21.15\nApache-2.0 LlamaIndex, Hugging Face\nLlamaIndex models, Hugging Face \nmodels\nContextualRelevancyMetric\n•\tSummarizationMetric\n•\tAnswerRelevancyMetric (RAGAS)\nFaithfulness (RAGAS)\n•\tContextualRecallMetric \n(RAGAS)\n•\tContextualPrecisionMetric \n(RAGAS)\n•\tGEval\n•\tHallucinationMetric\n•\tBiasMetric\n•\tToxicityMetric\n•\tKnowledgeRetentionMetric\n•\tLatency\n•\tCost\n•\tHallucinationMetric\n•\tBiasMetric\n•\tToxicityMetric\n•\tKnowledgeRetentionMetric\nagenta\n0.12.6\nMIT license\nLangchain, LlamaIndex, and \n“any others”\nOpenAI models, Cohere, or local \nmodels, and “any others”\n•\tExact match\n•\tRegex match\n•\tWebhook evaluator (Correctness)\n•\tSimilarity match (Jaccard)\n•\tAI Critic (LLM-based)\nTrulens\n0.27.0\nMIT license\nLangchain, LlamaIndex, NeMo, \nGuardrails\nOpenAI models, AzureOpenAI models, \nAmazon Bedrock models, LiteLLM \nmodels, Langchain models\n[Generation-based Stock \nFeedback Functions] \ncontext_relevance, qs_relevance\n[Generation-based Stock Feedback \nFunctions] \nrelevance\n[Generation-based Stock Feedback \nFunctions] \nCombinators: Groundedness\n[Stock Feedback Functions] \nHF language_match\n[Generation-based Stock Feedback \nFunctions] \ncomprehensiveness\n[Stock Feedback Functions] \nHF PII detection, HF positive \nsentiment, HF toxic, OpenAI \nmoderation_harassment \n[Generation-based Stock \nFeedback Functions] \ncoherence, conciseness, \ncorrectness, helpfulness\nCombinators: \nGround Truth \nAgreement \n(agreement_\nmeasure, \nbert_score, \nbleu, mae, \nrouge)\n•\tLatency\n•\tCost\n•\tToken \nCounts\n[Classification-based Stock Feedback Functions]\nHF toxic, OpenAI moderation_harassment, \nOpenAI moderation_harassment_threatening, \nOpenAI moderation_hate, OpenAI moderation_\nhatethreatening, OpenAI moderation_selfharm, \nOpenAI moderation_sexual, OpenAI moderation_\nsexualminors, OpenAI moderation_violence, \nOpenAI moderation_violencegraphic\n[Generation-based Stock Feedback Functions]\ncontroversiality, criminality, harmfulness, \ninsensitivity, maliciousness, misogyny, sentiment, \nstereotypes\nTonic Validate\n4.0.4\nMIT license\nLlamaIndex\nOpenAI models, AzureOpenAI models\nRetrieval precision\n•\tAugmentation precision\n•\tAugmentation accuracy\n•\tAnswer consistency (binary)\n•\tRetrieval k-recall\nAnswer Similarity\nContains Text\n•\tLatency\nLangChain\n0.1.13\nMIT license\nVarious\nVarious\n•\tString Evaluator \n•\tScoring Evaluator\n•\tString Evaluator \n•\tScoring Evaluator\n•\tString Evaluator \n•\tScoring Evaluator\n•\tString Evaluator \n•\tScoring Evaluator\n•\tString Evaluator \n•\tCriteria Evaluation (correctness)\n•\tString Evaluator \n•\tEmbedding Distance\n•\tString Evaluator \n•\tExact Match, Regex Match\n•\tString Evaluator \n•\tScoring Evaluator\n•\tComparison Evaluators \n•\tPairwise embedding distance\n•\tString Evaluator \n•\tCriteria Evaluation (conciseness \nor custom)\n•\tString Evaluator \n•\tCustom String Evaluator \n•\tHF evaluate library (perplexity \netc.)\n•\tString Evaluator \n•\tScoring EvaluatorW\n•\tComparison Evaluators \n•\tPairwise string comparison\n•\tComparison Evaluators \n•\tPairwise embedding distance\n•\tString \nEvaluator \n•\tScoring \nEvaluator\n•\tCallback \n•\tToken \ncounts\n•\tString Evaluator \n•\tCriteria Evaluation (constitutional principles)\n•\t \nLangSmith\n0.1.40\nClosed \nbeta\nVarious\nVarious\n•\tLangChain evaluators\n•\tLangChain evaluators\n•\tLangChain evaluators\n•\tLangChain evaluators\n•\tLangChain evaluators\n•\tLangChain evaluators\n•\tLangChain \nevaluators\n•\tLatency\n•\tCost\n•\tToken \nCounts\n•\tChat Bot Ben\n•\tchmarking using Simulation\nLlamaIndex\n0.10\nMIT license\nUpTrain, DeepEval, Ragas, Tonic \nvalidate, and various others\nVarious\n•\tRelevancyEvaluator\n•\tRetrieverEvaluator\n•\tContextRelevancyEvaluator\n•\tRetrieverEvaluator\n•\tContextRelevancyEvaluator\n•\tAnswerRelevancyEvaluator\n•\tGuidelineEvaluator\n•\tFaithfulnessEvaluator\n•\tRelevancyEvaluator\n•\tCorrectnessEvaluator\n•\tSemanticSimilarityEvaluator\nCost\nPairwiseEvaluator •\tGuidelineEvaluator\nHaystack\n1.25\nApache-2.0 Beir, Basic Agent Memory Tool, \nChainlit, Traceloop, and others\nOpenAI models, Anthropic models, \nCohere models, Hugging Face models, \nAmazon Bedrock models\n•\tRecall\n•\tMean Reciprocal Rank (MRR)\n•\tMean Average Precision \n(mAP)\n•\tExact Match (EM)\n•\tF1\n•\tSemantic Answer Similarity (SAS)\nHaystack\n2.0\nApache-2.0\nDeepEval, Context AI, Ragas, \nUpTrain, Gradient, fastRAG, \nTitan Takeoff Inference Server, \nand others\nOpenAI models, Azure, OpenAI \nmodels, Google AI modes, Google \nVertex AI models, Anthropic models, \nCohere models, Hugging Face \nmodels, Amazon Bedrock/Sagemaker \nmodels, vLLM Invocation Layer, \nFastEmbed, Jina AI embedding \nmodels, INSTRUCTOR embedding \nmodels, Voyage AI embedding models, \nLlama.cpp models, Mistral models, \nmixedbread models, Ollama models\n[RagasEvaluator]\n•\tCONTEXT_\nRELEVANCY[DeepEvalEvaluator]\n•\tCONTEXTUAL_\nRELEVANCE[UpTrainEvaluator]\n•\tCONTEXT_RELEVANCE\n[RagasEvaluator]\n•\tANSWER_\nRELEVANCY[DeepEvalEvaluator]\n•\tANSWER_\nRELEVANCY[UpTrainEvaluator]\n•\tRESPONSE_RELEVANCE\n•\tRESPONSE_COMPLETENESS\n•\tRESPONSE_COMPLETENESS_\nWRT_CONTEXT\n[RagasEvaluator]\n•\tFAITHFULNESS\n•\tCONTEXT_\nUTILIZATION[DeepEvalEvaluator]\n•\tFAITHFULNESS[UpTrainEvaluator]\n•\tRESPONSE_CONSISTENCY\n•\tFACTUAL_ACCURACY\n[RagasEvaluator]\n•\tCONTEXT_PRECISION\n•\tCONTEXT_\nRECALL[DeepEvalEvaluator]\n•\tCONTEXTUAL_PRECISION\n•\tCONTEXT_RECALL\n[RagasEvaluator]\n•\tANSWER_CORRECTNESS\n•\tANSWER_\nSIMILARITY[UpTrainEvaluator]\n•\tRESPONSE_MATCHING\n•\t[RagasEvaluator]\n•\t ASPECT_\nCRITIQUE[UpTrainEvaluator]\n•\tRESPONSE_CONCISENESS\n•\tCRITIQUE_LANGUAGE\n•\tCRITIQUE_TONE- GUIDELINE_\nADHERENCE\nDeepset Cloud 0.041\nClosed \nsource\nHaystack integrated frameworks Haystack models\nReference analysis (top-k \noptimization)\nGroundedness\nLatency\nUpTrain\n0.6.12\nApache-2.0\nOpenAI Evals, LlamaIndex, \nReplicate, Hugging Face, \nLangfuse, Helicone, Zeno, \nand others\nOpenAI models, Azure OpenAI models, \nClaude models, Mistral models, Ollama \nmodels, Together AI models, Anyscale \nmodels\n•\tContext Relevance\n•\tResponse Relevance\n•\tContext Utilization\n•\tFactual Accuracy\n•\tContext Conciseness\n•\tContext Reranking\n•\tResponse Matching\n•\tResponse Completeness\n•\tResponse Conciseness\n•\tResponse Validity\n•\tResponse Consistency\n•\tLanguage Features\n•\tTonality\n•\tCode Hallucination\n•\tSub-Query \nCompleteness\n•\tMulti-Query Accuracy\n•\tPrompt Injection\n•\tJailbreak Detection\n•\tUser \nSatisfaction\n•\tPrompt Injection\n•\tJailbreak Detection\nA Further Look into Existing RAG Evaluation Frameworks",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics/A Further Look into Existing RAG Evaluation Frameworks",
- "metadata": {
- "length": 12015,
- "summary": "The section titled 'A Further Look into Existing RAG Evaluation Frameworks' presents a comprehensive comparison table of various frameworks used for evaluating Retrieval-Augmented Generation (RAG) systems. The table details the features, capabilities, and metrics supported by each framework, including Giskard, promptfoo, Ragas, DeepEval, agenta, Trulens, Tonic Validate, LangChain, LangSmith, LlamaIndex, Haystack, Deepset Cloud, and UpTrain.",
- "page_nums": [
- 25,
- 26
- ],
- "keywords": [
- "Giskard",
- "promptfoo",
- "Ragas",
- "DeepEval",
- "agenta",
- "Trulens",
- "Tonic Validate",
- "LangChain",
- "LangSmith",
- "LlamaIndex",
- "Haystack",
- "Deepset Cloud",
- "UpTrain"
- ],
- "connect_to": [
- {
- "target": "asset_72af184c-53f5-5352-8d28-8736962902fe",
- "relation": "embeds",
- "ref": "[images/image_page_25_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 25,
- "artifact_ref": "page_citation_assets/page-25.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 26,
- "artifact_ref": "page_citation_assets/page-26.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 2382,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_b138705c-10c2-57a2-a40d-c0cb741e59dc",
- "type": "page",
- "content": "appliedAI \n28\nUnveiling the Mechanisms Behind These Metrics\nAn Intuitive View of Key RAG Metrics in Common Practice\nMetric\nContext \nRelevance\nContext \nEntities Recall\nAnswer \nCorrectness\nContext Recall\nMean \nReciprocal \nRank (MMR)\nAnswer \nRelevance\nPrecision@k\nMean Average \nPrecision \n(mAP)\nAnswer \nCompleteness\nContext \nPrecision@k\nAnswer \nSemantic \nSimilarity\nFaithfulness \nGroundedness\nWhat Does It Measure?\nRelevance of retrieved \ncontexts given the \nquestions\nAlignment between \nretrieved contexts and \nground truth in terms \nentities\nAccuracy of answers \nwhen compared to the \nground truth\nHow much ground truth \nappear in retrieved \ncontexts\nAn average of \neffectiveness of ranking \nthe first ground-truth \nrelevant context on top \nacross multipe queries\nRelevance of answers \ngiven the questions\nHow many top-ranked k \ncontexts are relevant to \nthe ground truth\nAn average of \neffectiveness of ranking \nground-truth relevant \ncontexts (at rank k) \nacross multipe queries\nCompleteness of \nresponses given the \nquestions\nEffectiveness of ranking \nground-truth relevant \ncontexts (at rank k)\nSimilarity between \nresponse and ground \ntruth\nFactual consistency \nbetween the answers \nand the given contexts\n# Relevant Sentences in Contexts\n# Sentences in Contexts\n# Entities in Both Contexts & Ground Truth\n# Entities in Ground Truth\nWeighted Average( Answer Semantic Similarity, \nFactual Correctness F1 (Response, Ground Truth) )\nAverage (Reciprocal Rank of the First Relevant \nContext from Multiple Queries)\nAverage( Similarity( Original Question, Artificial \nReverse-Engineered Question [i] ) )\nTrue Positives@k\nTrue Positives@k + False Positives@k\nAverage (Context Precision@k from Multiple \nQueries)\n# Aspects Asked in Question and Answered in \nResponse\n# Aspects Asked in Question\nSum( Precision@Each Relevant Items in Top-k)\n# Relevant Items in Top-k\nSimilarity( Response, Ground Truth )\n# Generated Claims Attributable to Contexts\n# Generated Claims in Response\nThe proportion of \nsentences in the retrieved \ncontexts that are relevant \nto the questions\nThe proportion of entities \nin the ground truth that \nappear also in the contexts\nWeighted average \nbetween answer semantic \nsimilarity and factual \ncorrectness (F1 based on \nfactual overlap)\nThe proportion of \nsentences in the ground \ntruth that are attributable \nto the contexts\nAveraging the reciprocal \nrank of the first relevant \ncontext over multiple \nqueries. The earlier the \nfirst relevant context \nappears, the better.\nAveraging cosine similarity \nof the original question \nto artificial questions \ngenerated based on the \nanswer\nThe proportion of top-\nranked k contexts that \nare relevant to the ground \ntruth\nAveraging Context \nPrecision@k over multiple \nqueries\nThe proportion of aspects \nasked in the question \nthat are answered in the \nresponse\nGet a cumulative view of \nprecision at each relevant \nitem till rank k, and then \naverage over the number \nof relevant items in top-k\nCosine similarity between \nthe response and the \nground truth in the \nembedding space\nThe proportion of \ngenerated claims in \nthe response that are \nattributable to the \ncontexts\nFormula in an Intuitive Form\nIntuitive Explanation of \nFormula\n# Ground Truth Sentences Attributable to Contexts\n# Sentences in Ground Truth",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics/Unveiling the Mechanisms Behind These Metrics",
- "metadata": {
- "length": 3282,
- "summary": "This page presents a comprehensive table titled 'An Intuitive View of Key RAG Metrics in Common Practice,' which details various evaluation metrics for Retrieval-Augmented Generation systems. The table lists 12 specific metrics, including Context Relevance, Precision@k, Mean Reciprocal Rank (MMR), and Answer Correctness. For each metric, it provides a description of what is measured, an intuitive formula representation, and a plain-language explanation of the calculation.",
- "page_nums": [
- 27
- ],
- "keywords": [
- "appliedAI"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 27,
- "artifact_ref": "page_citation_assets/page-27.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_d9a5f76c-5c13-575d-8a1b-e85286617cb5",
- "type": "page",
- "content": "RAG: Strategic & Technical Insights\n29\nReflections and Future Opportunities in RAG \nTechnology\nReflections\nFuture Opportunities in RAG Technology\nSeamless Integration\nThe future of RAG holds the potential \nfor seamless integration into a wide \narray of knowledge retrieval and \nquestion answering applications, such \nas search engines, customer service \nplatforms, in-car assistants, social \nmedia, and knowledge management \nsystems, enhancing user experiences \nwith context-aware responses.\nCoordinated Modules\nCurrently, LLM modules for retrieval \nmay not interpret user intent in the \nsame way as those for generation, \nleading to inconsistencies. Hence, \nthere is potential for these modules \nto achieve better communication and \ncoordination in the future, ensuring \na more unified understanding and \nresponse.\nReasoning & Agents\nIntegrating reasoning and agent \ncapabilities into future RAG solutions \nenhances precision and factuality \nby actively assessing content logic, \nquality, and consistency while \nautonomously adapting to user needs \nand taking suitable actions.\nCross-Modal Capabilities\nThe future of RAG envisions the \nintegration of multimodal data sources, \nincluding text, images, video, audio, \nand other types of (un)structured \ndata, to provide richer and more \ncomprehensive responses, leveraging \ndiverse data types to enhance \ninformation retrieval and generation.\nEvaluation Framework\nThe industry anticipates a \nstandardized and generalized \nframework to comprehensively \nevaluate various aspects of \nRAG systems across different \ndevelopment stages. Such a \nframework would ensure consistent \nassessment of quality, reliability, and \nscalability, guiding improvements \nthroughout the RAG industrialization \njourney.\nSustainability & Long Context \nWindow\nIn the near future, RAG solutions may \nremain a top option for industry due \nto factors like controllability (see \nsection 1). Ensuring long-term viability \nis crucial, especially as LLMs evolve \nwith longer context windows, likely \ncomplementing rather than replacing \nRAG. RAG systems should factor in \nthis evolution for future extensibility in \ntheir design.\n•\t From a sustainable perspective, RAG \nsolutions are crucial for industrial \nknowledge retrieval and question-\nanswering systems, considering \nfactors such as trustworthiness, \nconsistency, controllability, \nauditability, explanability, \ntransparency, process optimization, \nIP & data secrety, cost efficiency, and \nscalability.\n•\t To harness RAG solutions, it is \ncrucial to explore advanced, \nmodular techniques for optimizing \nretrieval, augmentation, and \ngeneration, such as HyDE or \nRAGAR. Finetuning models might \nbe necessary as a last resort, and \nshould be evaluated early on. The \ncomplexity and potential benefits \nmust be balanced against resource \nconstraints and development \nchallenges.\n•\t Recognizing realistic challenges \nearly in the RAG industrialization \njourney is essential for prioritizing \ndevelopment tasks and mitigating \nrisks. For example, limited resources \nat the start may hamper creating a \nthorough evaluation dataset based \non expert feedback.\n•\t We tackled realistic challenges \nsuch as initial scarcity of evaluation \ndata for determining chunking and \nembedding methods, adapting to \nmultifaceted contextual knowledge \nand domain-specific conventions, \nenhancing document relevance \nand consistency through metadata, \nconstructing SQL queries for \ntargeted search, task-specific \nfinetuning, and employing multimodal \nRAG-augmented reasoning (RAGAR) \nfor input verification.\n•\t Overall, retrieval quality emerges as \nthe key area requiring improvement \nin RAG development. Cost-\neffective strategies like metadata \nfiltering, query engineering, fusion, \nhybrid search, and HyDE are initial \nconsiderations. Subsequently, more \nsophisticated agentic approaches \ncould further enhance quality, \naligning with observations in the \ncomplexity/contribution map in \nSection 1.\n•\t Assessing RAG systems is complex \ndue to the interplay among \nquestions, contexts, ground truth, \nand responses, along with the need \nto evaluate these components \nindividually. Key metrics encompass \ncontext relevance, recall, precision, \nanswer semantic similarity, \ncorrectness, relevance, faithfulness, \netc.\n•\t We're observing a trend of growing \nLLM frameworks tailored for \nvarious aspects of RAG evaluation. \nHowever, while the metrics in these \nframeworks align with the mind map \npresented at the outset of Section \n3, there isn't a single framework that \nfully encompasses every aspect of \nRAG evaluation to date.\n•\t While these metrics capture \nspecific facets of RAG, it's \nimportant to acknowledge their \noccasional insufficiency. For \ninstance, evaluating the quality of \nopen-ended questions can pose \nchallenges as there's no fixed set of \ngolden references for such queries.\nSection 1: \nRAG Industrialization - \nLandscape & Strategy\nSection 2: \nRAG Recipes for Real-World \nChallenges\nSection 3: \nA Deep Dive into RAG \nEvaluation & Metrics",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics/Reflections and Future Opportunities in RAG Technology",
- "metadata": {
- "length": 5016,
- "summary": "This document page, titled 'Reflections and Future Opportunities in RAG Technology', outlines key strategic insights regarding Retrieval-Augmented Generation (RAG). The top section details three main areas: industrialization landscape and strategy, recipes for real-world challenges like data scarcity, and a deep dive into evaluation metrics. The bottom section, 'Future Opportunities in RAG Technology', highlights six potential advancements including seamless integration with other systems, reasoning agents, standardized evaluation frameworks, coordinated modules, cross-modal capabilities, and sustainability.",
- "page_nums": [
- 28
- ],
- "keywords": [
- "HyDE",
- "RAGAR"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 28,
- "artifact_ref": "page_citation_assets/page-28.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_38bc5a23-552b-5d58-95a5-52a0a813be35",
- "type": "page",
- "content": "appliedAI \n30\nReferences\n[1]\t\nH. Ye, T. Liu, A. Zhang, W. Hua, and W. Jia, “Cognitive Mirage: A Review of Hallucinations in Large \nLanguage Models.” arXiv, Sep. 13, 2023. Accessed: Sep. 19, 2023. [Online]. Available: http://arxiv.org/\nabs/2309.06794.\n[2]\t\nY. Gao et al., “Retrieval-Augmented Generation for Large Language Models: A Survey.” arXiv, Jan. 03, \n2024. Accessed: Jan. 06, 2024. [Online]. Available: http://arxiv.org/abs/2312.10997.\n[3]\t\nS. Barnett, S. Kurniawan, S. Thudumu, Z. Brannelly, and M. Abdelrazek, “Seven Failure Points When \nEngineering a Retrieval Augmented Generation System.” arXiv, Jan. 11, 2024. Accessed: Feb. 21, 2024. \n[Online]. Available: http://arxiv.org/abs/2401.05856.\n[4]\t\nZ. Wang, A. Liu, H. Lin, J. Li, X. Ma, and Y. Liang, “RAT: Retrieval Augmented Thoughts Elicit Context-\nAware Reasoning in Long-Horizon Generation.” arXiv, Mar. 08, 2024. Accessed: Mar. 14, 2024. [Online]. \nAvailable: http://arxiv.org/abs/2403.05313\n[5]\t\nM. A. Khaliq, P. Chang, M. Ma, B. Pflugfelder, and F. Miletić, “RAGAR, Your Falsehood RADAR: RAG-\nAugmented Reasoning for Political Fact-Checking using Multimodal Large Language Models.” arXiv, \nApr. 18, 2024. Accessed: Apr. 19, 2024. [Online]. Available: http://arxiv.org/abs/2404.12065\n[6]\t\nT. D. Nguyen et al., “AstroLLaMA: Towards Specialized Foundation Models in Astronomy.” arXiv, Sep. 12, \n2023. Accessed: Sep. 13, 2023. [Online]. Available: http://arxiv.org/abs/2309.06126.\n[7]\t\nM. Günther et al., “Jina Embeddings 2: 8192-Token General-Purpose Text Embeddings for Long \nDocuments.” arXiv, Feb. 04, 2024. doi: 10.48550/arXiv.2310.19923.\n\nRAG: Strategic & Technical Insights\n31\n“Generative AI's impressive \nnatural-language processing, \ncombined with RAG's \ncapabilities, revolutionizes \nknowledge management and \ndecision-making by allowing \nemployees to retrieve stored \ninternal knowledge and \nmanage information about \nproducts or processes swiftly \nand effectively, just as they \nwould when asking a human.”\n \n Bernhard Pflugfelder \nHead of Innovation Lab (GenAI), \napplied AI initiative",
- "path": "Retrieval-Augmented Generation Realized.pdf/References",
- "metadata": {
- "length": 2065,
- "summary": "This page features a quote discussing how Generative AI and RAG capabilities revolutionize knowledge management by enabling employees to retrieve internal information as effectively as asking a human. The quote is attributed to Bernhard Pflugfelder, Head of Innovation Lab (GenAI) for the applied AI initiative. The footer indicates this content is from 'RAG: Strategic & Technical Insights' on page 31.",
- "page_nums": [
- 29,
- 30
- ],
- "keywords": [
- "Bernhard Pflugfelder",
- "Innovation Lab (GenAI)"
- ],
- "connect_to": [
- {
- "target": "asset_664ab58a-7bba-537c-b891-49ea359813e5",
- "relation": "embeds",
- "ref": "[images/image_page_30_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 29,
- "artifact_ref": "page_citation_assets/page-29.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 30,
- "artifact_ref": "page_citation_assets/page-30.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_4822ecf6-beeb-5890-897f-ebbaf374d6f8",
- "type": "page",
- "content": "appliedAI \n32\nDr. Paul Yu-Chun Chang\nSenior AI Expert: Foundation Models - LLMs,\nappliedAI Initiative GmbH\np.chang@appliedai.de\nBernhard Pflugfelder works as Head of Innovation \nLab (GenAI) at the appliedAI Initiative GmbH. \nBernhard has 15 years of experience in the fields of \nData Science, Natural Language Processing (NLP), \nas well as data and AI across different companies \nsuch as BMW Group or Volkswagen Group. He is \nrenowned for his expertise especially in the field of \nAI in general, NLP and Generative AI in particular.\nBernhard Pflugfelder\nHead of Innovation Lab (GenAI), \nappliedAI Initiative GmbH\nb.pflugfelder@appliedai.de\nPaul Yu-Chun Chang works as an Senior AI Expert \nspecializing in Large Language Models at appliedAI \nInitiative GmbH. He has 10 years of interdisciplinary \nresearch experience in computational linguistics, \ncognitive neuroscience, and AI, and 6 years of \nindustrial experience in developing AI algorithms in \nlanguage modeling and image analytics. Paul holds \na PhD in Linguistics from LMU Munich, where he \nintegrated NLP and machine learning methods to \nstudy brain language cognition.\nAuthors",
- "path": "Retrieval-Augmented Generation Realized.pdf/Authors",
- "metadata": {
- "length": 1139,
- "summary": "This page serves as an 'Authors' section, presenting biographical profiles for two individuals: Dr. Paul Yu-Chun Chang and Bernhard Pflugfelder. Both are associated with the organization appliedAI Initiative GmbH. The text details their professional roles, areas of expertise in AI and NLP, years of experience, and educational backgrounds.",
- "page_nums": [
- 31
- ],
- "keywords": [
- "Dr. Paul Yu-Chun Chang",
- "Bernhard Pflugfelder",
- "appliedAI Initiative GmbH",
- "LMU Munich",
- "BMW Group",
- "Volkswagen Group"
- ],
- "connect_to": [
- {
- "target": "asset_03fc90cc-0291-566d-a56c-af3ff2b08bc0",
- "relation": "embeds",
- "ref": "[images/image_page_31_figure_1.png]"
- },
- {
- "target": "asset_f97527e6-ad1b-5b8a-be4d-a8715f5fe19f",
- "relation": "embeds",
- "ref": "[images/image_page_31_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 31,
- "artifact_ref": "page_citation_assets/page-31.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_f85bcabe-1aff-50e2-9e48-51e9ac25ab51",
- "type": "page",
- "content": "RAG: Strategic & Technical Insights\n33\nContributors\nContributing Companies\nJohannes Birk\nGenerative AI Engineer, \nappliedAI Initiative GmbH\nj.birk@appliedai.de\nHadara Steinberg\nAssociate Director, Engineering Team, \nSefaria\nhadara@sefaria.org\nEmre Demirci\nJun. AI Engineering LLM,\nappliedAI Initiative GmbH\ne.demirci@appliedai.de\nDr. Sebastian Husch Lee\nSolution Engineering Tech Lead, \ndeepset GmbH\nsebastian.huschlee@deepset.ai\nAntoine Leboyer \nManaging Director,\nTUM Venture Labs\nantoine.leboyer@\nunternehmertum.de\nDr. Saahil Ognawala\nHead of Product,\nJina AI GmbH\nsaahil.ognawala@jina.ai\nLev Eliezer Israel\nChief Product Officer, \nSefaria\nlev@sefaria.org\nMaximilian Werk\nHead of Engineering,\nJina AI GmbH\nmaximilian.werk@jina.ai\nNoah Santacruz\nSenior Research Engineer, \nSefaria\nnoah@sefaria.org\nDamian Depaoli \nAI Engineer,\nappliedAI Initiative GmbH\nd.depaoli@appliedai.de\nMingyang Ma \nPrincipal AI Strategist & Product \nManager,\nappliedAI Initiative GmbH\nm.ma@appliedai.de\nMohammed Abdul Khaliq\nInstitut für Maschinelle \nSprachverarbeitung, \nUniversity of Stuttgart\nmohammed.abdul-khaliq@ims.uni-\nstuttgart.de",
- "path": "Retrieval-Augmented Generation Realized.pdf/Contributors",
- "metadata": {
- "length": 1115,
- "summary": "This page lists the contributors to a document titled 'RAG: Strategic & Technical Insights'. It features headshots, names, job titles, and affiliations for ten individuals. The bottom section displays the logos of contributing companies, including TUM Venture Labs, Sefaria, Jina AI, and deepset.",
- "page_nums": [
- 32
- ],
- "keywords": [
- "Johannes Birk",
- "appliedAI Initiative GmbH",
- "Emre Demirci",
- "Antoine Leboyer",
- "TUM Venture Labs",
- "Lev Eliezer Israel",
- "Sefaria",
- "Noah Santacruz",
- "Damian Depaoli",
- "Hadara Steinberg",
- "Dr. Sebastian Husch Lee",
- "deepset GmbH",
- "Dr. Saahil Ognawala",
- "Jina AI GmbH",
- "Maximilian Werk",
- "Mohammed Abdul Khaliq",
- "University of Stuttgart",
- "Mingyang Ma"
- ],
- "connect_to": [
- {
- "target": "asset_d75b0c42-8a92-5a35-abad-106b216c692d",
- "relation": "embeds",
- "ref": "[images/image_page_32_figure_1.png]"
- },
- {
- "target": "asset_f78e041f-4b77-5495-8b2d-acf02de5188a",
- "relation": "embeds",
- "ref": "[images/image_page_32_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 32,
- "artifact_ref": "page_citation_assets/page-32.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_613f048e-0d01-535b-b183-44fcea7d9368",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation Realized.pdf/Contributors p32]",
- "path": "Retrieval-Augmented Generation Realized.pdf/Contributing Companies",
- "metadata": {
- "length": 70,
- "summary": "The section titled 'Contributing Companies' lists the organizations that contributed to the document. It displays the logos and names of five entities: TUM Venture Labs, Sefaria, Jina AI, deepset, and implicitly appliedAI Initiative GmbH (based on the contributors above).",
- "page_nums": [
- 32
- ],
- "keywords": [
- "TUM Venture Labs",
- "Sefaria",
- "Jina AI",
- "deepset"
- ],
- "connect_to": [
- {
- "target": "asset_d75b0c42-8a92-5a35-abad-106b216c692d",
- "relation": "related",
- "ref": "[images/image_page_32_figure_1.png]",
- "same_as_owner": "Retrieval-Augmented Generation Realized.pdf/Contributors"
- },
- {
- "target": "asset_f78e041f-4b77-5495-8b2d-acf02de5188a",
- "relation": "related",
- "ref": "[images/image_page_32_figure_2.png]",
- "same_as_owner": "Retrieval-Augmented Generation Realized.pdf/Contributors"
- }
- ],
- "page_assets": [
- {
- "page_num": 32,
- "artifact_ref": "page_citation_assets/page-32.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_7374b398-8da2-58d5-94f6-856e9a32fe66",
- "type": "page",
- "content": "appliedAI \n34\nAbout appliedAI Initiative GmbH\nappliedAI is Europe's largest initiative for the application of \ntrusted AI technology. It aims to advance Europe's industry to \nstay competitive in the Age of AI. The initiative was established \nin 2017 by Dr. Andreas Liebl as a division of UnternehmerTUM in \nMunich and transferred to a joint venture with Innovation Park \nArtificial Intelligence (IPAI) in Heilbronn in 2022.\nAt appliedAI, more than 160 employees work together with \n>20 companies in our Partnership to create best practices \non how to apply AI, in Professional Services and Accelerator \nPrograms to engineer AI powered solutions, develop AI \nstrategies and operating models as well as upskill thousands \nof employees and managers. Moreover, appliedAI offers an \nsoftware for managing the AI application portfolio to enhance \nAI Act compliance. appliedAI holistically supports international \ncorporations, like BMW, Porsche, or Siemens, as well as \nmedium-sized companies in their AI transformation.\nFor more information, please visit\nhttps://www.appliedai.de/en/",
- "path": "Retrieval-Augmented Generation Realized.pdf/About appliedAI Initiative GmbH",
- "metadata": {
- "length": 1081,
- "summary": "This page provides an overview of appliedAI Initiative GmbH, describing it as Europe's largest initiative for trusted AI technology. It details the organization's history, its establishment in 2017 by Dr. Andreas Liebl, and its current operations involving over 160 employees and a partnership with more than 20 companies. The lower section features a diagram illustrating the company's offerings, categorized into Professional Services, Partnership Programs, Software Solutions, and Ventures, mapped against maturity levels from 'Not started' to 'Shaper'.",
- "page_nums": [
- 33
- ],
- "keywords": [
- "appliedAI Initiative GmbH",
- "appliedAI",
- "Dr. Andreas Liebl",
- "UnternehmerTUM",
- "Munich",
- "Innovation Park Artificial Intelligence (IPAI)",
- "Heilbronn",
- "BMW",
- "Porsche",
- "Siemens"
- ],
- "connect_to": [
- {
- "target": "asset_685c9285-10f4-5f16-9004-99b1b774bfaa",
- "relation": "embeds",
- "ref": "[images/image_page_33_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 33,
- "artifact_ref": "page_citation_assets/page-33.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_ac78d3c9-8ca9-5174-abd8-344fda888c44",
- "type": "page",
- "content": "RAG: Strategic & Technical Insights\n35\nAcknowledgement \nWe express our sincere appreciation for the remarkable work \ncarried out by the authors, reviewers, and designer of this \nwhite paper. Their great expertise, motivation and dedication to \nevery detail made this white paper an exceptional contribution \nto the AI Community.\nFurthermore, we extend our gratitude to the contributors \nfrom Deepset, Jina.ai, Sefaria, and TUM Venture Labs for \ntheir valuable contributions. Their profound expertise and \ncommitment have played a pivotal role in shaping the ideas and \nknowledge presented in this paper.\nWe would also like to thank all the appliedAI partner companies \nthat actively participated in the appliedAI RAG Roundtables and \nbilateral interactions. Your involvement has been indispensable, \nproviding fundamental references and serving as a constant \nsource of inspiration.\nThe content presented in this white paper has been influenced \nand inspired by discussions and exchanges within the appliedAI \nRoundtables on Retrieval Augmented Generation in October \n2023 and April 2024, including the appliedAI industry partners \nAtruvia AG, BMW Group, Giesecke+Devrient GmbH, EnBW \nEnergie Baden-Württemberg AG, Infineon Technologies AG, \nLinde Group, Miele & Cie. KG, Munich Re, Group, MTU Aero \nEngines AG, NVIDIA, Roche Diagnostics GmbH, Rohde & \nSchwarz GmbH & Co. KG, Siemens AG, Telekom Deutschland \nGmbH and Wacker Chemie AG.\nThe collective expertise, exchange and dedication to \nadvancing the knowledge in Generative AI and RAG was a \ngreat inspiration throughout the process of creating this white \npaper.\n\nRetrieval-augmented Generation \nRealized: Strategic & Technical Insights \nfor Industrial Applications\nappliedAI Initiative GmbH\nAugust-Everding-Straße 25 \n81671 München \nGermany \nwww.appliedai.de",
- "path": "Retrieval-Augmented Generation Realized.pdf/Acknowledgement",
- "metadata": {
- "length": 1814,
- "summary": "This page is the cover of a report titled 'Retrieval-augmented Generation Realized: Strategic & Technical Insights for Industrial Applications'. It features the logo and name of the publisher, appliedAI Initiative GmbH, along with their contact address in Munich, Germany.",
- "page_nums": [
- 34,
- 35
- ],
- "keywords": [
- "appliedAI Initiative GmbH",
- "München",
- "Germany"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 34,
- "artifact_ref": "page_citation_assets/page-34.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 35,
- "artifact_ref": "page_citation_assets/page-35.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- }
- ]
-}
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/doc_nav.json b/apps/api/app/data/demo_documents/research-rag-realized/doc_nav.json
deleted file mode 100644
index db693e331..000000000
--- a/apps/api/app/data/demo_documents/research-rag-realized/doc_nav.json
+++ /dev/null
@@ -1,417 +0,0 @@
-{
- "version": "1.0",
- "file_name": "Retrieval-Augmented Generation Realized.pdf",
- "stats": {
- "total_chunks": 59,
- "text_chunks": 0,
- "image_chunks": 27,
- "table_chunks": 3,
- "page_chunks": 29,
- "max_depth": 4
- },
- "sections": [
- {
- "title": "Contents",
- "path": "Retrieval-Augmented Generation Realized.pdf/Contents",
- "level": 1,
- "summary": "The 'Contents' section outlines the structure of a white paper on Retrieval-Augmented Generation (RAG). It lists three main sections: Section 1 covers RAG industrialization, landscape, and strategy; Section 2 details 'RAG Recipes' for real-world challenges like data-driven chunking and multimodal reasoning; and Section 3 provides a deep dive into RAG evaluation and metrics. The table also includes preliminary sections like 'About This White Paper' and concluding sections such as 'References', 'Authors', and 'Acknowledgements'.",
- "chunk_count": 3,
- "children": []
- },
- {
- "title": "About This White Paper",
- "path": "Retrieval-Augmented Generation Realized.pdf/About This White Paper",
- "level": 1,
- "summary": "This section outlines the background, purpose, and design principles of the white paper. It explains that the document is based on studies and roundtable discussions regarding Retrieval-Augmented Generation (RAG), aiming to analyze industry demands and facilitate strategy development. The design principles emphasize simplifying complexity and fostering intuitiveness through visual illustrations. Additionally, it details how to use the white paper by describing three main sections: RAG Industrialization Landscape & Strategy, RAG Recipes for Real-World Challenges, and a Deep Dive into RAG Evaluation & Metrics.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Key Takeaways",
- "path": "Retrieval-Augmented Generation Realized.pdf/Key Takeaways",
- "level": 1,
- "summary": "The 'Key Takeaways' section summarizes the white paper's three main parts. It highlights that RAG solutions are essential for sustainable industrial knowledge retrieval due to their trustworthiness and cost efficiency, though resource constraints must be managed. It introduces five recipes for real-world challenges like limited data and complex contexts, emphasizing improved retrieval quality through strategies like metadata filtering and hybrid search. Finally, it notes the complexity of RAG evaluation involving multiple metrics and the industry's need for a standardized framework to ensure consistent quality and scalability.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "Section 1: RAG Industrialization - Landscape & Strategy",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy",
- "level": 1,
- "summary": "",
- "chunk_count": 8,
- "children": [
- {
- "title": "Let's Start with a Crash Course of RAG",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/Let's Start with a Crash Course of RAG",
- "level": 2,
- "summary": "This section provides an overview of the challenges associated with using Large Language Models (LLMs) in isolation, such as hallucinations, knowledge cutoffs, and lack of access to private data. It illustrates these issues with examples involving historical figures like Igor Stravinsky and Béla Bartók. The content then details the importance of Retrieval-Augmented Generation (RAG) for enterprises by listing key benefits including trustworthiness, consistency, controllability, auditability, process optimization, IP security, cost efficiency, scalability, and multi-step reasoning capabilities.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "Wait! Naive RAG Doesn't Work Well!",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/Wait! Naive RAG Doesn't Work Well!",
- "level": 2,
- "summary": "This page provides a crash course on Retrieval-Augmented Generation (RAG), defining it as a method that pairs LLMs with external databases. It details the four stages of RAG: Pre-retrieval, Retrieval, Post-retrieval, and Augmentation & Generation. A central diagram illustrates this workflow using a query about a debate between Elon Musk and Yann LeCun to demonstrate how RAG retrieves relevant context to generate an answer compared to a standard LLM response. The bottom section poses an 'Interesting Question' regarding whether long context windows can solve all data retrieval issues.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "So, What Are Some Techniques to Enhance RAG?",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/So, What Are Some Techniques to Enhance RAG?",
- "level": 2,
- "summary": "This section introduces 12 advanced or modular RAG techniques designed to enhance different components of a RAG system. It presents a chart that plots these techniques based on their 'Degree of Complexity' (considering resource consumption, cost, latency, etc.) versus their 'Projected Contribution to Overall Quality & Trustworthiness'. The identified techniques include Metadata Filtering, Hypothetical Document Embeddings (HyDE), Query Engineering & Fusion, Routing, Search Module & Hybrid Search, Task Adapter, Adaptive Retrieval & Self-RAG, Multiple Indexing Structures, Information or Context Compression, Extra Generation & Predicting, Memory, and Retrieval-augmented Reasoning (RAT & RAGAR).",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "And What Tools Can I Use to Develop a RAG System?",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/And What Tools Can I Use to Develop a RAG System?",
- "level": 2,
- "summary": "This page, titled 'So, What Are Some Techniques to Enhance RAG?', outlines twelve distinct strategies for improving Retrieval-Augmented Generation systems. The techniques are presented in a grid format and include Metadata Filtering, Query Engineering & Fusion, Task Adapter, Memory, Information or Context Compression, Extra Generation & Predicting, Routing, Multiple Indexing Structures, Hypothetical Document Embeddings (HyDE), Search Module & Hybrid Search, Adaptive Retrieval & Self-RAG, and Retrieval-augmented Reasoning (RAT & RAGAR). Each section details the main improvement areas (such as Retrieval, Augmentation, or Generation) and explains the specific methods used.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Alright, How Can I Strategically Prioritize My Development Focus to Industrialize a RAG Project?",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 1: RAG Industrialization - Landscape & Strategy/Alright, How Can I Strategically Prioritize My Development Focus to Industrialize a RAG Project?",
- "level": 2,
- "summary": "This section outlines the strategic framework for industrializing a RAG project, structured as 'The RAG Industrialization Journey'. It details four key dimensions—Core Development Focus, Key Challenges, Key Capabilities, and Key Metrics—across five distinct phases: Ideation, Prototyping, Proof of Concept (PoC), Best-known Methods (BKM), and Operation. The content provides specific tasks, potential obstacles, required skills, and performance indicators relevant to each phase of the development lifecycle.",
- "chunk_count": 2,
- "children": []
- }
- ]
- },
- {
- "title": "Section 2: RAG Recipes for Real-World Challenges",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges",
- "level": 1,
- "summary": "",
- "chunk_count": 14,
- "children": [
- {
- "title": "The Cold Start Recipe: Data-Driven Chunking & Embedding Strategy Without Evaluation Dataset",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The Cold Start Recipe: Data-Driven Chunking & Embedding Strategy Without Evaluation Dataset",
- "level": 2,
- "summary": "This section outlines a 'Cold Start Recipe' for establishing data-driven chunking and embedding strategies in Retrieval-Augmented Generation (RAG) systems when an evaluation dataset is unavailable. It addresses challenges such as the scarcity of expert-constructed datasets and resource constraints. The core methodology involves analyzing pairwise cosine similarity scores to assess embedding quality, drawing on research by Nguyen et al. that compares general LLMs (GPT-3) with domain-specific models (AstroLLaMA). A higher variance in similarity scores indicates better semantic representation. The section concludes with a practical application of this analytics approach to the 'appliedAI AI Act White Paper', utilizing Langchain tools like CharacterTextSplitter and QAGenerationChain to generate synthetic question-answer pairs for initial system assessment.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "The Virtual Havruta Recipe: Optimizing Queries for Multifaceted Contexts & Precise Citations",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The Virtual Havruta Recipe: Optimizing Queries for Multifaceted Contexts & Precise Citations",
- "level": 2,
- "summary": "This page details the results of a 'Cold Start Recipe' for data-driven chunking and embedding strategies. It compares two models, text-embedding-ada-002 and albert-small-v2, using tables to show pairwise cosine similarity distributions, RAGAS metrics (like Context Precision and Answer Relevance), and specific QA examples. The bottom section introduces a prototype RAG Cold Start Analytic App developed by appliedAI Initiative, which features grid search experiments, automatic evaluation, and exploratory data analysis visualizations.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "The Deepset Recipe: Unlocking the Mastery of Metadata for Filtering, Searching, and Reranking",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The Deepset Recipe: Unlocking the Mastery of Metadata for Filtering, Searching, and Reranking",
- "level": 2,
- "summary": "This document page outlines the 'Virtual Havruta Recipe,' a project designed to optimize queries for multifaceted contexts and precise citations in Jewish scripture study. It details the background, goals, and challenges addressed by this RAG-based system, such as incomplete queries and non-aligned reranking. The page further explains the query engineering process, including screening, editing, and adapting user inputs into primary and secondary source queries, alongside metadata filtering strategies.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "The Jina AI Recipe: Agent-Driven SQL Scoping and Task-Specific Finetuning for Patent Search",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The Jina AI Recipe: Agent-Driven SQL Scoping and Task-Specific Finetuning for Patent Search",
- "level": 2,
- "summary": "This section outlines 'The Jina AI Recipe' for patent search, which utilizes agent-driven SQL scoping and task-specific finetuning. It addresses challenges like inadequate precision in classic search methods by implementing LLM agents to construct dynamic SQL queries for filtering. The process involves establishing a baseline, implementing out-of-the-box improvements, and applying task-specific finetuning (e.g., LoRA) to optimize embedding models for domain-specific texts, ultimately improving retrieval quality.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking",
- "level": 2,
- "summary": "",
- "chunk_count": 8,
- "children": [
- {
- "title": "Background & Goals of this Recipe",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Background & Goals of this Recipe",
- "level": 3,
- "summary": "This section outlines the background and goals of the RAGAR recipe, which addresses the increasing prevalence of misinformation in political discourse. It introduces RAG-augmented reasoning (RAGAR) approaches, specifically Chain of RAG (CoRAG) and Tree of RAG (ToRAG), combined with Chain of Verification for evidence-based fact-checking. The text notes that while originally designed for political fact-checking, the method is repurposable for various industry applications requiring thorough verification.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Challenges Addressed",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Challenges Addressed",
- "level": 3,
- "summary": "The section titled 'Challenges Addressed' outlines three primary difficulties in multimodal fact-checking. These include Contextualized Multimodal Claim Generation, which involves generating claims from text while filtering relevant image details; Open-ended Multimodal Evidence Retrieval, focusing on retrieving text and image evidence with correct timeframes and source filtering; and Dynamic, Agentic Reasoning Actions, which requires managing prompts to determine logical reasoning paths and when to stop based on evidence accumulation.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "The RAGAR Approaches: Chain of RAG (CoRAG) & Tree of RAG (ToRAG)",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Approaches: Chain of RAG (CoRAG) & Tree of RAG (ToRAG)",
- "level": 3,
- "summary": "This section illustrates the RAGAR approaches, specifically comparing Chain of RAG (CoRAG) and Tree of RAG (ToRAG). It details a workflow starting with a 'Claim' and 'Multimodal Claim Generation' to produce context. The CoRAG approach is depicted as a linear sequence where questions are generated one by one, each followed by a RAG process to create QA pairs. In contrast, the ToRAG approach shows a branching structure where multiple questions are generated in parallel, processed through RAG, and then undergo 'QA Elimination' steps to filter results before proceeding to the next level. Both methods ultimately collect QA pairs for 'Veracity Prediction', resulting in a label and explanation.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking",
- "level": 3,
- "summary": "",
- "chunk_count": 2,
- "children": [
- {
- "title": "RAG-Augmented Reasoning Technique (Chain of RAG)",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-Augmented Reasoning Technique (Chain of RAG)",
- "level": 4,
- "summary": "The section titled 'RAG-Augmented Reasoning Technique (Chain of RAG)' outlines a linear workflow for fact-checking using multimodal data. It begins with a specific claim and image context regarding Mike Pompeo. The process involves a decision loop where the system determines if an image question is needed; if yes, it uses Google Reverse Image Search API to find relevant evidence, otherwise it performs standard Evidence Retrieval via DuckDuckGo Search. A follow-up check determines if further questioning is required before generating a final Veracity Label and Explanation.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "RAG-Augmented Reasoning Technique (Tree of RAG)",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-Augmented Reasoning Technique (Tree of RAG)",
- "level": 4,
- "summary": "This section details the 'RAG-Augmented Reasoning Technique (Tree of RAG)', a multimodal fact-checking workflow. It begins with a specific claim and image context, then branches into parallel processing paths ('Question/Followup Branch'). Each path independently evaluates an image using either Google Reverse Image Search or DuckDuckGo Evidence Retrieval to generate multiple QA pairs. These pairs undergo elimination before a final decision on whether follow-up is required, ultimately leading to a veracity label.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "Chain of RAG & Tree of RAG",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/Chain of RAG & Tree of RAG",
- "level": 3,
- "summary": "This section outlines two methodologies for RAG-augmented reasoning: Chain of RAG (CoRAG) and Tree of RAG (ToRAG). CoRAG utilizes sequential follow-up questions to retrieve further evidence and includes an early termination check to determine if sufficient information has been gathered. ToRAG creates question branches at each step of the reasoning process, eliminating pairs until only the best branch is chosen based on relevance, detail, additional information, and answer confidence.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "RAG-enhanced Agents for Enterprises",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 2: RAG Recipes for Real-World Challenges/The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking/RAG-enhanced Agents for Enterprises",
- "level": 3,
- "summary": "This section discusses the transformative potential of RAG-enhanced agents in enterprise solutions. It highlights how the combination of Generative AI, advanced information retrieval, and Large Language Models (LLMs) enables intelligent agents to handle complex applications such as decision-making and strategic planning.",
- "chunk_count": 2,
- "children": []
- }
- ]
- }
- ]
- },
- {
- "title": "Section 3: A Deep Dive into RAG Evaluation & Metrics",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics",
- "level": 1,
- "summary": "",
- "chunk_count": 5,
- "children": [
- {
- "title": "Now, how do we assess RAG systems?",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics/Now, how do we assess RAG systems?",
- "level": 2,
- "summary": "The page is a section header titled 'Section 3: A Deep Dive into RAG Evaluation & Metrics'. It introduces the third part of a document, which focuses on evaluating and measuring performance metrics for Retrieval-Augmented Generation (RAG) systems.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "A Further Look into Existing RAG Evaluation Frameworks",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics/A Further Look into Existing RAG Evaluation Frameworks",
- "level": 2,
- "summary": "The section titled 'A Further Look into Existing RAG Evaluation Frameworks' presents a comprehensive comparison table of various frameworks used for evaluating Retrieval-Augmented Generation (RAG) systems. The table details the features, capabilities, and metrics supported by each framework, including Giskard, promptfoo, Ragas, DeepEval, agenta, Trulens, Tonic Validate, LangChain, LangSmith, LlamaIndex, Haystack, Deepset Cloud, and UpTrain.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "Unveiling the Mechanisms Behind These Metrics",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics/Unveiling the Mechanisms Behind These Metrics",
- "level": 2,
- "summary": "This page presents a comprehensive table titled 'An Intuitive View of Key RAG Metrics in Common Practice,' which details various evaluation metrics for Retrieval-Augmented Generation systems. The table lists 12 specific metrics, including Context Relevance, Precision@k, Mean Reciprocal Rank (MMR), and Answer Correctness. For each metric, it provides a description of what is measured, an intuitive formula representation, and a plain-language explanation of the calculation.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Reflections and Future Opportunities in RAG Technology",
- "path": "Retrieval-Augmented Generation Realized.pdf/Section 3: A Deep Dive into RAG Evaluation & Metrics/Reflections and Future Opportunities in RAG Technology",
- "level": 2,
- "summary": "This document page, titled 'Reflections and Future Opportunities in RAG Technology', outlines key strategic insights regarding Retrieval-Augmented Generation (RAG). The top section details three main areas: industrialization landscape and strategy, recipes for real-world challenges like data scarcity, and a deep dive into evaluation metrics. The bottom section, 'Future Opportunities in RAG Technology', highlights six potential advancements including seamless integration with other systems, reasoning agents, standardized evaluation frameworks, coordinated modules, cross-modal capabilities, and sustainability.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "References",
- "path": "Retrieval-Augmented Generation Realized.pdf/References",
- "level": 1,
- "summary": "This page features a quote discussing how Generative AI and RAG capabilities revolutionize knowledge management by enabling employees to retrieve internal information as effectively as asking a human. The quote is attributed to Bernhard Pflugfelder, Head of Innovation Lab (GenAI) for the applied AI initiative. The footer indicates this content is from 'RAG: Strategic & Technical Insights' on page 31.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "Authors",
- "path": "Retrieval-Augmented Generation Realized.pdf/Authors",
- "level": 1,
- "summary": "This page serves as an 'Authors' section, presenting biographical profiles for two individuals: Dr. Paul Yu-Chun Chang and Bernhard Pflugfelder. Both are associated with the organization appliedAI Initiative GmbH. The text details their professional roles, areas of expertise in AI and NLP, years of experience, and educational backgrounds.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Contributors",
- "path": "Retrieval-Augmented Generation Realized.pdf/Contributors",
- "level": 1,
- "summary": "This page lists the contributors to a document titled 'RAG: Strategic & Technical Insights'. It features headshots, names, job titles, and affiliations for ten individuals. The bottom section displays the logos of contributing companies, including TUM Venture Labs, Sefaria, Jina AI, and deepset.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Contributing Companies",
- "path": "Retrieval-Augmented Generation Realized.pdf/Contributing Companies",
- "level": 1,
- "summary": "The section titled 'Contributing Companies' lists the organizations that contributed to the document. It displays the logos and names of five entities: TUM Venture Labs, Sefaria, Jina AI, deepset, and implicitly appliedAI Initiative GmbH (based on the contributors above).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "About appliedAI Initiative GmbH",
- "path": "Retrieval-Augmented Generation Realized.pdf/About appliedAI Initiative GmbH",
- "level": 1,
- "summary": "This page provides an overview of appliedAI Initiative GmbH, describing it as Europe's largest initiative for trusted AI technology. It details the organization's history, its establishment in 2017 by Dr. Andreas Liebl, and its current operations involving over 160 employees and a partnership with more than 20 companies. The lower section features a diagram illustrating the company's offerings, categorized into Professional Services, Partnership Programs, Software Solutions, and Ventures, mapped against maturity levels from 'Not started' to 'Shaper'.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Acknowledgement",
- "path": "Retrieval-Augmented Generation Realized.pdf/Acknowledgement",
- "level": 1,
- "summary": "This page is the cover of a report titled 'Retrieval-augmented Generation Realized: Strategic & Technical Insights for Industrial Applications'. It features the logo and name of the publisher, appliedAI Initiative GmbH, along with their contact address in Munich, Germany.",
- "chunk_count": 2,
- "children": []
- }
- ],
- "resources": {
- "images": [
- {
- "path": "images/image_page_1_figure_1.png",
- "summary": "[images/image_page_1_figure_1.png]"
- },
- {
- "path": "images/image_page_7_figure_1.png",
- "summary": "Challenges of LLM-Only Approaches diagram [images/image_page_7_figure_1.png]"
- },
- {
- "path": "images/image_page_7_figure_2.png",
- "summary": "What LLM Hallucination Looks Like example [images/image_page_7_figure_2.png]"
- },
- {
- "path": "images/image_page_7_figure_3.png",
- "summary": "Why is Retrieval-augmented Generation (RAG) Important for Enterprises? grid [images/image_page_7_figure_3.png]"
- },
- {
- "path": "images/image_page_8_figure_1.png",
- "summary": "[images/image_page_8_figure_1.png]"
- },
- {
- "path": "images/image_page_8_figure_2.png",
- "summary": "Interesting Question: Does Long Context Window Solve Everything? [images/image_page_8_figure_2.png]"
- },
- {
- "path": "images/image_page_9_figure_1.png",
- "summary": "Challenges of a Naive RAG System [images/image_page_9_figure_1.png]"
- },
- {
- "path": "images/image_page_9_figure_2.png",
- "summary": "From Naive RAG to Advanced and Modular RAG [images/image_page_9_figure_2.png]"
- },
- {
- "path": "images/image_page_9_figure_3.png",
- "summary": "RAG: Strategic & Technical Insights [images/image_page_9_figure_3.png]"
- },
- {
- "path": "images/image_page_15_figure_1.png",
- "summary": "[images/image_page_15_figure_1.png]"
- },
- {
- "path": "images/image_page_15_figure_2.png",
- "summary": "[images/image_page_15_figure_2.png]"
- },
- {
- "path": "images/image_page_16_figure_2.png",
- "summary": "[images/image_page_16_figure_2.png]"
- },
- {
- "path": "images/image_page_16_figure_4.png",
- "summary": "[images/image_page_16_figure_4.png]"
- },
- {
- "path": "images/image_page_17_figure_1.png",
- "summary": "Query Engineering, Fusion, & HyDE Process Flowchart [images/image_page_17_figure_1.png]"
- },
- {
- "path": "images/image_page_17_figure_2.png",
- "summary": "Metadata Filtering & Customized Reranking Logic Diagram [images/image_page_17_figure_2.png]"
- },
- {
- "path": "images/image_page_18_figure_1.png",
- "summary": "Metadata for Enhancement of Relevance and Consistency [images/image_page_18_figure_1.png]"
- },
- {
- "path": "images/image_page_20_figure_1.png",
- "summary": "The RAGAR Approaches: Chain of RAG (CoRAG) & Tree of RAG (ToRAG) [images/image_page_20_figure_1.png]"
- },
- {
- "path": "images/image_page_21_figure_1.png",
- "summary": "RAG-Augmented Reasoning Technique (Chain of RAG) [images/image_page_21_figure_1.png]"
- },
- {
- "path": "images/image_page_21_figure_2.png",
- "summary": "RAG-Augmented Reasoning Technique (Tree of RAG) [images/image_page_21_figure_2.png]"
- },
- {
- "path": "images/image_page_23_figure_1.png",
- "summary": "[images/image_page_23_figure_1.png]"
- },
- {
- "path": "images/image_page_25_figure_1.png",
- "summary": "A Mind Map of RAG Metrics: What do they Measure? [images/image_page_25_figure_1.png]"
- },
- {
- "path": "images/image_page_30_figure_1.png",
- "summary": "[images/image_page_30_figure_1.png]"
- },
- {
- "path": "images/image_page_31_figure_1.png",
- "summary": "[images/image_page_31_figure_1.png]"
- },
- {
- "path": "images/image_page_31_figure_2.png",
- "summary": "[images/image_page_31_figure_2.png]"
- },
- {
- "path": "images/image_page_32_figure_1.png",
- "summary": "Contributors [images/image_page_32_figure_1.png]"
- },
- {
- "path": "images/image_page_32_figure_2.png",
- "summary": "Contributing Companies [images/image_page_32_figure_2.png]"
- },
- {
- "path": "images/image_page_33_figure_1.png",
- "summary": "We offer a unique set of offerings to help companies on their way to becoming AI shapers [images/image_page_33_figure_1.png]"
- }
- ],
- "tables": [
- {
- "path": "tables/table_page_12_1.html",
- "summary": "tables/table_page_12_1.html"
- },
- {
- "path": "tables/table_page_16_1.html",
- "summary": "tables/table_page_16_1.html"
- },
- {
- "path": "tables/table_page_16_3.html",
- "summary": "tables/table_page_16_3.html"
- }
- ]
- }
-}
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_15_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_15_figure_1.png
deleted file mode 100644
index 848d971c2..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_15_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_15_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_15_figure_2.png
deleted file mode 100644
index 0c3e44d6a..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_15_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_16_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_16_figure_2.png
deleted file mode 100644
index 805ce2569..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_16_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_16_figure_4.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_16_figure_4.png
deleted file mode 100644
index f6436ef5d..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_16_figure_4.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_17_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_17_figure_1.png
deleted file mode 100644
index fcf8aadaa..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_17_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_17_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_17_figure_2.png
deleted file mode 100644
index 7ccc5d437..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_17_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_18_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_18_figure_1.png
deleted file mode 100644
index 33c1eeb1d..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_18_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_1_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_1_figure_1.png
deleted file mode 100644
index c837521d8..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_1_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_20_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_20_figure_1.png
deleted file mode 100644
index 05bb826a0..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_20_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_21_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_21_figure_1.png
deleted file mode 100644
index 64310e0d3..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_21_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_21_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_21_figure_2.png
deleted file mode 100644
index a47c9d833..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_21_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_23_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_23_figure_1.png
deleted file mode 100644
index e8eb55f22..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_23_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_25_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_25_figure_1.png
deleted file mode 100644
index 876796ec7..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_25_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_30_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_30_figure_1.png
deleted file mode 100644
index 7910f4748..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_30_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_31_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_31_figure_1.png
deleted file mode 100644
index 0ea111c0d..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_31_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_31_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_31_figure_2.png
deleted file mode 100644
index caa60c524..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_31_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_32_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_32_figure_1.png
deleted file mode 100644
index 859579908..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_32_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_32_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_32_figure_2.png
deleted file mode 100644
index 375724b9d..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_32_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_33_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_33_figure_1.png
deleted file mode 100644
index 699b748be..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_33_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_1.png
deleted file mode 100644
index 6990349f3..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_2.png
deleted file mode 100644
index 0997dbee3..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_3.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_3.png
deleted file mode 100644
index 9879c9fd8..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_7_figure_3.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_8_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_8_figure_1.png
deleted file mode 100644
index 4f912bcb4..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_8_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_8_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_8_figure_2.png
deleted file mode 100644
index 1c1f770da..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_8_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_1.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_1.png
deleted file mode 100644
index 2c1a042b6..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_2.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_2.png
deleted file mode 100644
index 1a534f9e8..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_3.png b/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_3.png
deleted file mode 100644
index f4660b04f..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/images/image_page_9_figure_3.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/manifest.json b/apps/api/app/data/demo_documents/research-rag-realized/manifest.json
deleted file mode 100644
index 2428042e4..000000000
--- a/apps/api/app/data/demo_documents/research-rag-realized/manifest.json
+++ /dev/null
@@ -1,251 +0,0 @@
-{
- "version": "2.0",
- "job_id": "job_e9c30a85c60b",
- "data_id": "demo-research-rag-realized",
- "source_file_name": "Retrieval-Augmented Generation Realized.pdf",
- "processing_date": "2026-08-18T18:04:44.041120Z",
- "processing": {
- "page_count": 35,
- "billing_status": "charged",
- "cost": {
- "micro_dollars": 525000,
- "credits": 0.525
- },
- "cost_estimate": {
- "currency": "USD",
- "pricing_version": "2026-08-18:874ab8b6c9e4",
- "total_input_cost": 0.0,
- "total_output_cost": 0.0,
- "total_cost": 0.0,
- "by_model": {
- "qwen3.5-flash": {
- "input_tokens": 42659,
- "output_tokens": 2012,
- "calls": 9,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 19770,
- "output_tokens": 240,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "calibration.react_loop": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 12602,
- "output_tokens": 1350,
- "calls": 4,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "calibration.inspect_pages": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 8206,
- "output_tokens": 366,
- "calls": 3,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "page_memory.page_locate": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 2081,
- "output_tokens": 56,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "timing": {
- "started_at": "2026-08-18T18:00:11.234156+00:00",
- "completed_at": "2026-08-18T18:04:44.034873+00:00",
- "duration_ms": 272800
- },
- "stages": {
- "timing_ms": {
- "page_memory.normalize_to_pdf": 0,
- "page_memory.profile": 140525,
- "page_memory.skeleton": 9,
- "page_memory.node_summary": 36197,
- "page_memory.node_rows": 74,
- "page_memory.node_assembly": 36276,
- "worker.parse.document": 272665
- },
- "token_usage": {
- "prompt_tokens": 42659,
- "completion_tokens": 2012,
- "total_tokens": 44671,
- "calls": 9,
- "by_model": {
- "qwen3.5-flash": {
- "prompt_tokens": 42659,
- "completion_tokens": 2012,
- "total_tokens": 44671,
- "calls": 9
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "prompt_tokens": 19770,
- "completion_tokens": 240,
- "total_tokens": 20010,
- "calls": 1,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 19770,
- "completion_tokens": 240,
- "total_tokens": 20010,
- "calls": 1
- }
- }
- },
- "calibration.react_loop": {
- "prompt_tokens": 12602,
- "completion_tokens": 1350,
- "total_tokens": 13952,
- "calls": 4,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 12602,
- "completion_tokens": 1350,
- "total_tokens": 13952,
- "calls": 4
- }
- }
- },
- "calibration.inspect_pages": {
- "prompt_tokens": 8206,
- "completion_tokens": 366,
- "total_tokens": 8572,
- "calls": 3,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 8206,
- "completion_tokens": 366,
- "total_tokens": 8572,
- "calls": 3
- }
- }
- },
- "page_memory.page_locate": {
- "prompt_tokens": 2081,
- "completion_tokens": 56,
- "total_tokens": 2137,
- "calls": 1,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 2081,
- "completion_tokens": 56,
- "total_tokens": 2137,
- "calls": 1
- }
- }
- }
- }
- }
- }
- },
- "statistics": {
- "total_chunks": 59,
- "text_chunks": 0,
- "image_chunks": 27,
- "table_chunks": 3,
- "page_chunks": 29,
- "total_pages": null
- },
- "HIERARCHY": {
- "Contents": {},
- "About This White Paper": {},
- "Key Takeaways": {},
- "Section 1: RAG Industrialization - Landscape & Strategy": {
- "Let's Start with a Crash Course of RAG": {},
- "Wait! Naive RAG Doesn't Work Well!": {},
- "So, What Are Some Techniques to Enhance RAG?": {},
- "And What Tools Can I Use to Develop a RAG System?": {},
- "Alright, How Can I Strategically Prioritize My Development Focus to Industrialize a RAG Project?": {}
- },
- "Section 2: RAG Recipes for Real-World Challenges": {
- "The Cold Start Recipe: Data-Driven Chunking & Embedding Strategy Without Evaluation Dataset": {},
- "The Virtual Havruta Recipe: Optimizing Queries for Multifaceted Contexts & Precise Citations": {},
- "The Deepset Recipe: Unlocking the Mastery of Metadata for Filtering, Searching, and Reranking": {},
- "The Jina AI Recipe: Agent-Driven SQL Scoping and Task-Specific Finetuning for Patent Search": {},
- "The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking": {
- "Background & Goals of this Recipe": {},
- "Challenges Addressed": {},
- "The RAGAR Approaches: Chain of RAG (CoRAG) & Tree of RAG (ToRAG)": {},
- "The RAGAR Recipe: Multimodal RAG-augmented Reasoning for Fact-checking": {
- "RAG-Augmented Reasoning Technique (Chain of RAG)": {},
- "RAG-Augmented Reasoning Technique (Tree of RAG)": {}
- },
- "Chain of RAG & Tree of RAG": {},
- "RAG-enhanced Agents for Enterprises": {}
- }
- },
- "Section 3: A Deep Dive into RAG Evaluation & Metrics": {
- "Now, how do we assess RAG systems?": {},
- "A Further Look into Existing RAG Evaluation Frameworks": {},
- "Unveiling the Mechanisms Behind These Metrics": {},
- "Reflections and Future Opportunities in RAG Technology": {}
- },
- "References": {},
- "Authors": {},
- "Contributors": {},
- "Contributing Companies": {},
- "About appliedAI Initiative GmbH": {},
- "Acknowledgement": {}
- }
-}
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-1.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-1.png
deleted file mode 100644
index 7e79f249f..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-10.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-10.png
deleted file mode 100644
index d016d207d..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-10.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-11.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-11.png
deleted file mode 100644
index 58996934f..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-11.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-12.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-12.png
deleted file mode 100644
index 0740ea363..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-12.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-13.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-13.png
deleted file mode 100644
index 0e0e6c6d5..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-13.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-14.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-14.png
deleted file mode 100644
index b2aaeb90e..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-14.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-15.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-15.png
deleted file mode 100644
index 9d628a36e..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-15.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-16.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-16.png
deleted file mode 100644
index 337d8988d..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-16.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-17.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-17.png
deleted file mode 100644
index 9109741cf..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-17.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-18.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-18.png
deleted file mode 100644
index c9f1233cf..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-18.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-19.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-19.png
deleted file mode 100644
index 7c054ec77..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-19.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-2.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-2.png
deleted file mode 100644
index 109be0da2..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-20.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-20.png
deleted file mode 100644
index e2d2d4720..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-20.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-21.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-21.png
deleted file mode 100644
index 6ad77e9b8..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-21.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-22.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-22.png
deleted file mode 100644
index 7d318a198..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-22.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-23.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-23.png
deleted file mode 100644
index 132ca4bc2..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-23.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-24.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-24.png
deleted file mode 100644
index 11dd5c03e..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-24.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-25.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-25.png
deleted file mode 100644
index 4785c3a20..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-25.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-26.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-26.png
deleted file mode 100644
index 6c4834bcb..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-26.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-27.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-27.png
deleted file mode 100644
index 2fb769c5a..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-27.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-28.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-28.png
deleted file mode 100644
index 894621e54..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-28.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-29.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-29.png
deleted file mode 100644
index de83a84bd..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-29.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-3.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-3.png
deleted file mode 100644
index bc11432f7..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-3.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-30.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-30.png
deleted file mode 100644
index 659b1afc3..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-30.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-31.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-31.png
deleted file mode 100644
index 4b0e2fc76..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-31.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-32.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-32.png
deleted file mode 100644
index 37b934110..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-32.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-33.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-33.png
deleted file mode 100644
index a1b2706a3..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-33.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-34.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-34.png
deleted file mode 100644
index a803f4a80..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-34.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-35.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-35.png
deleted file mode 100644
index 4222e7c61..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-35.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-4.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-4.png
deleted file mode 100644
index 2ac6d6c5e..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-4.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-5.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-5.png
deleted file mode 100644
index f2d96d90f..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-5.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-6.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-6.png
deleted file mode 100644
index 9763eccb9..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-6.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-7.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-7.png
deleted file mode 100644
index f2c638b32..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-7.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-8.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-8.png
deleted file mode 100644
index b0f5213b5..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-8.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-9.png b/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-9.png
deleted file mode 100644
index eb432fbc7..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-realized/page_citation_assets/page-9.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_12_1.html b/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_12_1.html
deleted file mode 100644
index 3e9807508..000000000
--- a/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_12_1.html
+++ /dev/null
@@ -1,249 +0,0 @@
-
-
-
-
Langchain LlamaIndex
-
Haystack
-
Haystack
-
-
-
(v0.01.13) (v0.10)
-
(v1.25)
-
(v2.0)
-
-
-
Unstructured Data Formats: Plain Text (HTML, MARKDOWN, TXT), Image (JPEG, PNG), Document (CSV, PDF)
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_16_1.html b/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_16_1.html
deleted file mode 100644
index 64a03b286..000000000
--- a/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_16_1.html
+++ /dev/null
@@ -1,88 +0,0 @@
-
-
-
-
-
-
ada-002
-
-
-
-
According to
-
18%
-
According to the
-
The study does not
-
-
-
the study, what
-
-
study, 18% of the AI
-
provide information
-
-
-
percentage of
-
-
systems examined
-
on the percentage of
-
-
-
the AI systems
-
-
were in the high-risk
-
AI systems examined
-
-
-
examined were in
-
-
class.
-
that were in the high-
-
-
-
the high-risk class?
-
-
-
risk class.
-
-
-
What are the three
-
The three risk classes
-
Prohibited AI
-
I'm sorry, but the
-
-
-
risk classes outlined
-
outlined in the AI
-
systems, High-risk
-
context provided
-
-
-
in the AI Regulation,
-
Regulation are high-risk,
-
AI systems, and
-
does not mention
-
-
-
and what are the
-
low-risk, and unclear. The
-
Low-risk AI systems.
-
the three risk classes
-
-
-
main requirements
-
main requirements for
-
-
outlined in the AI
-
-
-
for each?
-
each are not specified in
-
-
Regulation or their
-
-
-
-
the provided context.
-
-
main requirements.
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_16_3.html b/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_16_3.html
deleted file mode 100644
index 52e2814c3..000000000
--- a/apps/api/app/data/demo_documents/research-rag-realized/tables/table_page_16_3.html
+++ /dev/null
@@ -1,44 +0,0 @@
-
-
-
-
-
Context
-
Context
-
Context
-
-
-
-
-
-
-
-
Precision
-
Recall
-
Relevance
-
Faithfulness
-
Answer Correctness
-
-
Answer Relevance
-
-
-
text-embedding-
-
-
-
-
-
-
-
-
-
-
ada-002
-
0.85
-
0.58
-
0.10
-
0.67
-
0.53
-
-
0.89
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/chunks.json b/apps/api/app/data/demo_documents/research-rag-survey/chunks.json
deleted file mode 100644
index 888197d5f..000000000
--- a/apps/api/app/data/demo_documents/research-rag-survey/chunks.json
+++ /dev/null
@@ -1,1094 +0,0 @@
-{
- "chunks": [
- {
- "chunk_id": "asset_48894161-c0de-5c3e-9e8a-8b5d48d58840",
- "type": "image",
- "content": "Fig. 1. Technology tree of RAG research.\n[images/image_page_2_figure_1.png]",
- "path": "images/image_page_2_figure_1.png",
- "metadata": {
- "length": 75,
- "summary": "",
- "page_nums": [
- 2
- ],
- "file_path": "images/image_page_2_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_07431473-5219-5e23-bc95-303167528a21",
- "type": "image",
- "content": "Fig. 2\n[images/image_page_3_figure_1.png]",
- "path": "images/image_page_3_figure_1.png",
- "metadata": {
- "length": 41,
- "summary": "",
- "page_nums": [
- 3
- ],
- "file_path": "images/image_page_3_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_a108114a-c157-593e-bf49-4a24caea132b",
- "type": "image",
- "content": "Fig. 3. Comparison between the three paradigms of RAG.\n[images/image_page_4_figure_1.png]",
- "path": "images/image_page_4_figure_1.png",
- "metadata": {
- "length": 89,
- "summary": "",
- "page_nums": [
- 4
- ],
- "file_path": "images/image_page_4_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_5ab3ba23-ad7b-5fe5-b17f-e7c39765cc4a",
- "type": "image",
- "content": "Fig. 4\n[images/image_page_7_figure_1.png]",
- "path": "images/image_page_7_figure_1.png",
- "metadata": {
- "length": 41,
- "summary": "",
- "page_nums": [
- 7
- ],
- "file_path": "images/image_page_7_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_3c136a15-4895-5367-908f-8e2bc9eadce5",
- "type": "image",
- "content": "Fig. 5\n[images/image_page_11_figure_1.png]",
- "path": "images/image_page_11_figure_1.png",
- "metadata": {
- "length": 42,
- "summary": "",
- "page_nums": [
- 11
- ],
- "file_path": "images/image_page_11_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_c56f3f73-2c97-5893-9b64-49408dc47842",
- "type": "image",
- "content": "Summary of RAG ecosystem\n[images/image_page_16_figure_1.png]",
- "path": "images/image_page_16_figure_1.png",
- "metadata": {
- "length": 60,
- "summary": "",
- "page_nums": [
- 16
- ],
- "file_path": "images/image_page_16_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "node_81d15482-4af2-5f41-8011-3ed0a283dada",
- "type": "page",
- "content": "1\nRetrieval-Augmented Generation for Large\nLanguage Models: A Survey\nYunfan Gaoa, Yun Xiongb, Xinyu Gaob, Kangxiang Jiab, Jinliu Panb, Yuxi Bic, Yi Daia, Jiawei Suna, Meng\nWangc, and Haofen Wang a,c\naShanghai Research Institute for Intelligent Autonomous Systems, Tongji University\nbShanghai Key Laboratory of Data Science, School of Computer Science, Fudan University\ncCollege of Design and Innovation, Tongji University\nAbstract—Large Language Models (LLMs) showcase impres-\nsive capabilities but encounter challenges like hallucination,\noutdated knowledge, and non-transparent, untraceable reasoning\nprocesses. Retrieval-Augmented Generation (RAG) has emerged\nas a promising solution by incorporating knowledge from external\ndatabases. This enhances the accuracy and credibility of the\ngeneration, particularly for knowledge-intensive tasks, and allows\nfor continuous knowledge updates and integration of domain-\nspecific information. RAG synergistically merges LLMs’ intrin-\nsic knowledge with the vast, dynamic repositories of external\ndatabases. This comprehensive review paper offers a detailed\nexamination of the progression of RAG paradigms, encompassing\nthe Naive RAG, the Advanced RAG, and the Modular RAG.\nIt meticulously scrutinizes the tripartite foundation of RAG\nframeworks, which includes the retrieval, the generation and the\naugmentation techniques. The paper highlights the state-of-the-\nart technologies embedded in each of these critical components,\nproviding a profound understanding of the advancements in RAG\nsystems. Furthermore, this paper introduces up-to-date evalua-\ntion framework and benchmark. At the end, this article delineates\nthe challenges currently faced and points out prospective avenues\nfor research and development 1.\nIndex Terms—Large language model, retrieval-augmented gen-\neration, natural language processing, information retrieval\nI. INTRODUCTION\nL\nARGE language models (LLMs) have achieved remark-\nable success, though they still face significant limitations,\nespecially in domain-specific or knowledge-intensive tasks [1],\nnotably producing “hallucinations” [2] when handling queries\nbeyond their training data or requiring current information. To\novercome challenges, Retrieval-Augmented Generation (RAG)\nenhances LLMs by retrieving relevant document chunks from\nexternal knowledge base through semantic similarity calcu-\nlation. By referencing external knowledge, RAG effectively\nreduces the problem of generating factually incorrect content.\nIts integration into LLMs has resulted in widespread adoption,\nestablishing RAG as a key technology in advancing chatbots\nand enhancing the suitability of LLMs for real-world applica-\ntions.\nRAG technology has rapidly developed in recent years, and\nthe technology tree summarizing related research is shown\nCorresponding Author.Email:haofen.wang@tongji.edu.cn\n1Resources\nare\navailable\nat\nhttps://github.com/Tongji-KGLLM/\nRAG-Survey\nin Figure 1. The development trajectory of RAG in the era\nof large models exhibits several distinct stage characteristics.\nInitially, RAG’s inception coincided with the rise of the\nTransformer architecture, focusing on enhancing language\nmodels by incorporating additional knowledge through Pre-\nTraining Models (PTM). This early stage was characterized\nby foundational work aimed at refining pre-training techniques\n[3]–[5].The subsequent arrival of ChatGPT [6] marked a\npivotal moment, with LLM demonstrating powerful in context\nlearning (ICL) capabilities. RAG research shifted towards\nproviding better information for LLMs to answer more com-\nplex and knowledge-intensive tasks during the inference stage,\nleading to rapid development in RAG studies. As research\nprogressed, the enhancement of RAG was no longer limited\nto the inference stage but began to incorporate more with LLM\nfine-tuning techniques.\nThe burgeoning field of RAG has experienced swift growth,\nyet it has not been accompanied by a systematic synthesis that\ncould clarify its broader trajectory. This survey endeavors to\nfill this gap by mapping out the RAG process and charting\nits evolution and anticipated future paths, with a focus on the\nintegration of RAG within LLMs. This paper considers both\ntechnical paradigms and research methods, summarizing three\nmain research paradigms from over 100 RAG studies, and\nanalyzing key technologies in the core stages of “Retrieval,”\n“Generation,” and “Augmentation.” On the other hand, current\nresearch tends to focus more on methods, lacking analysis and\nsummarization of how to evaluate RAG. This paper compre-\nhensively reviews the downstream tasks, datasets, benchmarks,\nand evaluation methods applicable to RAG. Overall, this\npaper sets out to meticulously compile and categorize the\nfoundational technical concepts, historical progression, and\nthe spectrum of RAG methodologies and applications that\nhave emerged post-LLMs. It is designed to equip readers and\nprofessionals with a detailed and structured understanding of\nboth large models and RAG. It aims to illuminate the evolution\nof retrieval augmentation techniques, assess the strengths and\nweaknesses of various approaches in their respective contexts,\nand speculate on upcoming trends and innovations.\nOur contributions are as follows:\n• In this survey, we present a thorough and systematic\nreview of the state-of-the-art RAG methods, delineating\nits evolution through paradigms including naive RAG,\narXiv:2312.10997v5 [cs.CL] 27 Mar 2024",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/I. INTRODUCTION",
- "metadata": {
- "length": 5451,
- "summary": "This document is the first page of a survey paper titled \"Retrieval-Augmented Generation for Large Language Models: A Survey.\" It introduces the concept of Retrieval-Augmented Generation (RAG) as a solution to challenges like hallucination in Large Language Models (LLMs). The abstract outlines the paper's scope, which includes reviewing RAG paradigms (Naive, Advanced, Modular), frameworks, and evaluation methods. The introduction further details the rapid development of RAG technology and its integration into LLMs.",
- "page_nums": [
- 1
- ],
- "keywords": [
- "Yunfan Gao",
- "Yun Xiong",
- "Xinyu Gao",
- "Kangxiang Jia",
- "Jinliu Pan",
- "Yuxi Bi",
- "Yi Dai",
- "Jiawei Sun",
- "Meng Wang",
- "Haofen Wang",
- "Shanghai Research Institute for Intelligent Autonomous Systems",
- "Tongji University",
- "Shanghai Key Laboratory of Data Science",
- "School of Computer Science",
- "Fudan University",
- "College of Design and Innovation",
- "Transformer architecture",
- "ChatGPT"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 1,
- "artifact_ref": "page_citation_assets/page-1.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_40a73b86-ede7-5fa2-b284-24ac3b17d861",
- "type": "page",
- "content": "2\nFig. 1. Technology tree of RAG research. The stages of involving RAG mainly include pre-training, fine-tuning, and inference. With the emergence of LLMs,\nresearch on RAG initially focused on leveraging the powerful in context learning abilities of LLMs, primarily concentrating on the inference stage. Subsequent\nresearch has delved deeper, gradually integrating more with the fine-tuning of LLMs. Researchers have also been exploring ways to enhance language models\nin the pre-training stage through retrieval-augmented techniques.\nadvanced RAG, and modular RAG. This review contex-\ntualizes the broader scope of RAG research within the\nlandscape of LLMs.\n• We identify and discuss the central technologies integral\nto the RAG process, specifically focusing on the aspects\nof “Retrieval”, “Generation” and “Augmentation”, and\ndelve into their synergies, elucidating how these com-\nponents intricately collaborate to form a cohesive and\neffective RAG framework.\n• We have summarized the current assessment methods of\nRAG, covering 26 tasks, nearly 50 datasets, outlining\nthe evaluation objectives and metrics, as well as the\ncurrent evaluation benchmarks and tools. Additionally,\nwe anticipate future directions for RAG, emphasizing\npotential enhancements to tackle current challenges.\nThe paper unfolds as follows: Section II introduces the\nmain concept and current paradigms of RAG. The following\nthree sections explore core components—“Retrieval”, “Gen-\neration” and “Augmentation”, respectively. Section III focuses\non optimization methods in retrieval,including indexing, query\nand embedding optimization. Section IV concentrates on post-\nretrieval process and LLM fine-tuning in generation. Section V\nanalyzes the three augmentation processes. Section VI focuses\non RAG’s downstream tasks and evaluation system. Sec-\ntion VII mainly discusses the challenges that RAG currently\nfaces and its future development directions. At last, the paper\nconcludes in Section VIII.\nII. OVERVIEW OF RAG\nA typical application of RAG is illustrated in Figure 2.\nHere, a user poses a question to ChatGPT about a recent,\nwidely discussed news. Given ChatGPT’s reliance on pre-\ntraining data, it initially lacks the capacity to provide up-\ndates on recent developments. RAG bridges this information\ngap by sourcing and incorporating knowledge from external\ndatabases. In this case, it gathers relevant news articles related\nto the user’s query. These articles, combined with the original\nquestion, form a comprehensive prompt that empowers LLMs\nto generate a well-informed answer.\nThe RAG research paradigm is continuously evolving, and\nwe categorize it into three stages: Naive RAG, Advanced\nRAG, and Modular RAG, as showed in Figure 3. Despite\nRAG method are cost-effective and surpass the performance\nof the native LLM, they also exhibit several limitations.\nThe development of Advanced RAG and Modular RAG is\na response to these specific shortcomings in Naive RAG.\nA. Naive RAG\nThe Naive RAG research paradigm represents the earli-\nest methodology, which gained prominence shortly after the",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/A. Naive RAG",
- "metadata": {
- "length": 3082,
- "summary": "This page presents a technology tree of Retrieval-Augmented Generation (RAG) research, visualizing the evolution of methods from 2020 to 2024 across pre-training, fine-tuning, and inference stages. The text introduces the scope of RAG within Large Language Models (LLMs), outlines the paper's structure, and begins an overview of RAG paradigms including Naive, Advanced, and Modular RAG.",
- "page_nums": [
- 2
- ],
- "keywords": [
- "ChatGPT",
- "LLMs"
- ],
- "connect_to": [
- {
- "target": "asset_48894161-c0de-5c3e-9e8a-8b5d48d58840",
- "relation": "embeds",
- "ref": "[images/image_page_2_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 2,
- "artifact_ref": "page_citation_assets/page-2.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_fed9d070-08e5-5a77-9b48-f7c5e66a9683",
- "type": "page",
- "content": "3\nFig. 2. A representative instance of the RAG process applied to question answering. It mainly consists of 3 steps. 1) Indexing. Documents are split into chunks,\nencoded into vectors, and stored in a vector database. 2) Retrieval. Retrieve the Top k chunks most relevant to the question based on semantic similarity. 3)\nGeneration. Input the original question and the retrieved chunks together into LLM to generate the final answer.\nwidespread adoption of ChatGPT. The Naive RAG follows\na traditional process that includes indexing, retrieval, and\ngeneration, which is also characterized as a “Retrieve-Read”\nframework [7].\nIndexing starts with the cleaning and extraction of raw data\nin diverse formats like PDF, HTML, Word, and Markdown,\nwhich is then converted into a uniform plain text format. To\naccommodate the context limitations of language models, text\nis segmented into smaller, digestible chunks. Chunks are then\nencoded into vector representations using an embedding model\nand stored in vector database. This step is crucial for enabling\nefficient similarity searches in the subsequent retrieval phase.\nRetrieval. Upon receipt of a user query, the RAG system\nemploys the same encoding model utilized during the indexing\nphase to transform the query into a vector representation.\nIt then computes the similarity scores between the query\nvector and the vector of chunks within the indexed corpus.\nThe system prioritizes and retrieves the top K chunks that\ndemonstrate the greatest similarity to the query. These chunks\nare subsequently used as the expanded context in prompt.\nGeneration. The posed query and selected documents are\nsynthesized into a coherent prompt to which a large language\nmodel is tasked with formulating a response. The model’s\napproach to answering may vary depending on task-specific\ncriteria, allowing it to either draw upon its inherent parametric\nknowledge or restrict its responses to the information con-\ntained within the provided documents. In cases of ongoing\ndialogues, any existing conversational history can be integrated\ninto the prompt, enabling the model to engage in multi-turn\ndialogue interactions effectively.\nHowever, Naive RAG encounters notable drawbacks:\nRetrieval Challenges. The retrieval phase often struggles\nwith precision and recall, leading to the selection of misaligned\nor irrelevant chunks, and the missing of crucial information.\nGeneration Difficulties. In generating responses, the model\nmay face the issue of hallucination, where it produces con-\ntent not supported by the retrieved context. This phase can\nalso suffer from irrelevance, toxicity, or bias in the outputs,\ndetracting from the quality and reliability of the responses.\nAugmentation Hurdles. Integrating retrieved information\nwith the different task can be challenging, sometimes resulting\nin disjointed or incoherent outputs. The process may also\nencounter redundancy when similar information is retrieved\nfrom multiple sources, leading to repetitive responses. Deter-\nmining the significance and relevance of various passages and\nensuring stylistic and tonal consistency add further complexity.\nFacing complex issues, a single retrieval based on the original\nquery may not suffice to acquire adequate context information.\nMoreover, there’s a concern that generation models might\noverly rely on augmented information, leading to outputs that\nsimply echo retrieved content without adding insightful or\nsynthesized information.\nB. Advanced RAG\nAdvanced RAG introduces specific improvements to over-\ncome the limitations of Naive RAG. Focusing on enhancing re-\ntrieval quality, it employs pre-retrieval and post-retrieval strate-\ngies. To tackle the indexing issues, Advanced RAG refines\nits indexing techniques through the use of a sliding window\napproach, fine-grained segmentation, and the incorporation of\nmetadata. Additionally, it incorporates several optimization\nmethods to streamline the retrieval process [8].",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/B. Advanced RAG",
- "metadata": {
- "length": 3950,
- "summary": "This page details the architecture and process of Retrieval-Augmented Generation (RAG), illustrated by a flowchart in Figure 2 that outlines the steps of indexing, retrieval, and generation. The text describes how Naive RAG functions through these three phases—converting documents into vector chunks, retrieving relevant context based on user queries, and generating answers using a Large Language Model (LLM). It also discusses challenges such as hallucination and redundancy, and introduces Advanced RAG strategies to improve performance.",
- "page_nums": [
- 3
- ],
- "keywords": [
- "ChatGPT",
- "OpenAI",
- "Sam Altman"
- ],
- "connect_to": [
- {
- "target": "asset_07431473-5219-5e23-bc95-303167528a21",
- "relation": "embeds",
- "ref": "[images/image_page_3_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_b5997b3e-9664-5755-98bd-5c695cc3209b",
- "type": "page",
- "content": "4\nFig. 3.\nComparison between the three paradigms of RAG. (Left) Naive RAG mainly consists of three parts: indexing, retrieval and generation. (Middle)\nAdvanced RAG proposes multiple optimization strategies around pre-retrieval and post-retrieval, with a process similar to the Naive RAG, still following a\nchain-like structure. (Right) Modular RAG inherits and develops from the previous paradigm, showcasing greater flexibility overall. This is evident in the\nintroduction of multiple specific functional modules and the replacement of existing modules. The overall process is not limited to sequential retrieval and\ngeneration; it includes methods such as iterative and adaptive retrieval.\nPre-retrieval process. In this stage, the primary focus is\non optimizing the indexing structure and the original query.\nThe goal of optimizing indexing is to enhance the quality of\nthe content being indexed. This involves strategies: enhancing\ndata granularity, optimizing index structures, adding metadata,\nalignment optimization, and mixed retrieval. While the goal\nof query optimization is to make the user’s original question\nclearer and more suitable for the retrieval task. Common\nmethods include query rewriting query transformation, query\nexpansion and other techniques [7], [9]–[11].\nPost-Retrieval Process. Once relevant context is retrieved,\nit’s crucial to integrate it effectively with the query. The main\nmethods in post-retrieval process include rerank chunks and\ncontext compressing. Re-ranking the retrieved information to\nrelocate the most relevant content to the edges of the prompt is\na key strategy. This concept has been implemented in frame-\nworks such as LlamaIndex2, LangChain3, and HayStack [12].\nFeeding all relevant documents directly into LLMs can lead\nto information overload, diluting the focus on key details with\nirrelevant content.To mitigate this, post-retrieval efforts con-\ncentrate on selecting the essential information, emphasizing\ncritical sections, and shortening the context to be processed.\n2https://www.llamaindex.ai\n3https://www.langchain.com/\nC. Modular RAG\nThe modular RAG architecture advances beyond the for-\nmer two RAG paradigms, offering enhanced adaptability and\nversatility. It incorporates diverse strategies for improving its\ncomponents, such as adding a search module for similarity\nsearches and refining the retriever through fine-tuning. Inno-\nvations like restructured RAG modules [13] and rearranged\nRAG pipelines [14] have been introduced to tackle specific\nchallenges. The shift towards a modular RAG approach is\nbecoming prevalent, supporting both sequential processing and\nintegrated end-to-end training across its components. Despite\nits distinctiveness, Modular RAG builds upon the foundational\nprinciples of Advanced and Naive RAG, illustrating a progres-\nsion and refinement within the RAG family.\n1) New Modules: The Modular RAG framework introduces\nadditional specialized components to enhance retrieval and\nprocessing capabilities. The Search module adapts to spe-\ncific scenarios, enabling direct searches across various data\nsources like search engines, databases, and knowledge graphs,\nusing LLM-generated code and query languages [15]. RAG-\nFusion addresses traditional search limitations by employing\na multi-query strategy that expands user queries into diverse\nperspectives, utilizing parallel vector searches and intelligent\nre-ranking to uncover both explicit and transformative knowl-\nedge [16]. The Memory module leverages the LLM’s memory\nto guide retrieval, creating an unbounded memory pool that",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/C. Modular RAG",
- "metadata": {
- "length": 3574,
- "summary": "This page presents Figure 3, which visually compares three paradigms of Retrieval-Augmented Generation (RAG): Naive RAG, Advanced RAG, and Modular RAG. The text below the figure elaborates on pre-retrieval and post-retrieval processes, highlighting optimization strategies like query rewriting and reranking. It also details the Modular RAG architecture, emphasizing its flexibility through specialized components such as search modules and memory pools.",
- "page_nums": [
- 4
- ],
- "keywords": [
- "Naive RAG",
- "Advanced RAG",
- "Modular RAG",
- "LlamaIndex",
- "LangChain",
- "HayStack",
- "RAG-Fusion"
- ],
- "connect_to": [
- {
- "target": "asset_a108114a-c157-593e-bf49-4a24caea132b",
- "relation": "embeds",
- "ref": "[images/image_page_4_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 4,
- "artifact_ref": "page_citation_assets/page-4.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_2fc2a4ae-5b00-572a-886d-933e90468996",
- "type": "page",
- "content": "5\naligns the text more closely with data distribution through iter-\native self-enhancement [17], [18]. Routing in the RAG system\nnavigates through diverse data sources, selecting the optimal\npathway for a query, whether it involves summarization,\nspecific database searches, or merging different information\nstreams [19]. The Predict module aims to reduce redundancy\nand noise by generating context directly through the LLM,\nensuring relevance and accuracy [13]. Lastly, the Task Adapter\nmodule tailors RAG to various downstream tasks, automating\nprompt retrieval for zero-shot inputs and creating task-specific\nretrievers through few-shot query generation [20], [21] .This\ncomprehensive approach not only streamlines the retrieval pro-\ncess but also significantly improves the quality and relevance\nof the information retrieved, catering to a wide array of tasks\nand queries with enhanced precision and flexibility.\n2) New Patterns: Modular RAG offers remarkable adapt-\nability by allowing module substitution or reconfiguration\nto address specific challenges. This goes beyond the fixed\nstructures of Naive and Advanced RAG, characterized by a\nsimple “Retrieve” and “Read” mechanism. Moreover, Modular\nRAG expands this flexibility by integrating new modules or\nadjusting interaction flow among existing ones, enhancing its\napplicability across different tasks.\nInnovations such as the Rewrite-Retrieve-Read [7]model\nleverage the LLM’s capabilities to refine retrieval queries\nthrough a rewriting module and a LM-feedback mechanism\nto update rewriting model., improving task performance.\nSimilarly, approaches like Generate-Read [13] replace tradi-\ntional retrieval with LLM-generated content, while Recite-\nRead [22] emphasizes retrieval from model weights, enhanc-\ning the model’s ability to handle knowledge-intensive tasks.\nHybrid retrieval strategies integrate keyword, semantic, and\nvector searches to cater to diverse queries. Additionally, em-\nploying sub-queries and hypothetical document embeddings\n(HyDE) [11] seeks to improve retrieval relevance by focusing\non embedding similarities between generated answers and real\ndocuments.\nAdjustments in module arrangement and interaction, such\nas the Demonstrate-Search-Predict (DSP) [23] framework\nand the iterative Retrieve-Read-Retrieve-Read flow of ITER-\nRETGEN [14], showcase the dynamic use of module out-\nputs to bolster another module’s functionality, illustrating a\nsophisticated understanding of enhancing module synergy.\nThe flexible orchestration of Modular RAG Flow showcases\nthe benefits of adaptive retrieval through techniques such as\nFLARE [24] and Self-RAG [25]. This approach transcends\nthe fixed RAG retrieval process by evaluating the necessity\nof retrieval based on different scenarios. Another benefit of\na flexible architecture is that the RAG system can more\neasily integrate with other technologies (such as fine-tuning\nor reinforcement learning) [26]. For example, this can involve\nfine-tuning the retriever for better retrieval results, fine-tuning\nthe generator for more personalized outputs, or engaging in\ncollaborative fine-tuning [27].\nD. RAG vs Fine-tuning\nThe augmentation of LLMs has attracted considerable atten-\ntion due to their growing prevalence. Among the optimization\nmethods for LLMs, RAG is often compared with Fine-tuning\n(FT) and prompt engineering. Each method has distinct charac-\nteristics as illustrated in Figure 4. We used a quadrant chart to\nillustrate the differences among three methods in two dimen-\nsions: external knowledge requirements and model adaption\nrequirements. Prompt engineering leverages a model’s inherent\ncapabilities with minimum necessity for external knowledge\nand model adaption. RAG can be likened to providing a model\nwith a tailored textbook for information retrieval, ideal for pre-\ncise information retrieval tasks. In contrast, FT is comparable\nto a student internalizing knowledge over time, suitable for\nscenarios requiring replication of specific structures, styles, or\nformats.\nRAG excels in dynamic environments by offering real-\ntime knowledge updates and effective utilization of external\nknowledge sources with high interpretability. However, it\ncomes with higher latency and ethical considerations regarding\ndata retrieval. On the other hand, FT is more static, requiring\nretraining for updates but enabling deep customization of the\nmodel’s behavior and style. It demands significant compu-\ntational resources for dataset preparation and training, and\nwhile it can reduce hallucinations, it may face challenges with\nunfamiliar data.\nIn multiple evaluations of their performance on various\nknowledge-intensive tasks across different topics, [28] re-\nvealed that while unsupervised fine-tuning shows some im-\nprovement, RAG consistently outperforms it, for both exist-\ning knowledge encountered during training and entirely new\nknowledge. Additionally, it was found that LLMs struggle\nto learn new factual information through unsupervised fine-\ntuning. The choice between RAG and FT depends on the\nspecific needs for data dynamics, customization, and com-\nputational capabilities in the application context. RAG and\nFT are not mutually exclusive and can complement each\nother, enhancing a model’s capabilities at different levels.\nIn some instances, their combined use may lead to optimal\nperformance. The optimization process involving RAG and FT\nmay require multiple iterations to achieve satisfactory results.\nIII. RETRIEVAL\nIn the context of RAG, it is crucial to efficiently retrieve\nrelevant documents from the data source. There are several\nkey issues involved, such as the retrieval source, retrieval\ngranularity, pre-processing of the retrieval, and selection of\nthe corresponding embedding model.\nA. Retrieval Source\nRAG relies on external knowledge to enhance LLMs, while\nthe type of retrieval source and the granularity of retrieval\nunits both affect the final generation results.\n1) Data Structure: Initially, text is s the mainstream source\nof retrieval. Subsequently, the retrieval source expanded to in-\nclude semi-structured data (PDF) and structured data (Knowl-\nedge Graph, KG) for enhancement. In addition to retrieving\nfrom original external sources, there is also a growing trend in\nrecent researches towards utilizing content generated by LLMs\nthemselves for retrieval and enhancement purposes.",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/D. RAG vs Fine-tuning",
- "metadata": {
- "length": 6362,
- "summary": "The section 'D. RAG vs Fine-tuning' compares Retrieval-Augmented Generation (RAG) and Fine-tuning (FT) as optimization methods for Large Language Models (LLMs). It describes RAG as a method that leverages external knowledge sources, offering real-time updates and interpretability but with higher latency. In contrast, FT involves retraining the model to internalize knowledge, allowing for deep customization of behavior and style but requiring significant computational resources. The text notes that while unsupervised fine-tuning shows some improvement, RAG consistently outperforms it in handling new knowledge. The section concludes by stating that RAG and FT are not mutually exclusive and can complement each other.",
- "page_nums": [
- 5
- ],
- "keywords": [
- "RAG",
- "Fine-tuning",
- "LLMs"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 5,
- "artifact_ref": "page_citation_assets/page-5.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_bff6aa56-61a7-5ea2-a396-b71a0ee59036",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/D. RAG vs Fine-tuning p5]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/A. Retrieval Source",
- "metadata": {
- "length": 130,
- "summary": "This section, titled 'A. Retrieval Source', discusses the types of data sources used in Retrieval-Augmented Generation (RAG) systems to enhance Large Language Models (LLMs). It outlines the evolution from text as the primary source to include semi-structured data like PDFs and structured data such as Knowledge Graphs (KG). Additionally, it highlights a growing trend of utilizing content generated by LLMs themselves for retrieval and enhancement purposes.",
- "page_nums": [
- 5
- ],
- "keywords": [
- "RAG",
- "LLMs",
- "PDF",
- "Knowledge Graph"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 5,
- "artifact_ref": "page_citation_assets/page-5.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_78538662-ce62-5a5c-b55d-63d9364f3600",
- "type": "page",
- "content": "6\nTABLE I\nSUMMARY OF RAG METHODS\nMethod\nRetrieval Source\nRetrieval\nData Type\nRetrieval\nGranularity\nAugmentation\nStage\nRetrieval\nprocess\nCoG [29]\nWikipedia\nText\nPhrase\nPre-training\nIterative\nDenseX [30]\nFactoidWiki\nText\nProposition\nInference\nOnce\nEAR [31]\nDataset-base\nText\nSentence\nTuning\nOnce\nUPRISE [20]\nDataset-base\nText\nSentence\nTuning\nOnce\nRAST [32]\nDataset-base\nText\nSentence\nTuning\nOnce\nSelf-Mem [17]\nDataset-base\nText\nSentence\nTuning\nIterative\nFLARE [24]\nSearch Engine,Wikipedia\nText\nSentence\nTuning\nAdaptive\nPGRA [33]\nWikipedia\nText\nSentence\nInference\nOnce\nFILCO [34]\nWikipedia\nText\nSentence\nInference\nOnce\nRADA [35]\nDataset-base\nText\nSentence\nInference\nOnce\nFilter-rerank [36]\nSynthesized dataset\nText\nSentence\nInference\nOnce\nR-GQA [37]\nDataset-base\nText\nSentence Pair\nTuning\nOnce\nLLM-R [38]\nDataset-base\nText\nSentence Pair\nInference\nIterative\nTIGER [39]\nDataset-base\nText\nItem-base\nPre-training\nOnce\nLM-Indexer [40]\nDataset-base\nText\nItem-base\nTuning\nOnce\nBEQUE [9]\nDataset-base\nText\nItem-base\nTuning\nOnce\nCT-RAG [41]\nSynthesized dataset\nText\nItem-base\nTuning\nOnce\nAtlas [42]\nWikipedia, Common Crawl\nText\nChunk\nPre-training\nIterative\nRAVEN [43]\nWikipedia\nText\nChunk\nPre-training\nOnce\nRETRO++ [44]\nPre-training Corpus\nText\nChunk\nPre-training\nIterative\nINSTRUCTRETRO [45]\nPre-training corpus\nText\nChunk\nPre-training\nIterative\nRRR [7]\nSearch Engine\nText\nChunk\nTuning\nOnce\nRA-e2e [46]\nDataset-base\nText\nChunk\nTuning\nOnce\nPROMPTAGATOR [21]\nBEIR\nText\nChunk\nTuning\nOnce\nAAR [47]\nMSMARCO,Wikipedia\nText\nChunk\nTuning\nOnce\nRA-DIT [27]\nCommon Crawl,Wikipedia\nText\nChunk\nTuning\nOnce\nRAG-Robust [48]\nWikipedia\nText\nChunk\nTuning\nOnce\nRA-Long-Form [49]\nDataset-base\nText\nChunk\nTuning\nOnce\nCoN [50]\nWikipedia\nText\nChunk\nTuning\nOnce\nSelf-RAG [25]\nWikipedia\nText\nChunk\nTuning\nAdaptive\nBGM [26]\nWikipedia\nText\nChunk\nInference\nOnce\nCoQ [51]\nWikipedia\nText\nChunk\nInference\nIterative\nToken-Elimination [52]\nWikipedia\nText\nChunk\nInference\nOnce\nPaperQA [53]\nArxiv,Online Database,PubMed\nText\nChunk\nInference\nIterative\nNoiseRAG [54]\nFactoidWiki\nText\nChunk\nInference\nOnce\nIAG [55]\nSearch Engine,Wikipedia\nText\nChunk\nInference\nOnce\nNoMIRACL [56]\nWikipedia\nText\nChunk\nInference\nOnce\nToC [57]\nSearch Engine,Wikipedia\nText\nChunk\nInference\nRecursive\nSKR [58]\nDataset-base,Wikipedia\nText\nChunk\nInference\nAdaptive\nITRG [59]\nWikipedia\nText\nChunk\nInference\nIterative\nRAG-LongContext [60]\nDataset-base\nText\nChunk\nInference\nOnce\nITER-RETGEN [14]\nWikipedia\nText\nChunk\nInference\nIterative\nIRCoT [61]\nWikipedia\nText\nChunk\nInference\nRecursive\nLLM-Knowledge-Boundary [62]\nWikipedia\nText\nChunk\nInference\nOnce\nRAPTOR [63]\nDataset-base\nText\nChunk\nInference\nRecursive\nRECITE [22]\nLLMs\nText\nChunk\nInference\nOnce\nICRALM [64]\nPile,Wikipedia\nText\nChunk\nInference\nIterative\nRetrieve-and-Sample [65]\nDataset-base\nText\nDoc\nTuning\nOnce\nZemi [66]\nC4\nText\nDoc\nTuning\nOnce\nCRAG [67]\nArxiv\nText\nDoc\nInference\nOnce\n1-PAGER [68]\nWikipedia\nText\nDoc\nInference\nIterative\nPRCA [69]\nDataset-base\nText\nDoc\nInference\nOnce\nQLM-Doc-ranking [70]\nDataset-base\nText\nDoc\nInference\nOnce\nRecomp [71]\nWikipedia\nText\nDoc\nInference\nOnce\nDSP [23]\nWikipedia\nText\nDoc\nInference\nIterative\nRePLUG [72]\nPile\nText\nDoc\nInference\nOnce\nARM-RAG [73]\nDataset-base\nText\nDoc\nInference\nIterative\nGenRead [13]\nLLMs\nText\nDoc\nInference\nIterative\nUniMS-RAG [74]\nDataset-base\nText\nMulti\nTuning\nOnce\nCREA-ICL [19]\nDataset-base\nCrosslingual,Text\nSentence\nInference\nOnce\nPKG [75]\nLLM\nTabular,Text\nChunk\nInference\nOnce\nSANTA [76]\nDataset-base\nCode,Text\nItem\nPre-training\nOnce\nSURGE [77]\nFreebase\nKG\nSub-Graph\nTuning\nOnce\nMK-ToD [78]\nDataset-base\nKG\nEntity\nTuning\nOnce\nDual-Feedback-ToD [79]\nDataset-base\nKG\nEntity Sequence\nTuning\nOnce\nKnowledGPT [15]\nDataset-base\nKG\nTriplet\nInference\nMuti-time\nFABULA [80]\nDataset-base,Graph\nKG\nEntity\nInference\nOnce\nHyKGE [81]\nCMeKG\nKG\nEntity\nInference\nOnce\nKALMV [82]\nWikipedia\nKG\nTriplet\nInference\nIterative\nRoG [83]\nFreebase\nKG\nTriplet\nInference\nIterative\nG-Retriever [84]\nDataset-base\nTextGraph\nSub-Graph\nInference\nOnce",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/A. Retrieval Source/TABLE I",
- "metadata": {
- "length": 3967,
- "summary": "",
- "page_nums": [
- 6
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 6,
- "artifact_ref": "page_citation_assets/page-6.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_c772c463-4557-5591-85c9-d83ae8272062",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/A. Retrieval Source/TABLE I p6]\n\n7\nFig. 4. RAG compared with other model optimization methods in the aspects of “External Knowledge Required” and “Model Adaption Required”. Prompt\nEngineering requires low modifications to the model and external knowledge, focusing on harnessing the capabilities of LLMs themselves. Fine-tuning, on\nthe other hand, involves further training the model. In the early stages of RAG (Naive RAG), there is a low demand for model modifications. As research\nprogresses, Modular RAG has become more integrated with fine-tuning techniques.\nUnstructured Data, such as text, is the most widely used\nretrieval source, which are mainly gathered from corpus. For\nopen-domain question-answering (ODQA) tasks, the primary\nretrieval sources are Wikipedia Dump with the current major\nversions including HotpotQA 4 (1st October , 2017), DPR5 (20\nDecember, 2018). In addition to encyclopedic data, common\nunstructured data includes cross-lingual text [19] and domain-\nspecific data (such as medical [67]and legal domains [29]).\nSemi-structured data. typically refers to data that contains a\ncombination of text and table information, such as PDF. Han-\ndling semi-structured data poses challenges for conventional\nRAG systems due to two main reasons. Firstly, text splitting\nprocesses may inadvertently separate tables, leading to data\ncorruption during retrieval. Secondly, incorporating tables into\nthe data can complicate semantic similarity searches. When\ndealing with semi-structured data, one approach involves lever-\naging the code capabilities of LLMs to execute Text-2-SQL\nqueries on tables within databases, such as TableGPT [85].\nAlternatively, tables can be transformed into text format for\nfurther analysis using text-based methods [75]. However, both\nof these methods are not optimal solutions, indicating substan-\ntial research opportunities in this area.\nStructured data, such as knowledge graphs (KGs) [86] ,\nwhich are typically verified and can provide more precise in-\nformation. KnowledGPT [15] generates KB search queries and\nstores knowledge in a personalized base, enhancing the RAG\nmodel’s knowledge richness. In response to the limitations of\nLLMs in understanding and answering questions about textual\ngraphs, G-Retriever [84] integrates Graph Neural Networks\n4https://hotpotqa.github.io/wiki-readme.html\n5https://github.com/facebookresearch/DPR\n(GNNs), LLMs and RAG, enhancing graph comprehension\nand question-answering capabilities through soft prompting\nof the LLM, and employs the Prize-Collecting Steiner Tree\n(PCST) optimization problem for targeted graph retrieval. On\nthe contrary, it requires additional effort to build, validate,\nand maintain structured databases. On the contrary, it requires\nadditional effort to build, validate, and maintain structured\ndatabases.\nLLMs-Generated Content. Addressing the limitations of\nexternal auxiliary information in RAG, some research has\nfocused on exploiting LLMs’ internal knowledge. SKR [58]\nclassifies questions as known or unknown, applying retrieval\nenhancement selectively. GenRead [13] replaces the retriever\nwith an LLM generator, finding that LLM-generated contexts\noften contain more accurate answers due to better alignment\nwith the pre-training objectives of causal language modeling.\nSelfmem [17] iteratively creates an unbounded memory pool\nwith a retrieval-enhanced generator, using a memory selec-\ntor to choose outputs that serve as dual problems to the\noriginal question, thus self-enhancing the generative model.\nThese methodologies underscore the breadth of innovative\ndata source utilization in RAG, striving to improve model\nperformance and task effectiveness.\n2) Retrieval Granularity: Another important factor besides\nthe data format of the retrieval source is the granularity of\nthe retrieved data. Coarse-grained retrieval units theoretically\ncan provide more relevant information for the problem, but\nthey may also contain redundant content, which could distract\nthe retriever and language models in downstream tasks [50],\n[87]. On the other hand, fine-grained retrieval unit granularity\nincreases the burden of retrieval and does not guarantee seman-\ntic integrity and meeting the required knowledge. Choosing",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/A. Retrieval Source/SUMMARY OF RAG METHODS",
- "metadata": {
- "length": 4326,
- "summary": "The page features Figure 4, a chart comparing RAG (Retrieval-Augmented Generation) with other model optimization methods based on the axes of 'External Knowledge Required' and 'Model Adaptation Required'. The figure illustrates that Prompt Engineering requires low external knowledge and adaptation, while Fine-tuning requires high adaptation. The text below discusses various data sources for retrieval systems, including unstructured data (e.g., Wikipedia, HotpotQA), semi-structured data (e.g., PDFs, tables via TableGPT), and structured data like knowledge graphs (e.g., G-Retriever). It also covers LLM-generated content and retrieval granularity.",
- "page_nums": [
- 6,
- 7
- ],
- "keywords": [
- "RAG",
- "Wikipedia Dump",
- "HotpotQA",
- "DPR",
- "TableGPT",
- "G-Retriever",
- "LLMs"
- ],
- "connect_to": [
- {
- "target": "asset_5ab3ba23-ad7b-5fe5-b17f-e7c39765cc4a",
- "relation": "embeds",
- "ref": "[images/image_page_7_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 6,
- "artifact_ref": "page_citation_assets/page-6.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 7,
- "artifact_ref": "page_citation_assets/page-7.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_80ddce91-66bc-58e4-8b91-2534d66e8ec1",
- "type": "page",
- "content": "8\nthe appropriate retrieval granularity during inference can be\na simple and effective strategy to improve the retrieval and\ndownstream task performance of dense retrievers.\nIn text, retrieval granularity ranges from fine to coarse,\nincluding Token, Phrase, Sentence, Proposition, Chunks, Doc-\nument. Among them, DenseX [30]proposed the concept of\nusing propositions as retrieval units. Propositions are defined\nas atomic expressions in the text, each encapsulating a unique\nfactual segment and presented in a concise, self-contained nat-\nural language format. This approach aims to enhance retrieval\nprecision and relevance. On the Knowledge Graph (KG),\nretrieval granularity includes Entity, Triplet, and sub-Graph.\nThe granularity of retrieval can also be adapted to downstream\ntasks, such as retrieving Item IDs [40]in recommendation tasks\nand Sentence pairs [38]. Detailed information is illustrated in\nTable I.\nB. Indexing Optimization\nIn the Indexing phase, documents will be processed, seg-\nmented, and transformed into Embeddings to be stored in a\nvector database. The quality of index construction determines\nwhether the correct context can be obtained in the retrieval\nphase.\n1) Chunking Strategy: The most common method is to split\nthe document into chunks on a fixed number of tokens (e.g.,\n100, 256, 512) [88]. Larger chunks can capture more context,\nbut they also generate more noise, requiring longer processing\ntime and higher costs. While smaller chunks may not fully\nconvey the necessary context, they do have less noise. How-\never, chunks leads to truncation within sentences, prompting\nthe optimization of a recursive splits and sliding window meth-\nods, enabling layered retrieval by merging globally related\ninformation across multiple retrieval processes [89]. Never-\ntheless, these approaches still cannot strike a balance between\nsemantic completeness and context length. Therefore, methods\nlike Small2Big have been proposed, where sentences (small)\nare used as the retrieval unit, and the preceding and following\nsentences are provided as (big) context to LLMs [90].\n2) Metadata Attachments: Chunks can be enriched with\nmetadata information such as page number, file name, au-\nthor,category timestamp. Subsequently, retrieval can be filtered\nbased on this metadata, limiting the scope of the retrieval.\nAssigning different weights to document timestamps during\nretrieval can achieve time-aware RAG, ensuring the freshness\nof knowledge and avoiding outdated information.\nIn addition to extracting metadata from the original doc-\numents, metadata can also be artificially constructed. For\nexample, adding summaries of paragraph, as well as intro-\nducing hypothetical questions. This method is also known as\nReverse HyDE. Specifically, using LLM to generate questions\nthat can be answered by the document, then calculating the\nsimilarity between the original question and the hypothetical\nquestion during retrieval to reduce the semantic gap between\nthe question and the answer.\n3) Structural Index: One effective method for enhancing\ninformation retrieval is to establish a hierarchical structure for\nthe documents. By constructing In structure, RAG system can\nexpedite the retrieval and processing of pertinent data.\nHierarchical index structure. File are arranged in parent-\nchild relationships, with chunks linked to them. Data sum-\nmaries are stored at each node, aiding in the swift traversal\nof data and assisting the RAG system in determining which\nchunks to extract. This approach can also mitigate the illusion\ncaused by block extraction issues.\nKnowledge Graph index. Utilize KG in constructing the\nhierarchical structure of documents contributes to maintaining\nconsistency. It delineates the connections between different\nconcepts and entities, markedly reducing the potential for\nillusions. Another advantage is the transformation of the\ninformation retrieval process into instructions that LLM can\ncomprehend, thereby enhancing the accuracy of knowledge\nretrieval and enabling LLM to generate contextually coherent\nresponses, thus improving the overall efficiency of the RAG\nsystem. To capture the logical relationship between document\ncontent and structure, KGP [91] proposed a method of building\nan index between multiple documents using KG. This KG\nconsists of nodes (representing paragraphs or structures in the\ndocuments, such as pages and tables) and edges (indicating\nsemantic/lexical similarity between paragraphs or relationships\nwithin the document structure), effectively addressing knowl-\nedge retrieval and reasoning problems in a multi-document\nenvironment.\nC. Query Optimization\nOne of the primary challenges with Naive RAG is its\ndirect reliance on the user’s original query as the basis for\nretrieval. Formulating a precise and clear question is difficult,\nand imprudent queries result in subpar retrieval effectiveness.\nSometimes, the question itself is complex, and the language\nis not well-organized. Another difficulty lies in language\ncomplexity ambiguity. Language models often struggle when\ndealing with specialized vocabulary or ambiguous abbrevi-\nations with multiple meanings. For instance, they may not\ndiscern whether “LLM” refers to large language model or a\nMaster of Laws in a legal context.\n1) Query Expansion: Expanding a single query into mul-\ntiple queries enriches the content of the query, providing\nfurther context to address any lack of specific nuances, thereby\nensuring the optimal relevance of the generated answers.\nMulti-Query. By employing prompt engineering to expand\nqueries via LLMs, these queries can then be executed in\nparallel. The expansion of queries is not random, but rather\nmeticulously designed.\nSub-Query. The process of sub-question planning represents\nthe generation of the necessary sub-questions to contextualize\nand fully answer the original question when combined. This\nprocess of adding relevant context is, in principle, similar\nto query expansion. Specifically, a complex question can be\ndecomposed into a series of simpler sub-questions using the\nleast-to-most prompting method [92].\nChain-of-Verification(CoVe). The expanded queries undergo\nvalidation by LLM to achieve the effect of reducing halluci-\nnations. Validated expanded queries typically exhibit higher\nreliability [93].",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/B. Indexing Optimization",
- "metadata": {
- "length": 6289,
- "summary": "The section 'B. Indexing Optimization' discusses strategies for processing and segmenting documents into embeddings for storage in a vector database. It details three main approaches: Chunking Strategy (splitting text into fixed or recursive chunks, mentioning methods like Small2Big), Metadata Attachments (enriching chunks with page numbers, file names, or artificial metadata like hypothetical questions via Reverse HyDE), and Structural Index (establishing hierarchical structures like parent-child relationships or Knowledge Graphs to improve retrieval consistency and efficiency).",
- "page_nums": [
- 8
- ],
- "keywords": [
- "DenseX",
- "Small2Big",
- "Reverse HyDE"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_d966d18c-d38d-5b29-9172-c1630b9dea76",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/B. Indexing Optimization p8]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/C. Query Optimization",
- "metadata": {
- "length": 128,
- "summary": "The section 'Query Optimization' addresses the challenges of Naive RAG, such as imprecise queries and language ambiguity. It outlines several strategies to improve retrieval effectiveness: Query Expansion (enriching a single query with more context), Multi-Query (executing multiple expanded queries in parallel), Sub-Query (decomposing complex questions into simpler sub-questions), and Chain-of-Verification (CoVe) (validating expanded queries via an LLM to reduce hallucinations).",
- "page_nums": [
- 8
- ],
- "keywords": [
- "Naive RAG",
- "LLM",
- "RAG system"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_e3903385-26d9-5797-aece-072b9b1d0aae",
- "type": "page",
- "content": "9\n2) Query Transformation: The core concept is to retrieve\nchunks based on a transformed query instead of the user’s\noriginal query.\nQuery Rewrite.The original queries are not always optimal\nfor LLM retrieval, especially in real-world scenarios. There-\nfore, we can prompt LLM to rewrite the queries. In addition to\nusing LLM for query rewriting, specialized smaller language\nmodels, such as RRR (Rewrite-retrieve-read) [7]. The imple-\nmentation of the query rewrite method in the Taobao, known\nas BEQUE [9] has notably enhanced recall effectiveness for\nlong-tail queries, resulting in a rise in GMV.\nAnother query transformation method is to use prompt\nengineering to let LLM generate a query based on the original\nquery for subsequent retrieval. HyDE [11] construct hypothet-\nical documents (assumed answers to the original query). It\nfocuses on embedding similarity from answer to answer rather\nthan seeking embedding similarity for the problem or query.\nUsing the Step-back Prompting method [10], the original\nquery is abstracted to generate a high-level concept question\n(step-back question). In the RAG system, both the step-back\nquestion and the original query are used for retrieval, and both\nthe results are utilized as the basis for language model answer\ngeneration.\n3) Query Routing: Based on varying queries, routing to\ndistinct RAG pipeline,which is suitable for a versatile RAG\nsystem designed to accommodate diverse scenarios.\nMetadata Router/ Filter. The first step involves extracting\nkeywords (entity) from the query, followed by filtering based\non the keywords and metadata within the chunks to narrow\ndown the search scope.\nSemantic Router is another method of routing involves\nleveraging the semantic information of the query. Specific\napprach see Semantic Router 6. Certainly, a hybrid routing\napproach can also be employed, combining both semantic and\nmetadata-based methods for enhanced query routing.\nD. Embedding\nIn RAG, retrieval is achieved by calculating the similarity\n(e.g. cosine similarity) between the embeddings of the ques-\ntion and document chunks, where the semantic representation\ncapability of embedding models plays a key role. This mainly\nincludes a sparse encoder (BM25) and a dense retriever (BERT\narchitecture Pre-training language models). Recent research\nhas introduced prominent embedding models such as AngIE,\nVoyage, BGE,etc [94]–[96], which are benefit from multi-task\ninstruct tuning. Hugging Face’s MTEB leaderboard 7 evaluates\nembedding models across 8 tasks, covering 58 datasests. Ad-\nditionally, C-MTEB focuses on Chinese capability, covering\n6 tasks and 35 datasets. There is no one-size-fits-all answer\nto “which embedding model to use.” However, some specific\nmodels are better suited for particular use cases.\n1) Mix/hybrid Retrieval : Sparse and dense embedding\napproaches capture different relevance features and can ben-\nefit from each other by leveraging complementary relevance\ninformation. For instance, sparse retrieval models can be used\n6https://github.com/aurelio-labs/semantic-router\n7https://huggingface.co/spaces/mteb/leaderboard\nto provide initial search results for training dense retrieval\nmodels. Additionally, pre-training language models (PLMs)\ncan be utilized to learn term weights to enhance sparse\nretrieval. Specifically, it also demonstrates that sparse retrieval\nmodels can enhance the zero-shot retrieval capability of dense\nretrieval models and assist dense retrievers in handling queries\ncontaining rare entities, thereby improving robustness.\n2) Fine-tuning Embedding Model: In instances where the\ncontext significantly deviates from pre-training corpus, partic-\nularly within highly specialized disciplines such as healthcare,\nlegal practice, and other sectors replete with proprietary jargon,\nfine-tuning the embedding model on your own domain dataset\nbecomes essential to mitigate such discrepancies.\nIn addition to supplementing domain knowledge, another\npurpose of fine-tuning is to align the retriever and generator,\nfor example, using the results of LLM as the supervision signal\nfor fine-tuning, known as LSR (LM-supervised Retriever).\nPROMPTAGATOR [21] utilizes the LLM as a few-shot query\ngenerator to create task-specific retrievers, addressing chal-\nlenges in supervised fine-tuning, particularly in data-scarce\ndomains. Another approach, LLM-Embedder [97], exploits\nLLMs to generate reward signals across multiple downstream\ntasks. The retriever is fine-tuned with two types of supervised\nsignals: hard labels for the dataset and soft rewards from\nthe LLMs. This dual-signal approach fosters a more effective\nfine-tuning process, tailoring the embedding model to diverse\ndownstream applications. REPLUG [72] utilizes a retriever\nand an LLM to calculate the probability distributions of the\nretrieved documents and then performs supervised training\nby computing the KL divergence. This straightforward and\neffective training method enhances the performance of the\nretrieval model by using an LM as the supervisory signal,\neliminating the need for specific cross-attention mechanisms.\nMoreover, inspired by RLHF (Reinforcement Learning from\nHuman Feedback), utilizing LM-based feedback to reinforce\nthe retriever through reinforcement learning.\nE. Adapter\nFine-tuning models may present challenges, such as in-\ntegrating functionality through an API or addressing con-\nstraints arising from limited local computational resources.\nConsequently, some approaches opt to incorporate an external\nadapter to aid in alignment.\nTo optimize the multi-task capabilities of LLM, UP-\nRISE [20] trained a lightweight prompt retriever that can\nautomatically retrieve prompts from a pre-built prompt pool\nthat are suitable for a given zero-shot task input. AAR\n(Augmentation-Adapted Retriver) [47] introduces a universal\nadapter designed to accommodate multiple downstream tasks.\nWhile PRCA [69] add a pluggable reward-driven contextual\nadapter to enhance performance on specific tasks. BGM [26]\nkeeps the retriever and LLM fixed,and trains a bridge Seq2Seq\nmodel in between. The bridge model aims to transform the\nretrieved information into a format that LLMs can work with\neffectively, allowing it to not only rerank but also dynami-\ncally select passages for each query, and potentially employ\nmore advanced strategies like repetition. Furthermore, PKG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/D. Embedding",
- "metadata": {
- "length": 6341,
- "summary": "The section discusses the role of embedding models in RAG systems, which calculate similarity between question and document chunks. It highlights prominent models like AnglE, Voyage, and BGE that benefit from multi-task instruction tuning. The text also mentions evaluation benchmarks such as Hugging Face's MTEB leaderboard (covering 8 tasks and 58 datasets) and C-MTEB for Chinese capability. Finally, it introduces 'Mix/hybrid Retrieval' as a method combining sparse and dense embedding approaches to capture different relevance features.",
- "page_nums": [
- 9
- ],
- "keywords": [
- "RAG",
- "AnglE",
- "Voyage",
- "BGE",
- "Hugging Face",
- "MTEB",
- "C-MTEB"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_70cbd426-3e38-56f8-a75f-ed1b047bb381",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/D. Embedding p9]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/E. Adapter",
- "metadata": {
- "length": 116,
- "summary": "The section titled 'E. Adapter' discusses the use of external adapters to address challenges in fine-tuning large language models (LLMs), such as limited computational resources or API integration constraints. It describes several approaches including UPRISE, which trains a lightweight prompt retriever; AAR (Augmentation-Adapted Retriever), a universal adapter for multiple downstream tasks; PRCA, a pluggable reward-driven contextual adapter; BGM, which uses a bridge Seq2Seq model between the retriever and LLM; and PKG, which employs advanced strategies like repetition.",
- "page_nums": [
- 9
- ],
- "keywords": [
- "UPRISE",
- "AAR",
- "PRCA",
- "BGM",
- "PKG"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_d794b0d2-eb82-5436-8cd4-f49ff6d75398",
- "type": "page",
- "content": "10\nintroduces an innovative method for integrating knowledge\ninto white-box models via directive fine-tuning [75]. In this\napproach, the retriever module is directly substituted to gen-\nerate relevant documents according to a query. This method\nassists in addressing the difficulties encountered during the\nfine-tuning process and enhances model performance.\nIV. GENERATION\nAfter retrieval, it is not a good practice to directly input all\nthe retrieved information to the LLM for answering questions.\nFollowing will introduce adjustments from two perspectives:\nadjusting the retrieved content and adjusting the LLM.\nA. Context Curation\nRedundant information can interfere with the final gener-\nation of LLM, and overly long contexts can also lead LLM\nto the “Lost in the middle” problem [98]. Like humans, LLM\ntends to only focus on the beginning and end of long texts,\nwhile forgetting the middle portion. Therefore, in the RAG\nsystem, we typically need to further process the retrieved\ncontent.\n1) Reranking: Reranking fundamentally reorders document\nchunks to highlight the most pertinent results first, effectively\nreducing the overall document pool, severing a dual purpose\nin information retrieval, acting as both an enhancer and a\nfilter, delivering refined inputs for more precise language\nmodel processing [70]. Reranking can be performed using\nrule-based methods that depend on predefined metrics like\nDiversity, Relevance, and MRR, or model-based approaches\nlike Encoder-Decoder models from the BERT series (e.g.,\nSpanBERT), specialized reranking models such as Cohere\nrerank or bge-raranker-large, and general large language mod-\nels like GPT [12], [99].\n2) Context Selection/Compression: A common misconcep-\ntion in the RAG process is the belief that retrieving as many\nrelevant documents as possible and concatenating them to form\na lengthy retrieval prompt is beneficial. However, excessive\ncontext can introduce more noise, diminishing the LLM’s\nperception of key information .\n(Long) LLMLingua [100], [101] utilize small language\nmodels (SLMs) such as GPT-2 Small or LLaMA-7B, to\ndetect and remove unimportant tokens, transforming it into\na form that is challenging for humans to comprehend but\nwell understood by LLMs. This approach presents a direct\nand practical method for prompt compression, eliminating the\nneed for additional training of LLMs while balancing language\nintegrity and compression ratio. PRCA tackled this issue by\ntraining an information extractor [69]. Similarly, RECOMP\nadopts a comparable approach by training an information\ncondenser using contrastive learning [71]. Each training data\npoint consists of one positive sample and five negative sam-\nples, and the encoder undergoes training using contrastive loss\nthroughout this process [102] .\nIn addition to compressing the context, reducing the num-\nber of documents aslo helps improve the accuracy of the\nmodel’s answers. Ma et al. [103] propose the “Filter-Reranker”\nparadigm, which combines the strengths of LLMs and SLMs.\nIn this paradigm, SLMs serve as filters, while LLMs function\nas reordering agents. The research shows that instructing\nLLMs to rearrange challenging samples identified by SLMs\nleads to significant improvements in various Information\nExtraction (IE) tasks. Another straightforward and effective\napproach involves having the LLM evaluate the retrieved\ncontent before generating the final answer. This allows the\nLLM to filter out documents with poor relevance through LLM\ncritique. For instance, in Chatlaw [104], the LLM is prompted\nto self-suggestion on the referenced legal provisions to assess\ntheir relevance.\nB. LLM Fine-tuning\nTargeted fine-tuning based on the scenario and data char-\nacteristics on LLMs can yield better results. This is also one\nof the greatest advantages of using on-premise LLMs. When\nLLMs lack data in a specific domain, additional knowledge can\nbe provided to the LLM through fine-tuning. Huggingface’s\nfine-tuning data can also be used as an initial step.\nAnother benefit of fine-tuning is the ability to adjust the\nmodel’s input and output. For example, it can enable LLM to\nadapt to specific data formats and generate responses in a par-\nticular style as instructed [37]. For retrieval tasks that engage\nwith structured data, the SANTA framework [76] implements\na tripartite training regimen to effectively encapsulate both\nstructural and semantic nuances. The initial phase focuses on\nthe retriever, where contrastive learning is harnessed to refine\nthe query and document embeddings.\nAligning LLM outputs with human or retriever preferences\nthrough reinforcement learning is a potential approach. For\ninstance, manually annotating the final generated answers\nand then providing feedback through reinforcement learning.\nIn addition to aligning with human preferences, it is also\npossible to align with the preferences of fine-tuned models\nand retrievers [79]. When circumstances prevent access to\npowerful proprietary models or larger parameter open-source\nmodels, a simple and effective method is to distill the more\npowerful models(e.g. GPT-4). Fine-tuning of LLM can also\nbe coordinated with fine-tuning of the retriever to align pref-\nerences. A typical approach, such as RA-DIT [27], aligns the\nscoring functions between Retriever and Generator using KL\ndivergence.\nV. AUGMENTATION PROCESS IN RAG\nIn the domain of RAG, the standard practice often involves\na singular (once) retrieval step followed by generation, which\ncan lead to inefficiencies and sometimes is typically insuffi-\ncient for complex problems demanding multi-step reasoning,\nas it provides a limited scope of information [105]. Many\nstudies have optimized the retrieval process in response to this\nissue, and we have summarised them in Figure 5.\nA. Iterative Retrieval\nIterative retrieval is a process where the knowledge base\nis repeatedly searched based on the initial query and the text\ngenerated so far, providing a more comprehensive knowledge",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/IV. GENERATION/A. Context Curation",
- "metadata": {
- "length": 5970,
- "summary": "The section 'A. Context Curation' discusses methods to process retrieved information before it is fed into a Large Language Model (LLM) to avoid issues like the 'Lost in the middle' problem. It details two main approaches: 1) Reranking, which reorders document chunks based on relevance using metrics or models like BERT and GPT; and 2) Context Selection/Compression, which reduces context length by removing unimportant tokens or summarizing content using techniques like contrastive learning or small language models (SLMs).",
- "page_nums": [
- 10
- ],
- "keywords": [
- "LLM",
- "RAG",
- "BERT",
- "SpanBERT",
- "Cohere rerank",
- "bge-ranker-large",
- "GPT",
- "LLMLingua",
- "GPT-2 Small",
- "LLaMA-7B",
- "PRCA",
- "RECOMP",
- "Ma et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_7d03b19f-e5c9-5475-a896-c71a26b994f3",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/IV. GENERATION/A. Context Curation p10]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/IV. GENERATION/B. LLM Fine-tuning",
- "metadata": {
- "length": 124,
- "summary": "This section discusses targeted fine-tuning of Large Language Models (LLMs) to improve performance in specific domains or data formats. It highlights benefits such as incorporating additional knowledge, adapting to structured data (e.g., via the SANTA framework), and aligning outputs with human or retriever preferences through reinforcement learning. The text also covers distilling powerful proprietary models into smaller ones and coordinating fine-tuning between retrievers and generators using methods like RA-DIT.",
- "page_nums": [
- 10
- ],
- "keywords": [
- "Huggingface",
- "SANTA framework",
- "Chatlaw",
- "GPT-4",
- "RA-DIT"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_44c4ccb5-e484-5e7a-98a7-c83661a853b6",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/IV. GENERATION/A. Context Curation p10]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG/A. Iterative Retrieval",
- "metadata": {
- "length": 124,
- "summary": "The section titled 'A. Iterative Retrieval' introduces a process where the knowledge base is repeatedly searched based on an initial query and previously generated text to provide more comprehensive knowledge.",
- "page_nums": [
- 10
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_63a5dff1-6f8e-5e89-aa00-3b0058dc5e07",
- "type": "page",
- "content": "11\nFig. 5. In addition to the most common once retrieval, RAG also includes three types of retrieval augmentation processes. (left) Iterative retrieval involves\nalternating between retrieval and generation, allowing for richer and more targeted context from the knowledge base at each step. (Middle) Recursive retrieval\ninvolves gradually refining the user query and breaking down the problem into sub-problems, then continuously solving complex problems through retrieval\nand generation. (Right) Adaptive retrieval focuses on enabling the RAG system to autonomously determine whether external knowledge retrieval is necessary\nand when to stop retrieval and generation, often utilizing LLM-generated special tokens for control.\nbase for LLMs. This approach has been shown to enhance\nthe robustness of subsequent answer generation by offering\nadditional contextual references through multiple retrieval\niterations. However, it may be affected by semantic discon-\ntinuity and the accumulation of irrelevant information. ITER-\nRETGEN [14] employs a synergistic approach that lever-\nages “retrieval-enhanced generation” alongside “generation-\nenhanced retrieval” for tasks that necessitate the reproduction\nof specific information. The model harnesses the content\nrequired to address the input task as a contextual basis for\nretrieving pertinent knowledge, which in turn facilitates the\ngeneration of improved responses in subsequent iterations.\nB. Recursive Retrieval\nRecursive retrieval is often used in information retrieval and\nNLP to improve the depth and relevance of search results.\nThe process involves iteratively refining search queries based\non the results obtained from previous searches. Recursive\nRetrieval aims to enhance the search experience by gradu-\nally converging on the most pertinent information through a\nfeedback loop. IRCoT [61] uses chain-of-thought to guide\nthe retrieval process and refines the CoT with the obtained\nretrieval results. ToC [57] creates a clarification tree that\nsystematically optimizes the ambiguous parts in the Query. It\ncan be particularly useful in complex search scenarios where\nthe user’s needs are not entirely clear from the outset or where\nthe information sought is highly specialized or nuanced. The\nrecursive nature of the process allows for continuous learning\nand adaptation to the user’s requirements, often resulting in\nimproved satisfaction with the search outcomes.\nTo address specific data scenarios, recursive retrieval and\nmulti-hop retrieval techniques are utilized together. Recursive\nretrieval involves a structured index to process and retrieve\ndata in a hierarchical manner, which may include summarizing\nsections of a document or lengthy PDF before performing a\nretrieval based on this summary. Subsequently, a secondary\nretrieval within the document refines the search, embodying\nthe recursive nature of the process. In contrast, multi-hop\nretrieval is designed to delve deeper into graph-structured data\nsources, extracting interconnected information [106].\nC. Adaptive Retrieval\nAdaptive retrieval methods, exemplified by Flare [24] and\nSelf-RAG [25], refine the RAG framework by enabling LLMs\nto actively determine the optimal moments and content for\nretrieval, thus enhancing the efficiency and relevance of the\ninformation sourced.\nThese methods are part of a broader trend wherein\nLLMs employ active judgment in their operations, as seen\nin model agents like AutoGPT, Toolformer, and Graph-\nToolformer [107]–[109]. Graph-Toolformer, for instance, di-\nvides its retrieval process into distinct steps where LLMs\nproactively use retrievers, apply Self-Ask techniques, and em-\nploy few-shot prompts to initiate search queries. This proactive\nstance allows LLMs to decide when to search for necessary\ninformation, akin to how an agent utilizes tools.\nWebGPT [110] integrates a reinforcement learning frame-\nwork to train the GPT-3 model in autonomously using a\nsearch engine during text generation. It navigates this process\nusing special tokens that facilitate actions such as search\nengine queries, browsing results, and citing references, thereby\nexpanding GPT-3’s capabilities through the use of external\nsearch engines. Flare automates timing retrieval by monitoring\nthe confidence of the generation process, as indicated by the",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG/B. Recursive Retrieval",
- "metadata": {
- "length": 4304,
- "summary": "The section 'B. Recursive Retrieval' describes a process used in information retrieval and NLP to improve search depth and relevance by iteratively refining queries based on previous results. It highlights methods like IRCoT, which uses chain-of-thought for guidance, and ToC, which creates clarification trees to optimize ambiguous queries. The text also notes that recursive retrieval is often combined with multi-hop retrieval techniques.",
- "page_nums": [
- 11
- ],
- "keywords": [
- "IRCoT",
- "ToC"
- ],
- "connect_to": [
- {
- "target": "asset_3c136a15-4895-5367-908f-8e2bc9eadce5",
- "relation": "embeds",
- "ref": "[images/image_page_11_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_80485953-7213-5e6e-ac94-3a9ac2c09a6c",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG/B. Recursive Retrieval p11]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG/C. Adaptive Retrieval",
- "metadata": {
- "length": 143,
- "summary": "This section discusses adaptive retrieval methods, which enable Large Language Models (LLMs) to actively determine the optimal timing and content for retrieval. It highlights Flare and Self-RAG as examples of this approach. The text also contextualizes these methods within a broader trend of LLMs employing active judgment, similar to model agents like AutoGPT, Toolformer, and Graph-Toolformer. Additionally, it mentions WebGPT's use of reinforcement learning to train GPT-3 for autonomous search engine usage.",
- "page_nums": [
- 11
- ],
- "keywords": [
- "Flare",
- "Self-RAG",
- "AutoGPT",
- "Toolformer",
- "Graph-Toolformer",
- "WebGPT",
- "GPT-3"
- ],
- "connect_to": [
- {
- "target": "asset_3c136a15-4895-5367-908f-8e2bc9eadce5",
- "relation": "related",
- "ref": "[images/image_page_11_figure_1.png]",
- "same_as_owner": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG/B. Recursive Retrieval"
- }
- ],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_ff48c89a-b01b-50e3-b555-94fb84a65a2a",
- "type": "page",
- "content": "12\nprobability of generated terms [24]. When the probability falls\nbelow a certain threshold would activates the retrieval system\nto collect relevant information, thus optimizing the retrieval\ncycle. Self-RAG [25] introduces “reflection tokens” that allow\nthe model to introspect its outputs. These tokens come in\ntwo varieties: “retrieve” and “critic”. The model autonomously\ndecides when to activate retrieval, or alternatively, a predefined\nthreshold may trigger the process. During retrieval, the gen-\nerator conducts a fragment-level beam search across multiple\nparagraphs to derive the most coherent sequence. Critic scores\nare used to update the subdivision scores, with the flexibility\nto adjust these weights during inference, tailoring the model’s\nbehavior. Self-RAG’s design obviates the need for additional\nclassifiers or reliance on Natural Language Inference (NLI)\nmodels, thus streamlining the decision-making process for\nwhen to engage retrieval mechanisms and improving the\nmodel’s autonomous judgment capabilities in generating ac-\ncurate responses.\nVI. TASK AND EVALUATION\nThe rapid advancement and growing adoption of RAG\nin the field of NLP have propelled the evaluation of RAG\nmodels to the forefront of research in the LLMs community.\nThe primary objective of this evaluation is to comprehend\nand optimize the performance of RAG models across diverse\napplication scenarios.This chapter will mainly introduce the\nmain downstream tasks of RAG, datasets, and how to evaluate\nRAG systems.\nA. Downstream Task\nThe core task of RAG remains Question Answering (QA),\nincluding\ntraditional\nsingle-hop/multi-hop\nQA,\nmultiple-\nchoice, domain-specific QA as well as long-form scenarios\nsuitable for RAG. In addition to QA, RAG is continuously\nbeing expanded into multiple downstream tasks, such as Infor-\nmation Extraction (IE), dialogue generation, code search, etc.\nThe main downstream tasks of RAG and their corresponding\ndatasets are summarized in Table II.\nB. Evaluation Target\nHistorically, RAG models assessments have centered on\ntheir execution in specific downstream tasks. These evaluations\nemploy established metrics suitable to the tasks at hand. For\ninstance, question answering evaluations might rely on EM\nand F1 scores [7], [45], [59], [72], whereas fact-checking\ntasks often hinge on Accuracy as the primary metric [4],\n[14], [42]. BLEU and ROUGE metrics are also commonly\nused to evaluate answer quality [26], [32], [52], [78]. Tools\nlike RALLE, designed for the automatic evaluation of RAG\napplications, similarly base their assessments on these task-\nspecific metrics [160]. Despite this, there is a notable paucity\nof research dedicated to evaluating the distinct characteristics\nof RAG models.The main evaluation objectives include:\nRetrieval Quality. Evaluating the retrieval quality is crucial\nfor determining the effectiveness of the context sourced by\nthe retriever component. Standard metrics from the domains\nof search engines, recommendation systems, and information\nretrieval systems are employed to measure the performance of\nthe RAG retrieval module. Metrics such as Hit Rate, MRR, and\nNDCG are commonly utilized for this purpose [161], [162].\nGeneration Quality. The assessment of generation quality\ncenters on the generator’s capacity to synthesize coherent and\nrelevant answers from the retrieved context. This evaluation\ncan be categorized based on the content’s objectives: unlabeled\nand labeled content. For unlabeled content, the evaluation\nencompasses the faithfulness, relevance, and non-harmfulness\nof the generated answers. In contrast, for labeled content,\nthe focus is on the accuracy of the information produced by\nthe model [161]. Additionally, both retrieval and generation\nquality assessments can be conducted through manual or\nautomatic evaluation methods [29], [161], [163].\nC. Evaluation Aspects\nContemporary evaluation practices of RAG models empha-\nsize three primary quality scores and four essential abilities,\nwhich collectively inform the evaluation of the two principal\ntargets of the RAG model: retrieval and generation.\n1) Quality Scores: Quality scores include context rele-\nvance, answer faithfulness, and answer relevance. These qual-\nity scores evaluate the efficiency of the RAG model from\ndifferent perspectives in the process of information retrieval\nand generation [164]–[166].\nContext Relevance evaluates the precision and specificity\nof the retrieved context, ensuring relevance and minimizing\nprocessing costs associated with extraneous content.\nAnswer Faithfulness ensures that the generated answers\nremain true to the retrieved context, maintaining consistency\nand avoiding contradictions.\nAnswer Relevance requires that the generated answers are\ndirectly pertinent to the posed questions, effectively addressing\nthe core inquiry.\n2) Required Abilities: RAG evaluation also encompasses\nfour abilities indicative of its adaptability and efficiency:\nnoise robustness, negative rejection, information integration,\nand counterfactual robustness [167], [168]. These abilities are\ncritical for the model’s performance under various challenges\nand complex scenarios, impacting the quality scores.\nNoise Robustness appraises the model’s capability to man-\nage noise documents that are question-related but lack sub-\nstantive information.\nNegative Rejection assesses the model’s discernment in\nrefraining from responding when the retrieved documents do\nnot contain the necessary knowledge to answer a question.\nInformation Integration evaluates the model’s proficiency in\nsynthesizing information from multiple documents to address\ncomplex questions.\nCounterfactual Robustness tests the model’s ability to rec-\nognize and disregard known inaccuracies within documents,\neven when instructed about potential misinformation.\nContext relevance and noise robustness are important for\nevaluating the quality of retrieval, while answer faithfulness,\nanswer relevance, negative rejection, information integration,\nand counterfactual robustness are important for evaluating the\nquality of generation.",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/A. Downstream Task",
- "metadata": {
- "length": 6066,
- "summary": "The section 'A. Downstream Task' outlines the primary applications of Retrieval-Augmented Generation (RAG) models. It identifies Question Answering (QA) as the core task, covering various formats like single-hop/multi-hop QA and domain-specific scenarios. Additionally, it notes that RAG is expanding into other tasks such as Information Extraction (IE), dialogue generation, and code search. The text concludes by stating that these main downstream tasks and their corresponding datasets are summarized in Table II.",
- "page_nums": [
- 12
- ],
- "keywords": [
- "RAG",
- "LLMs",
- "Table II"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 12,
- "artifact_ref": "page_citation_assets/page-12.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_1b784b91-ad88-589a-9e40-5c7ef410c203",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/A. Downstream Task p12]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/B. Evaluation Target",
- "metadata": {
- "length": 132,
- "summary": "This section discusses the historical focus of RAG model assessments on specific downstream tasks and the metrics used for them, such as EM, F1, Accuracy, BLEU, and ROUGE. It introduces the main evaluation objectives: Retrieval Quality (measuring context effectiveness) and Generation Quality (assessing the synthesis of coherent answers).",
- "page_nums": [
- 12
- ],
- "keywords": [
- "RAG",
- "EM",
- "F1",
- "BLEU",
- "ROUGE",
- "RALLE"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 12,
- "artifact_ref": "page_citation_assets/page-12.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_60bbdce7-e982-5b16-a863-e39f08c95b4d",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/A. Downstream Task p12]\n\n13\nTABLE II\nDOWNSTREAM TASKS AND DATASETS OF RAG\nTask\nSub Task\nDataset\nMethod\nQA\nSingle-hop\nNatural Qustion(NQ) [111]\n[26], [30], [34], [42], [45], [50], [52], [59], [64], [82]\n[3], [4], [22], [27], [40], [43], [54], [62], [71], [112]\n[20], [44], [72]\nTriviaQA(TQA) [113]\n[13], [30], [34], [45], [50], [64]\n[4], [27], [59], [62], [112]\n[22], [25], [43], [44], [71], [72]\nSQuAD [114]\n[20], [23], [30], [32], [45], [69], [112]\nWeb Questions(WebQ) [115]\n[3], [4], [13], [30], [50], [68]\nPopQA [116]\n[7], [25], [67]\nMS MARCO [117]\n[4], [40], [52]\nMulti-hop\nHotpotQA [118]\n[23], [26], [31], [34], [47], [51], [61], [82]\n[7], [14], [22], [27], [59], [62], [69], [71], [91]\n2WikiMultiHopQA [119]\n[14], [24], [48], [59], [61], [91]\nMuSiQue [120]\n[14], [51], [61], [91]\nLong-form QA\nELI5 [121]\n[27], [34], [43], [49], [51]\nNarrativeQA(NQA) [122]\n[45], [60], [63], [123]\nASQA [124]\n[24], [57]\nQMSum(QM) [125]\n[60], [123]\nDomain QA\nQasper [126]\n[60], [63]\nCOVID-QA [127]\n[35], [46]\nCMB [128],MMCU Medical [129]\n[81]\nMulti-Choice QA\nQuALITY [130]\n[60], [63]\nARC [131]\n[25], [67]\nCommonsenseQA [132]\n[58], [66]\nGraph QA\nGraphQA [84]\n[84]\nDialog\nDialog Generation\nWizard of Wikipedia (WoW) [133]\n[13], [27], [34], [42]\nPersonal Dialog\nKBP [134]\n[74], [135]\nDuleMon [136]\n[74]\nTask-oriented Dialog\nCamRest [137]\n[78], [79]\nRecommendation\nAmazon(Toys,Sport,Beauty) [138]\n[39], [40]\nIE\nEvent Argument Extraction\nWikiEvent [139]\n[13], [27], [37], [42]\nRAMS [140]\n[36], [37]\nRelation Extraction\nT-REx [141],ZsRE [142]\n[27], [51]\nReasoning\nCommonsense Reasoning\nHellaSwag [143]\n[20], [66]\nCoT Reasoning\nCoT Reasoning [144]\n[27]\nComplex Reasoning\nCSQA [145]\n[55]\nOthers\nLanguage Understanding\nMMLU [146]\n[7], [27], [28], [42], [43], [47], [72]\nLanguage Modeling\nWikiText-103 [147]\n[5], [29], [64], [71]\nStrategyQA [148]\n[14], [24], [48], [51], [55], [58]\nFact Checking/Verification\nFEVER [149]\n[4], [13], [27], [34], [42], [50]\nPubHealth [150]\n[25], [67]\nText Generation\nBiography [151]\n[67]\nText Summarization\nWikiASP [152]\n[24]\nXSum [153]\n[17]\nText Classification\nVioLens [154]\n[19]\nTREC [155]\n[33]\nSentiment\nSST-2 [156]\n[20], [33], [38]\nCode Search\nCodeSearchNet [157]\n[76]\nRobustness Evaluation\nNoMIRACL [56]\n[56]\nMath\nGSM8K [158]\n[73]\nMachine Translation\nJRC-Acquis [159]\n[17]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/C. Evaluation Aspects",
- "metadata": {
- "length": 2394,
- "summary": "The section titled 'C. Evaluation Aspects' outlines the primary quality scores and essential abilities used to evaluate RAG models. It details three quality scores: Context Relevance, Answer Faithfulness, and Answer Relevance. Additionally, it describes four required abilities indicative of a model's adaptability and efficiency: Noise Robustness, Negative Rejection, Information Integration, and Counterfactual Robustness.",
- "page_nums": [
- 12,
- 13
- ],
- "keywords": [
- "Context Relevance",
- "Answer Faithfulness",
- "Answer Relevance",
- "Noise Robustness",
- "Negative Rejection",
- "Information Integration",
- "Counterfactual Robustness"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 12,
- "artifact_ref": "page_citation_assets/page-12.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 13,
- "artifact_ref": "page_citation_assets/page-13.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_d76651f6-6373-50f7-876b-ddd3ddb15900",
- "type": "page",
- "content": "14\nTABLE III\nSUMMARY OF METRICS APPLICABLE FOR EVALUATION ASPECTS OF RAG\nContext\nRelevance\nFaithfulness\nAnswer\nRelevance\nNoise\nRobustness\nNegative\nRejection\nInformation\nIntegration\nCounterfactual\nRobustness\nAccuracy\n✓\n✓\n✓\n✓\n✓\n✓\n✓\nEM\n✓\nRecall\n✓\nPrecision\n✓\n✓\nR-Rate\n✓\nCosine Similarity\n✓\nHit Rate\n✓\nMRR\n✓\nNDCG\n✓\nBLEU\n✓\n✓\n✓\nROUGE/ROUGE-L\n✓\n✓\n✓\nThe specific metrics for each evaluation aspect are sum-\nmarized in Table III. It is essential to recognize that these\nmetrics, derived from related work, are traditional measures\nand do not yet represent a mature or standardized approach for\nquantifying RAG evaluation aspects. Custom metrics tailored\nto the nuances of RAG models, though not included here, have\nalso been developed in some evaluation studies.\nD. Evaluation Benchmarks and Tools\nA series of benchmark tests and tools have been proposed\nto facilitate the evaluation of RAG.These instruments furnish\nquantitative metrics that not only gauge RAG model perfor-\nmance but also enhance comprehension of the model’s capabil-\nities across various evaluation aspects. Prominent benchmarks\nsuch as RGB, RECALL and CRUD\n[167]–[169] focus on\nappraising the essential abilities of RAG models. Concur-\nrently, state-of-the-art automated tools like RAGAS [164],\nARES [165], and TruLens8 employ LLMs to adjudicate the\nquality scores. These tools and benchmarks collectively form\na robust framework for the systematic evaluation of RAG\nmodels, as summarized in Table IV.\nVII. DISCUSSION AND FUTURE PROSPECTS\nDespite the considerable progress in RAG technology, sev-\neral challenges persist that warrant in-depth research.This\nchapter will mainly introduce the current challenges and future\nresearch directions faced by RAG.\nA. RAG vs Long Context\nWith the deepening of related research, the context of LLMs\nis continuously expanding [170]–[172]. Presently, LLMs can\neffortlessly manage contexts exceeding 200,000 tokens 9. This\ncapability signifies that long-document question answering,\npreviously reliant on RAG, can now incorporate the entire\ndocument directly into the prompt. This has also sparked\ndiscussions on whether RAG is still necessary when LLMs\n8https://www.trulens.org/trulens eval/core concepts rag triad/\n9https://kimi.moonshot.cn\nare not constrained by context. In fact, RAG still plays an\nirreplaceable role. On one hand, providing LLMs with a\nlarge amount of context at once will significantly impact its\ninference speed, while chunked retrieval and on-demand input\ncan significantly improve operational efficiency. On the other\nhand, RAG-based generation can quickly locate the original\nreferences for LLMs to help users verify the generated an-\nswers. The entire retrieval and reasoning process is observable,\nwhile generation solely relying on long context remains a\nblack box. Conversely, the expansion of context provides new\nopportunities for the development of RAG, enabling it to\naddress more complex problems and integrative or summary\nquestions that require reading a large amount of material to\nanswer [49]. Developing new RAG methods in the context of\nsuper-long contexts is one of the future research trends.\nB. RAG Robustness\nThe presence of noise or contradictory information during\nretrieval can detrimentally affect RAG’s output quality. This\nsituation is figuratively referred to as “Misinformation can\nbe worse than no information at all”. Improving RAG’s\nresistance to such adversarial or counterfactual inputs is gain-\ning research momentum and has become a key performance\nmetric [48], [50], [82]. Cuconasu et al. [54] analyze which\ntype of documents should be retrieved, evaluate the relevance\nof the documents to the prompt, their position, and the\nnumber included in the context. The research findings reveal\nthat including irrelevant documents can unexpectedly increase\naccuracy by over 30%, contradicting the initial assumption\nof reduced quality. These results underscore the importance\nof developing specialized strategies to integrate retrieval with\nlanguage generation models, highlighting the need for further\nresearch and exploration into the robustness of RAG.\nC. Hybrid Approaches\nCombining RAG with fine-tuning is emerging as a leading\nstrategy. Determining the optimal integration of RAG and\nfine-tuning whether sequential, alternating, or through end-to-\nend joint training—and how to harness both parameterized",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/D. Evaluation Benchmarks and Tools",
- "metadata": {
- "length": 4362,
- "summary": "This section discusses benchmark tests and tools proposed for evaluating Retrieval-Augmented Generation (RAG) models. It highlights prominent benchmarks such as RGB, RECALL, and CRUD, which focus on appraising essential RAG abilities. Additionally, it mentions state-of-the-art automated tools like RAGAS, ARES, and TruLens that employ Large Language Models (LLMs) to adjudicate quality scores.",
- "page_nums": [
- 14
- ],
- "keywords": [
- "RGB",
- "RECALL",
- "CRUD",
- "RAGAS",
- "ARES",
- "TruLens"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 14,
- "artifact_ref": "page_citation_assets/page-14.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_7156a295-a51e-50a2-9261-62714a95fdb4",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/D. Evaluation Benchmarks and Tools p14]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/A. RAG vs Long Context",
- "metadata": {
- "length": 148,
- "summary": "This section discusses the impact of expanding Large Language Model (LLM) contexts, which now exceed 200,000 tokens. It explores whether Retrieval-Augmented Generation (RAG) remains necessary when LLMs can ingest entire documents directly. The text contrasts RAG with long-context approaches, noting that while long context improves efficiency and allows for complex problem solving, RAG offers benefits like faster inference speed, operational efficiency through chunked retrieval, and verifiable references. The section concludes by identifying the development of new RAG methods for super-long contexts as a future research trend.",
- "page_nums": [
- 14
- ],
- "keywords": [
- "LLMs",
- "RAG"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 14,
- "artifact_ref": "page_citation_assets/page-14.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_1370474d-8250-5ba2-8d77-ab140e965bb6",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/D. Evaluation Benchmarks and Tools p14]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/B. RAG Robustness",
- "metadata": {
- "length": 148,
- "summary": "The section 'B. RAG Robustness' discusses the impact of noise or contradictory information on Retrieval-Augmented Generation (RAG) output quality, a problem described as 'Misinformation can be worse than no information at all'. It highlights that improving resistance to such adversarial inputs is a key performance metric. The text cites research by Cuconasu et al. [54], which found that including irrelevant documents can unexpectedly increase accuracy by over 30%, contradicting initial assumptions about reduced quality. This underscores the need for specialized strategies to integrate retrieval with language generation models.",
- "page_nums": [
- 14
- ],
- "keywords": [
- "Cuconasu"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 14,
- "artifact_ref": "page_citation_assets/page-14.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_b04de59e-dce2-5078-9c23-5040049ae346",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/D. Evaluation Benchmarks and Tools p14]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/C. Hybrid Approaches",
- "metadata": {
- "length": 148,
- "summary": "The section titled 'C. Hybrid Approaches' discusses the emerging strategy of combining Retrieval-Augmented Generation (RAG) with fine-tuning. It highlights the importance of determining the optimal integration method, whether sequential, alternating, or through end-to-end joint training, to effectively harness both parameterized and non-parameterized knowledge.",
- "page_nums": [
- 14
- ],
- "keywords": [
- "RAG",
- "LLMs"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 14,
- "artifact_ref": "page_citation_assets/page-14.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_322bdad5-7b4f-5941-8907-e28ca87a8766",
- "type": "page",
- "content": "15\nTABLE IV\nSUMMARY OF EVALUATION FRAMEWORKS\nEvaluation Framework\nEvaluation Targets\nEvaluation Aspects\nQuantitative Metrics\nRGB†\nRetrieval Quality\nGeneration Quality\nNoise Robustness\nNegative Rejection\nInformation Integration\nCounterfactual Robustness\nAccuracy\nEM\nAccuracy\nAccuracy\nRECALL†\nGeneration Quality\nCounterfactual Robustness\nR-Rate (Reappearance Rate)\nRAGAS‡\nRetrieval Quality\nGeneration Quality\nContext Relevance\nFaithfulness\nAnswer Relevance\n*\n*\nCosine Similarity\nARES‡\nRetrieval Quality\nGeneration Quality\nContext Relevance\nFaithfulness\nAnswer Relevance\nAccuracy\nAccuracy\nAccuracy\nTruLens‡\nRetrieval Quality\nGeneration Quality\nContext Relevance\nFaithfulness\nAnswer Relevance\n*\n*\n*\nCRUD†\nRetrieval Quality\nGeneration Quality\nCreative Generation\nKnowledge-intensive QA\nError Correction\nSummarization\nBLEU\nROUGE-L\nBertScore\nRAGQuestEval\n† represents a benchmark, and ‡ represents a tool. * denotes customized quantitative metrics, which deviate from traditional\nmetrics. Readers are encouraged to consult pertinent literature for the specific quantification formulas associated with these\nmetrics, as required.\nand non-parameterized advantages are areas ripe for explo-\nration [27]. Another trend is to introduce SLMs with specific\nfunctionalities into RAG and fine-tuned by the results of RAG\nsystem. For example, CRAG [67] trains a lightweight retrieval\nevaluator to assess the overall quality of the retrieved docu-\nments for a query and triggers different knowledge retrieval\nactions based on confidence levels.\nD. Scaling laws of RAG\nEnd-to-end RAG models and pre-trained models based\non\nRAG\nare\nstill\none\nof\nthe\nfocuses\nof\ncurrent\nre-\nsearchers [173].The parameters of these models are one of\nthe key factors.While scaling laws [174] are established for\nLLMs, their applicability to RAG remains uncertain. Initial\nstudies like RETRO++ [44] have begun to address this, yet the\nparameter count in RAG models still lags behind that of LLMs.\nThe possibility of an Inverse Scaling Law 10, where smaller\nmodels outperform larger ones, is particularly intriguing and\nmerits further investigation.\nE. Production-Ready RAG\nRAG’s practicality and alignment with engineering require-\nments have facilitated its adoption. However, enhancing re-\ntrieval efficiency, improving document recall in large knowl-\nedge bases, and ensuring data security—such as preventing\n10https://github.com/inverse-scaling/prize\ninadvertent disclosure of document sources or metadata by\nLLMs—are critical engineering challenges that remain to be\naddressed [175].\nThe development of the RAG ecosystem is greatly impacted\nby the progression of its technical stack. Key tools like\nLangChain and LLamaIndex have quickly gained popularity\nwith the emergence of ChatGPT, providing extensive RAG-\nrelated APIs and becoming essential in the realm of LLMs.The\nemerging technology stack, while not as rich in features as\nLangChain and LLamaIndex, stands out through its specialized\nproducts. For example, Flowise AI prioritizes a low-code\napproach, allowing users to deploy AI applications, including\nRAG, through a user-friendly drag-and-drop interface. Other\ntechnologies like HayStack, Meltano, and Cohere Coral are\nalso gaining attention for their unique contributions to the field.\nIn addition to AI-focused vendors, traditional software and\ncloud service providers are expanding their offerings to include\nRAG-centric services. Weaviate’s Verba 11 is designed for\npersonal assistant applications, while Amazon’s Kendra\n12\noffers intelligent enterprise search services, enabling users to\nbrowse various content repositories using built-in connectors.\nIn the development of RAG technology, there is a clear\ntrend towards different specialization directions, such as: 1)\nCustomization - tailoring RAG to meet specific requirements.\n2) Simplification - making RAG easier to use to reduce the\n11https://github.com/weaviate/Verba\n12https://aws.amazon.com/cn/kendra/",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/D. Scaling laws of RAG",
- "metadata": {
- "length": 3939,
- "summary": "This section discusses the scaling laws of RAG (Retrieval-Augmented Generation) models. It notes that while scaling laws are established for LLMs, their applicability to RAG remains uncertain. The text highlights that initial studies like RETRO++ have begun addressing this, but parameter counts in RAG models still lag behind those of LLMs. It also mentions the intriguing possibility of an Inverse Scaling Law, where smaller models might outperform larger ones.",
- "page_nums": [
- 15
- ],
- "keywords": [
- "RAG",
- "LLMs",
- "RETRO++"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 15,
- "artifact_ref": "page_citation_assets/page-15.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_e2a7d0fb-94fd-5991-86de-8febb707debe",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/D. Scaling laws of RAG p15]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/E. Production-Ready RAG",
- "metadata": {
- "length": 149,
- "summary": "This section discusses the practicality and engineering requirements of production-ready RAG systems. It highlights key challenges such as enhancing retrieval efficiency, improving document recall in large knowledge bases, and ensuring data security to prevent inadvertent disclosure. The text also notes that the RAG ecosystem is significantly influenced by technical stack advancements, mentioning popular tools like LangChain and LlamaIndex, low-code platforms like Flowise AI, and services from traditional vendors like Weaviate's Verba and Amazon's Kendra. Finally, it identifies a trend towards specialization in RAG technology, focusing on customization and simplification.",
- "page_nums": [
- 15
- ],
- "keywords": [
- "LangChain",
- "LlamaIndex",
- "ChatGPT",
- "Flowise AI",
- "HayStack",
- "Meltano",
- "Cohere Coral",
- "Weaviate",
- "Verba",
- "Amazon",
- "Kendra"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 15,
- "artifact_ref": "page_citation_assets/page-15.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_46a9ffb4-cc81-501b-9f5b-0d3d0847afa9",
- "type": "page",
- "content": "16\nFig. 6. Summary of RAG ecosystem\ninitial learning curve. 3) Specialization - optimizing RAG to\nbetter serve production environments.\nThe mutual growth of RAG models and their technology\nstacks is evident; technological advancements continuously\nestablish new standards for existing infrastructure. In turn,\nenhancements to the technology stack drive the development\nof RAG capabilities. RAG toolkits are converging into a\nfoundational technology stack, laying the groundwork for\nadvanced enterprise applications. However, a fully integrated,\ncomprehensive platform concept is still in the future, requiring\nfurther innovation and development.\nF. Multi-modal RAG\nRAG\nhas\ntranscended\nits\ninitial\ntext-based\nquestion-\nanswering confines, embracing a diverse array of modal data.\nThis expansion has spawned innovative multimodal models\nthat integrate RAG concepts across various domains:\nImage. RA-CM3 [176] stands as a pioneering multimodal\nmodel of both retrieving and generating text and images.\nBLIP-2 [177] leverages frozen image encoders alongside\nLLMs for efficient visual language pre-training, enabling zero-\nshot image-to-text conversions. The “Visualize Before You\nWrite” method [178] employs image generation to steer the\nLM’s text generation, showing promise in open-ended text\ngeneration tasks.\nAudio and Video. The GSS method retrieves and stitches\ntogether audio clips to convert machine-translated data into\nspeech-translated data [179]. UEOP marks a significant ad-\nvancement in end-to-end automatic speech recognition by\nincorporating external, offline strategies for voice-to-text con-\nversion [180]. Additionally, KNN-based attention fusion lever-\nages audio embeddings and semantically related text embed-\ndings to refine ASR, thereby accelerating domain adaptation.\nVid2Seq augments language models with specialized temporal\nmarkers, facilitating the prediction of event boundaries and\ntextual descriptions within a unified output sequence [181].\nCode. RBPS [182] excels in small-scale learning tasks by\nretrieving code examples that align with developers’ objectives\nthrough encoding and frequency analysis. This approach has\ndemonstrated efficacy in tasks such as test assertion genera-\ntion and program repair. For structured knowledge, the CoK\nmethod [106] first extracts facts pertinent to the input query\nfrom a knowledge graph, then integrates these facts as hints\nwithin the input, enhancing performance in knowledge graph\nquestion-answering tasks.\nVIII. CONCLUSION\nThe summary of this paper, as depicted in Figure 6, empha-\nsizes RAG’s significant advancement in enhancing the capa-\nbilities of LLMs by integrating parameterized knowledge from\nlanguage models with extensive non-parameterized data from\nexternal knowledge bases. The survey showcases the evolution\nof RAG technologies and their application on many different\ntasks. The analysis outlines three developmental paradigms\nwithin the RAG framework: Naive, Advanced, and Modu-\nlar RAG, each representing a progressive enhancement over\nits predecessors. RAG’s technical integration with other AI\nmethodologies, such as fine-tuning and reinforcement learning,\nhas further expanded its capabilities. Despite the progress in\nRAG technology, there are research opportunities to improve\nits robustness and its ability to handle extended contexts.\nRAG’s application scope is expanding into multimodal do-\nmains, adapting its principles to interpret and process diverse\ndata forms like images, videos, and code. This expansion high-\nlights RAG’s significant practical implications for AI deploy-\nment, attracting interest from academic and industrial sectors.",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/F. Multi-modal RAG",
- "metadata": {
- "length": 3641,
- "summary": "Section F discusses the expansion of Retrieval-Augmented Generation (RAG) beyond text-based tasks to include multimodal data. It details specific models and methods for handling images, audio, and video. For images, it mentions RA-CM3, BLIP-2, and the 'Visualize Before You Write' method. For audio and video, it describes the GSS method, UEOP, KNN-based attention fusion, Vid2Seq, RBPS, and the CoK method.",
- "page_nums": [
- 16
- ],
- "keywords": [
- "RA-CM3",
- "BLIP-2",
- "GSS",
- "UEOP"
- ],
- "connect_to": [
- {
- "target": "asset_c56f3f73-2c97-5893-9b64-49408dc47842",
- "relation": "embeds",
- "ref": "[images/image_page_16_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 16,
- "artifact_ref": "page_citation_assets/page-16.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_5ed2b6d2-8df5-5c61-97d6-15580cbf94e7",
- "type": "page",
- "content": "[SAME-AS Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/F. Multi-modal RAG p16]",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VIII. CONCLUSION",
- "metadata": {
- "length": 145,
- "summary": "The conclusion summarizes the paper's emphasis on RAG's advancement in enhancing LLM capabilities by integrating parameterized knowledge with external data. It outlines three developmental paradigms (Naive, Advanced, and Modular RAG) and highlights technical integrations like fine-tuning. The section concludes by noting RAG's expanding scope into multimodal domains such as images, videos, and code, underscoring its practical implications for AI deployment.",
- "page_nums": [
- 16
- ],
- "keywords": [
- "RAG",
- "LLMs"
- ],
- "connect_to": [
- {
- "target": "asset_c56f3f73-2c97-5893-9b64-49408dc47842",
- "relation": "related",
- "ref": "[images/image_page_16_figure_1.png]",
- "same_as_owner": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/F. Multi-modal RAG"
- }
- ],
- "page_assets": [
- {
- "page_num": 16,
- "artifact_ref": "page_citation_assets/page-16.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- },
- {
- "chunk_id": "node_c108906b-0327-5365-90da-20cd799fa8b4",
- "type": "page",
- "content": "17\nThe growing ecosystem of RAG is evidenced by the rise in\nRAG-centric AI applications and the continuous development\nof supportive tools. As RAG’s application landscape broadens,\nthere is a need to refine evaluation methodologies to keep\npace with its evolution. Ensuring accurate and representative\nperformance assessments is crucial for fully capturing RAG’s\ncontributions to the AI research and development community.\nREFERENCES\n[1] N. Kandpal, H. Deng, A. Roberts, E. Wallace, and C. Raffel, “Large\nlanguage models struggle to learn long-tail knowledge,” in Interna-\ntional Conference on Machine Learning.\nPMLR, 2023, pp. 15 696–\n15 707.\n[2] Y. Zhang, Y. Li, L. Cui, D. Cai, L. Liu, T. Fu, X. Huang, E. Zhao,\nY. Zhang, Y. Chen et al., “Siren’s song in the ai ocean: A survey on hal-\nlucination in large language models,” arXiv preprint arXiv:2309.01219,\n2023.\n[3] D. Arora, A. Kini, S. R. Chowdhury, N. Natarajan, G. Sinha, and\nA. Sharma, “Gar-meets-rag paradigm for zero-shot information re-\ntrieval,” arXiv preprint arXiv:2310.20158, 2023.\n[4] P. Lewis, E. Perez, A. Piktus, F. Petroni, V. Karpukhin, N. Goyal,\nH. K¨uttler, M. Lewis, W.-t. Yih, T. Rockt¨aschel et al., “Retrieval-\naugmented generation for knowledge-intensive nlp tasks,” Advances in\nNeural Information Processing Systems, vol. 33, pp. 9459–9474, 2020.\n[5] S. Borgeaud, A. Mensch, J. Hoffmann, T. Cai, E. Rutherford, K. Milli-\ncan, G. B. Van Den Driessche, J.-B. Lespiau, B. Damoc, A. Clark et al.,\n“Improving language models by retrieving from trillions of tokens,”\nin International conference on machine learning.\nPMLR, 2022, pp.\n2206–2240.\n[6] L. Ouyang, J. Wu, X. Jiang, D. Almeida, C. Wainwright, P. Mishkin,\nC. Zhang, S. Agarwal, K. Slama, A. Ray et al., “Training language\nmodels to follow instructions with human feedback,” Advances in\nneural information processing systems, vol. 35, pp. 27 730–27 744,\n2022.\n[7] X. Ma, Y. Gong, P. He, H. Zhao, and N. Duan, “Query rewrit-\ning for retrieval-augmented large language models,” arXiv preprint\narXiv:2305.14283, 2023.\n[8] I.\nILIN,\n“Advanced\nrag\ntechniques:\nan\nil-\nlustrated\noverview,”\nhttps://pub.towardsai.net/\nadvanced-rag-techniques-an-illustrated-overview-04d193d8fec6,\n2023.\n[9] W. Peng, G. Li, Y. Jiang, Z. Wang, D. Ou, X. Zeng, E. Chen et al.,\n“Large language model based long-tail query rewriting in taobao\nsearch,” arXiv preprint arXiv:2311.03758, 2023.\n[10] H. S. Zheng, S. Mishra, X. Chen, H.-T. Cheng, E. H. Chi, Q. V. Le,\nand D. Zhou, “Take a step back: Evoking reasoning via abstraction in\nlarge language models,” arXiv preprint arXiv:2310.06117, 2023.\n[11] L. Gao, X. Ma, J. Lin, and J. Callan, “Precise zero-shot dense retrieval\nwithout relevance labels,” arXiv preprint arXiv:2212.10496, 2022.\n[12] V. Blagojevi, “Enhancing rag pipelines in haystack: Introducing diver-\nsityranker and lostinthemiddleranker,” https://towardsdatascience.com/\nenhancing-rag-pipelines-in-haystack-45f14e2bc9f5, 2023.\n[13] W. Yu, D. Iter, S. Wang, Y. Xu, M. Ju, S. Sanyal, C. Zhu, M. Zeng,\nand M. Jiang, “Generate rather than retrieve: Large language models\nare strong context generators,” arXiv preprint arXiv:2209.10063, 2022.\n[14] Z. Shao, Y. Gong, Y. Shen, M. Huang, N. Duan, and W. Chen,\n“Enhancing retrieval-augmented large language models with iterative\nretrieval-generation synergy,” arXiv preprint arXiv:2305.15294, 2023.\n[15] X. Wang, Q. Yang, Y. Qiu, J. Liang, Q. He, Z. Gu, Y. Xiao,\nand W. Wang, “Knowledgpt: Enhancing large language models with\nretrieval and storage access on knowledge bases,” arXiv preprint\narXiv:2308.11761, 2023.\n[16] A.\nH.\nRaudaschl,\n“Forget\nrag,\nthe\nfuture\nis\nrag-fusion,”\nhttps://towardsdatascience.com/\nforget-rag-the-future-is-rag-fusion-1147298d8ad1, 2023.\n[17] X. Cheng, D. Luo, X. Chen, L. Liu, D. Zhao, and R. Yan, “Lift\nyourself up: Retrieval-augmented text generation with self memory,”\narXiv preprint arXiv:2305.02437, 2023.\n[18] S. Wang, Y. Xu, Y. Fang, Y. Liu, S. Sun, R. Xu, C. Zhu, and\nM. Zeng, “Training data is more valuable than you think: A simple\nand effective method by retrieving from training data,” arXiv preprint\narXiv:2203.08773, 2022.\n[19] X. Li, E. Nie, and S. Liang, “From classification to generation:\nInsights into crosslingual retrieval augmented icl,” arXiv preprint\narXiv:2311.06595, 2023.\n[20] D. Cheng, S. Huang, J. Bi, Y. Zhan, J. Liu, Y. Wang, H. Sun,\nF. Wei, D. Deng, and Q. Zhang, “Uprise: Universal prompt retrieval\nfor improving zero-shot evaluation,” arXiv preprint arXiv:2303.08518,\n2023.\n[21] Z. Dai, V. Y. Zhao, J. Ma, Y. Luan, J. Ni, J. Lu, A. Bakalov, K. Guu,\nK. B. Hall, and M.-W. Chang, “Promptagator: Few-shot dense retrieval\nfrom 8 examples,” arXiv preprint arXiv:2209.11755, 2022.\n[22] Z. Sun, X. Wang, Y. Tay, Y. Yang, and D. Zhou, “Recitation-augmented\nlanguage models,” arXiv preprint arXiv:2210.01296, 2022.\n[23] O. Khattab, K. Santhanam, X. L. Li, D. Hall, P. Liang, C. Potts,\nand M. Zaharia, “Demonstrate-search-predict: Composing retrieval\nand language models for knowledge-intensive nlp,” arXiv preprint\narXiv:2212.14024, 2022.\n[24] Z. Jiang, F. F. Xu, L. Gao, Z. Sun, Q. Liu, J. Dwivedi-Yu, Y. Yang,\nJ. Callan, and G. Neubig, “Active retrieval augmented generation,”\narXiv preprint arXiv:2305.06983, 2023.\n[25] A. Asai, Z. Wu, Y. Wang, A. Sil, and H. Hajishirzi, “Self-rag:\nLearning to retrieve, generate, and critique through self-reflection,”\narXiv preprint arXiv:2310.11511, 2023.\n[26] Z. Ke, W. Kong, C. Li, M. Zhang, Q. Mei, and M. Bendersky,\n“Bridging the preference gap between retrievers and llms,” arXiv\npreprint arXiv:2401.06954, 2024.\n[27] X. V. Lin, X. Chen, M. Chen, W. Shi, M. Lomeli, R. James, P. Ro-\ndriguez, J. Kahn, G. Szilvasy, M. Lewis et al., “Ra-dit: Retrieval-\naugmented dual instruction tuning,” arXiv preprint arXiv:2310.01352,\n2023.\n[28] O. Ovadia, M. Brief, M. Mishaeli, and O. Elisha, “Fine-tuning or\nretrieval? comparing knowledge injection in llms,” arXiv preprint\narXiv:2312.05934, 2023.\n[29] T. Lan, D. Cai, Y. Wang, H. Huang, and X.-L. Mao, “Copy is all\nyou need,” in The Eleventh International Conference on Learning\nRepresentations, 2022.\n[30] T. Chen, H. Wang, S. Chen, W. Yu, K. Ma, X. Zhao, D. Yu, and\nH. Zhang, “Dense x retrieval: What retrieval granularity should we\nuse?” arXiv preprint arXiv:2312.06648, 2023.\n[31] F. Luo and M. Surdeanu, “Divide & conquer for entailment-aware\nmulti-hop evidence retrieval,” arXiv preprint arXiv:2311.02616, 2023.\n[32] Q. Gou, Z. Xia, B. Yu, H. Yu, F. Huang, Y. Li, and N. Cam-Tu,\n“Diversify question generation with retrieval-augmented style transfer,”\narXiv preprint arXiv:2310.14503, 2023.\n[33] Z. Guo, S. Cheng, Y. Wang, P. Li, and Y. Liu, “Prompt-guided re-\ntrieval augmentation for non-knowledge-intensive tasks,” arXiv preprint\narXiv:2305.17653, 2023.\n[34] Z. Wang, J. Araki, Z. Jiang, M. R. Parvez, and G. Neubig, “Learning\nto filter context for retrieval-augmented generation,” arXiv preprint\narXiv:2311.08377, 2023.\n[35] M. Seo, J. Baek, J. Thorne, and S. J. Hwang, “Retrieval-augmented\ndata augmentation for low-resource domain tasks,” arXiv preprint\narXiv:2402.13482, 2024.\n[36] Y. Ma, Y. Cao, Y. Hong, and A. Sun, “Large language model is not\na good few-shot information extractor, but a good reranker for hard\nsamples!” arXiv preprint arXiv:2303.08559, 2023.\n[37] X. Du and H. Ji, “Retrieval-augmented generative question answering\nfor event argument extraction,” arXiv preprint arXiv:2211.07067, 2022.\n[38] L. Wang, N. Yang, and F. Wei, “Learning to retrieve in-context\nexamples for large language models,” arXiv preprint arXiv:2307.07164,\n2023.\n[39] S. Rajput, N. Mehta, A. Singh, R. H. Keshavan, T. Vu, L. Heldt,\nL. Hong, Y. Tay, V. Q. Tran, J. Samost et al., “Recommender systems\nwith generative retrieval,” arXiv preprint arXiv:2305.05065, 2023.\n[40] B. Jin, H. Zeng, G. Wang, X. Chen, T. Wei, R. Li, Z. Wang, Z. Li,\nY. Li, H. Lu et al., “Language models as semantic indexers,” arXiv\npreprint arXiv:2310.07815, 2023.\n[41] R. Anantha, T. Bethi, D. Vodianik, and S. Chappidi, “Context tuning\nfor retrieval augmented generation,” arXiv preprint arXiv:2312.05708,\n2023.\n[42] G. Izacard, P. Lewis, M. Lomeli, L. Hosseini, F. Petroni, T. Schick,\nJ. Dwivedi-Yu, A. Joulin, S. Riedel, and E. Grave, “Few-shot\nlearning with retrieval augmented language models,” arXiv preprint\narXiv:2208.03299, 2022.\n[43] J. Huang, W. Ping, P. Xu, M. Shoeybi, K. C.-C. Chang, and B. Catan-\nzaro, “Raven: In-context learning with retrieval augmented encoder-\ndecoder language models,” arXiv preprint arXiv:2308.07922, 2023.\n\n18\n[44] B. Wang, W. Ping, P. Xu, L. McAfee, Z. Liu, M. Shoeybi, Y. Dong,\nO. Kuchaiev, B. Li, C. Xiao et al., “Shall we pretrain autoregressive\nlanguage models with retrieval? a comprehensive study,” arXiv preprint\narXiv:2304.06762, 2023.\n[45] B. Wang, W. Ping, L. McAfee, P. Xu, B. Li, M. Shoeybi, and B. Catan-\nzaro, “Instructretro: Instruction tuning post retrieval-augmented pre-\ntraining,” arXiv preprint arXiv:2310.07713, 2023.\n[46] S. Siriwardhana, R. Weerasekera, E. Wen, T. Kaluarachchi, R. Rana,\nand S. Nanayakkara, “Improving the domain adaptation of retrieval\naugmented generation (rag) models for open domain question answer-\ning,” Transactions of the Association for Computational Linguistics,\nvol. 11, pp. 1–17, 2023.\n[47] Z. Yu, C. Xiong, S. Yu, and Z. Liu, “Augmentation-adapted retriever\nimproves generalization of language models as generic plug-in,” arXiv\npreprint arXiv:2305.17331, 2023.\n[48] O. Yoran, T. Wolfson, O. Ram, and J. Berant, “Making retrieval-\naugmented language models robust to irrelevant context,” arXiv\npreprint arXiv:2310.01558, 2023.\n[49] H.-T. Chen, F. Xu, S. A. Arora, and E. Choi, “Understanding re-\ntrieval augmentation for long-form question answering,” arXiv preprint\narXiv:2310.12150, 2023.\n[50] W. Yu, H. Zhang, X. Pan, K. Ma, H. Wang, and D. Yu, “Chain-of-note:\nEnhancing robustness in retrieval-augmented language models,” arXiv\npreprint arXiv:2311.09210, 2023.\n[51] S. Xu, L. Pang, H. Shen, X. Cheng, and T.-S. Chua, “Search-in-the-\nchain: Towards accurate, credible and traceable large language models\nfor knowledgeintensive tasks,” CoRR, vol. abs/2304.14732, 2023.\n[52] M. Berchansky, P. Izsak, A. Caciularu, I. Dagan, and M. Wasserblat,\n“Optimizing retrieval-augmented reader models via token elimination,”\narXiv preprint arXiv:2310.13682, 2023.\n[53] J. L´ala, O. O’Donoghue, A. Shtedritski, S. Cox, S. G. Rodriques,\nand A. D. White, “Paperqa: Retrieval-augmented generative agent for\nscientific research,” arXiv preprint arXiv:2312.07559, 2023.\n[54] F. Cuconasu, G. Trappolini, F. Siciliano, S. Filice, C. Campagnano,\nY. Maarek, N. Tonellotto, and F. Silvestri, “The power of noise:\nRedefining retrieval for rag systems,” arXiv preprint arXiv:2401.14887,\n2024.\n[55] Z. Zhang, X. Zhang, Y. Ren, S. Shi, M. Han, Y. Wu, R. Lai, and\nZ. Cao, “Iag: Induction-augmented generation framework for answer-\ning reasoning questions,” in Proceedings of the 2023 Conference on\nEmpirical Methods in Natural Language Processing, 2023, pp. 1–14.\n[56] N. Thakur, L. Bonifacio, X. Zhang, O. Ogundepo, E. Kamalloo,\nD. Alfonso-Hermelo, X. Li, Q. Liu, B. Chen, M. Rezagholizadeh et al.,\n“Nomiracl: Knowing when you don’t know for robust multilingual\nretrieval-augmented generation,” arXiv preprint arXiv:2312.11361,\n2023.\n[57] G. Kim, S. Kim, B. Jeon, J. Park, and J. Kang, “Tree of clarifica-\ntions: Answering ambiguous questions with retrieval-augmented large\nlanguage models,” arXiv preprint arXiv:2310.14696, 2023.\n[58] Y. Wang, P. Li, M. Sun, and Y. Liu, “Self-knowledge guided\nretrieval augmentation for large language models,” arXiv preprint\narXiv:2310.05002, 2023.\n[59] Z. Feng, X. Feng, D. Zhao, M. Yang, and B. Qin, “Retrieval-\ngeneration synergy augmented large language models,” arXiv preprint\narXiv:2310.05149, 2023.\n[60] P. Xu, W. Ping, X. Wu, L. McAfee, C. Zhu, Z. Liu, S. Subramanian,\nE. Bakhturina, M. Shoeybi, and B. Catanzaro, “Retrieval meets long\ncontext large language models,” arXiv preprint arXiv:2310.03025,\n2023.\n[61] H. Trivedi, N. Balasubramanian, T. Khot, and A. Sabharwal, “Interleav-\ning retrieval with chain-of-thought reasoning for knowledge-intensive\nmulti-step questions,” arXiv preprint arXiv:2212.10509, 2022.\n[62] R. Ren, Y. Wang, Y. Qu, W. X. Zhao, J. Liu, H. Tian, H. Wu, J.-\nR. Wen, and H. Wang, “Investigating the factual knowledge boundary\nof large language models with retrieval augmentation,” arXiv preprint\narXiv:2307.11019, 2023.\n[63] P. Sarthi, S. Abdullah, A. Tuli, S. Khanna, A. Goldie, and C. D.\nManning, “Raptor: Recursive abstractive processing for tree-organized\nretrieval,” arXiv preprint arXiv:2401.18059, 2024.\n[64] O. Ram, Y. Levine, I. Dalmedigos, D. Muhlgay, A. Shashua, K. Leyton-\nBrown, and Y. Shoham, “In-context retrieval-augmented language\nmodels,” arXiv preprint arXiv:2302.00083, 2023.\n[65] Y. Ren, Y. Cao, P. Guo, F. Fang, W. Ma, and Z. Lin, “Retrieve-and-\nsample: Document-level event argument extraction via hybrid retrieval\naugmentation,” in Proceedings of the 61st Annual Meeting of the\nAssociation for Computational Linguistics (Volume 1: Long Papers),\n2023, pp. 293–306.\n[66] Z. Wang, X. Pan, D. Yu, D. Yu, J. Chen, and H. Ji, “Zemi: Learning\nzero-shot semi-parametric language models from multiple tasks,” arXiv\npreprint arXiv:2210.00185, 2022.\n[67] S.-Q. Yan, J.-C. Gu, Y. Zhu, and Z.-H. Ling, “Corrective retrieval\naugmented generation,” arXiv preprint arXiv:2401.15884, 2024.\n[68] P. Jain, L. B. Soares, and T. Kwiatkowski, “1-pager: One pass answer\ngeneration and evidence retrieval,” arXiv preprint arXiv:2310.16568,\n2023.\n[69] H. Yang, Z. Li, Y. Zhang, J. Wang, N. Cheng, M. Li, and J. Xiao, “Prca:\nFitting black-box large language models for retrieval question answer-\ning via pluggable reward-driven contextual adapter,” arXiv preprint\narXiv:2310.18347, 2023.\n[70] S. Zhuang, B. Liu, B. Koopman, and G. Zuccon, “Open-source large\nlanguage models are strong zero-shot query likelihood models for\ndocument ranking,” arXiv preprint arXiv:2310.13243, 2023.\n[71] F. Xu, W. Shi, and E. Choi, “Recomp: Improving retrieval-augmented\nlms with compression and selective augmentation,” arXiv preprint\narXiv:2310.04408, 2023.\n[72] W. Shi, S. Min, M. Yasunaga, M. Seo, R. James, M. Lewis, L. Zettle-\nmoyer, and W.-t. Yih, “Replug: Retrieval-augmented black-box lan-\nguage models,” arXiv preprint arXiv:2301.12652, 2023.\n[73] E. Melz, “Enhancing llm intelligence with arm-rag: Auxiliary ra-\ntionale memory for retrieval augmented generation,” arXiv preprint\narXiv:2311.04177, 2023.\n[74] H. Wang, W. Huang, Y. Deng, R. Wang, Z. Wang, Y. Wang, F. Mi,\nJ. Z. Pan, and K.-F. Wong, “Unims-rag: A unified multi-source\nretrieval-augmented generation for personalized dialogue systems,”\narXiv preprint arXiv:2401.13256, 2024.\n[75] Z. Luo, C. Xu, P. Zhao, X. Geng, C. Tao, J. Ma, Q. Lin, and D. Jiang,\n“Augmented large language models with parametric knowledge guid-\ning,” arXiv preprint arXiv:2305.04757, 2023.\n[76] X. Li, Z. Liu, C. Xiong, S. Yu, Y. Gu, Z. Liu, and G. Yu, “Structure-\naware language model pretraining improves dense retrieval on struc-\ntured data,” arXiv preprint arXiv:2305.19912, 2023.\n[77] M. Kang, J. M. Kwak, J. Baek, and S. J. Hwang, “Knowledge\ngraph-augmented language models for knowledge-grounded dialogue\ngeneration,” arXiv preprint arXiv:2305.18846, 2023.\n[78] W. Shen, Y. Gao, C. Huang, F. Wan, X. Quan, and W. Bi, “Retrieval-\ngeneration alignment for end-to-end task-oriented dialogue system,”\narXiv preprint arXiv:2310.08877, 2023.\n[79] T. Shi, L. Li, Z. Lin, T. Yang, X. Quan, and Q. Wang, “Dual-feedback\nknowledge retrieval for task-oriented dialogue systems,” arXiv preprint\narXiv:2310.14528, 2023.\n[80] P. Ranade and A. Joshi, “Fabula: Intelligence report generation\nusing retrieval-augmented narrative construction,” arXiv preprint\narXiv:2310.13848, 2023.\n[81] X. Jiang, R. Zhang, Y. Xu, R. Qiu, Y. Fang, Z. Wang, J. Tang,\nH. Ding, X. Chu, J. Zhao et al., “Think and retrieval: A hypothesis\nknowledge graph enhanced medical large language models,” arXiv\npreprint arXiv:2312.15883, 2023.\n[82] J. Baek, S. Jeong, M. Kang, J. C. Park, and S. J. Hwang,\n“Knowledge-augmented language model verification,” arXiv preprint\narXiv:2310.12836, 2023.\n[83] L. Luo, Y.-F. Li, G. Haffari, and S. Pan, “Reasoning on graphs: Faithful\nand interpretable large language model reasoning,” arXiv preprint\narXiv:2310.01061, 2023.\n[84] X. He, Y. Tian, Y. Sun, N. V. Chawla, T. Laurent, Y. LeCun,\nX. Bresson, and B. Hooi, “G-retriever: Retrieval-augmented generation\nfor textual graph understanding and question answering,” arXiv preprint\narXiv:2402.07630, 2024.\n[85] L. Zha, J. Zhou, L. Li, R. Wang, Q. Huang, S. Yang, J. Yuan, C. Su,\nX. Li, A. Su et al., “Tablegpt: Towards unifying tables, nature language\nand commands into one gpt,” arXiv preprint arXiv:2307.08674, 2023.\n[86] M. Gaur, K. Gunaratna, V. Srinivasan, and H. Jin, “Iseeq: Information\nseeking question generation using dynamic meta-information retrieval\nand knowledge graphs,” in Proceedings of the AAAI Conference on\nArtificial Intelligence, vol. 36, no. 10, 2022, pp. 10 672–10 680.\n[87] F. Shi, X. Chen, K. Misra, N. Scales, D. Dohan, E. H. Chi, N. Sch¨arli,\nand D. Zhou, “Large language models can be easily distracted by\nirrelevant context,” in International Conference on Machine Learning.\nPMLR, 2023, pp. 31 210–31 227.\n[88] R.\nTeja,\n“Evaluating\nthe\nideal\nchunk\nsize\nfor\na\nrag\nsystem\nusing\nllamaindex,”\nhttps://www.llamaindex.ai/blog/\nevaluating-the-ideal-chunk-size-for-a-rag-system-using-llamaindex-6207e5d3fec5,\n2023.\n\n19\n[89] Langchain, “Recursively split by character,” https://python.langchain.\ncom/docs/modules/data connection/document transformers/recursive\ntext splitter, 2023.\n[90] S.\nYang,\n“Advanced\nrag\n01:\nSmall-to-\nbig\nretrieval,”\nhttps://towardsdatascience.com/\nadvanced-rag-01-small-to-big-retrieval-172181b396d4, 2023.\n[91] Y. Wang, N. Lipka, R. A. Rossi, A. Siu, R. Zhang, and T. Derr,\n“Knowledge graph prompting for multi-document question answering,”\narXiv preprint arXiv:2308.11730, 2023.\n[92] D. Zhou, N. Sch¨arli, L. Hou, J. Wei, N. Scales, X. Wang, D. Schu-\nurmans, C. Cui, O. Bousquet, Q. Le et al., “Least-to-most prompting\nenables complex reasoning in large language models,” arXiv preprint\narXiv:2205.10625, 2022.\n[93] S. Dhuliawala, M. Komeili, J. Xu, R. Raileanu, X. Li, A. Celikyilmaz,\nand J. Weston, “Chain-of-verification reduces hallucination in large\nlanguage models,” arXiv preprint arXiv:2309.11495, 2023.\n[94] X. Li and J. Li, “Angle-optimized text embeddings,” arXiv preprint\narXiv:2309.12871, 2023.\n[95] VoyageAI, “Voyage’s embedding models,” https://docs.voyageai.com/\nembeddings/, 2023.\n[96] BAAI,\n“Flagembedding,”\nhttps://github.com/FlagOpen/\nFlagEmbedding, 2023.\n[97] P. Zhang, S. Xiao, Z. Liu, Z. Dou, and J.-Y. Nie, “Retrieve anything\nto augment large language models,” arXiv preprint arXiv:2310.07554,\n2023.\n[98] N. F. Liu, K. Lin, J. Hewitt, A. Paranjape, M. Bevilacqua, F. Petroni,\nand P. Liang, “Lost in the middle: How language models use long\ncontexts,” arXiv preprint arXiv:2307.03172, 2023.\n[99] Y. Gao, T. Sheng, Y. Xiang, Y. Xiong, H. Wang, and J. Zhang, “Chat-\nrec: Towards interactive and explainable llms-augmented recommender\nsystem,” arXiv preprint arXiv:2303.14524, 2023.\n[100] N. Anderson, C. Wilson, and S. D. Richardson, “Lingua: Addressing\nscenarios for live interpretation and automatic dubbing,” in Proceedings\nof the 15th Biennial Conference of the Association for Machine\nTranslation in the Americas (Volume 2: Users and Providers Track\nand Government Track), J. Campbell, S. Larocca, J. Marciano,\nK. Savenkov, and A. Yanishevsky, Eds.\nOrlando, USA: Association\nfor Machine Translation in the Americas, Sep. 2022, pp. 202–209.\n[Online]. Available: https://aclanthology.org/2022.amta-upg.14\n[101] H. Jiang, Q. Wu, X. Luo, D. Li, C.-Y. Lin, Y. Yang, and L. Qiu,\n“Longllmlingua: Accelerating and enhancing llms in long context\nscenarios via prompt compression,” arXiv preprint arXiv:2310.06839,\n2023.\n[102] V. Karpukhin, B. O˘guz, S. Min, P. Lewis, L. Wu, S. Edunov, D. Chen,\nand W.-t. Yih, “Dense passage retrieval for open-domain question\nanswering,” arXiv preprint arXiv:2004.04906, 2020.\n[103] Y. Ma, Y. Cao, Y. Hong, and A. Sun, “Large language model is\nnot a good few-shot information extractor, but a good reranker for\nhard samples!” ArXiv, vol. abs/2303.08559, 2023. [Online]. Available:\nhttps://api.semanticscholar.org/CorpusID:257532405\n[104] J. Cui, Z. Li, Y. Yan, B. Chen, and L. Yuan, “Chatlaw: Open-source\nlegal large language model with integrated external knowledge bases,”\narXiv preprint arXiv:2306.16092, 2023.\n[105] O. Yoran, T. Wolfson, O. Ram, and J. Berant, “Making retrieval-\naugmented language models robust to irrelevant context,” arXiv\npreprint arXiv:2310.01558, 2023.\n[106] X. Li, R. Zhao, Y. K. Chia, B. Ding, L. Bing, S. Joty, and S. Poria,\n“Chain of knowledge: A framework for grounding large language mod-\nels with structured knowledge bases,” arXiv preprint arXiv:2305.13269,\n2023.\n[107] H.\nYang,\nS.\nYue,\nand\nY.\nHe,\n“Auto-gpt\nfor\nonline\ndecision\nmaking:\nBenchmarks\nand\nadditional\nopinions,”\narXiv\npreprint\narXiv:2306.02224, 2023.\n[108] T. Schick, J. Dwivedi-Yu, R. Dess`ı, R. Raileanu, M. Lomeli, L. Zettle-\nmoyer, N. Cancedda, and T. Scialom, “Toolformer: Language models\ncan teach themselves to use tools,” arXiv preprint arXiv:2302.04761,\n2023.\n[109] J. Zhang, “Graph-toolformer: To empower llms with graph rea-\nsoning ability via prompt augmented by chatgpt,” arXiv preprint\narXiv:2304.11116, 2023.\n[110] R. Nakano, J. Hilton, S. Balaji, J. Wu, L. Ouyang, C. Kim,\nC. Hesse, S. Jain, V. Kosaraju, W. Saunders et al., “Webgpt: Browser-\nassisted question-answering with human feedback,” arXiv preprint\narXiv:2112.09332, 2021.\n[111] T. Kwiatkowski, J. Palomaki, O. Redfield, M. Collins, A. Parikh,\nC. Alberti, D. Epstein, I. Polosukhin, J. Devlin, K. Lee et al., “Natural\nquestions: a benchmark for question answering research,” Transactions\nof the Association for Computational Linguistics, vol. 7, pp. 453–466,\n2019.\n[112] Y. Liu, S. Yavuz, R. Meng, M. Moorthy, S. Joty, C. Xiong, and Y. Zhou,\n“Exploring the integration strategies of retriever and large language\nmodels,” arXiv preprint arXiv:2308.12574, 2023.\n[113] M. Joshi, E. Choi, D. S. Weld, and L. Zettlemoyer, “Triviaqa: A large\nscale distantly supervised challenge dataset for reading comprehen-\nsion,” arXiv preprint arXiv:1705.03551, 2017.\n[114] P. Rajpurkar, J. Zhang, K. Lopyrev, and P. Liang, “Squad: 100,000+\nquestions\nfor\nmachine\ncomprehension\nof\ntext,”\narXiv\npreprint\narXiv:1606.05250, 2016.\n[115] J. Berant, A. Chou, R. Frostig, and P. Liang, “Semantic parsing on\nfreebase from question-answer pairs,” in Proceedings of the 2013\nconference on empirical methods in natural language processing, 2013,\npp. 1533–1544.\n[116] A. Mallen, A. Asai, V. Zhong, R. Das, H. Hajishirzi, and D. Khashabi,\n“When not to trust language models: Investigating effectiveness and\nlimitations of parametric and non-parametric memories,” arXiv preprint\narXiv:2212.10511, 2022.\n[117] T. Nguyen, M. Rosenberg, X. Song, J. Gao, S. Tiwary, R. Majumder,\nand L. Deng, “Ms marco: A human-generated machine reading com-\nprehension dataset,” 2016.\n[118] Z. Yang, P. Qi, S. Zhang, Y. Bengio, W. W. Cohen, R. Salakhutdi-\nnov, and C. D. Manning, “Hotpotqa: A dataset for diverse, explain-\nable multi-hop question answering,” arXiv preprint arXiv:1809.09600,\n2018.\n[119] X. Ho, A.-K. D. Nguyen, S. Sugawara, and A. Aizawa, “Constructing a\nmulti-hop qa dataset for comprehensive evaluation of reasoning steps,”\narXiv preprint arXiv:2011.01060, 2020.\n[120] H. Trivedi, N. Balasubramanian, T. Khot, and A. Sabharwal, “Musique:\nMultihop questions via single-hop question composition,” Transactions\nof the Association for Computational Linguistics, vol. 10, pp. 539–554,\n2022.\n[121] A. Fan, Y. Jernite, E. Perez, D. Grangier, J. Weston, and M. Auli, “Eli5:\nLong form question answering,” arXiv preprint arXiv:1907.09190,\n2019.\n[122] T. Koˇcisk`y, J. Schwarz, P. Blunsom, C. Dyer, K. M. Hermann, G. Melis,\nand E. Grefenstette, “The narrativeqa reading comprehension chal-\nlenge,” Transactions of the Association for Computational Linguistics,\nvol. 6, pp. 317–328, 2018.\n[123] K.-H. Lee, X. Chen, H. Furuta, J. Canny, and I. Fischer, “A human-\ninspired reading agent with gist memory of very long contexts,” arXiv\npreprint arXiv:2402.09727, 2024.\n[124] I. Stelmakh, Y. Luan, B. Dhingra, and M.-W. Chang, “Asqa: Factoid\nquestions meet long-form answers,” arXiv preprint arXiv:2204.06092,\n2022.\n[125] M. Zhong, D. Yin, T. Yu, A. Zaidi, M. Mutuma, R. Jha, A. H.\nAwadallah, A. Celikyilmaz, Y. Liu, X. Qiu et al., “Qmsum: A new\nbenchmark for query-based multi-domain meeting summarization,”\narXiv preprint arXiv:2104.05938, 2021.\n[126] P. Dasigi, K. Lo, I. Beltagy, A. Cohan, N. A. Smith, and M. Gardner,\n“A dataset of information-seeking questions and answers anchored in\nresearch papers,” arXiv preprint arXiv:2105.03011, 2021.\n[127] T. M¨oller, A. Reina, R. Jayakumar, and M. Pietsch, “Covid-qa: A\nquestion answering dataset for covid-19,” in ACL 2020 Workshop on\nNatural Language Processing for COVID-19 (NLP-COVID), 2020.\n[128] X. Wang, G. H. Chen, D. Song, Z. Zhang, Z. Chen, Q. Xiao, F. Jiang,\nJ. Li, X. Wan, B. Wang et al., “Cmb: A comprehensive medical\nbenchmark in chinese,” arXiv preprint arXiv:2308.08833, 2023.\n[129] H. Zeng, “Measuring massive multitask chinese understanding,” arXiv\npreprint arXiv:2304.12986, 2023.\n[130] R. Y. Pang, A. Parrish, N. Joshi, N. Nangia, J. Phang, A. Chen, V. Pad-\nmakumar, J. Ma, J. Thompson, H. He et al., “Quality: Question an-\nswering with long input texts, yes!” arXiv preprint arXiv:2112.08608,\n2021.\n[131] P. Clark, I. Cowhey, O. Etzioni, T. Khot, A. Sabharwal, C. Schoenick,\nand O. Tafjord, “Think you have solved question answering? try arc,\nthe ai2 reasoning challenge,” arXiv preprint arXiv:1803.05457, 2018.\n[132] A. Talmor, J. Herzig, N. Lourie, and J. Berant, “Commonsenseqa:\nA question answering challenge targeting commonsense knowledge,”\narXiv preprint arXiv:1811.00937, 2018.\n[133] E. Dinan, S. Roller, K. Shuster, A. Fan, M. Auli, and J. Weston,\n“Wizard of wikipedia: Knowledge-powered conversational agents,”\narXiv preprint arXiv:1811.01241, 2018.\n[134] H. Wang, M. Hu, Y. Deng, R. Wang, F. Mi, W. Wang, Y. Wang, W.-\nC. Kwan, I. King, and K.-F. Wong, “Large language models as source\n\n20\nplanner for personalized knowledge-grounded dialogue,” arXiv preprint\narXiv:2310.08840, 2023.\n[135] ——, “Large language models as source planner for personal-\nized knowledge-grounded dialogue,” arXiv preprint arXiv:2310.08840,\n2023.\n[136] X. Xu, Z. Gou, W. Wu, Z.-Y. Niu, H. Wu, H. Wang, and S. Wang,\n“Long time no see! open-domain conversation with long-term persona\nmemory,” arXiv preprint arXiv:2203.05797, 2022.\n[137] T.-H. Wen, M. Gasic, N. Mrksic, L. M. Rojas-Barahona, P.-H.\nSu, S. Ultes, D. Vandyke, and S. Young, “Conditional generation\nand snapshot learning in neural dialogue systems,” arXiv preprint\narXiv:1606.03352, 2016.\n[138] R. He and J. McAuley, “Ups and downs: Modeling the visual evolution\nof fashion trends with one-class collaborative filtering,” in proceedings\nof the 25th international conference on world wide web, 2016, pp.\n507–517.\n[139] S. Li, H. Ji, and J. Han, “Document-level event argument extraction\nby conditional generation,” arXiv preprint arXiv:2104.05919, 2021.\n[140] S. Ebner, P. Xia, R. Culkin, K. Rawlins, and B. Van Durme, “Multi-\nsentence argument linking,” arXiv preprint arXiv:1911.03766, 2019.\n[141] H. Elsahar, P. Vougiouklis, A. Remaci, C. Gravier, J. Hare, F. Laforest,\nand E. Simperl, “T-rex: A large scale alignment of natural language\nwith knowledge base triples,” in Proceedings of the Eleventh Inter-\nnational Conference on Language Resources and Evaluation (LREC\n2018), 2018.\n[142] O. Levy, M. Seo, E. Choi, and L. Zettlemoyer, “Zero-shot relation ex-\ntraction via reading comprehension,” arXiv preprint arXiv:1706.04115,\n2017.\n[143] R. Zellers, A. Holtzman, Y. Bisk, A. Farhadi, and Y. Choi, “Hel-\nlaswag: Can a machine really finish your sentence?” arXiv preprint\narXiv:1905.07830, 2019.\n[144] S. Kim, S. J. Joo, D. Kim, J. Jang, S. Ye, J. Shin, and M. Seo,\n“The cot collection: Improving zero-shot and few-shot learning of\nlanguage models via chain-of-thought fine-tuning,” arXiv preprint\narXiv:2305.14045, 2023.\n[145] A. Saha, V. Pahuja, M. Khapra, K. Sankaranarayanan, and S. Chandar,\n“Complex sequential question answering: Towards learning to converse\nover linked question answer pairs with a knowledge graph,” in Proceed-\nings of the AAAI conference on artificial intelligence, vol. 32, no. 1,\n2018.\n[146] D. Hendrycks, C. Burns, S. Basart, A. Zou, M. Mazeika, D. Song, and\nJ. Steinhardt, “Measuring massive multitask language understanding,”\narXiv preprint arXiv:2009.03300, 2020.\n[147] S. Merity, C. Xiong, J. Bradbury, and R. Socher, “Pointer sentinel\nmixture models,” arXiv preprint arXiv:1609.07843, 2016.\n[148] M. Geva, D. Khashabi, E. Segal, T. Khot, D. Roth, and J. Berant,\n“Did aristotle use a laptop? a question answering benchmark with\nimplicit reasoning strategies,” Transactions of the Association for\nComputational Linguistics, vol. 9, pp. 346–361, 2021.\n[149] J. Thorne, A. Vlachos, C. Christodoulopoulos, and A. Mittal, “Fever: a\nlarge-scale dataset for fact extraction and verification,” arXiv preprint\narXiv:1803.05355, 2018.\n[150] N. Kotonya and F. Toni, “Explainable automated fact-checking for\npublic health claims,” arXiv preprint arXiv:2010.09926, 2020.\n[151] R. Lebret, D. Grangier, and M. Auli, “Neural text generation from\nstructured data with application to the biography domain,” arXiv\npreprint arXiv:1603.07771, 2016.\n[152] H. Hayashi, P. Budania, P. Wang, C. Ackerson, R. Neervannan,\nand G. Neubig, “Wikiasp: A dataset for multi-domain aspect-based\nsummarization,” Transactions of the Association for Computational\nLinguistics, vol. 9, pp. 211–225, 2021.\n[153] S. Narayan, S. B. Cohen, and M. Lapata, “Don’t give me the details,\njust the summary! topic-aware convolutional neural networks for ex-\ntreme summarization,” arXiv preprint arXiv:1808.08745, 2018.\n[154] S. Saha, J. A. Junaed, M. Saleki, A. S. Sharma, M. R. Rifat, M. Rahouti,\nS. I. Ahmed, N. Mohammed, and M. R. Amin, “Vio-lens: A novel\ndataset of annotated social network posts leading to different forms\nof communal violence and its evaluation,” in Proceedings of the First\nWorkshop on Bangla Language Processing (BLP-2023), 2023, pp. 72–\n84.\n[155] X. Li and D. Roth, “Learning question classifiers,” in COLING 2002:\nThe 19th International Conference on Computational Linguistics, 2002.\n[156] R. Socher, A. Perelygin, J. Wu, J. Chuang, C. D. Manning, A. Y. Ng,\nand C. Potts, “Recursive deep models for semantic compositionality\nover a sentiment treebank,” in Proceedings of the 2013 conference on\nempirical methods in natural language processing, 2013, pp. 1631–\n1642.\n[157] H. Husain, H.-H. Wu, T. Gazit, M. Allamanis, and M. Brockschmidt,\n“Codesearchnet challenge: Evaluating the state of semantic code\nsearch,” arXiv preprint arXiv:1909.09436, 2019.\n[158] K. Cobbe, V. Kosaraju, M. Bavarian, M. Chen, H. Jun, L. Kaiser,\nM. Plappert, J. Tworek, J. Hilton, R. Nakano et al., “Training verifiers\nto solve math word problems,” arXiv preprint arXiv:2110.14168, 2021.\n[159] R. Steinberger, B. Pouliquen, A. Widiger, C. Ignat, T. Erjavec, D. Tufis,\nand D. Varga, “The jrc-acquis: A multilingual aligned parallel corpus\nwith 20+ languages,” arXiv preprint cs/0609058, 2006.\n[160] Y. Hoshi, D. Miyashita, Y. Ng, K. Tatsuno, Y. Morioka, O. Torii,\nand J. Deguchi, “Ralle: A framework for developing and eval-\nuating retrieval-augmented large language models,” arXiv preprint\narXiv:2308.10633, 2023.\n[161] J. Liu, “Building production-ready rag applications,” https://www.ai.\nengineer/summit/schedule/building-production-ready-rag-applications,\n2023.\n[162] I. Nguyen, “Evaluating rag part i: How to evaluate document retrieval,”\nhttps://www.deepset.ai/blog/rag-evaluation-retrieval, 2023.\n[163] Q. Leng, K. Uhlenhuth, and A. Polyzotis, “Best practices for\nllm evaluation of rag applications,” https://www.databricks.com/blog/\nLLM-auto-eval-best-practices-RAG, 2023.\n[164] S. Es, J. James, L. Espinosa-Anke, and S. Schockaert, “Ragas: Au-\ntomated evaluation of retrieval augmented generation,” arXiv preprint\narXiv:2309.15217, 2023.\n[165] J. Saad-Falcon, O. Khattab, C. Potts, and M. Zaharia, “Ares: An\nautomated evaluation framework for retrieval-augmented generation\nsystems,” arXiv preprint arXiv:2311.09476, 2023.\n[166] C.\nJarvis\nand\nJ.\nAllard,\n“A\nsurvey\nof\ntechniques\nfor\nmaximizing\nllm\nperformance,”\nhttps://community.openai.\ncom/t/openai-dev-day-2023-breakout-sessions/505213#\na-survey-of-techniques-for-maximizing-llm-performance-2, 2023.\n[167] J. Chen, H. Lin, X. Han, and L. Sun, “Benchmarking large lan-\nguage models in retrieval-augmented generation,” arXiv preprint\narXiv:2309.01431, 2023.\n[168] Y. Liu, L. Huang, S. Li, S. Chen, H. Zhou, F. Meng, J. Zhou, and\nX. Sun, “Recall: A benchmark for llms robustness against external\ncounterfactual knowledge,” arXiv preprint arXiv:2311.08147, 2023.\n[169] Y. Lyu, Z. Li, S. Niu, F. Xiong, B. Tang, W. Wang, H. Wu, H. Liu,\nT. Xu, and E. Chen, “Crud-rag: A comprehensive chinese benchmark\nfor retrieval-augmented generation of large language models,” arXiv\npreprint arXiv:2401.17043, 2024.\n[170] P. Xu, W. Ping, X. Wu, L. McAfee, C. Zhu, Z. Liu, S. Subramanian,\nE. Bakhturina, M. Shoeybi, and B. Catanzaro, “Retrieval meets long\ncontext large language models,” arXiv preprint arXiv:2310.03025,\n2023.\n[171] C. Packer, V. Fang, S. G. Patil, K. Lin, S. Wooders, and J. E. Gon-\nzalez, “Memgpt: Towards llms as operating systems,” arXiv preprint\narXiv:2310.08560, 2023.\n[172] G. Xiao, Y. Tian, B. Chen, S. Han, and M. Lewis, “Efficient\nstreaming language models with attention sinks,” arXiv preprint\narXiv:2309.17453, 2023.\n[173] T. Zhang, S. G. Patil, N. Jain, S. Shen, M. Zaharia, I. Stoica, and J. E.\nGonzalez, “Raft: Adapting language model to domain specific rag,”\narXiv preprint arXiv:2403.10131, 2024.\n[174] J. Kaplan, S. McCandlish, T. Henighan, T. B. Brown, B. Chess,\nR. Child, S. Gray, A. Radford, J. Wu, and D. Amodei, “Scaling laws\nfor neural language models,” arXiv preprint arXiv:2001.08361, 2020.\n[175] U. Alon, F. Xu, J. He, S. Sengupta, D. Roth, and G. Neubig, “Neuro-\nsymbolic language modeling with automaton-augmented retrieval,” in\nInternational Conference on Machine Learning.\nPMLR, 2022, pp.\n468–485.\n[176] M. Yasunaga, A. Aghajanyan, W. Shi, R. James, J. Leskovec, P. Liang,\nM. Lewis, L. Zettlemoyer, and W.-t. Yih, “Retrieval-augmented multi-\nmodal language modeling,” arXiv preprint arXiv:2211.12561, 2022.\n[177] J. Li, D. Li, S. Savarese, and S. Hoi, “Blip-2: Bootstrapping language-\nimage pre-training with frozen image encoders and large language\nmodels,” arXiv preprint arXiv:2301.12597, 2023.\n[178] W. Zhu, A. Yan, Y. Lu, W. Xu, X. E. Wang, M. Eckstein, and W. Y.\nWang, “Visualize before you write: Imagination-guided open-ended\ntext generation,” arXiv preprint arXiv:2210.03765, 2022.\n[179] J. Zhao, G. Haffar, and E. Shareghi, “Generating synthetic speech from\nspokenvocab for speech translation,” arXiv preprint arXiv:2210.08174,\n2022.\n[180] D. M. Chan, S. Ghosh, A. Rastrow, and B. Hoffmeister, “Using external\noff-policy speech-to-text mappings in contextual end-to-end automated\nspeech recognition,” arXiv preprint arXiv:2301.02736, 2023.\n\n21\n[181] A. Yang, A. Nagrani, P. H. Seo, A. Miech, J. Pont-Tuset, I. Laptev,\nJ. Sivic, and C. Schmid, “Vid2seq: Large-scale pretraining of a visual\nlanguage model for dense video captioning,” in Proceedings of the\nIEEE/CVF Conference on Computer Vision and Pattern Recognition,\n2023, pp. 10 714–10 726.\n[182] N. Nashid, M. Sintaha, and A. Mesbah, “Retrieval-based prompt\nselection for code-related few-shot learning,” in 2023 IEEE/ACM 45th\nInternational Conference on Software Engineering (ICSE), 2023, pp.\n2450–2462.",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/REFERENCES",
- "metadata": {
- "length": 35954,
- "summary": "This section is a comprehensive bibliography of academic papers and technical reports related to Retrieval-Augmented Generation (RAG) and large language models. The references cover topics such as retrieval methods, hallucination mitigation, evaluation benchmarks, and specific model architectures. The list includes publications from major conferences like NeurIPS, ACL, EMNLP, and ICSE, as well as arXiv preprints.",
- "page_nums": [
- 17,
- 18,
- 19,
- 20,
- 21
- ],
- "keywords": [
- "IEEE/CVF Conference on Computer Vision and Pattern Recognition",
- "IEEE/ACM 45th International Conference on Software Engineering"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 18,
- "artifact_ref": "page_citation_assets/page-18.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 19,
- "artifact_ref": "page_citation_assets/page-19.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 20,
- "artifact_ref": "page_citation_assets/page-20.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- },
- {
- "page_num": 21,
- "artifact_ref": "page_citation_assets/page-21.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1224,
- "height": 1584
- }
- ]
- }
- }
- ]
-}
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/doc_nav.json b/apps/api/app/data/demo_documents/research-rag-survey/doc_nav.json
deleted file mode 100644
index e2bb54ddc..000000000
--- a/apps/api/app/data/demo_documents/research-rag-survey/doc_nav.json
+++ /dev/null
@@ -1,339 +0,0 @@
-{
- "version": "1.0",
- "file_name": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf",
- "stats": {
- "total_chunks": 35,
- "text_chunks": 0,
- "image_chunks": 6,
- "table_chunks": 0,
- "page_chunks": 29,
- "max_depth": 4
- },
- "sections": [
- {
- "title": "Root",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root",
- "level": 1,
- "summary": "",
- "chunk_count": 35,
- "children": [
- {
- "title": "I. INTRODUCTION",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/I. INTRODUCTION",
- "level": 2,
- "summary": "This document is the first page of a survey paper titled \"Retrieval-Augmented Generation for Large Language Models: A Survey.\" It introduces the concept of Retrieval-Augmented Generation (RAG) as a solution to challenges like hallucination in Large Language Models (LLMs). The abstract outlines the paper's scope, which includes reviewing RAG paradigms (Naive, Advanced, Modular), frameworks, and evaluation methods. The introduction further details the rapid development of RAG technology and its integration into LLMs.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "II. OVERVIEW OF RAG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG",
- "level": 2,
- "summary": "",
- "chunk_count": 4,
- "children": [
- {
- "title": "A. Naive RAG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/A. Naive RAG",
- "level": 3,
- "summary": "This page presents a technology tree of Retrieval-Augmented Generation (RAG) research, visualizing the evolution of methods from 2020 to 2024 across pre-training, fine-tuning, and inference stages. The text introduces the scope of RAG within Large Language Models (LLMs), outlines the paper's structure, and begins an overview of RAG paradigms including Naive, Advanced, and Modular RAG.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "B. Advanced RAG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/B. Advanced RAG",
- "level": 3,
- "summary": "This page details the architecture and process of Retrieval-Augmented Generation (RAG), illustrated by a flowchart in Figure 2 that outlines the steps of indexing, retrieval, and generation. The text describes how Naive RAG functions through these three phases—converting documents into vector chunks, retrieving relevant context based on user queries, and generating answers using a Large Language Model (LLM). It also discusses challenges such as hallucination and redundancy, and introduces Advanced RAG strategies to improve performance.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C. Modular RAG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/C. Modular RAG",
- "level": 3,
- "summary": "This page presents Figure 3, which visually compares three paradigms of Retrieval-Augmented Generation (RAG): Naive RAG, Advanced RAG, and Modular RAG. The text below the figure elaborates on pre-retrieval and post-retrieval processes, highlighting optimization strategies like query rewriting and reranking. It also details the Modular RAG architecture, emphasizing its flexibility through specialized components such as search modules and memory pools.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "D. RAG vs Fine-tuning",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/II. OVERVIEW OF RAG/D. RAG vs Fine-tuning",
- "level": 3,
- "summary": "The section 'D. RAG vs Fine-tuning' compares Retrieval-Augmented Generation (RAG) and Fine-tuning (FT) as optimization methods for Large Language Models (LLMs). It describes RAG as a method that leverages external knowledge sources, offering real-time updates and interpretability but with higher latency. In contrast, FT involves retraining the model to internalize knowledge, allowing for deep customization of behavior and style but requiring significant computational resources. The text notes that while unsupervised fine-tuning shows some improvement, RAG consistently outperforms it in handling new knowledge. The section concludes by stating that RAG and FT are not mutually exclusive and can complement each other.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "III. RETRIEVAL",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL",
- "level": 2,
- "summary": "",
- "chunk_count": 8,
- "children": [
- {
- "title": "A. Retrieval Source",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/A. Retrieval Source",
- "level": 3,
- "summary": "This section, titled 'A. Retrieval Source', discusses the types of data sources used in Retrieval-Augmented Generation (RAG) systems to enhance Large Language Models (LLMs). It outlines the evolution from text as the primary source to include semi-structured data like PDFs and structured data such as Knowledge Graphs (KG). Additionally, it highlights a growing trend of utilizing content generated by LLMs themselves for retrieval and enhancement purposes.",
- "chunk_count": 4,
- "children": [
- {
- "title": "TABLE I",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/A. Retrieval Source/TABLE I",
- "level": 4,
- "summary": "6 TABLE I SUMMARY OF RAG METHODS Method Retrieval Source Retrieval Data Type Retrieval Granularity Augmentation Stage Retrieval process CoG [29] Wikipedia Text Phrase Pre-training Iterative DenseX [30] FactoidWiki Text Proposition Inference Once EAR [31] Dataset-base Text Sentence Tuning Once UPRISE [20] Dataset-base Text Sentence Tuning Once RAST [32] Dataset-base Text Sentence Tuning Once Self-Mem [17] Dataset-base Text Sentence Tuning Iterative FLARE [24] Search Engine,Wikipedia Text Sentence Tuning Adaptive PGRA [33] Wikipedia Text Sentence Inference Once FILCO [34] Wikipedia Text Sentence Inference Once RADA [35] Dataset-base Text Sentence Inference Once Filter-rerank [36] Synthesized dataset Text Sentence Inference Once R-GQA [37] Dataset-base Text Sentence Pair Tuning Once LLM-R [38] Dataset-base Text Sentence Pair Inference Iterative TIGER [39] Dataset-base Text Item-base Pre-training Once LM-Indexer [40] Dataset-base Text Item-base Tuning Once BEQUE [9] Dataset-base Text Item-base Tuning Once CT-RAG [41] Synthesized dataset Text Item-base Tuning Once Atlas [42] Wikipedia, Common Crawl Text Chunk Pre-training Iterative RAVEN [43] Wikipedia Text Chunk Pre-training Once RETRO++ [44] Pre-training Corpus Text Chunk Pre-training Iterative INSTRUCTRETRO [45] Pre-training corpus Text...Once KnowledGPT [15] Dataset-base KG Triplet Inference Muti-time FABULA [80] Dataset-base,Graph KG Entity Inference Once HyKGE [81] CMeKG KG Entity Inference Once KALMV [82] Wikipedia KG Triplet Inference Iterative RoG [83] Freebase KG Triplet Inference Iterative G-Retriever [84] Dataset-base TextGraph Sub-Graph Inference Once",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "SUMMARY OF RAG METHODS",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/A. Retrieval Source/SUMMARY OF RAG METHODS",
- "level": 4,
- "summary": "The page features Figure 4, a chart comparing RAG (Retrieval-Augmented Generation) with other model optimization methods based on the axes of 'External Knowledge Required' and 'Model Adaptation Required'. The figure illustrates that Prompt Engineering requires low external knowledge and adaptation, while Fine-tuning requires high adaptation. The text below discusses various data sources for retrieval systems, including unstructured data (e.g., Wikipedia, HotpotQA), semi-structured data (e.g., PDFs, tables via TableGPT), and structured data like knowledge graphs (e.g., G-Retriever). It also covers LLM-generated content and retrieval granularity.",
- "chunk_count": 2,
- "children": []
- }
- ]
- },
- {
- "title": "B. Indexing Optimization",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/B. Indexing Optimization",
- "level": 3,
- "summary": "The section 'B. Indexing Optimization' discusses strategies for processing and segmenting documents into embeddings for storage in a vector database. It details three main approaches: Chunking Strategy (splitting text into fixed or recursive chunks, mentioning methods like Small2Big), Metadata Attachments (enriching chunks with page numbers, file names, or artificial metadata like hypothetical questions via Reverse HyDE), and Structural Index (establishing hierarchical structures like parent-child relationships or Knowledge Graphs to improve retrieval consistency and efficiency).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C. Query Optimization",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/C. Query Optimization",
- "level": 3,
- "summary": "The section 'Query Optimization' addresses the challenges of Naive RAG, such as imprecise queries and language ambiguity. It outlines several strategies to improve retrieval effectiveness: Query Expansion (enriching a single query with more context), Multi-Query (executing multiple expanded queries in parallel), Sub-Query (decomposing complex questions into simpler sub-questions), and Chain-of-Verification (CoVe) (validating expanded queries via an LLM to reduce hallucinations).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "D. Embedding",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/D. Embedding",
- "level": 3,
- "summary": "The section discusses the role of embedding models in RAG systems, which calculate similarity between question and document chunks. It highlights prominent models like AnglE, Voyage, and BGE that benefit from multi-task instruction tuning. The text also mentions evaluation benchmarks such as Hugging Face's MTEB leaderboard (covering 8 tasks and 58 datasets) and C-MTEB for Chinese capability. Finally, it introduces 'Mix/hybrid Retrieval' as a method combining sparse and dense embedding approaches to capture different relevance features.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "E. Adapter",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/III. RETRIEVAL/E. Adapter",
- "level": 3,
- "summary": "The section titled 'E. Adapter' discusses the use of external adapters to address challenges in fine-tuning large language models (LLMs), such as limited computational resources or API integration constraints. It describes several approaches including UPRISE, which trains a lightweight prompt retriever; AAR (Augmentation-Adapted Retriever), a universal adapter for multiple downstream tasks; PRCA, a pluggable reward-driven contextual adapter; BGM, which uses a bridge Seq2Seq model between the retriever and LLM; and PKG, which employs advanced strategies like repetition.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "IV. GENERATION",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/IV. GENERATION",
- "level": 2,
- "summary": "",
- "chunk_count": 2,
- "children": [
- {
- "title": "A. Context Curation",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/IV. GENERATION/A. Context Curation",
- "level": 3,
- "summary": "The section 'A. Context Curation' discusses methods to process retrieved information before it is fed into a Large Language Model (LLM) to avoid issues like the 'Lost in the middle' problem. It details two main approaches: 1) Reranking, which reorders document chunks based on relevance using metrics or models like BERT and GPT; and 2) Context Selection/Compression, which reduces context length by removing unimportant tokens or summarizing content using techniques like contrastive learning or small language models (SLMs).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "B. LLM Fine-tuning",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/IV. GENERATION/B. LLM Fine-tuning",
- "level": 3,
- "summary": "This section discusses targeted fine-tuning of Large Language Models (LLMs) to improve performance in specific domains or data formats. It highlights benefits such as incorporating additional knowledge, adapting to structured data (e.g., via the SANTA framework), and aligning outputs with human or retriever preferences through reinforcement learning. The text also covers distilling powerful proprietary models into smaller ones and coordinating fine-tuning between retrievers and generators using methods like RA-DIT.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "V. AUGMENTATION PROCESS IN RAG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG",
- "level": 2,
- "summary": "",
- "chunk_count": 3,
- "children": [
- {
- "title": "A. Iterative Retrieval",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG/A. Iterative Retrieval",
- "level": 3,
- "summary": "The section titled 'A. Iterative Retrieval' introduces a process where the knowledge base is repeatedly searched based on an initial query and previously generated text to provide more comprehensive knowledge.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "B. Recursive Retrieval",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG/B. Recursive Retrieval",
- "level": 3,
- "summary": "The section 'B. Recursive Retrieval' describes a process used in information retrieval and NLP to improve search depth and relevance by iteratively refining queries based on previous results. It highlights methods like IRCoT, which uses chain-of-thought for guidance, and ToC, which creates clarification trees to optimize ambiguous queries. The text also notes that recursive retrieval is often combined with multi-hop retrieval techniques.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C. Adaptive Retrieval",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/V. AUGMENTATION PROCESS IN RAG/C. Adaptive Retrieval",
- "level": 3,
- "summary": "This section discusses adaptive retrieval methods, which enable Large Language Models (LLMs) to actively determine the optimal timing and content for retrieval. It highlights Flare and Self-RAG as examples of this approach. The text also contextualizes these methods within a broader trend of LLMs employing active judgment, similar to model agents like AutoGPT, Toolformer, and Graph-Toolformer. Additionally, it mentions WebGPT's use of reinforcement learning to train GPT-3 for autonomous search engine usage.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "VI. TASK AND EVALUATION",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION",
- "level": 2,
- "summary": "",
- "chunk_count": 5,
- "children": [
- {
- "title": "A. Downstream Task",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/A. Downstream Task",
- "level": 3,
- "summary": "The section 'A. Downstream Task' outlines the primary applications of Retrieval-Augmented Generation (RAG) models. It identifies Question Answering (QA) as the core task, covering various formats like single-hop/multi-hop QA and domain-specific scenarios. Additionally, it notes that RAG is expanding into other tasks such as Information Extraction (IE), dialogue generation, and code search. The text concludes by stating that these main downstream tasks and their corresponding datasets are summarized in Table II.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "B. Evaluation Target",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/B. Evaluation Target",
- "level": 3,
- "summary": "This section discusses the historical focus of RAG model assessments on specific downstream tasks and the metrics used for them, such as EM, F1, Accuracy, BLEU, and ROUGE. It introduces the main evaluation objectives: Retrieval Quality (measuring context effectiveness) and Generation Quality (assessing the synthesis of coherent answers).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C. Evaluation Aspects",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/C. Evaluation Aspects",
- "level": 3,
- "summary": "The section titled 'C. Evaluation Aspects' outlines the primary quality scores and essential abilities used to evaluate RAG models. It details three quality scores: Context Relevance, Answer Faithfulness, and Answer Relevance. Additionally, it describes four required abilities indicative of a model's adaptability and efficiency: Noise Robustness, Negative Rejection, Information Integration, and Counterfactual Robustness.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "D. Evaluation Benchmarks and Tools",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VI. TASK AND EVALUATION/D. Evaluation Benchmarks and Tools",
- "level": 3,
- "summary": "This section discusses benchmark tests and tools proposed for evaluating Retrieval-Augmented Generation (RAG) models. It highlights prominent benchmarks such as RGB, RECALL, and CRUD, which focus on appraising essential RAG abilities. Additionally, it mentions state-of-the-art automated tools like RAGAS, ARES, and TruLens that employ Large Language Models (LLMs) to adjudicate quality scores.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "VII. DISCUSSION AND FUTURE PROSPECTS",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS",
- "level": 2,
- "summary": "",
- "chunk_count": 6,
- "children": [
- {
- "title": "A. RAG vs Long Context",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/A. RAG vs Long Context",
- "level": 3,
- "summary": "This section discusses the impact of expanding Large Language Model (LLM) contexts, which now exceed 200,000 tokens. It explores whether Retrieval-Augmented Generation (RAG) remains necessary when LLMs can ingest entire documents directly. The text contrasts RAG with long-context approaches, noting that while long context improves efficiency and allows for complex problem solving, RAG offers benefits like faster inference speed, operational efficiency through chunked retrieval, and verifiable references. The section concludes by identifying the development of new RAG methods for super-long contexts as a future research trend.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "B. RAG Robustness",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/B. RAG Robustness",
- "level": 3,
- "summary": "The section 'B. RAG Robustness' discusses the impact of noise or contradictory information on Retrieval-Augmented Generation (RAG) output quality, a problem described as 'Misinformation can be worse than no information at all'. It highlights that improving resistance to such adversarial inputs is a key performance metric. The text cites research by Cuconasu et al. [54], which found that including irrelevant documents can unexpectedly increase accuracy by over 30%, contradicting initial assumptions about reduced quality. This underscores the need for specialized strategies to integrate retrieval with language generation models.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C. Hybrid Approaches",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/C. Hybrid Approaches",
- "level": 3,
- "summary": "The section titled 'C. Hybrid Approaches' discusses the emerging strategy of combining Retrieval-Augmented Generation (RAG) with fine-tuning. It highlights the importance of determining the optimal integration method, whether sequential, alternating, or through end-to-end joint training, to effectively harness both parameterized and non-parameterized knowledge.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "D. Scaling laws of RAG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/D. Scaling laws of RAG",
- "level": 3,
- "summary": "This section discusses the scaling laws of RAG (Retrieval-Augmented Generation) models. It notes that while scaling laws are established for LLMs, their applicability to RAG remains uncertain. The text highlights that initial studies like RETRO++ have begun addressing this, but parameter counts in RAG models still lag behind those of LLMs. It also mentions the intriguing possibility of an Inverse Scaling Law, where smaller models might outperform larger ones.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "E. Production-Ready RAG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/E. Production-Ready RAG",
- "level": 3,
- "summary": "This section discusses the practicality and engineering requirements of production-ready RAG systems. It highlights key challenges such as enhancing retrieval efficiency, improving document recall in large knowledge bases, and ensuring data security to prevent inadvertent disclosure. The text also notes that the RAG ecosystem is significantly influenced by technical stack advancements, mentioning popular tools like LangChain and LlamaIndex, low-code platforms like Flowise AI, and services from traditional vendors like Weaviate's Verba and Amazon's Kendra. Finally, it identifies a trend towards specialization in RAG technology, focusing on customization and simplification.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "F. Multi-modal RAG",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VII. DISCUSSION AND FUTURE PROSPECTS/F. Multi-modal RAG",
- "level": 3,
- "summary": "Section F discusses the expansion of Retrieval-Augmented Generation (RAG) beyond text-based tasks to include multimodal data. It details specific models and methods for handling images, audio, and video. For images, it mentions RA-CM3, BLIP-2, and the 'Visualize Before You Write' method. For audio and video, it describes the GSS method, UEOP, KNN-based attention fusion, Vid2Seq, RBPS, and the CoK method.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "VIII. CONCLUSION",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/VIII. CONCLUSION",
- "level": 2,
- "summary": "The conclusion summarizes the paper's emphasis on RAG's advancement in enhancing LLM capabilities by integrating parameterized knowledge with external data. It outlines three developmental paradigms (Naive, Advanced, and Modular RAG) and highlights technical integrations like fine-tuning. The section concludes by noting RAG's expanding scope into multimodal domains such as images, videos, and code, underscoring its practical implications for AI deployment.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "REFERENCES",
- "path": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf/Root/REFERENCES",
- "level": 2,
- "summary": "This section is a comprehensive bibliography of academic papers and technical reports related to Retrieval-Augmented Generation (RAG) and large language models. The references cover topics such as retrieval methods, hallucination mitigation, evaluation benchmarks, and specific model architectures. The list includes publications from major conferences like NeurIPS, ACL, EMNLP, and ICSE, as well as arXiv preprints.",
- "chunk_count": 5,
- "children": []
- }
- ]
- }
- ],
- "resources": {
- "images": [
- {
- "path": "images/image_page_2_figure_1.png",
- "summary": "Fig. 1. Technology tree of RAG research. [images/image_page_2_figure_1.png]"
- },
- {
- "path": "images/image_page_3_figure_1.png",
- "summary": "Fig. 2 [images/image_page_3_figure_1.png]"
- },
- {
- "path": "images/image_page_4_figure_1.png",
- "summary": "Fig. 3. Comparison between the three paradigms of RAG. [images/image_page_4_figure_1.png]"
- },
- {
- "path": "images/image_page_7_figure_1.png",
- "summary": "Fig. 4 [images/image_page_7_figure_1.png]"
- },
- {
- "path": "images/image_page_11_figure_1.png",
- "summary": "Fig. 5 [images/image_page_11_figure_1.png]"
- },
- {
- "path": "images/image_page_16_figure_1.png",
- "summary": "Summary of RAG ecosystem [images/image_page_16_figure_1.png]"
- }
- ],
- "tables": []
- }
-}
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_11_figure_1.png b/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_11_figure_1.png
deleted file mode 100644
index f016c1440..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_11_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_16_figure_1.png b/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_16_figure_1.png
deleted file mode 100644
index d171fb5f9..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_16_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_2_figure_1.png b/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_2_figure_1.png
deleted file mode 100644
index 2a0f5ec98..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_2_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_3_figure_1.png b/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_3_figure_1.png
deleted file mode 100644
index a81d8ee76..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_3_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_4_figure_1.png b/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_4_figure_1.png
deleted file mode 100644
index 9c3e00c13..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_4_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_7_figure_1.png b/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_7_figure_1.png
deleted file mode 100644
index e3a034aa3..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/images/image_page_7_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/manifest.json b/apps/api/app/data/demo_documents/research-rag-survey/manifest.json
deleted file mode 100644
index 6a1f95424..000000000
--- a/apps/api/app/data/demo_documents/research-rag-survey/manifest.json
+++ /dev/null
@@ -1,291 +0,0 @@
-{
- "version": "2.0",
- "job_id": "job_9f032493aca9",
- "data_id": "demo-research-rag-survey",
- "source_file_name": "Retrieval-Augmented Generation for Large Language Models: A Survey.pdf",
- "processing_date": "2026-08-18T18:01:34.860565Z",
- "processing": {
- "page_count": 21,
- "billing_status": "charged",
- "cost": {
- "micro_dollars": 315000,
- "credits": 0.315
- },
- "cost_estimate": {
- "currency": "USD",
- "pricing_version": "2026-08-18:874ab8b6c9e4",
- "total_input_cost": 0.004314,
- "total_output_cost": 0.00087646,
- "total_cost": 0.00519046,
- "by_model": {
- "qwen3.5-flash": {
- "input_tokens": 16662,
- "output_tokens": 255,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- },
- "deepseek-chat": {
- "input_tokens": 1950,
- "output_tokens": 457,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.000273,
- "output_cost": 0.00012796,
- "total_cost": 0.00040096,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- },
- "qwen3.6-flash": {
- "input_tokens": 16164,
- "output_tokens": 499,
- "calls": 6,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.004041,
- "output_cost": 0.0007485,
- "total_cost": 0.0047895,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 16662,
- "output_tokens": 255,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "page_memory.hierarchy": {
- "input_cost": 0.000273,
- "output_cost": 0.00012796,
- "total_cost": 0.00040096,
- "models": {
- "deepseek-chat": {
- "input_tokens": 1950,
- "output_tokens": 457,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.000273,
- "output_cost": 0.00012796,
- "total_cost": 0.00040096,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- },
- "page_memory.asset_detect": {
- "input_cost": 0.004041,
- "output_cost": 0.0007485,
- "total_cost": 0.0047895,
- "models": {
- "qwen3.6-flash": {
- "input_tokens": 16164,
- "output_tokens": 499,
- "calls": 6,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.004041,
- "output_cost": 0.0007485,
- "total_cost": 0.0047895,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "timing": {
- "started_at": "2026-08-18T17:58:44.066877+00:00",
- "completed_at": "2026-08-18T18:01:34.855331+00:00",
- "duration_ms": 170788
- },
- "stages": {
- "timing_ms": {
- "page_memory.normalize_to_pdf": 0,
- "page_memory.profile": 28423,
- "page_memory.skeleton": 0,
- "page_memory.title_render": 3264,
- "page_memory.title_detection": 24489,
- "page_memory.fine_hierarchy": 2748,
- "page_memory.render": 6307,
- "page_memory.tag": 36761,
- "page_memory.assets": 19839,
- "page_memory.node_summary": 48679,
- "page_memory.node_rows": 72,
- "page_memory.node_assembly": 48771,
- "worker.parse.document": 170640
- },
- "token_usage": {
- "prompt_tokens": 34776,
- "completion_tokens": 1211,
- "total_tokens": 35987,
- "calls": 8,
- "by_model": {
- "qwen3.5-flash": {
- "prompt_tokens": 16662,
- "completion_tokens": 255,
- "total_tokens": 16917,
- "calls": 1
- },
- "deepseek-chat": {
- "prompt_tokens": 1950,
- "completion_tokens": 457,
- "total_tokens": 2407,
- "calls": 1
- },
- "qwen3.6-flash": {
- "prompt_tokens": 16164,
- "completion_tokens": 499,
- "total_tokens": 16663,
- "calls": 6
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "prompt_tokens": 16662,
- "completion_tokens": 255,
- "total_tokens": 16917,
- "calls": 1,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 16662,
- "completion_tokens": 255,
- "total_tokens": 16917,
- "calls": 1
- }
- }
- },
- "page_memory.hierarchy": {
- "prompt_tokens": 1950,
- "completion_tokens": 457,
- "total_tokens": 2407,
- "calls": 1,
- "models": {
- "deepseek-chat": {
- "prompt_tokens": 1950,
- "completion_tokens": 457,
- "total_tokens": 2407,
- "calls": 1
- }
- }
- },
- "page_memory.asset_detect": {
- "prompt_tokens": 16164,
- "completion_tokens": 499,
- "total_tokens": 16663,
- "calls": 6,
- "models": {
- "qwen3.6-flash": {
- "prompt_tokens": 16164,
- "completion_tokens": 499,
- "total_tokens": 16663,
- "calls": 6
- }
- }
- }
- }
- }
- }
- },
- "statistics": {
- "total_chunks": 35,
- "text_chunks": 0,
- "image_chunks": 6,
- "table_chunks": 0,
- "page_chunks": 29,
- "total_pages": null
- },
- "HIERARCHY": {
- "Root": {
- "I. INTRODUCTION": {},
- "II. OVERVIEW OF RAG": {
- "A. Naive RAG": {},
- "B. Advanced RAG": {},
- "C. Modular RAG": {},
- "D. RAG vs Fine-tuning": {}
- },
- "III. RETRIEVAL": {
- "A. Retrieval Source": {
- "TABLE I": {},
- "SUMMARY OF RAG METHODS": {}
- },
- "B. Indexing Optimization": {},
- "C. Query Optimization": {},
- "D. Embedding": {},
- "E. Adapter": {}
- },
- "IV. GENERATION": {
- "A. Context Curation": {},
- "B. LLM Fine-tuning": {}
- },
- "V. AUGMENTATION PROCESS IN RAG": {
- "A. Iterative Retrieval": {},
- "B. Recursive Retrieval": {},
- "C. Adaptive Retrieval": {}
- },
- "VI. TASK AND EVALUATION": {
- "A. Downstream Task": {},
- "B. Evaluation Target": {},
- "C. Evaluation Aspects": {},
- "D. Evaluation Benchmarks and Tools": {}
- },
- "VII. DISCUSSION AND FUTURE PROSPECTS": {
- "A. RAG vs Long Context": {},
- "B. RAG Robustness": {},
- "C. Hybrid Approaches": {},
- "D. Scaling laws of RAG": {},
- "E. Production-Ready RAG": {},
- "F. Multi-modal RAG": {}
- },
- "VIII. CONCLUSION": {},
- "REFERENCES": {}
- }
- }
-}
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-1.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-1.png
deleted file mode 100644
index 64adf8d03..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-10.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-10.png
deleted file mode 100644
index 141061639..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-10.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-11.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-11.png
deleted file mode 100644
index fc9b23512..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-11.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-12.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-12.png
deleted file mode 100644
index 77ce3ea30..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-12.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-13.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-13.png
deleted file mode 100644
index 38fbba053..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-13.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-14.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-14.png
deleted file mode 100644
index ec817b7da..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-14.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-15.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-15.png
deleted file mode 100644
index ad0ec2b9f..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-15.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-16.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-16.png
deleted file mode 100644
index bdf78eb6f..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-16.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-17.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-17.png
deleted file mode 100644
index ff8bc51da..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-17.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-18.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-18.png
deleted file mode 100644
index ce4c58d58..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-18.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-19.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-19.png
deleted file mode 100644
index 3d820db17..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-19.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-2.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-2.png
deleted file mode 100644
index d1d0caadb..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-20.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-20.png
deleted file mode 100644
index 2f3d7aeba..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-20.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-21.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-21.png
deleted file mode 100644
index 44c63cbe2..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-21.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-3.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-3.png
deleted file mode 100644
index e4e7037e0..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-3.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-4.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-4.png
deleted file mode 100644
index 58a080c65..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-4.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-5.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-5.png
deleted file mode 100644
index fce682c35..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-5.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-6.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-6.png
deleted file mode 100644
index 7550f9956..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-6.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-7.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-7.png
deleted file mode 100644
index 8689cff15..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-7.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-8.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-8.png
deleted file mode 100644
index f98a95c2a..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-8.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-9.png b/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-9.png
deleted file mode 100644
index 6b0b5abb5..000000000
Binary files a/apps/api/app/data/demo_documents/research-rag-survey/page_citation_assets/page-9.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/chunks.json b/apps/api/app/data/demo_documents/research-toolformer/chunks.json
deleted file mode 100644
index fb7f4b6c6..000000000
--- a/apps/api/app/data/demo_documents/research-toolformer/chunks.json
+++ /dev/null
@@ -1,1260 +0,0 @@
-{
- "chunks": [
- {
- "chunk_id": "asset_a57ea806-1513-572e-8417-f97247057849",
- "type": "image",
- "content": "Figure 1: Exemplary predictions of Toolformer.\n[images/image_page_1_figure_1.png]",
- "path": "images/image_page_1_figure_1.png",
- "metadata": {
- "length": 81,
- "summary": "",
- "page_nums": [
- 1
- ],
- "file_path": "images/image_page_1_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_09c69238-408d-5906-b7b8-3e14c8d5271e",
- "type": "image",
- "content": "Figure 2: Key steps in our approach, illustrated for a question answering tool\n[images/image_page_2_figure_1.png]",
- "path": "images/image_page_2_figure_1.png",
- "metadata": {
- "length": 113,
- "summary": "",
- "page_nums": [
- 2
- ],
- "file_path": "images/image_page_2_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_a22412e2-f71d-53e8-89f9-b4459f266dc6",
- "type": "image",
- "content": "An exemplary prompt P(x) used to generate API calls for the question answering tool.\n[images/image_page_3_figure_1.png]",
- "path": "images/image_page_3_figure_1.png",
- "metadata": {
- "length": 119,
- "summary": "",
- "page_nums": [
- 3
- ],
- "file_path": "images/image_page_3_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_194baf47-eacf-50fd-bc26-49bd50f75db0",
- "type": "image",
- "content": "Figure 4: Average performance on LAMA, our math benchmarks and our QA benchmarks for GPT-2 models of different sizes and GPT-J finetuned with our approach, both with and without API calls.\n[images/image_page_9_figure_1.png]",
- "path": "images/image_page_9_figure_1.png",
- "metadata": {
- "length": 223,
- "summary": "",
- "page_nums": [
- 9
- ],
- "file_path": "images/image_page_9_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_68be5006-73df-5a58-8636-707711d572ac",
- "type": "table",
- "content": "tables/table_page_9_2.html",
- "path": "tables/table_page_9_2.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 9
- ],
- "file_path": "tables/table_page_9_2.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "node_0bd66d9f-c8e0-5e99-8f56-899ba181adc8",
- "type": "page",
- "content": "Toolformer: Language Models Can Teach Themselves to Use Tools\nTimo Schick\nJane Dwivedi-Yu\nRoberto Dessì†\nRoberta Raileanu\nMaria Lomeli\nLuke Zettlemoyer\nNicola Cancedda\nThomas Scialom\nMeta AI Research †Universitat Pompeu Fabra\nAbstract\nLanguage models (LMs) exhibit remarkable\nabilities to solve new tasks from just a few\nexamples or textual instructions, especially at\nscale. They also, paradoxically, struggle with\nbasic functionality, such as arithmetic or fac-\ntual lookup, where much simpler and smaller\nmodels excel.\nIn this paper, we show that\nLMs can teach themselves to use external tools\nvia simple APIs and achieve the best of both\nworlds.\nWe introduce Toolformer, a model\ntrained to decide which APIs to call, when to\ncall them, what arguments to pass, and how to\nbest incorporate the results into future token\nprediction. This is done in a self-supervised\nway, requiring nothing more than a handful of\ndemonstrations for each API. We incorporate\na range of tools, including a calculator, a Q&A\nsystem, a search engine, a translation system,\nand a calendar. Toolformer achieves substan-\ntially improved zero-shot performance across\na variety of downstream tasks, often competi-\ntive with much larger models, without sacrific-\ning its core language modeling abilities.\n1\nIntroduction\nLarge language models achieve impressive zero-\nand few-shot results on a variety of natural lan-\nguage processing tasks (Brown et al., 2020; Chowd-\nhery et al., 2022, i.a.) and show several emergent\ncapabilities (Wei et al., 2022). However, all of\nthese models have several inherent limitations that\ncan at best be partially addressed by further scal-\ning. These limitations include an inability to access\nup-to-date information on recent events (Komeili\net al., 2022) and the related tendency to hallucinate\nfacts (Maynez et al., 2020; Ji et al., 2022), difficul-\nties in understanding low-resource languages (Lin\net al., 2021), a lack of mathematical skills to per-\nform precise calculations (Patel et al., 2021) and an\nunawareness of the progression of time (Dhingra\net al., 2022).\nThe New England Journal of Medicine is a registered \ntrademark of [QA(“Who is the publisher of The New \nEngland Journal of Medicine?”) → Massachusetts \nMedical Society] the MMS.\nOut of 1400 participants, 400 (or [Calculator(400 / 1400) \n→ 0.29] 29%) passed the test. \nThe name derives from “la tortuga”, the Spanish word for \n[MT(“tortuga”) → turtle] turtle.\nThe Brown Act is California’s law [WikiSearch(“Brown \nAct”) → The Ralph M. Brown Act is an act of the \nCalifornia State Legislature that guarantees the public's \nright to attend and participate in meetings of local \nlegislative bodies.] that requires legislative bodies, like \ncity councils, to hold their meetings open to the public.\nFigure 1: Exemplary predictions of Toolformer. The\nmodel autonomously decides to call different APIs\n(from top to bottom: a question answering system,\na calculator, a machine translation system, and a\nWikipedia search engine) to obtain information that is\nuseful for completing a piece of text.\nA simple way to overcome these limitations of\ntoday’s language models is to give them the abil-\nity to use external tools such as search engines,\ncalculators, or calendars. However, existing ap-\nproaches either rely on large amounts of human\nannotations (Komeili et al., 2022; Thoppilan et al.,\n2022) or limit tool use to task-specific settings only\n(e.g., Gao et al., 2022; Parisi et al., 2022), hinder-\ning a more widespread adoption of tool use in LMs.\nTherefore, we propose Toolformer, a model that\nlearns to use tools in a novel way, which fulfills the\nfollowing desiderata:\n• The use of tools should be learned in a\nself-supervised way without requiring large\namounts of human annotations. This is impor-\narXiv:2302.04761v1 [cs.CL] 9 Feb 2023",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Abstract",
- "metadata": {
- "length": 3814,
- "summary": "The abstract introduces Toolformer, a model trained to decide which APIs to call and how to incorporate their results into future token prediction. It is trained in a self-supervised way using only a handful of demonstrations per API. The model incorporates tools such as a calculator, Q&A system, search engine, translation system, and calendar, achieving substantially improved zero-shot performance across various downstream tasks.",
- "page_nums": [
- 1
- ],
- "keywords": [
- "Toolformer",
- "Meta AI Research"
- ],
- "connect_to": [
- {
- "target": "asset_a57ea806-1513-572e-8417-f97247057849",
- "relation": "embeds",
- "ref": "[images/image_page_1_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 1,
- "artifact_ref": "page_citation_assets/page-1.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_5aa54c2a-d0de-5cf2-b37b-60ddab4bb0dc",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Abstract p1]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/1 Introduction",
- "metadata": {
- "length": 92,
- "summary": "The Introduction section outlines the limitations of current large language models, such as their inability to access up-to-date information, tendency to hallucinate facts, and lack of mathematical skills. It proposes 'Toolformer', a model that learns to use external tools like search engines and calculators in a self-supervised manner without requiring extensive human annotations.",
- "page_nums": [
- 1
- ],
- "keywords": [
- "Brown et al., 2020",
- "Chowdhery et al., 2022",
- "Wei et al., 2022",
- "Komeili et al., 2022",
- "Maynez et al., 2020",
- "Ji et al., 2022",
- "Lin et al., 2021",
- "Patel et al., 2021",
- "Dhingra et al., 2022"
- ],
- "connect_to": [
- {
- "target": "asset_a57ea806-1513-572e-8417-f97247057849",
- "relation": "related",
- "ref": "[images/image_page_1_figure_1.png]",
- "same_as_owner": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Abstract"
- }
- ],
- "page_assets": [
- {
- "page_num": 1,
- "artifact_ref": "page_citation_assets/page-1.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_03cdb0ef-34e3-5f2e-90d9-50452629d6e2",
- "type": "page",
- "content": "x1:i-1 = Pittsburgh is \n also known as\n xi:n = the Steel City\nx* = Pittsburgh is \n also known as\n [QA(What …? \n → Steel City)] \n the Steel City.\nci\n1 = What other name is \n Pittsburgh known by?\nci\n2 = Which country is\n Pittsburgh in?\nri\n1 = Steel City \nri\n2 = United States\nLi(ci\n1 → Steel City)\n < min(Li(ci\n1 → ε), Li(ε))\nLi(ci\n2 → United States)\n > min(Li(ci\n2 → ε), Li(ε))\n1 \nSample API Calls\n2 \nExecute API Calls\n3 \nFilter API Calls\nLM Dataset\nLM Dataset \nwith API Calls\nFigure 2: Key steps in our approach, illustrated for a question answering tool: Given an input text x, we first\nsample a position i and corresponding API call candidates c1\ni , c2\ni , . . . , ck\ni . We then execute these API calls and\nfilter out all calls which do not reduce the loss Li over the next tokens. All remaining API calls are interleaved\nwith the original text, resulting in a new text x∗.\ntant not only because of the costs associated\nwith such annotations, but also because what\nhumans find useful may be different from\nwhat a model finds useful.\n• The LM should not lose any of its generality\nand should be able to decide for itself when\nand how to use which tool. In contrast to\nexisting approaches, this enables a much more\ncomprehensive use of tools that is not tied to\nspecific tasks.\nOur approach for achieving these goals is based\non the recent idea of using large LMs with in-\ncontext learning (Brown et al., 2020) to generate\nentire datasets from scratch (Schick and Schütze,\n2021b; Honovich et al., 2022; Wang et al., 2022):\nGiven just a handful of human-written examples\nof how an API can be used, we let a LM annotate\na huge language modeling dataset with potential\nAPI calls. We then use a self-supervised loss to\ndetermine which of these API calls actually help\nthe model in predicting future tokens. Finally, we\nfinetune the LM itself on the API calls that it con-\nsiders useful. As illustrated in Figure 1, through\nthis simple approach, LMs can learn to control a va-\nriety of tools, and to choose for themselves which\ntool to use when and how.\nAs our approach is agnostic of the dataset be-\ning used, we can apply it to the exact same dataset\nthat was used to pretrain a model in the first place.\nThis ensures that the model does not lose any\nof its generality and language modeling abilities.\nWe conduct experiments on a variety of differ-\nent downstream tasks, demonstrating that after\nlearning to use tools, Toolformer, which is based\non a pretrained GPT-J model (Wang and Komat-\nsuzaki, 2021) with 6.7B parameters, achieves much\nstronger zero-shot results, clearly outperforming a\nmuch larger GPT-3 model (Brown et al., 2020) and\nseveral other baselines on various tasks.\n2\nApproach\nOur aim is to equip a language model M with the\nability to use different tools by means of API calls.\nWe require that inputs and outputs for each API\ncan be represented as text sequences. This allows\nseamless insertion of API calls into any given text,\nusing special tokens to mark the start and end of\neach such call.\nWe represent each API call as a tuple c = (ac, ic)\nwhere ac is the name of the API and ic is the cor-\nresponding input. Given an API call c with a cor-\nresponding result r, we denote the linearized se-\nquences of the API call not including and including\nits result, respectively, as:\ne(c) = ac(ic) \ne(c, r) = ac(ic) →r \nwhere “”, “” and “→” are special\ntokens.1 Some examples of linearized API calls\ninserted into text sequences are shown in Figure 1.\nGiven a dataset C = {x1, . . . , x|C|} of plain\ntexts, we first convert this dataset into a dataset\nC∗augmented with API calls. This is done in three\nsteps, illustrated in Figure 2: First, we exploit the\nin-context learning ability of M to sample a large\nnumber of potential API calls. We then execute\nthese API calls and finally check whether the ob-\ntained responses are helpful for predicting future\ntokens; this is used as a filtering criterion. After\nfiltering, we merge API calls for different tools,\nresulting in the augmented dataset C∗, and finetune\n1In practice, we use the token sequences “ [”, “]” and\n“->” to represent “”, “” and “→”, respec-\ntively. This enables our approach to work without modifying\nthe existing LM’s vocabulary. For reasons of readability, we\nstill refer to them as “”, “” and “→” through-\nout this section.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach",
- "metadata": {
- "length": 4408,
- "summary": "The page presents a method for equipping language models with tool-use capabilities via API calls. It details an approach using in-context learning to generate datasets of potential API calls, which are then filtered based on their utility for predicting future tokens. The text includes Figure 2, illustrating the three-step process: sampling API calls, executing them, and filtering them. It also references prior work such as Toolformer (Wang and Komatsuzaki, 2021) and GPT-3 (Brown et al., 2020).",
- "page_nums": [
- 2
- ],
- "keywords": [
- "Pittsburgh",
- "Steel City",
- "United States",
- "Toolformer",
- "GPT-J",
- "GPT-3"
- ],
- "connect_to": [
- {
- "target": "asset_09c69238-408d-5906-b7b8-3e14c8d5271e",
- "relation": "embeds",
- "ref": "[images/image_page_2_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 2,
- "artifact_ref": "page_citation_assets/page-2.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_cad20b8d-96e8-5fcb-a030-b672992ed358",
- "type": "page",
- "content": "Your task is to add calls to a Question Answering API to a \npiece of text. The questions should help you get \ninformation required to complete the text. You can call the \nAPI by writing \"[QA(question)]\" where \"question\" is the \nquestion you want to ask. Here are some examples of API \ncalls:\nInput: Joe Biden was born in Scranton, Pennsylvania.\nOutput: Joe Biden was born in [QA(\"Where was Joe \nBiden born?\")] Scranton, [QA(\"In which state is \nScranton?\")] Pennsylvania.\nInput: Coca-Cola, or Coke, is a carbonated soft drink \nmanufactured by the Coca-Cola Company.\nOutput: Coca-Cola, or [QA(\"What other name is \nCoca-Cola known by?\")] Coke, is a carbonated soft drink \nmanufactured by [QA(\"Who manufactures Coca-Cola?\")] \nthe Coca-Cola Company.\nInput: x\nOutput:\nFigure 3: An exemplary prompt P(x) used to generate\nAPI calls for the question answering tool.\nM itself on this dataset. Each of these steps is\ndescribed in more detail below.\nSampling API Calls\nFor each API, we write a\nprompt P(x) that encourages the LM to anno-\ntate an example x = x1, . . . , xn with API calls.\nAn example of such a prompt for a question an-\nswering tool is shown in Figure 3; all prompts\nused are shown in Appendix A.2. Let pM(zn+1 |\nz1, . . . , zn) be the probability that M assigns to\ntoken zn+1 as a continuation for the sequence\nz1, . . . , zn. We first sample up to k candidate posi-\ntions for doing API calls by computing, for each\ni ∈{1, . . . , n}, the probability\npi = pM( | P(x), x1:i−1)\nthat M assigns to starting an API call at position\ni. Given a sampling threshold τs, we keep all po-\nsitions I = {i | pi > τs}; if there are more than k\nsuch positions, we only keep the top k.\nFor each position i ∈I, we then obtain up to m\nAPI calls c1\ni , . . . , cm\ni by sampling from M given the\nsequence [P(x), x1, . . . , xi−1, ] as a prefix\nand as an end-of-sequence token.2\n2We discard all examples where M does not generate the\n token.\nExecuting API Calls\nAs a next step, we execute\nall API calls generated by M to obtain the corre-\nsponding results. How this is done depends entirely\non the API itself – for example, it can involve call-\ning another neural network, executing a Python\nscript or using a retrieval system to perform search\nover a large corpus. The response for each API call\nci needs to be a single text sequence ri.\nFiltering API Calls\nLet i be the position of the\nAPI call ci in the sequence x = x1, . . . , xn, and let\nri be the response from the API. Further, given a\nsequence (wi | i ∈N) of weights, let\nLi(z) = −\nn\nX\nj=i\nwj−i · log pM(xj | z, x1:j−1)\nbe the weighted cross entropy loss for M over the\ntokens xi, . . . , xn if the model is prefixed with z.\nWe compare two different instantiations of this loss:\nL+\ni = Li(e(ci, ri))\nL−\ni = min (Li(ε), Li(e(ci, ε)))\nwhere ε denotes an empty sequence. The former is\nthe weighted loss over all tokens xi, . . . , xn if the\nAPI call and its result are given to M as a prefix;3\nthe latter is the minimum of the losses obtained\nfrom (i) doing no API call at all and (ii) doing an\nAPI call, but not providing the response. Intuitively,\nan API call is helpful to M if providing it with both\nthe input and the output of this call makes it easier\nfor the model to predict future tokens, compared to\nnot receiving the API call at all, or receiving only\nits input. Given a filtering threshold τf, we thus\nonly keep API calls for which\nL−\ni −L+\ni ≥τf\nholds, i.e., adding the API call and its result reduces\nthe loss by at least τf, compared to not doing any\nAPI call or obtaining no result from it.\nModel Finetuning\nAfter sampling and filtering\ncalls for all APIs, we finally merge the remaining\nAPI calls and interleave them with the original\ninputs. That is, for an input text x = x1, . . . , xn\nwith a corresponding API call and result (ci, ri) at\nposition i, we construct the new sequence x∗=\n3We provide e(ci, ri) as a prefix instead of inserting it at\nposition i because M is not yet finetuned on any examples\ncontaining API calls, so inserting it in the middle of x would\ninterrupt the flow and not align with patterns in the pretraining\ncorpus, thus hurting perplexity.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Sampling API Calls",
- "metadata": {
- "length": 4137,
- "summary": "The section describes the process of sampling API calls for a Question Answering task. It involves writing a prompt P(x) to encourage a Language Model (LM) to annotate text with API calls. The method calculates the probability p_i of starting an API call at each position i in the sequence. A sampling threshold τ_s is used to filter positions, keeping those where p_i > τ_s. If more than k positions are kept, only the top k are selected. Finally, up to m API calls are sampled for each selected position using the model M.",
- "page_nums": [
- 3
- ],
- "keywords": [
- "Question Answering",
- "Figure 3",
- "Appendix A.2"
- ],
- "connect_to": [
- {
- "target": "asset_a22412e2-f71d-53e8-89f9-b4459f266dc6",
- "relation": "embeds",
- "ref": "[images/image_page_3_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_6d48e8f5-fbaf-5e25-b2f3-952fe17501f9",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Sampling API Calls p3]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Executing API Calls",
- "metadata": {
- "length": 113,
- "summary": "This section describes the process of executing API calls generated by a model M. It explains that execution methods vary depending on the specific API, such as calling another neural network, running a Python script, or using a retrieval system. The output for each call must be formatted as a single text sequence.",
- "page_nums": [
- 3
- ],
- "keywords": [
- "Python"
- ],
- "connect_to": [
- {
- "target": "asset_a22412e2-f71d-53e8-89f9-b4459f266dc6",
- "relation": "related",
- "ref": "[images/image_page_3_figure_1.png]",
- "same_as_owner": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Sampling API Calls"
- }
- ],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_0595d1d4-2c72-5647-a711-213191d95f3f",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Sampling API Calls p3]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Filtering API Calls",
- "metadata": {
- "length": 113,
- "summary": "The section 'Filtering API Calls' describes a method to determine whether an API call is helpful for the model. It defines a weighted cross-entropy loss function and compares two scenarios: one where the API call and its result are provided as a prefix, and another where no API call is made or only the input is provided. The text explains that an API call is considered helpful if providing it reduces the loss by at least a specific threshold (τf).",
- "page_nums": [
- 3
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_a22412e2-f71d-53e8-89f9-b4459f266dc6",
- "relation": "related",
- "ref": "[images/image_page_3_figure_1.png]",
- "same_as_owner": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Sampling API Calls"
- }
- ],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_835fdafc-5bde-5965-95c6-e5eb4fc9764c",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Sampling API Calls p3]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Model Finetuning",
- "metadata": {
- "length": 113,
- "summary": "The section describes the final step of the process, which is Model Finetuning. It explains that after sampling and filtering API calls for all APIs, the remaining calls are merged with the original text to create a new training sequence. This involves interleaving the API call and its result into the input text at the specific position where the call was made.",
- "page_nums": [
- 3
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_a22412e2-f71d-53e8-89f9-b4459f266dc6",
- "relation": "related",
- "ref": "[images/image_page_3_figure_1.png]",
- "same_as_owner": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Sampling API Calls"
- }
- ],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_95c2f8ca-16d6-5d09-a7b4-4fa580f9fb3b",
- "type": "page",
- "content": "x1:i−1, e(ci, ri), xi:n; we proceed analogously for\ntexts with multiple API calls. Doing this for all x ∈\nC results in the new dataset C∗augmented with API\ncalls. We use this new dataset to finetune M, using\na standard language modeling objective. Crucially,\napart from inserted API calls the augmented dataset\nC∗contains the exact same texts as C, the original\ndataset. As a consequence, finetuning M on C∗\nexposes it to the same content as finetuning on C.\nMoreover, as API calls are inserted in exactly those\npositions and with exactly those inputs that help\nM predict future tokens, finetuning on C∗enables\nthe language model to decide when and how to use\nwhich tool, based purely on its own feedback.\nInference\nWhen generating text with M after\nfinetuning with our approach, we perform regular\ndecoding until M produces the “→” token, indicat-\ning that it next expects the response for an API call.\nAt this point, we interrupt the decoding process,\ncall the appropriate API to get a response, and con-\ntinue the decoding process after inserting both the\nresponse and the token.\n3\nTools\nWe explore a variety of tools to address different\nshortcomings of regular LMs. The only constraints\nwe impose on these tools is that (i) both their inputs\nand outputs can be represented as text sequences,\nand (ii) we can obtain a few demonstrations of\ntheir intended use. Concretely, we explore the fol-\nlowing five tools: a question answering system, a\nWikipedia search engine, a calculator, a calendar,\nand a machine translation system. Some examples\nof potential calls and return strings for the APIs\nassociated with each of these tools are shown in\nTable 1. We briefly discuss all tools below; further\ndetails can be found in Appendix A.\nQuestion Answering\nOur first tool is a question\nanswering system based on another LM that can an-\nswer simple factoid questions. Specifically, we use\nAtlas (Izacard et al., 2022), a retrieval-augmented\nLM finetuned on Natural Questions (Kwiatkowski\net al., 2019).\nCalculator\nAs a second tool, we use a calculator\nthat can perform simple numeric calculations; we\nonly support the four basic arithmetic operations.\nResults are always rounded to two decimal places.\nWikipedia Search\nOur third tool is a search en-\ngine that, given a search term, returns short text\nsnippets from Wikipedia. Compared to our ques-\ntion answering tool, this search enables a model\nto get more comprehensive information on a sub-\nject, but requires it to extract the relevant parts by\nitself. As our search engine, we use a BM25 re-\ntriever (Robertson et al., 1995; Baeza-Yates et al.,\n1999) that indexes the Wikipedia dump from KILT\n(Petroni et al., 2021).\nMachine Translation System\nOur fourth tool is\na machine translation system based on a LM that\ncan translate a phrase from any language into En-\nglish. More concretely, we use the 600M parameter\nNLLB (Costa-jussà et al., 2022) as our multilingual\nmachine translation model that works for 200 lan-\nguages (including low-resource ones). The source\nlanguage is automatically detected using the fast-\nText classifier (Joulin et al., 2016), while the target\nlanguage is always set to English.\nCalendar\nOur final tool is a calendar API that,\nwhen queried, returns the current date without tak-\ning any input. This provides temporal context for\npredictions that require some awareness of time.\n4\nExperiments\nWe investigate whether our approach enables a\nmodel to use tools without any further supervision\nand to decide for itself when and how to call which\nof the available tools. To test this, we select a vari-\nety of downstream tasks where we assume at least\none of the considered tools to be useful, and evalu-\nate performance in zero-shot settings (Section 4.2).\nBeyond that, we also ensure that our approach does\nnot hurt the model’s core language modeling abili-\nties; we verify this by looking at perplexity on two\nlanguage modeling datasets (Section 4.3). Finally,\nwe investigate how the ability to learn using tools\nis affected by model size (Section 4.4).\n4.1\nExperimental Setup\nDataset Generation\nThroughout all of our ex-\nperiments, we use a subset of CCNet (Wenzek et al.,\n2020) as our language modeling dataset C and GPT-\nJ (Wang and Komatsuzaki, 2021) as our language\nmodel M. To reduce the computational cost of\nannotating C with API calls, we define heuristics\nfor some APIs to get a subset of C for which API\ncalls are more likely to be helpful than for an av-\nerage text. For example, we only consider texts\nfor the calculator tool if they contain at least three\nnumbers. Details of the heuristics used are given in",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/3 Tools",
- "metadata": {
- "length": 4589,
- "summary": "Section 3 describes five tools used to augment regular language models: a question answering system (Atlas), a calculator, a Wikipedia search engine (BM25), a machine translation system (NLLB), and a calendar API. The section details the specific implementations and constraints for each tool.",
- "page_nums": [
- 4
- ],
- "keywords": [
- "Atlas",
- "Natural Questions",
- "Kwiatkowski et al.",
- "Wikipedia",
- "BM25",
- "Robertson et al.",
- "Baeza-Yates et al.",
- "KILT",
- "Petroni et al.",
- "NLLB",
- "Costa-jussà et al.",
- "fastText",
- "Joulin et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 4,
- "artifact_ref": "page_citation_assets/page-4.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_551f3321-64b7-5799-b8a4-334923e5ac35",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/3 Tools p4]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/4 Experiments",
- "metadata": {
- "length": 91,
- "summary": "Section 4.1 Experimental Setup describes the dataset generation process for the experiments. It specifies that a subset of CCNet is used as the language modeling dataset (C) and GPT-J is used as the language model (M). To reduce annotation costs, heuristics are defined to select subsets of C where API calls are likely to be helpful.",
- "page_nums": [
- 4
- ],
- "keywords": [
- "CCNet",
- "GPT-J"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 4,
- "artifact_ref": "page_citation_assets/page-4.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c90a77cd-53f8-5291-8d94-14f3ca28142d",
- "type": "page",
- "content": "API Name\nExample Input\nExample Output\nQuestion Answering\nWhere was the Knights\nof Columbus founded?\nNew Haven, Connecticut\nWikipedia Search\nFishing Reel Types\nSpin fishing > Spin fishing is distinguished between fly fishing and bait\ncast fishing by the type of rod and reel used. There are two types of reels\nused when spin fishing, the open faced reel and the closed faced reel.\nCalculator\n27 + 4 * 2\n35\nCalendar\nε\nToday is Monday, January 30, 2023.\nMachine Translation\nsûreté nucléaire\nnuclear safety\nTable 1: Examples of inputs and outputs for all APIs used.\nNumber of Examples\nAPI\nτf = 0.5\nτf = 1.0\nτf = 2.0\nQuestion Answering\n51,987\n18,526\n5,135\nWikipedia Search\n207,241\n60,974\n13,944\nCalculator\n3,680\n994\n138\nCalendar\n61,811\n20,587\n3,007\nMachine Translation\n3,156\n1,034\n229\nTable 2: Number of examples with API calls in C∗for\ndifferent values of our filtering threshold τf.\nAppendix A. For obtaining C∗from C, we perform\nall steps described in Section 2 and additionally\nfilter out all examples for which all API calls were\neliminated in the filtering step.4 For the weighting\nfunction, we use\nwt =\n˜wt\nP\ns∈N ˜ws\nwith ˜wt = max(0, 1 −0.2 · t)\nto make sure that API calls happen close to where\nthe information provided by the API is actually\nhelpful for the model. The thresholds τs and τf are\nchosen individually for each tool to ensure a suffi-\nciently larger number of examples; see Appendix A\nfor details. Table 2 shows relevant statistics of our\nfinal dataset augmented with API calls.\nModel Finetuning\nWe finetune M on C∗using\na batch size of 128 and a learning rate of 1 · 10−5\nwith linear warmup for the first 10% of training.\nDetails of our finetuning procedure are given in\nAppendix B.\nBaseline Models\nThroughout the remainder of\nthis section, we mainly compare the following mod-\nels:\n4While this filtering alters the distribution of training exam-\nples, we assume that the remaining examples are close enough\nto the original distribution so that M’s language modeling\nabilities remain unaffected. This assumption is empirically\nvalidated in Section 4.3.\n• GPT-J: A regular GPT-J model without any\nfinetuning.\n• GPT-J + CC: GPT-J finetuned on C, our sub-\nset of CCNet without any API calls.\n• Toolformer: GPT-J finetuned on C∗, our sub-\nset of CCNet augmented with API calls.\n• Toolformer (disabled): The same model as\nToolformer, but API calls are disabled during\ndecoding.5\nFor most tasks, we additionally compare to OPT\n(66B) (Zhang et al., 2022) and GPT-36 (175B)\n(Brown et al., 2020), two models that are about\n10 and 25 times larger than our other baseline mod-\nels, respectively.\n4.2\nDownstream Tasks\nWe evaluate all models on a variety of downstream\ntasks. In all cases, we consider a prompted zero-\nshot setup – i.e., models are instructed to solve\neach task in natural language, but we do not pro-\nvide any in-context examples. This is in contrast\nto prior work on tool use (e.g., Gao et al., 2022;\nParisi et al., 2022), where models are provided\nwith dataset-specific examples of how a tool can be\nused to solve a concrete task. We choose the more\nchallenging zero-shot setup as we are interested\nin seeing whether Toolformer works in precisely\nthose cases where a user does not specify in ad-\nvance which tools should be used in which way for\nsolving a specific problem.\nWe use standard greedy decoding, but with one\nmodification for Toolformer: We let the model start\nan API call not just when is the most likely\n5This is achieved by manually setting the probability of\nthe token to 0.\n6We use the original davinci variant that is not finetuned\non any instructions.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/Baseline Models",
- "metadata": {
- "length": 3584,
- "summary": "The section describes the baseline models used for comparison, including GPT-J (unfinetuned), GPT-J + CC (finetuned on a subset of CCNet without API calls), Toolformer (finetuned on a subset of CCNet augmented with API calls), and Toolformer (disabled) where API calls are disabled during decoding. It also mentions comparisons to larger models OPT (66B) and GPT-3 (175B).",
- "page_nums": [
- 5
- ],
- "keywords": [
- "GPT-J",
- "CCNet",
- "Toolformer",
- "OPT",
- "Zhang et al.",
- "GPT-3",
- "Brown et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 5,
- "artifact_ref": "page_citation_assets/page-5.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_4fbdcb70-4761-5d00-b9cc-bdb023e8c1db",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/Baseline Models p5]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks",
- "metadata": {
- "length": 110,
- "summary": "Section 4.2 describes the evaluation setup for downstream tasks, focusing on a prompted zero-shot approach where models are instructed to solve tasks in natural language without provided examples. It contrasts this with prior work that uses dataset-specific examples and mentions comparisons against larger baseline models like OPT (66B) and GPT-3 (175B). The section also details the use of standard greedy decoding with a specific modification for Toolformer.",
- "page_nums": [
- 5
- ],
- "keywords": [
- "OPT",
- "GPT-3",
- "Toolformer"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 5,
- "artifact_ref": "page_citation_assets/page-5.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_6805b5b0-787b-55ce-b36d-e3164355d152",
- "type": "page",
- "content": "token, but whenever it is one of the k most likely\ntokens. For k = 1, this corresponds to regular\ngreedy decoding; we instead use k = 10 to in-\ncrease the disposition of our model to make use of\nthe APIs that it has access to. At the same time,\nwe only at most one API call per input to make\nsure the model does not get stuck in a loop where\nit constantly calls APIs without producing any ac-\ntual output. The effect of these modifications is\nexplored in Section 5.\n4.2.1\nLAMA\nWe evaluate our models on the SQuAD, Google-\nRE and T-REx subsets of the LAMA benchmark\n(Petroni et al., 2019). For each of these subsets, the\ntask is to complete a short statement with a miss-\ning fact (e.g., a date or a place). As LAMA was\noriginally designed to evaluate masked language\nmodels (e.g., Devlin et al., 2019), we filter out ex-\namples where the mask token is not the final token,\nso that the remaining examples can be processed\nin a left-to-right fashion. To account for different\ntokenizations and added complexity from not in-\nforming the model that a single word is required,\nwe use a slightly more lenient evaluation criterion\nthan exact match and simply check whether the\ncorrect word is within the first five words predicted\nby the model. As LAMA is based on statements\nobtained directly from Wikipedia, we prevent Tool-\nformer from using the Wikipedia Search API to\navoid giving it an unfair advantage.\nResults for all models can be seen in Table 3.\nAll GPT-J models without tool use achieve similar\nperformance. Crucially, Toolformer clearly outper-\nforms these baseline models, improving upon the\nbest baseline by 11.7, 5.2 and 18.6 points, respec-\ntively. It also clearly outperforms OPT (66B) and\nGPT-3 (175B), despite both models being much\nlarger. This is achieved because the model inde-\npendently decides to ask the question answering\ntool for the required information in almost all cases\n(98.1%); for only very few examples, it uses a dif-\nferent tool (0.7%) or no tool at all (1.2%).\n4.2.2\nMath Datasets\nWe test mathematical reasoning abilities on ASDiv\n(Miao et al., 2020), SVAMP (Patel et al., 2021) and\nthe MAWPS benchmark (Koncel-Kedziorski et al.,\n2016). We again account for the fact that we test\nall models in a zero-shot setup by using a more\nlenient evaluation criterion: As the required output\nis always a number, we simply check for the first\nModel\nSQuAD Google-RE T-REx\nGPT-J\n17.8\n4.9\n31.9\nGPT-J + CC\n19.2\n5.6\n33.2\nToolformer (disabled)\n22.1\n6.3\n34.9\nToolformer\n33.8\n11.5\n53.5\nOPT (66B)\n21.6\n2.9\n30.1\nGPT-3 (175B)\n26.8\n7.0\n39.8\nTable 3: Results on subsets of LAMA. Toolformer uses\nthe question answering tool for most examples, clearly\noutperforming all baselines of the same size and achiev-\ning results competitive with GPT-3 (175B).\nModel\nASDiv\nSVAMP\nMAWPS\nGPT-J\n7.5\n5.2\n9.9\nGPT-J + CC\n9.6\n5.0\n9.3\nToolformer (disabled)\n14.8\n6.3\n15.0\nToolformer\n40.4\n29.4\n44.0\nOPT (66B)\n6.0\n4.9\n7.9\nGPT-3 (175B)\n14.0\n10.0\n19.8\nTable 4:\nResults for various benchmarks requiring\nmathematical reasoning. Toolformer makes use of the\ncalculator tool for most examples, clearly outperform-\ning even OPT (66B) and GPT-3 (175B).\nnumber predicted by the model.7\nTable 4 shows results for all benchmarks. While\nGPT-J and GPT-J + CC perform about the same,\nToolformer achieves stronger results even when\nAPI calls are disabled. We surmise that this is be-\ncause the model is finetuned on many examples\nof API calls and their results, improving its own\nmathematical capabilities. Nonetheless, allowing\nthe model to make API calls more than doubles per-\nformance for all tasks, and also clearly outperforms\nthe much larger OPT and GPT-3 models. This is\nbecause across all benchmarks, for 97.9% of all\nexamples the model decides to ask the calculator\ntool for help.\n4.2.3\nQuestion Answering\nWe look at Web Questions (Berant et al., 2013),\nNatural Questions (Kwiatkowski et al., 2019) and\nTriviaQA (Joshi et al., 2017), the three question an-\nswering datasets considered by Brown et al. (2020).\nFor evaluation, we check whether the first 20 words\npredicted by a model contain the correct answer\ninstead of requiring an exact match. For Tool-\nformer, we disable the question answering tool as\n7An exception to this is if the model’s prediction contains\nan equation (e.g., “The correct answer is 5+3=8”), in which\ncase we consider the first number after the “=” sign to be its\nprediction.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.1 LAMA",
- "metadata": {
- "length": 4370,
- "summary": "Section 4.2.1 evaluates the Toolformer model on the LAMA benchmark, specifically using the SQuAD, Google-RE, and T-REx subsets to test fact completion tasks. The text explains that examples were filtered to ensure the mask token is the final token and that the Wikipedia Search API was disabled for Toolformer to prevent unfair advantages. Table 3 presents results showing that Toolformer significantly outperforms baseline models like GPT-J and larger models like OPT (66B) and GPT-3 (175B), achieving competitive scores with GPT-3 (175B). The section attributes this success to the model's ability to independently decide to use a question answering tool in almost all cases.",
- "page_nums": [
- 6
- ],
- "keywords": [
- "LAMA",
- "SQuAD",
- "Google-RE",
- "T-REx",
- "Petroni et al.",
- "Devlin et al.",
- "Toolformer",
- "Wikipedia",
- "Table 3",
- "GPT-J",
- "OPT",
- "GPT-3"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 6,
- "artifact_ref": "page_citation_assets/page-6.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_f37ea67a-2586-565f-9dd1-8f05e2f959cb",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.1 LAMA p6]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.2 Math Datasets",
- "metadata": {
- "length": 126,
- "summary": "Section 4.2.2 Math Datasets evaluates the mathematical reasoning abilities of models on three benchmarks: ASDiv, SVAMP, and MAWPS. The section describes a zero-shot evaluation setup using a lenient criterion where the model's prediction is checked for containing the first number after an equals sign. Table 4 presents results showing that Toolformer significantly outperforms baseline models (GPT-J, OPT, GPT-3) by leveraging a calculator tool in 97.9% of cases.",
- "page_nums": [
- 6
- ],
- "keywords": [
- "ASDiv",
- "SVAMP",
- "MAWPS"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 6,
- "artifact_ref": "page_citation_assets/page-6.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_0b92535b-5eb5-59d5-8c6c-533d701c83a0",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.1 LAMA p6]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.3 Question Answering",
- "metadata": {
- "length": 126,
- "summary": "Section 4.2.3 introduces the evaluation of question answering capabilities using three datasets: Web Questions, Natural Questions, and TriviaQA. The section specifies that for evaluation, the system checks if the correct answer is contained within the first 20 words predicted by the model rather than requiring an exact match. Additionally, it notes that for the Toolformer model, the question answering tool is disabled during this specific evaluation.",
- "page_nums": [
- 6
- ],
- "keywords": [
- "Web Questions",
- "Natural Questions",
- "TriviaQA"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 6,
- "artifact_ref": "page_citation_assets/page-6.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_d5dfa098-7682-5d55-bb6d-610b92060865",
- "type": "page",
- "content": "Model\nWebQS\nNQ\nTriviaQA\nGPT-J\n18.5\n12.8\n43.9\nGPT-J + CC\n18.4\n12.2\n45.6\nToolformer (disabled)\n18.9\n12.6\n46.7\nToolformer\n26.3\n17.7\n48.8\nOPT (66B)\n18.6\n11.4\n45.7\nGPT-3 (175B)\n29.0\n22.6\n65.9\nTable 5: Results for various question answering dataset.\nUsing the Wikipedia search tool for most examples,\nToolformer clearly outperforms baselines of the same\nsize, but falls short of GPT-3 (175B).\nthis would make solving the tasks trivial, especially\ngiven that the underlying QA system was finetuned\non Natural Questions.\nResults are shown in Table 5.\nOnce again,\nToolformer clearly outperforms all other models\nbased on GPT-J, this time mostly relying on the\nWikipedia search API (99.3%) to find relevant in-\nformation. However, Toolformer still lags behind\nthe much larger GPT-3 (175B) model. This is likely\ndue to both the simplicity of our search engine (in\nmany cases, it returns results that are clearly not\na good match for a given query) and the inability\nof Toolformer to interact with it, e.g., by refor-\nmulating its query if results are not helpful or by\nbrowsing through multiple of the top results. We\nbelieve that adding this functionality is an exciting\ndirection for future work.\n4.2.4\nMultilingual Question Answering\nWe evaluate Toolformer and all baseline models\non MLQA (Lewis et al., 2019), a multilingual\nquestion-answering benchmark. A context para-\ngraph for each question is provided in English,\nwhile the question can be in Arabic, German, Span-\nish, Hindi, Vietnamese, or Simplified Chinese. In\norder to solve the task, the model needs to be able\nto understand both the paragraph and the question,\nso it may benefit from translating the question into\nEnglish. Our evaluation metric is the percentage of\ntimes the model’s generation, capped at 10 words,\ncontains the correct answer.\nResults are shown in Table 6. Using API calls\nconsistently improves Toolformer’s performance\nfor all languages, suggesting that it has learned to\nmake use of the machine translation tool. Depend-\ning on the language, this tool is used for 63.8%\nto 94.9% of all examples; the only exception to\nthis is Hindi, for which the machine translation\ntool is used in only 7.3% of cases. However, Tool-\nModel\nEs\nDe\nHi\nVi\nZh\nAr\nGPT-J\n15.2 16.5\n1.3\n8.2 18.2\n8.2\nGPT-J + CC\n15.7 14.9\n0.5\n8.3 13.7\n4.6\nToolformer (disabled)\n19.8 11.9\n1.2 10.1 15.0\n3.1\nToolformer\n20.6 13.5\n1.4 10.6 16.8\n3.7\nOPT (66B)\n0.3\n0.1\n1.1\n0.2\n0.7\n0.1\nGPT-3 (175B)\n3.4\n1.1\n0.1\n1.7 17.7\n0.1\nGPT-J (All En)\n24.3 27.0 23.9 23.3 23.1 23.6\nGPT-3 (All En)\n24.7 27.2 26.1 24.9 23.6 24.0\nTable 6: Results on MLQA for Spanish (Es), German\n(De), Hindi (Hi), Vietnamese (Vi), Chinese (Zh) and\nArabic (Ar). While using the machine translation tool\nto translate questions is helpful across all languages,\nfurther pretraining on CCNet deteriorates performance;\nconsequently, Toolformer does not consistently outper-\nform GPT-J. The final two rows correspond to models\nthat are given contexts and questions in English.\nformer does not consistently outperform vanilla\nGPT-J. This is mainly because for some languages,\nfinetuning on CCNet deteriorates performance; this\nmight be due to a distribution shift compared to\nGPT-J’s original pretraining data.\nOPT and GPT-3 perform surprisingly weak\nacross all languages, mostly because they fail to\nprovide an answer in English despite being in-\nstructed to do so. A potential reason for GPT-J not\nsuffering from this problem is that it was trained on\nmore multilingual data than both OPT and GPT-3,\nincluding the EuroParl corpus (Koehn, 2005; Gao\net al., 2020). As an upper bound, we also evaluate\nGPT-J and GPT-3 on a variant of MLQA where\nboth the context and the question are provided in\nEnglish. In this setup, GPT-3 performs better than\nall other models, supporting our hypothesis that\nits subpar performance on MLQA is due to the\nmultilingual aspect of the task.\n4.2.5\nTemporal Datasets\nTo investigate the calendar API’s utility, we eval-\nuate all models on TEMPLAMA (Dhingra et al.,\n2022) and a new dataset that we call DATESET.\nTEMPLAMA is a dataset built from Wikidata that\ncontains cloze queries about facts that change with\ntime (e.g., “Cristiano Ronaldo plays for ___”)\nas well as the correct answer for the years be-\ntween 2010 and 2020. DATESET, described in\nAppendix D, is also generated through a series\nof templates, but populated using a combination\nof random dates/durations (e.g., “What day of the\nweek was it 30 days ago?”). Critically, knowing the\ncurrent date is required to answer these questions.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.4 Multilingual Question Answering",
- "metadata": {
- "length": 4503,
- "summary": "Section 4.2.4 evaluates Toolformer and baseline models on the MLQA multilingual question-answering benchmark, where context is in English but questions are in Arabic, German, Spanish, Hindi, Vietnamese, or Simplified Chinese. Results (Table 6) show that using a machine translation API consistently improves performance across languages, with Toolformer outperforming GPT-J baselines in most cases except for some languages where finetuning on CCNet causes deterioration. OPT and GPT-3 perform poorly due to lack of multilingual training data.",
- "page_nums": [
- 7
- ],
- "keywords": [
- "Toolformer",
- "MLQA",
- "Lewis et al.",
- "GPT-J",
- "CCNet",
- "OPT",
- "GPT-3",
- "EuroParl corpus",
- "Koehn",
- "Gao et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 7,
- "artifact_ref": "page_citation_assets/page-7.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_61abf750-1842-5bcd-8087-82409f8c5f8d",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.4 Multilingual Question Answering p7]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.5 Temporal Datasets",
- "metadata": {
- "length": 153,
- "summary": "Section 4.2.5 Temporal Datasets introduces the evaluation of models on temporal datasets to investigate the utility of a calendar API. It describes two datasets: TEMPLAMA, which contains cloze queries about facts that change over time (e.g., 'Cristiano Ronaldo plays for ___') based on Wikidata for years between 2010 and 2020; and DATESET, a new dataset generated using templates with random dates/durations (e.g., 'What day of the week was it 30 days ago?'). Both require knowledge of the current date to answer correctly.",
- "page_nums": [
- 7
- ],
- "keywords": [
- "TEMPLAMA",
- "DATESET",
- "Wikidata",
- "Cristiano Ronaldo"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 7,
- "artifact_ref": "page_citation_assets/page-7.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_9dd0f12b-a007-50d2-b5fe-eaa7602d2232",
- "type": "page",
- "content": "Model\nTEMPLAMA\nDATESET\nGPT-J\n13.7\n3.9\nGPT-J + CC\n12.9\n2.9\nToolformer (disabled)\n12.7\n5.9\nToolformer\n16.3\n27.3\nOPT (66B)\n14.5\n1.3\nGPT-3 (175B)\n15.5\n0.8\nTable 7: Results for the temporal datasets. Toolformer\noutperforms all baselines, but does not make use of the\ncalendar tool for TEMPLAMA.\nFor both tasks, we use the same evaluation as for\nthe original LAMA dataset.\nResults shown in Table 7 illustrate that Tool-\nformer outperforms all baselines for both TEM-\nPLAMA and DATESET. However, closer inspec-\ntion shows that improvements on TEMPLAMA\ncan not be attributed to the calendar tool, which is\nonly used for 0.2% of all examples, but mostly to\nthe Wikipedia search and question answering tools,\nwhich Toolformer calls the most. This makes sense\ngiven that named entities in TEMPLAMA are often\nso specific and rare that even knowing the exact\ndate alone would be of little help. The best course\nof action for this dataset – first querying the calen-\ndar API to get the current date, and then querying\nthe question answering system with this date – is\nnot only prohibited by our restriction of using at\nmost one API call per example, but also hard to\nlearn for Toolformer given that all API calls in its\ntraining data are sampled independently.\nFor DATESET, on the other hand, the consider-\nable improvement of Toolformer compared to other\nmodels can be fully accredited to the calendar tool,\nwhich it makes use of for 54.8% of all examples.\n4.3\nLanguage Modeling\nIn addition to verifying improved performance on\nvarious downstream tasks, we also want to ensure\nthat language modeling performance of Toolformer\ndoes not degrade through our finetuning with API\ncalls.\nTo this end, we evaluate our models on\ntwo language modeling datasets: WikiText (Mer-\nity et al., 2017) and a subset of 10,000 randomly\nselected documents from CCNet (Wenzek et al.,\n2020) that were not used during training. Perplex-\nities of various models are shown in Table 8. As\none would expect, finetuning on CCNet leads to\nslightly improved performance on a different CC-\nNet subset, but it slightly deteriorates performance\non WikiText, presumably because the original pre-\nModel\nWikiText\nCCNet\nGPT-J\n9.9\n10.6\nGPT-J + CC\n10.3\n10.5\nToolformer (disabled)\n10.3\n10.5\nTable 8: Perplexities of different models on WikiText\nand our validation subset of CCNet. Adding API calls\ncomes without a cost in terms of perplexity for lan-\nguage modeling without any API calls.\ntraining data for GPT-J is more similar to Wiki-\nText than our randomly selected subset of CCNet.\nMost importantly, however, training on C∗(our\ndataset annotated with API calls) does not lead to\nan increase in perplexity compared to training on\nC when API calls are disabled at inference time.8\n4.4\nScaling Laws\nWe investigate how the ability to ask external tools\nfor help affects performance as we vary the size\nof our LM. To this end, we apply our approach\nnot just to GPT-J, but also to four smaller mod-\nels from the GPT-2 family (Radford et al., 2019),\nwith 124M, 355M, 775M and 1.6B parameters, re-\nspectively. We do so using only a subset of three\ntools: the question answering system, the calcula-\ntor, and the Wikipedia search engine. Apart from\nthis, we follow the experimental setup described in\nSection 4.1.\nFigure 4 shows that the ability to leverage the\nprovided tools only emerges at around 775M pa-\nrameters: smaller models achieve similar perfor-\nmance both with and without tools. An exception\nto this is the Wikipedia search engine used mostly\nfor QA benchmarks; we hypothesize that this is\nbecause the API is comparably easy to use. While\nmodels become better at solving tasks without API\ncalls as they grow in size, their ability to make good\nuse of the provided API improves at the same time.\nAs a consequence, there remains a large gap be-\ntween predictions with and without API calls even\nfor our biggest model.\n5\nAnalysis\nDecoding Strategy\nWe investigate the effect of\nour modified decoding strategy introduced in Sec-\ntion 4.2, where instead of always generating the\n8We do not evaluate the perplexity of Toolformer with\nAPI calls enabled as computing the probability pM(xt |\nx1, . . . , xt−1) of token xt given x1, . . . , xt−1 would require\nmarginalizing over all potential API calls that the model could\nmake at position t, which is intractable.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.3 Language Modeling",
- "metadata": {
- "length": 4307,
- "summary": "Section 4.3 discusses the language modeling performance of the Toolformer model to ensure it does not degrade during finetuning with API calls. The authors evaluate their models on two datasets: WikiText and a subset of CCNet. Table 8 presents perplexity results for various models (GPT-J, GPT-J + CC, Toolformer) on these datasets, showing that adding API calls does not increase perplexity compared to training without them.",
- "page_nums": [
- 8
- ],
- "keywords": [
- "Toolformer",
- "WikiText",
- "CCNet",
- "GPT-J"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c306b7a4-2d69-55f1-b82a-cc94001133a6",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.3 Language Modeling p8]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.4 Scaling Laws",
- "metadata": {
- "length": 116,
- "summary": "Section 4.4 investigates how the ability to use external tools affects performance as the size of the language model (LM) increases. The authors apply their approach to four smaller models from the GPT-2 family (124M, 355M, 775M, and 1.6B parameters) alongside GPT-J, using a subset of three tools: question answering, calculator, and Wikipedia search. The section reports that leveraging provided tools only emerges at around 775M parameters; smaller models perform similarly with or without tools. It is hypothesized that the Wikipedia search API is an exception because it is easy to use. As models grow larger, they improve at solving tasks both with and without API calls, but a large gap remains between predictions made with and without API calls even for the largest model.",
- "page_nums": [
- 8
- ],
- "keywords": [
- "GPT-J",
- "GPT-2",
- "Radford et al., 2019"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_44383b2c-3bae-57e3-bdcd-70e5d9ae3c94",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.3 Language Modeling p8]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/5 Analysis",
- "metadata": {
- "length": 116,
- "summary": "The section titled '5 Analysis' begins by introducing a subsection on 'Decoding Strategy', which investigates the effects of a modified decoding strategy previously introduced in Section 4.2. The text explains that this strategy involves generating tokens differently than always generating the most probable one, though the full explanation is cut off at the bottom of the page.",
- "page_nums": [
- 8
- ],
- "keywords": [
- "Section 4.2"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_8a1aed2b-ebf6-5792-8260-97641c6dd2f4",
- "type": "page",
- "content": "0\n5\n10\n15\n20\n25\n30\n0\n2000\n4000\n6000\nModel Parameters (M)\nLAMA\n Toolformer\n Toolformer (disabled)\n GPT3\n0\n5\n10\n15\n20\n25\n30\n35\n40\n0\n2000\n4000\n6000\nModel Parameters (M)\nQA Benchmarks\n0\n5\n10\n15\n20\n25\n30\n0\n2000\n4000\n6000\nModel Parameters (M)\nMath Benchmarks\nFigure 4: Average performance on LAMA, our math benchmarks and our QA benchmarks for GPT-2 models of\ndifferent sizes and GPT-J finetuned with our approach, both with and without API calls. While API calls are not\nhelpful to the smallest models, larger models learn how to make good use of them. Even for bigger models, the\ngap between model predictions with and without API calls remains high.\nmost likely token, we generate the token\nif it is one of the k most likely tokens. Table 9\nshows performance on the T-REx subset of LAMA\nand on WebQS for different values of k. As ex-\npected, increasing k leads to the model doing API\ncalls for more examples – from 40.3% and 8.5%\nwith k = 1 (i.e., regular greedy decoding) to 98.1%\nand 100% for k = 10. While for T-REx, there is\nalready a clear improvement in performance with\ngreedy decoding, on WebQS our model only starts\nto make a substantial number of API calls as we\nslightly increase k. Interestingly, for k = 1 the\nmodel is calibrated to some extent: It decides to\ncall APIs for examples that it would perform partic-\nularly badly on without making API calls. This can\nbe seen from the fact that performance on examples\nwhere it decides not to make an API call (44.3 and\n19.9) is higher than average performance if no API\ncalls are made at all (34.9 and 18.9). However, this\ncalibration is lost for higher values of k.\nData Quality\nWe qualitatively analyze some\nAPI calls generated with our approach for different\nAPIs. Table 10 shows some examples of texts from\nCCNet augmented with API calls, as well as the\ncorresponding score L−\ni −L+\ni that is used as a fil-\ntering criterion, and whether the API calls made by\nthe model are intuitively useful in the given context.\nAs can be seen, high values of L−\ni −L+\ni typically\ncorrespond to useful API calls, whereas low values\ncorrespond to API calls that do not provide any in-\nformation that is useful for predicting future tokens.\nThere are some exceptions, e.g., an API call for\nT-REx\nWebQS\nk\nAll\nAC\nNC\n%\nAll\nAC\nNC\n%\n0\n34.9\n–\n34.9\n0.0\n18.9\n–\n18.9\n0.0\n1\n47.8 53.0 44.3 40.3\n19.3 17.1 19.9\n8.5\n3\n52.9 58.0 29.0 82.8\n26.3 26.5\n6.6\n99.3\n10\n53.5 54.0 22.5 98.1\n26.3 26.4\n–\n100.0\nTable 9: Toolformer results on the T-REx subset of\nLAMA and on WebQS for different values of k used\nduring decoding. Numbers shown are overall perfor-\nmance (All), performance on the subset where the\nmodel decides to make an API call (AC) and all re-\nmaining examples (NC), as well as the percentage of\nexamples for which the model decides to call an API\n(%).\n“Fast train success” in the fourth example that does\nnot give any relevant information but still reduces\nperplexity. However, some amount of noise in the\nAPI calls that are not filtered can actually be useful\nas it forces the model finetuned on C∗to not always\nblindly follow the results of each call it makes.\n6\nRelated Work\nLanguage Model Pretraining\nThere are various\napproaches that augment language models with\nsome form of additional textual information during\npretraining, including various forms of metadata\n(Keskar et al., 2019), HTML tags (Aghajanyan\net al., 2021), Wikipedia markup (Schick et al.,\n2022), or related texts obtained from an informa-\ntion retrieval system (Guu et al., 2020; Borgeaud\net al., 2021; Izacard et al., 2022). For all of these",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/6 Related Work",
- "metadata": {
- "length": 3554,
- "summary": "This page presents experimental results and analysis regarding a model called Toolformer. Figure 4 displays line charts comparing the performance of Toolformer, Toolformer with disabled API calls, and GPT3 across LAMA, Math Benchmarks, and QA Benchmarks as model parameters increase. Table 9 details Toolformer's performance on T-REx and WebQS datasets for different values of k (the number of most likely tokens considered), showing metrics for overall performance, API call success, and non-call accuracy. The text discusses how increasing k leads to more API calls and improved performance, particularly on WebQS. A section titled 'Data Quality' qualitatively analyzes API calls generated by the approach, referencing Table 10 (not fully visible) and discussing filtering criteria based on score differences. The page concludes with the beginning of Section 6, 'Related Work,' which introduces approaches to augment language models with textual information during pretraining.",
- "page_nums": [
- 9
- ],
- "keywords": [
- "Toolformer",
- "GPT3",
- "LAMA",
- "T-REx",
- "WebQS",
- "CCNet",
- "Keskar et al., 2019",
- "Aghajanyan et al., 2021",
- "Schick et al., 2022",
- "Guu et al., 2020",
- "Borgeaud et al., 2021",
- "Izacard et al., 2022"
- ],
- "connect_to": [
- {
- "target": "asset_194baf47-eacf-50fd-bc26-49bd50f75db0",
- "relation": "embeds",
- "ref": "[images/image_page_9_figure_1.png]"
- },
- {
- "target": "asset_68be5006-73df-5a58-8636-707711d572ac",
- "relation": "embeds",
- "ref": "[tables/table_page_9_2.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_6e64b52f-6225-57e1-a8e0-e22d6082953a",
- "type": "page",
- "content": "Example\nL−\ni −L+\ni\nUseful\nThe Flodden Window (a war memorial dedicated to The Middleton Archers), in the Grade I-listed\nChurch of St Leonard in Middleton is said to be the oldest war memorial in the United King-\ndom. WikiSearch(War memorial Flodden) →Battle of Flodden > Commemoration >\nThe stained-glass Flodden Window in Middleton Parish Church [. . . ] was constructed by Sir\nRichard Assheton in memory of the Battle of Flodden and the archers from Middleton who\nfought in it. Sir Richard Assheton of Middleton (who built St Leonard) was granted\nknighthood [. ..]\n5.49\n\u0013\nNote: The WL will be open on Friday, Calendar() →Today is Thursday, March 9, 2017.\n March 10, and Sunday, March 19 for regular hours.\n2.11\n\u0013\nThe Nile has an approximate length of QA(What is the approximate length of the Nile?)\n→6,853 km 6,853 kilometers, the White Nile being its main source.\n2.08\n\u0013\nIf Venus had an atmosphere similar to Earth’s then you would expect Venus’ mean temperature to be\n499 K (1.74 x 287) rather than 735 K which is Calculator(735 / 499) →1.47 1.47\n(735 / 499) times hotter than it should be.\n1.59\n\u0013\nYou are here: Home / Featured / Catch this fast train to success! WikiSearch(Fast train\nsuccess) →Fast Train > It also peaked at #23 on the Canadian CHUM singles chart, on June\n26, 1971. The success of this single established Myles Goodwyn as the band’s main songwriter,\nand made it possible for April Wine to record a second album. Don’t wait weeks and\n[. . .]\n0.92\n\u0017\nOs Melhores Escolas em Jersey 2020 MT(Os Melhores Escolas em Jersey) →The Best\nSchools in Jersey On this page you can search for Universities, Colleges and Business\nschools in Jersey\n0.70\n\u0013\nEnjoy these pictures from the Calendar() →Today is Friday, April 19, 2013. \nEaster Egg Hunt.\n0.33\n\u0013\n85 patients (23%) were hospitalised alive and admitted to a hospital ward. Of them, Calcula-\ntor(85 / 23) →3.70 65% had a cardiac aetiology [. . .]\n−0.02\n\u0017\nBut hey, after the Calendar() →Today is Saturday, June 25, 2011. Disneyland\nfiasco with the fire drill, I think it’s safe to say Chewey won’t let anyone die in a fire.\n−0.41\n\u0017\nThe last time I was with QA(Who was last time I was with?) →The Last Time \nhim I asked what he likes about me and he said he would tell me one day.\n−1.23\n\u0017\nTable 10: Examples of API calls for different tools, sorted by the value of L−\ni −L+\ni that is used as a filtering\ncriterion. High values typically correspond to API calls that are intuitively useful for predicting future tokens.\napproaches, additional information is always pro-\nvided, regardless of whether it is helpful or not. In\ncontrast, Toolformer learns for itself to explicitly\nasks for the right information.\nTool Use\nSeveral approaches aim to equip LMs\nwith the ability to use external tools such as search\nengines (Komeili et al., 2022; Thoppilan et al.,\n2022; Lazaridou et al., 2022; Shuster et al., 2022;\nYao et al., 2022), web browsers (Nakano et al.,\n2021), calculators (Cobbe et al., 2021; Thoppilan\net al., 2022), translation systems (Thoppilan et al.,\n2022) and Python interpreters (Gao et al., 2022).\nThe way these models learn to use tools can roughly\nbe divided into two approaches: Either they rely on\nlarge amounts of human supervision (Komeili et al.,\n2022; Nakano et al., 2021; Thoppilan et al., 2022)\nor they work by prompting the language model in\na few-shot setup tailored towards a specific task\nwhere it is known a priori which tools needs to be\nused (Gao et al., 2022; Lazaridou et al., 2022; Yao\net al., 2022). In contrast, the self-supervised nature\nof Toolformer enables it to learn how and when to\nuse tools without requiring a specific prompt that\nshows task-specific examples of how a tool could\nbe used. Perhaps most closely related to our work\nis TALM (Parisi et al., 2022), an approach that\nuses a similar self-supervised objective for teach-\ning a model to use a calculator and a search engine,\nbut explores this only in settings where a model is\nfinetuned for downstream tasks.\nBootstrapping\nThe idea of using self-training\nand bootstrapping techniques to improve models\nhas been investigated in various contexts, rang-\ning from word sense disambiguation (Yarowsky,\n1995), relation extraction (Brin, 1999; Agichtein\nand Gravano, 2000), parsing (McClosky et al.,\n2006; Reichart and Rappoport, 2007), sequence\ngeneration (He et al., 2020), few-shot text classi-",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/6 Related Work/Tool Use",
- "metadata": {
- "length": 4470,
- "summary": "The section discusses approaches to equip language models (LMs) with the ability to use external tools such as search engines, web browsers, calculators, translation systems, and Python interpreters. It categorizes learning methods into those relying on large amounts of human supervision versus few-shot prompting setups. It also mentions TALM as a related self-supervised approach.",
- "page_nums": [
- 10
- ],
- "keywords": [
- "Komeili et al.",
- "Thoppilan et al.",
- "Lazaridou et al.",
- "Shuster et al.",
- "Yao et al.",
- "Nakano et al.",
- "Cobbe et al.",
- "Gao et al.",
- "TALM",
- "Parisi et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_a27b905b-3ee2-57f4-a2d5-519f6f10b6a8",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/6 Related Work/Tool Use p10]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/6 Related Work/Bootstrapping",
- "metadata": {
- "length": 108,
- "summary": "The section titled 'Bootstrapping' discusses the concept of using self-training and bootstrapping techniques to improve models. It lists various contexts where these methods have been investigated, including word sense disambiguation, relation extraction, parsing, sequence generation, and few-shot text classification.",
- "page_nums": [
- 10
- ],
- "keywords": [
- "TALM",
- "Parisi et al.",
- "Yarowsky",
- "Brin",
- "Agichtein",
- "Gravano",
- "McClosky et al.",
- "Reichart",
- "Rappoport",
- "He et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_0d2ddbdc-7ca2-5bec-8026-909f35d25101",
- "type": "page",
- "content": "fication (Schick and Schütze, 2021a) and retrieval\n(Izacard and Grave, 2021) to reasoning (Zelikman\net al., 2022). In a similar spirit to these approaches,\nToolformer is trained on its own predictions after\napplying a perplexity-based filtering step.\n7\nLimitations\nWhile our approach enables LMs to learn how to\nuse a variety of tools in a self-supervised way, there\nare some clear limitations to what can be achieved\nwith our method in its current form. One such limi-\ntation is the inability of Toolformer to use tools in a\nchain (i.e., using the output of one tool as an input\nfor another tool). This is due to the fact that API\ncalls for each tool are generated independently; as a\nconsequence, there are no examples of chained tool\nuse in the finetuning dataset. Our current approach\nalso does not allow the LM to use a tool in an in-\nteractive way – especially for tools such as search\nengines, that could potentially return hundreds of\ndifferent results, enabling a LM to browse through\nthese results or to refine its search query in a simi-\nlar spirit to Nakano et al. (2021) can be crucial for\ncertain applications. Beyond this, we found models\ntrained with Toolformer to often be sensitive to the\nexact wording of their input when deciding whether\nor not to call an API; this is perhaps unsurprising\ngiven that LMs are known to be very sensitive to\nthe prompt they are provided with in both zero-\nand few-shot settings (Jiang et al., 2020; Schick\nand Schütze, 2021a). Depending on the tool, our\nmethod is also very sample-inefficient; for example,\nprocessing more than a million documents results\nin only a few thousand examples of useful calls\nto the calculator API. A potential solution to this\nproblem might be to iteratively apply our approach,\nsimilar to how this is done in related bootstrapping\napproaches (Schick and Schütze, 2021a; Izacard\nand Grave, 2021; Parisi et al., 2022). Finally, when\ndeciding whether or not to make an API call, Tool-\nformer currently does not take into account the\ntool-dependent, computational cost incurred from\nmaking an API call.\n8\nConclusion\nWe have introduced Toolformer, a language model\nthat learns in a self-supervised way how to use\ndifferent tools such as search engines, calculators,\nand translation systems via simple API calls. This\nis done by finetuning on a large number of sampled\nAPI calls that are filtered based on whether they\nreduce perplexity on future tokens. Toolformer\nconsiderably improves zero-shot performance of a\n6.7B parameter GPT-J model, enabling it to even\noutperform a much larger GPT-3 model on a range\nof different downstream tasks.\nReferences\nArmen Aghajanyan, Dmytro Okhonko, Mike Lewis,\nMandar Joshi, Hu Xu, Gargi Ghosh, and Luke Zettle-\nmoyer. 2021.\nHtlm: Hyper-text pre-training and\nprompting of language models.\nEugene Agichtein and Luis Gravano. 2000. Snowball:\nExtracting relations from large plain-text collections.\nIn Proceedings of the Fifth ACM Conference on Dig-\nital Libraries, DL ’00, page 85–94, New York, NY,\nUSA. Association for Computing Machinery.\nRicardo Baeza-Yates, Berthier Ribeiro-Neto, et al.\n1999.\nModern information retrieval, volume 463.\nACM press New York.\nJonathan Berant, Andrew Chou, Roy Frostig, and Percy\nLiang. 2013.\nSemantic parsing on Freebase from\nquestion-answer pairs. In Proceedings of the 2013\nConference on Empirical Methods in Natural Lan-\nguage Processing, pages 1533–1544, Seattle, Wash-\nington, USA. Association for Computational Lin-\nguistics.\nSebastian Borgeaud, Arthur Mensch, Jordan Hoff-\nmann, Trevor Cai, Eliza Rutherford, Katie Millican,\nGeorge van den Driessche, Jean-Baptiste Lespiau,\nBogdan Damoc, Aidan Clark, Diego de Las Casas,\nAurelia Guy, Jacob Menick, Roman Ring, Tom Hen-\nnigan, Saffron Huang, Loren Maggiore, Chris Jones,\nAlbin Cassirer, Andy Brock, Michela Paganini, Ge-\noffrey Irving, Oriol Vinyals, Simon Osindero, Karen\nSimonyan, Jack W. Rae, Erich Elsen, and Laurent\nSifre. 2021. Improving language models by retriev-\ning from trillions of tokens.\nSergey Brin. 1999. Extracting patterns and relations\nfrom the world wide web. In The World Wide Web\nand Databases, pages 172–183, Berlin, Heidelberg.\nSpringer Berlin Heidelberg.\nTom Brown, Benjamin Mann, Nick Ryder, Melanie\nSubbiah,\nJared\nD\nKaplan,\nPrafulla\nDhariwal,\nArvind Neelakantan, Pranav Shyam, Girish Sastry,\nAmanda Askell, Sandhini Agarwal, Ariel Herbert-\nVoss, Gretchen Krueger, Tom Henighan, Rewon\nChild, Aditya Ramesh, Daniel Ziegler, Jeffrey Wu,\nClemens Winter, Chris Hesse, Mark Chen, Eric\nSigler, Mateusz Litwin, Scott Gray, Benjamin Chess,\nJack Clark, Christopher Berner, Sam McCandlish,\nAlec Radford, Ilya Sutskever, and Dario Amodei.\n2020. Language models are few-shot learners. In\nAdvances in Neural Information Processing Systems,\nvolume 33, pages 1877–1901. Curran Associates,\nInc.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/7 Limitations",
- "metadata": {
- "length": 4798,
- "summary": "The section outlines several limitations of the Toolformer approach. Key issues include the inability to use tools in a chain (using one tool's output as another's input) because API calls are generated independently, and the lack of support for interactive tool usage like search engines. Additionally, models are sensitive to prompt wording, the method is sample-inefficient (requiring large datasets for few examples), and it does not account for the computational cost of making API calls.",
- "page_nums": [
- 11
- ],
- "keywords": [
- "Toolformer",
- "Nakano et al.",
- "Jiang et al.",
- "Schick and Schütze",
- "Parisi et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_a5c75288-8fdc-5907-a144-abbf32b51151",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/7 Limitations p11]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/8 Conclusion",
- "metadata": {
- "length": 98,
- "summary": "The section introduces Toolformer, a language model designed to learn self-supervised usage of various tools like search engines and calculators through simple API calls. The model is finetuned on sampled API calls filtered by their ability to reduce perplexity on future tokens. It is noted that Toolformer significantly improves zero-shot performance compared to larger models.",
- "page_nums": [
- 11
- ],
- "keywords": [
- "Toolformer",
- "GPT-J",
- "GPT-3"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c77f33d1-378a-5579-937b-adf76ae0841d",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/7 Limitations p11]\n\nAakanksha Chowdhery, Sharan Narang, Jacob Devlin,\nMaarten Bosma, Gaurav Mishra, Adam Roberts,\nPaul Barham, Hyung Won Chung, Charles Sutton,\nSebastian Gehrmann, Parker Schuh, Kensen Shi,\nSasha Tsvyashchenko, Joshua Maynez, Abhishek\nRao, Parker Barnes, Yi Tay, Noam Shazeer, Vin-\nodkumar Prabhakaran, Emily Reif, Nan Du, Ben\nHutchinson, Reiner Pope, James Bradbury, Jacob\nAustin, Michael Isard, Guy Gur-Ari, Pengcheng\nYin, Toju Duke, Anselm Levskaya, Sanjay Ghe-\nmawat, Sunipa Dev, Henryk Michalewski, Xavier\nGarcia, Vedant Misra, Kevin Robinson, Liam Fe-\ndus, Denny Zhou, Daphne Ippolito, David Luan,\nHyeontaek Lim, Barret Zoph, Alexander Spiridonov,\nRyan Sepassi, David Dohan, Shivani Agrawal, Mark\nOmernick, Andrew M. Dai, Thanumalayan Sankara-\nnarayana Pillai, Marie Pellat, Aitor Lewkowycz,\nErica Moreira, Rewon Child, Oleksandr Polozov,\nKatherine Lee, Zongwei Zhou, Xuezhi Wang, Bren-\nnan Saeta, Mark Diaz, Orhan Firat, Michele Catasta,\nJason Wei, Kathy Meier-Hellstern, Douglas Eck,\nJeff Dean, Slav Petrov, and Noah Fiedel. 2022.\nPalm: Scaling language modeling with pathways.\nKarl Cobbe, Vineet Kosaraju, Mohammad Bavarian,\nMark Chen, Heewoo Jun, Lukasz Kaiser, Matthias\nPlappert, Jerry Tworek, Jacob Hilton, Reiichiro\nNakano, et al. 2021. Training verifiers to solve math\nword problems. arXiv preprint arXiv:2110.14168.\nMarta R Costa-jussà, James Cross, Onur Çelebi, Maha\nElbayad, Kenneth Heafield, Kevin Heffernan, Elahe\nKalbassi, Janice Lam, Daniel Licht, Jean Maillard,\net al. 2022.\nNo language left behind:\nScaling\nhuman-centered machine translation. arXiv preprint\narXiv:2207.04672.\nJacob Devlin, Ming-Wei Chang, Kenton Lee, and\nKristina Toutanova. 2019.\nBERT: Pre-training of\ndeep bidirectional transformers for language under-\nstanding.\nIn Proceedings of the 2019 Conference\nof the North American Chapter of the Association\nfor Computational Linguistics: Human Language\nTechnologies, Volume 1 (Long and Short Papers),\npages 4171–4186, Minneapolis, Minnesota. Associ-\nation for Computational Linguistics.\nBhuwan Dhingra, Jeremy R. Cole, Julian Martin\nEisenschlos, Daniel Gillick, Jacob Eisenstein, and\nWilliam W. Cohen. 2022.\nTime-aware language\nmodels as temporal knowledge bases. Transactions\nof the Association for Computational Linguistics,\n10:257–273.\nLeo Gao, Stella Biderman, Sid Black, Laurence Gold-\ning, Travis Hoppe, Charles Foster, Jason Phang, Ho-\nrace He, Anish Thite, Noa Nabeshima, et al. 2020.\nThe pile: An 800gb dataset of diverse text for lan-\nguage modeling. arXiv preprint arXiv:2101.00027.\nLuyu Gao, Aman Madaan, Shuyan Zhou, Uri Alon,\nPengfei Liu, Yiming Yang, Jamie Callan, and Gra-\nham Neubig. 2022.\nPal: Program-aided language\nmodels.\nKelvin Guu, Kenton Lee, Zora Tung, Panupong Pasu-\npat, and Ming-Wei Chang. 2020. Realm: Retrieval-\naugmented language model pre-training.\nJunxian He, Jiatao Gu, Jiajun Shen, and Marc’Aurelio\nRanzato. 2020.\nRevisiting self-training for neural\nsequence generation.\nIn International Conference\non Learning Representations.\nOr Honovich, Thomas Scialom, Omer Levy, and Timo\nSchick. 2022. Unnatural instructions: Tuning lan-\nguage models with (almost) no human labor.\nGautier Izacard and Edouard Grave. 2021. Distilling\nknowledge from reader to retriever for question an-\nswering. In International Conference on Learning\nRepresentations.\nGautier Izacard, Patrick Lewis, Maria Lomeli, Lucas\nHosseini, Fabio Petroni, Timo Schick, Jane Dwivedi-\nYu, Armand Joulin, Sebastian Riedel, and Edouard\nGrave. 2022. Atlas: Few-shot learning with retrieval\naugmented language models.\nZiwei Ji, Nayeon Lee, Rita Frieske, Tiezheng Yu,\nDan Su, Yan Xu, Etsuko Ishii, Yejin Bang, Andrea\nMadotto, and Pascale Fung. 2022. Survey of hallu-\ncination in natural language generation. ACM Com-\nputing Surveys.\nZhengbao Jiang, Frank F. Xu, Jun Araki, and Graham\nNeubig. 2020.\nHow can we know what language\nmodels know? Transactions of the Association for\nComputational Linguistics, 8:423–438.\nMandar Joshi, Eunsol Choi, Daniel Weld, and Luke\nZettlemoyer. 2017.\nTriviaQA: A large scale dis-\ntantly supervised challenge dataset for reading com-\nprehension. In Proceedings of the 55th Annual Meet-\ning of the Association for Computational Linguistics\n(Volume 1: Long Papers), pages 1601–1611, Van-\ncouver, Canada. Association for Computational Lin-\nguistics.\nArmand Joulin, Edouard Grave, Piotr Bojanowski,\nMatthijs Douze, Hérve Jégou, and Tomas Mikolov.\n2016. Fasttext. zip: Compressing text classification\nmodels. arXiv preprint arXiv:1612.03651.\nNitish Shirish Keskar, Bryan McCann, Lav R. Varsh-\nney, Caiming Xiong, and Richard Socher. 2019.\nCtrl: A conditional transformer language model for\ncontrollable generation.\nPhilipp Koehn. 2005. Europarl: A parallel corpus for\nstatistical machine translation.\nIn Proceedings of\nmachine translation summit x: papers, pages 79–86.\nMojtaba Komeili, Kurt Shuster, and Jason Weston.\n2022. Internet-augmented dialogue generation. In\nProceedings of the 60th Annual Meeting of the As-\nsociation for Computational Linguistics (Volume 1:\nLong Papers), pages 8460–8478, Dublin, Ireland.\nAssociation for Computational Linguistics.\n\nRik Koncel-Kedziorski, Subhro Roy, Aida Amini,\nNate Kushman, and Hannaneh Hajishirzi. 2016.\nMAWPS: A math word problem repository. In Pro-\nceedings of the 2016 Conference of the North Amer-\nican Chapter of the Association for Computational\nLinguistics: Human Language Technologies, pages\n1152–1157, San Diego, California. Association for\nComputational Linguistics.\nTom Kwiatkowski, Jennimaria Palomaki, Olivia Red-\nfield, Michael Collins, Ankur Parikh, Chris Al-\nberti, Danielle Epstein, Illia Polosukhin, Jacob De-\nvlin, Kenton Lee, Kristina Toutanova, Llion Jones,\nMatthew Kelcey, Ming-Wei Chang, Andrew M. Dai,\nJakob Uszkoreit, Quoc Le, and Slav Petrov. 2019.\nNatural questions: A benchmark for question an-\nswering research. Transactions of the Association\nfor Computational Linguistics, 7:452–466.\nAngeliki Lazaridou, Elena Gribovskaya, Wojciech\nStokowiec, and Nikolai Grigorev. 2022.\nInternet-\naugmented\nlanguage\nmodels\nthrough\nfew-shot\nprompting for open-domain question answering.\narXiv preprint arXiv:2203.05115.\nPatrick Lewis, Barlas O˘guz, Ruty Rinott, Sebastian\nRiedel, and Holger Schwenk. 2019.\nMlqa: Eval-\nuating cross-lingual extractive question answering.\narXiv preprint arXiv:1910.07475.\nXi Victoria Lin, Todor Mihaylov, Mikel Artetxe, Tianlu\nWang, Shuohui Chen, Daniel Simig, Myle Ott, Na-\nman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth\nPasunuru, Sam Shleifer, Punit Singh Koura, Vishrav\nChaudhary, Brian O’Horo, Jeff Wang, Luke Zettle-\nmoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoy-\nanov, and Xian Li. 2021.\nFew-shot learning with\nmultilingual language models.\nJoshua Maynez, Shashi Narayan, Bernd Bohnet, and\nRyan McDonald. 2020. On faithfulness and factual-\nity in abstractive summarization.\nDavid McClosky, Eugene Charniak, and Mark Johnson.\n2006.\nEffective self-training for parsing.\nIn Pro-\nceedings of the Human Language Technology Con-\nference of the NAACL, Main Conference, pages 152–\n159, New York City, USA. Association for Compu-\ntational Linguistics.\nStephen Merity, Caiming Xiong, James Bradbury, and\nRichard Socher. 2017. Pointer sentinel mixture mod-\nels. In International Conference on Learning Repre-\nsentations.\nShen-yun Miao, Chao-Chun Liang, and Keh-Yih Su.\n2020. A diverse corpus for evaluating and develop-\ning English math word problem solvers. In Proceed-\nings of the 58th Annual Meeting of the Association\nfor Computational Linguistics, pages 975–984, On-\nline. Association for Computational Linguistics.\nReiichiro Nakano, Jacob Hilton, Suchir Balaji, Jeff Wu,\nLong Ouyang, Christina Kim, Christopher Hesse,\nShantanu Jain, Vineet Kosaraju, William Saunders,\nXu Jiang, Karl Cobbe, Tyna Eloundou, Gretchen\nKrueger, Kevin Button, Matthew Knight, Benjamin\nChess, and John Schulman. 2021. Webgpt: Browser-\nassisted question-answering with human feedback.\nAaron Parisi, Yao Zhao, and Noah Fiedel. 2022. Talm:\nTool augmented language models.\nArkil Patel, Satwik Bhattamishra, and Navin Goyal.\n2021. Are NLP models really able to solve simple\nmath word problems?\nIn Proceedings of the 2021\nConference of the North American Chapter of the\nAssociation for Computational Linguistics: Human\nLanguage Technologies, pages 2080–2094, Online.\nAssociation for Computational Linguistics.\nFabio Petroni, Aleksandra Piktus, Angela Fan, Patrick\nLewis, Majid Yazdani, Nicola De Cao, James\nThorne, Yacine Jernite, Vladimir Karpukhin, Jean\nMaillard, Vassilis Plachouras, Tim Rocktäschel, and\nSebastian Riedel. 2021.\nKILT: a benchmark for\nknowledge intensive language tasks. In Proceedings\nof the 2021 Conference of the North American Chap-\nter of the Association for Computational Linguistics:\nHuman Language Technologies, pages 2523–2544,\nOnline. Association for Computational Linguistics.\nFabio Petroni, Tim Rocktäschel, Sebastian Riedel,\nPatrick Lewis, Anton Bakhtin, Yuxiang Wu, and\nAlexander Miller. 2019. Language models as knowl-\nedge bases?\nIn Proceedings of the 2019 Confer-\nence on Empirical Methods in Natural Language\nProcessing and the 9th International Joint Confer-\nence on Natural Language Processing (EMNLP-\nIJCNLP), pages 2463–2473, Hong Kong, China. As-\nsociation for Computational Linguistics.\nAlec Radford, Jeffrey Wu, Rewon Child, David Luan,\nDario Amodei, Ilya Sutskever, et al. 2019.\nLan-\nguage models are unsupervised multitask learners.\nOpenAI blog, 1(8):9.\nRoi Reichart and Ari Rappoport. 2007. Self-training\nfor enhancement and domain adaptation of statisti-\ncal parsers trained on small datasets.\nIn Proceed-\nings of the 45th Annual Meeting of the Association of\nComputational Linguistics, pages 616–623, Prague,\nCzech Republic. Association for Computational Lin-\nguistics.\nStephen E Robertson, Steve Walker, Susan Jones,\nMicheline M Hancock-Beaulieu, Mike Gatford, et al.\n1995. Okapi at trec-3. Nist Special Publication Sp,\n109:109.\nTimo Schick, Jane Dwivedi-Yu, Zhengbao Jiang, Fabio\nPetroni, Patrick Lewis, Gautier Izacard, Qingfei You,\nChristoforos Nalmpantis, Edouard Grave, and Se-\nbastian Riedel. 2022.\nPeer: A collaborative lan-\nguage model.\nTimo Schick and Hinrich Schütze. 2021a. Exploiting\ncloze-questions for few-shot text classification and\nnatural language inference. In Proceedings of the\n\n16th Conference of the European Chapter of the As-\nsociation for Computational Linguistics: Main Vol-\nume, pages 255–269, Online. Association for Com-\nputational Linguistics.\nTimo Schick and Hinrich Schütze. 2021b. Generating\ndatasets with pretrained language models. In Pro-\nceedings of the 2021 Conference on Empirical Meth-\nods in Natural Language Processing, pages 6943–\n6951, Online and Punta Cana, Dominican Republic.\nAssociation for Computational Linguistics.\nKurt Shuster, Jing Xu, Mojtaba Komeili, Da Ju,\nEric Michael Smith, Stephen Roller, Megan Ung,\nMoya Chen, Kushal Arora, Joshua Lane, Morteza\nBehrooz, William Ngan, Spencer Poff, Naman\nGoyal, Arthur Szlam, Y-Lan Boureau, Melanie Kam-\nbadur, and Jason Weston. 2022. Blenderbot 3: a de-\nployed conversational agent that continually learns\nto responsibly engage.\nRomal Thoppilan, Daniel De Freitas, Jamie Hall,\nNoam Shazeer, Apoorv Kulshreshtha, Heng-Tze\nCheng, Alicia Jin, Taylor Bos, Leslie Baker, Yu Du,\nYaGuang Li, Hongrae Lee, Huaixiu Steven Zheng,\nAmin Ghafouri, Marcelo Menegali, Yanping Huang,\nMaxim Krikun, Dmitry Lepikhin, James Qin, De-\nhao Chen, Yuanzhong Xu, Zhifeng Chen, Adam\nRoberts, Maarten Bosma, Vincent Zhao, Yanqi\nZhou, Chung-Ching Chang, Igor Krivokon, Will\nRusch, Marc Pickett, Pranesh Srinivasan, Laichee\nMan, Kathleen Meier-Hellstern, Meredith Ringel\nMorris, Tulsee Doshi, Renelito Delos Santos, Toju\nDuke, Johnny Soraker, Ben Zevenbergen, Vinod-\nkumar Prabhakaran, Mark Diaz, Ben Hutchinson,\nKristen Olson, Alejandra Molina, Erin Hoffman-\nJohn, Josh Lee, Lora Aroyo, Ravi Rajakumar,\nAlena Butryna, Matthew Lamm, Viktoriya Kuzmina,\nJoe Fenton, Aaron Cohen, Rachel Bernstein, Ray\nKurzweil, Blaise Aguera-Arcas, Claire Cui, Marian\nCroak, Ed Chi, and Quoc Le. 2022. Lamda: Lan-\nguage models for dialog applications.\nBen Wang and Aran Komatsuzaki. 2021.\nGPT-\nJ-6B:\nA\n6\nBillion\nParameter\nAutoregressive\nLanguage Model.\nhttps://github.com/\nkingoflolz/mesh-transformer-jax.\nYizhong Wang, Yeganeh Kordi, Swaroop Mishra, Al-\nisa Liu, Noah A. Smith, Daniel Khashabi, and Han-\nnaneh Hajishirzi. 2022. Self-instruct: Aligning lan-\nguage model with self generated instructions.\nJason Wei, Yi Tay, Rishi Bommasani, Colin Raf-\nfel, Barret Zoph, Sebastian Borgeaud, Dani Yo-\ngatama, Maarten Bosma, Denny Zhou, Donald Met-\nzler, Ed H. Chi, Tatsunori Hashimoto, Oriol Vinyals,\nPercy Liang, Jeff Dean, and William Fedus. 2022.\nEmergent abilities of large language models.\nGuillaume Wenzek, Marie-Anne Lachaux, Alexis Con-\nneau, Vishrav Chaudhary, Francisco Guzmán, Ar-\nmand Joulin, and Edouard Grave. 2020.\nCCNet:\nExtracting high quality monolingual datasets from\nweb crawl data. In Proceedings of the Twelfth Lan-\nguage Resources and Evaluation Conference, pages\n4003–4012, Marseille, France. European Language\nResources Association.\nShunyu Yao, Jeffrey Zhao, Dian Yu, Nan Du, Izhak\nShafran, Karthik Narasimhan, and Yuan Cao. 2022.\nReact: Synergizing reasoning and acting in language\nmodels.\nDavid Yarowsky. 1995. Unsupervised word sense dis-\nambiguation rivaling supervised methods. In 33rd\nAnnual Meeting of the Association for Computa-\ntional Linguistics, pages 189–196, Cambridge, Mas-\nsachusetts, USA. Association for Computational\nLinguistics.\nEric Zelikman, Yuhuai Wu, Jesse Mu, and Noah D.\nGoodman. 2022.\nStar: Bootstrapping reasoning\nwith reasoning.\nSusan Zhang, Stephen Roller, Naman Goyal, Mikel\nArtetxe, Moya Chen, Shuohui Chen, Christopher De-\nwan, Mona Diab, Xian Li, Xi Victoria Lin, Todor Mi-\nhaylov, Myle Ott, Sam Shleifer, Kurt Shuster, Daniel\nSimig, Punit Singh Koura, Anjali Sridhar, Tianlu\nWang, and Luke Zettlemoyer. 2022. Opt: Open pre-\ntrained transformer language models.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/References",
- "metadata": {
- "length": 14027,
- "summary": "",
- "page_nums": [
- 11,
- 12,
- 13,
- 14
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 12,
- "artifact_ref": "page_citation_assets/page-12.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 13,
- "artifact_ref": "page_citation_assets/page-13.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 14,
- "artifact_ref": "page_citation_assets/page-14.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_ea6f3e96-8cf7-5921-84b1-cabd1233e20f",
- "type": "page",
- "content": "A\nAPI Details\nWhen sampling and filtering API calls, by default\nwe use values of τs = 0.05 and τf = 1.0 – i.e.,\nwe only make API calls at positions where the\nprobability of the token is at least 5%, and\nwe keep API calls if they reduce the loss by at least\n1.0. We only keep the top k = 5 such positions and\nsample up to m = 5 API calls for each position\nidentified in a piece of text. Due to the heuristic\nfiltering described below, we generate API calls for\nthe calculator and machine translation system on\nonly a small subset of C; to compensate for this,\nwe set τs = 0.0, k = 20 and m = 10 for these\ntools. As the resulting sets of API calls are still\ncomparably small, we additionally set τf = 0.5.\nA.1\nImplementation\nQuestion Answering\nWe use the Atlas model of\nIzacard et al. (2022) finetuned on Natural Ques-\ntions (Kwiatkowski et al., 2019) as our question\nanswering system. For creating C∗we use Atlas-\nlarge, enabling us to efficiently process millions\nof API calls; during inference, we use the larger\nAtlas-xxl model.\nCalculator\nOur calculator is based on a simple\nPython script and only supports the operators “+”,\n“−”, “∗”, and “/”. It does not return any result\nfor syntactically invalid equations. For sampling\nAPI calls, we apply heuristic filters to our subset of\nCCNet and only process documents that either (i)\ncontain at least three numbers within a window of\n100 tokens, where one of these numbers is the result\nof applying a mathematical operation to the other\ntwo, (ii) contain one of the sequences “=”, “equals”,\n“equal to”, “total of”, “average of” followed by a\nnumber, or (iii) contain at least three numbers; for\ntexts that only match the last criterion, we only\nkeep a random subset of 1%.\nCalendar\nFor creating our dataset C∗, we operate\nunder the assumption that the calendar date in such\ncases should be the date that the document was\ncreated. We approximate this by extracting the date\nfrom the URL, if it is present. We filter out texts for\nwhich a date cannot be extracted, leaving around\n18% of the documents.\nMachine Translation\nFor both training and in-\nference, we use the 600M parameter NLLB (Costa-\njussà et al., 2022) as our machine translation (MT)\nmodel. The source language is automatically de-\ntected using the fastText classifier (Joulin et al.,\n2016), while the target language is always set to\nEnglish. Since most of the CCNet dataset is in\nEnglish, we filter out the parts that contain only\nEnglish text before generating API calls. More\nspecifically, we only keep those paragraphs which\ncontain text chunks in a language other than En-\nglish preceded and followed by English text. We\nuse text chunks of size 10 tokens. To determine\nwhether the middle text chunk is in a language\ndifferent than English we again use the fastText\nclassifier with a confidence greater than 0.8. We\nalso filter out any text chunks that contain only\nnumbers or special symbols. This filtering mecha-\nnism allows us to generate data more efficiently by\nfocusing our API call generations in places where\nthe MT tool is likely to be helpful. After generating\nthe MT API calls, we additionally remove from our\ntraining set those where the input to the MT tool\nappears after the API call but not before it. While\nduring data generation the model can look ahead\nto generate API calls, this is not possible at infer-\nence time, so we want to dissuade the model from\ncalling the API in such cases.\nA.2\nPrompts\nBelow, we list the prompts used to sample API\ncalls for each tool considered.\nQuestion Answering\nWe use the following\nprompt for the question answering tool:\nYour task is to add calls to a Question\nAnswering API to a piece of text.\nThe questions should help you get\ninformation required to complete the\ntext. You can call the API by writing\n\"[QA(question)]\" where \"question\" is the\nquestion you want to ask. Here are some\nexamples of API calls:\nInput: Joe Biden was born in Scranton,\nPennsylvania.\nOutput: Joe Biden was born in [QA(\"Where\nwas Joe Biden born?\")] Scranton,\n[QA(\"In which state is Scranton?\")]\nPennsylvania.\nInput: Coca-Cola, or Coke, is a\ncarbonated soft drink manufactured by\nthe Coca-Cola Company.\nOutput: Coca-Cola, or [QA(\"What other\nname is Coca-Cola known by?\")] Coke, is\na carbonated soft drink manufactured by\n[QA(\"Who manufactures Coca-Cola?\")] the\nCoca-Cola Company.\nInput: x\nOutput:\nCalculator\nWe use the following prompt for the\ncalculator:\nYour task is to add calls to a\nCalculator API to a piece of text.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.1 Implementation",
- "metadata": {
- "length": 4457,
- "summary": "Section A.1 Implementation details the specific models and tools used for four tasks: Question Answering (Atlas model), Calculator (Python script with basic operators), Calendar (date extraction from URLs), and Machine Translation (600M parameter NLLB). It also describes data filtering heuristics applied to the CCNet dataset, such as language detection using fastText and token chunking.",
- "page_nums": [
- 15
- ],
- "keywords": [
- "Atlas",
- "Izacard et al.",
- "Natural Questions",
- "Kwiatkowski et al.",
- "CCNet",
- "NLLB",
- "Costa-jussà et al.",
- "fastText",
- "Joulin et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 15,
- "artifact_ref": "page_citation_assets/page-15.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_760bc05a-0958-5cd7-8445-6458dc6b2e04",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.1 Implementation p15]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts",
- "metadata": {
- "length": 117,
- "summary": "Section A.2 lists the specific prompts used to sample API calls for the Question Answering and Calculator tools. For Question Answering, the prompt instructs the model to add API calls to text to answer questions, providing examples of input/output pairs involving Joe Biden and Coca-Cola. For the Calculator tool, the prompt begins by defining the task as adding calls to a Calculator API.",
- "page_nums": [
- 15
- ],
- "keywords": [
- "Question Answering",
- "Calculator"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 15,
- "artifact_ref": "page_citation_assets/page-15.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_a0f1606c-fdec-5bef-aedc-7b6f5b3e7f50",
- "type": "page",
- "content": "The calls should help you get\ninformation required to complete the\ntext. You can call the API by writing\n\"[Calculator(expression)]\" where\n\"expression\" is the expression to be\ncomputed. Here are some examples of API\ncalls:\nInput: The number in the next term is 18\n+ 12 x 3 = 54.\nOutput: The number in the next term is\n18 + 12 x 3 = [Calculator(18 + 12 * 3)]\n54.\nInput: The population is 658,893 people.\nThis is 11.4% of the national average of\n5,763,868 people.\nOutput: The population is 658,893 people.\nThis is 11.4% of the national average of\n[Calculator(658,893 / 11.4%)] 5,763,868\npeople.\nInput: A total of 252 qualifying matches\nwere played, and 723 goals were scored\n(an average of 2.87 per match). This is\nthree times less than the 2169 goals\nlast year.\nOutput: A total of 252 qualifying\nmatches were played, and 723 goals were\nscored (an average of [Calculator(723\n/ 252)] 2.87 per match). This is twenty\ngoals more than the [Calculator(723 -\n20)] 703 goals last year.\nInput: I went to Paris in 1994 and\nstayed there until 2011, so in total,\nit was 17 years.\nOutput: I went to Paris in 1994 and\nstayed there until 2011, so in total, it\nwas [Calculator(2011 - 1994)] 17 years.\nInput: From this, we have 4 * 30 minutes\n= 120 minutes.\nOutput: From this, we have 4 * 30\nminutes = [Calculator(4 * 30)] 120\nminutes.\nInput: x\nOutput:\nWikipedia Search\nWe use the following prompt\nfor the Wikipedia search tool:\nYour task is to complete a given piece\nof text. You can use a Wikipedia Search\nAPI to look up information. You can do\nso by writing \"[WikiSearch(term)]\" where\n\"term\" is the search term you want to\nlook up. Here are some examples of API\ncalls:\nInput: The colors on the flag of Ghana\nhave the following meanings: red is for\nthe blood of martyrs, green for forests,\nand gold for mineral wealth.\nOutput: The colors on the flag of Ghana\nhave the following meanings: red is for\n[WikiSearch(\"Ghana flag red meaning\")]\nthe blood of martyrs, green for forests,\nand gold for mineral wealth.\nInput: But what are the risks during\nproduction of nanomaterials? Some\nnanomaterials may give rise to various\nkinds of lung damage.\nOutput: But what are the risks\nduring production of nanomaterials?\n[WikiSearch(\"nanomaterial production\nrisks\")] Some nanomaterials may give\nrise to various kinds of lung damage.\nInput: Metformin is the first-line drug\nfor patients with type 2 diabetes and\nobesity.\nOutput: Metformin is the first-line drug\nfor [WikiSearch(\"Metformin first-line\ndrug\")] patients with type 2 diabetes\nand obesity.\nInput: x\nOutput:\nMachine Translation\nWe use the following\nprompt for the machine translation tool:\nYour task is to complete a given piece\nof text by using a Machine Translation\nAPI.\nYou can do so by writing \"[MT(text)]\"\nwhere text is the text to be translated\ninto English.\nHere are some examples:\nInput: He has published one book: O\nhomem suprimido (“The Supressed Man”)\nOutput: He has published one book: O\nhomem suprimido [MT(O homem suprimido)]\n(“The Supressed Man”)\nInput: In Morris de Jonge’s Jeschuah,\nder klassische jüdische Mann, there is a\ndescription of a Jewish writer\nOutput: In Morris de Jonge’s Jeschuah,\nder klassische jüdische Mann [MT(der\nklassische jüdische Mann)], there is a\ndescription of a Jewish writer\nInput: 南京高淳县住房和城乡建设局城市新\n区\n设\n计a plane of reference Gaochun is\none of seven districts of the provincial\ncapital Nanjing\nOutput: [MT(南京高淳县住房和城乡建设局城市新\n区设计)] a plane of reference Gaochun is\none of seven districts of the provincial\ncapital Nanjing\nInput: x\nOutput:\nCalendar\nWe use the following prompt for the\ncalendar tool:\nYour task is to add calls to a Calendar\nAPI to a piece of text. The API calls\nshould help you get information required\nto complete the text. You can call the\nAPI by writing \"[Calendar()]\" Here are\nsome examples of API calls:\nInput: Today is the first Friday of the\nyear.\nOutput: Today is the first [Calendar()]\nFriday of the year.",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts/Wikipedia Search",
- "metadata": {
- "length": 3895,
- "summary": "The section titled 'Wikipedia Search' explains how to use a Wikipedia Search API to look up information. It provides examples of input prompts and corresponding outputs, such as searching for the meaning of colors on the Ghana flag and risks associated with nanomaterial production.",
- "page_nums": [
- 16
- ],
- "keywords": [
- "Ghana"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 16,
- "artifact_ref": "page_citation_assets/page-16.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_7e1795af-47e8-55cf-923e-f94019512f8e",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts/Wikipedia Search p16]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts/Machine Translation",
- "metadata": {
- "length": 127,
- "summary": "The section titled 'Machine Translation' explains how to use a Machine Translation API. It instructs the user to complete text by using the tool with the syntax '[MT(text)]', where 'text' is the input to be translated into English. The section provides three examples: translating a book title from Portuguese, translating a German phrase about Morris de Jonge's work, and translating Chinese text regarding Gaochun district in Nanjing.",
- "page_nums": [
- 16
- ],
- "keywords": [
- "Morris de Jonge",
- "Gaochun",
- "Nanjing"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 16,
- "artifact_ref": "page_citation_assets/page-16.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_d55ab6aa-f444-5fb6-b9cf-fb02d4b9b94c",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts/Wikipedia Search p16]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts/Calendar",
- "metadata": {
- "length": 127,
- "summary": "This section explains the usage of a Calendar API tool designed to add calls for processing text. It provides instructions on how to invoke the API using the syntax \"[Calendar()]\" and includes an example where the input 'Today is the first Friday of the year' is processed.",
- "page_nums": [
- 16
- ],
- "keywords": [
- "Friday"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 16,
- "artifact_ref": "page_citation_assets/page-16.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c013d919-67e8-54d0-899f-4133b7164920",
- "type": "page",
- "content": "Input: The president of the United\nStates is Joe Biden.\nOutput: The president of the United\nStates is [Calendar()] Joe Biden.\nInput: The current day of the week is\nWednesday.\nOutput: The current day of the week is\n[Calendar()] Wednesday.\nInput: The number of days from now until\nChristmas is 30.\nOutput: The number of days from now\nuntil Christmas is [Calendar()] 30.\nInput: The store is never open on the\nweekend, so today it is closed.\nOutput: The store is never open on the\nweekend, so today [Calendar()] it is\nclosed.\nInput: x\nOutput:\nB\nToolformer Training\nWe use up to 25k examples per API. Max sequence\nlength 1,024. Effective batch size of 128. All mod-\nels are trained using DeepSpeed’s ZeRO-3 (Rasley\net al., 2020). We used 8 NVIDIA A100 40GB\nGPUs with BF16. Training up to 2k steps, where\nwe evaluate PPL on a small development set from\nCCNet containing 1,000 examples every 500 steps.\nWe pick the checkpoint that performs best.\nC\nZero-Shot Prompts\nC.1\nLAMA and TEMPLAMA\nFor both LAMA and TEMPLAMA, given an input\ntext x, we use the following prompt: Please\ncomplete the following text so\nthat it is factually correct:\nx.\nC.2\nMath Benchmarks\nFor all math benchmarks, given a context x and\na question q, our prompt is: x q The answer\nis.\nC.3\nQuestion Answering\nFor all question answering datasets, including\nDATESET, we simply prefix the question with\nAnswer the following question:. We\nappend a question mark if the question does not\nalready end with one.\nC.4\nMultilingual Question Answering\nFor MLQA, given a context x and a ques-\ntion\nq,\nour\nprompt\nis:\nYour task is\nTemplate\nSize\nHow many days {ago was, are there until}\n{past_date, future_date}?\n400\nWhat {day of the week, day of the month, month,\nyear} was it (current_date – past_date) {days,\nweeks, months, years} ago?\n800\nWhat {day of the week, day of the month, month,\nyear} will it be in (future_date – current_date)\ndays?\n800\nWhat day of the week {is, was} it on {past_date,\nfuture_date}?\n400\nWhat {day of the week, day of the month, month,\nyear} {is, was} it {the day before yesterday, yes-\nterday, today, tomorrow, the day after tomorrow}?\n4,000\nWhat {day of the week, day of the month, month}\n{is, was} holiday this year?\n1,800\nHow many {days, weeks, months, years} {ago\nwas, are there until} holiday this year?\n1,200\nTotal\n9,400\nTable 11: Templates used to create DATESET where\na current_date is randomly selected.\nFor each cur-\nrent_date, a random past_date and future_date is gen-\nerated and used to fill each template, if relevant. The\nfederal holidays in the United States (e.g., Thanksgiv-\ning) were used in the templates involving holidays.\nto answer a question based on\nthe following paragraph:\nx Now\nanswer the following question in\nEnglish:\nq.\nD\nDATESET\nDATESET is created by first randomly selecting 500\n“current dates”. For each current date, another rela-\ntively past/future date is randomly selected within\na four-year range, and the two dates are used to fill\nthe query templates in Table 11. An example of one\nsuch query using the first template would be, “How\nmany days ago was August 14, 2020?” If called,\nthe Calendar tool would return the presumed cur-\nrent date (e.g., “Today is Sunday, November 20,\n2020”).",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/B Toolformer Training",
- "metadata": {
- "length": 3196,
- "summary": "The section details the training configuration for Toolformer, specifying a maximum of 25k examples per API, a sequence length of 1,024, and an effective batch size of 128. It notes that models are trained using DeepSpeed's ZeRO-3 on 8 NVIDIA A100 40GB GPUs with BF16 precision. The training runs for up to 2k steps, evaluating perplexity (PPL) every 500 steps on a small development set from CCNet containing 1,000 examples.",
- "page_nums": [
- 17
- ],
- "keywords": [
- "Toolformer",
- "DeepSpeed",
- "ZeRO-3",
- "Rasley et al.",
- "NVIDIA",
- "A100",
- "CCNet"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_cb621028-2177-5533-b20c-cf5ee4043c65",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/B Toolformer Training p17]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts/C.1 LAMA and TEMPLAMA",
- "metadata": {
- "length": 106,
- "summary": "Section C.1 describes the zero-shot prompt used for both LAMA and TEMPLAMA tasks. The prompt instructs the model to complete an input text 'x' so that it is factually correct.",
- "page_nums": [
- 17
- ],
- "keywords": [
- "LAMA",
- "TEMPLAMA"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_1c0d3f63-0207-5bbd-b97d-8cfe897c7a35",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/B Toolformer Training p17]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts/C.2 Math Benchmarks",
- "metadata": {
- "length": 106,
- "summary": "Section C.2 Math Benchmarks describes the prompt format used for math benchmarks, which involves providing a context (x) and a question (q) followed by 'The answer is.'",
- "page_nums": [
- 17
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_dae1ac9c-4b8f-5081-a68d-2f49984b5ec7",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/B Toolformer Training p17]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts/C.3 Question Answering",
- "metadata": {
- "length": 106,
- "summary": "Section C.3 describes the prompt format used for question answering datasets, including DATESET. The method involves prefixing the question with 'Answer the following question:' and appending a question mark if one is not already present.",
- "page_nums": [
- 17
- ],
- "keywords": [
- "DATESET"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_db4f86d6-4b13-514c-82d7-ca5cced1155d",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/B Toolformer Training p17]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts/C.4 Multilingual Question Answering",
- "metadata": {
- "length": 106,
- "summary": "Section C.4 describes the prompt format used for Multilingual Question Answering (MLQA). It specifies that given a context x and a question q, the task is to answer the question in English.",
- "page_nums": [
- 17
- ],
- "keywords": [
- "MLQA"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_e73a6729-1841-54c7-b4eb-cd080eb04dd6",
- "type": "page",
- "content": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/B Toolformer Training p17]",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/D DATESET",
- "metadata": {
- "length": 106,
- "summary": "Section D describes the creation of the DATESET dataset. It involves randomly selecting 500 'current dates' and generating relative past/future dates within a four-year range to fill query templates found in Table 11. The section provides an example query ('How many days ago was August 14, 2020?') and explains that a Calendar tool is used to return the current date (e.g., 'Today is Sunday, November 20, 2020').",
- "page_nums": [
- 17
- ],
- "keywords": [
- "DATESET",
- "Table 11",
- "August 14, 2020",
- "Calendar",
- "Sunday",
- "November 20, 2020"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- }
- ]
-}
diff --git a/apps/api/app/data/demo_documents/research-toolformer/doc_nav.json b/apps/api/app/data/demo_documents/research-toolformer/doc_nav.json
deleted file mode 100644
index 5b41d1ad1..000000000
--- a/apps/api/app/data/demo_documents/research-toolformer/doc_nav.json
+++ /dev/null
@@ -1,368 +0,0 @@
-{
- "version": "1.0",
- "file_name": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf",
- "stats": {
- "total_chunks": 41,
- "text_chunks": 0,
- "image_chunks": 4,
- "table_chunks": 1,
- "page_chunks": 36,
- "max_depth": 4
- },
- "sections": [
- {
- "title": "Root",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root",
- "level": 1,
- "summary": "",
- "chunk_count": 39,
- "children": [
- {
- "title": "Abstract",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Abstract",
- "level": 2,
- "summary": "The abstract introduces Toolformer, a model trained to decide which APIs to call and how to incorporate their results into future token prediction. It is trained in a self-supervised way using only a handful of demonstrations per API. The model incorporates tools such as a calculator, Q&A system, search engine, translation system, and calendar, achieving substantially improved zero-shot performance across various downstream tasks.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "1 Introduction",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/1 Introduction",
- "level": 2,
- "summary": "The Introduction section outlines the limitations of current large language models, such as their inability to access up-to-date information, tendency to hallucinate facts, and lack of mathematical skills. It proposes 'Toolformer', a model that learns to use external tools like search engines and calculators in a self-supervised manner without requiring extensive human annotations.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "2 Approach",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach",
- "level": 2,
- "summary": "The page presents a method for equipping language models with tool-use capabilities via API calls. It details an approach using in-context learning to generate datasets of potential API calls, which are then filtered based on their utility for predicting future tokens. The text includes Figure 2, illustrating the three-step process: sampling API calls, executing them, and filtering them. It also references prior work such as Toolformer (Wang and Komatsuzaki, 2021) and GPT-3 (Brown et al., 2020).",
- "chunk_count": 5,
- "children": [
- {
- "title": "Sampling API Calls",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Sampling API Calls",
- "level": 3,
- "summary": "The section describes the process of sampling API calls for a Question Answering task. It involves writing a prompt P(x) to encourage a Language Model (LM) to annotate text with API calls. The method calculates the probability p_i of starting an API call at each position i in the sequence. A sampling threshold τ_s is used to filter positions, keeping those where p_i > τ_s. If more than k positions are kept, only the top k are selected. Finally, up to m API calls are sampled for each selected position using the model M.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Executing API Calls",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Executing API Calls",
- "level": 3,
- "summary": "This section describes the process of executing API calls generated by a model M. It explains that execution methods vary depending on the specific API, such as calling another neural network, running a Python script, or using a retrieval system. The output for each call must be formatted as a single text sequence.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Filtering API Calls",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Filtering API Calls",
- "level": 3,
- "summary": "The section 'Filtering API Calls' describes a method to determine whether an API call is helpful for the model. It defines a weighted cross-entropy loss function and compares two scenarios: one where the API call and its result are provided as a prefix, and another where no API call is made or only the input is provided. The text explains that an API call is considered helpful if providing it reduces the loss by at least a specific threshold (τf).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Model Finetuning",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/2 Approach/Model Finetuning",
- "level": 3,
- "summary": "The section describes the final step of the process, which is Model Finetuning. It explains that after sampling and filtering API calls for all APIs, the remaining calls are merged with the original text to create a new training sequence. This involves interleaving the API call and its result into the input text at the specific position where the call was made.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "3 Tools",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/3 Tools",
- "level": 2,
- "summary": "Section 3 describes five tools used to augment regular language models: a question answering system (Atlas), a calculator, a Wikipedia search engine (BM25), a machine translation system (NLLB), and a calendar API. The section details the specific implementations and constraints for each tool.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4 Experiments",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/4 Experiments",
- "level": 2,
- "summary": "Section 4.1 Experimental Setup describes the dataset generation process for the experiments. It specifies that a subset of CCNet is used as the language modeling dataset (C) and GPT-J is used as the language model (M). To reduce annotation costs, heuristics are defined to select subsets of C where API calls are likely to be helpful.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Appendix A",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A",
- "level": 2,
- "summary": "",
- "chunk_count": 9,
- "children": [
- {
- "title": "Baseline Models",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/Baseline Models",
- "level": 3,
- "summary": "The section describes the baseline models used for comparison, including GPT-J (unfinetuned), GPT-J + CC (finetuned on a subset of CCNet without API calls), Toolformer (finetuned on a subset of CCNet augmented with API calls), and Toolformer (disabled) where API calls are disabled during decoding. It also mentions comparisons to larger models OPT (66B) and GPT-3 (175B).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.2 Downstream Tasks",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks",
- "level": 3,
- "summary": "Section 4.2 describes the evaluation setup for downstream tasks, focusing on a prompted zero-shot approach where models are instructed to solve tasks in natural language without provided examples. It contrasts this with prior work that uses dataset-specific examples and mentions comparisons against larger baseline models like OPT (66B) and GPT-3 (175B). The section also details the use of standard greedy decoding with a specific modification for Toolformer.",
- "chunk_count": 6,
- "children": [
- {
- "title": "4.2.1 LAMA",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.1 LAMA",
- "level": 4,
- "summary": "Section 4.2.1 evaluates the Toolformer model on the LAMA benchmark, specifically using the SQuAD, Google-RE, and T-REx subsets to test fact completion tasks. The text explains that examples were filtered to ensure the mask token is the final token and that the Wikipedia Search API was disabled for Toolformer to prevent unfair advantages. Table 3 presents results showing that Toolformer significantly outperforms baseline models like GPT-J and larger models like OPT (66B) and GPT-3 (175B), achieving competitive scores with GPT-3 (175B). The section attributes this success to the model's ability to independently decide to use a question answering tool in almost all cases.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.2.2 Math Datasets",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.2 Math Datasets",
- "level": 4,
- "summary": "Section 4.2.2 Math Datasets evaluates the mathematical reasoning abilities of models on three benchmarks: ASDiv, SVAMP, and MAWPS. The section describes a zero-shot evaluation setup using a lenient criterion where the model's prediction is checked for containing the first number after an equals sign. Table 4 presents results showing that Toolformer significantly outperforms baseline models (GPT-J, OPT, GPT-3) by leveraging a calculator tool in 97.9% of cases.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.2.3 Question Answering",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.3 Question Answering",
- "level": 4,
- "summary": "Section 4.2.3 introduces the evaluation of question answering capabilities using three datasets: Web Questions, Natural Questions, and TriviaQA. The section specifies that for evaluation, the system checks if the correct answer is contained within the first 20 words predicted by the model rather than requiring an exact match. Additionally, it notes that for the Toolformer model, the question answering tool is disabled during this specific evaluation.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.2.4 Multilingual Question Answering",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.4 Multilingual Question Answering",
- "level": 4,
- "summary": "Section 4.2.4 evaluates Toolformer and baseline models on the MLQA multilingual question-answering benchmark, where context is in English but questions are in Arabic, German, Spanish, Hindi, Vietnamese, or Simplified Chinese. Results (Table 6) show that using a machine translation API consistently improves performance across languages, with Toolformer outperforming GPT-J baselines in most cases except for some languages where finetuning on CCNet causes deterioration. OPT and GPT-3 perform poorly due to lack of multilingual training data.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.2.5 Temporal Datasets",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.2 Downstream Tasks/4.2.5 Temporal Datasets",
- "level": 4,
- "summary": "Section 4.2.5 Temporal Datasets introduces the evaluation of models on temporal datasets to investigate the utility of a calendar API. It describes two datasets: TEMPLAMA, which contains cloze queries about facts that change over time (e.g., 'Cristiano Ronaldo plays for ___') based on Wikidata for years between 2010 and 2020; and DATESET, a new dataset generated using templates with random dates/durations (e.g., 'What day of the week was it 30 days ago?'). Both require knowledge of the current date to answer correctly.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "4.3 Language Modeling",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.3 Language Modeling",
- "level": 3,
- "summary": "Section 4.3 discusses the language modeling performance of the Toolformer model to ensure it does not degrade during finetuning with API calls. The authors evaluate their models on two datasets: WikiText and a subset of CCNet. Table 8 presents perplexity results for various models (GPT-J, GPT-J + CC, Toolformer) on these datasets, showing that adding API calls does not increase perplexity compared to training without them.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.4 Scaling Laws",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/Appendix A/4.4 Scaling Laws",
- "level": 3,
- "summary": "Section 4.4 investigates how the ability to use external tools affects performance as the size of the language model (LM) increases. The authors apply their approach to four smaller models from the GPT-2 family (124M, 355M, 775M, and 1.6B parameters) alongside GPT-J, using a subset of three tools: question answering, calculator, and Wikipedia search. The section reports that leveraging provided tools only emerges at around 775M parameters; smaller models perform similarly with or without tools. It is hypothesized that the Wikipedia search API is an exception because it is easy to use. As models grow larger, they improve at solving tasks both with and without API calls, but a large gap remains between predictions made with and without API calls even for the largest model.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "5 Analysis",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/5 Analysis",
- "level": 2,
- "summary": "The section titled '5 Analysis' begins by introducing a subsection on 'Decoding Strategy', which investigates the effects of a modified decoding strategy previously introduced in Section 4.2. The text explains that this strategy involves generating tokens differently than always generating the most probable one, though the full explanation is cut off at the bottom of the page.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "6 Related Work",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/6 Related Work",
- "level": 2,
- "summary": "This page presents experimental results and analysis regarding a model called Toolformer. Figure 4 displays line charts comparing the performance of Toolformer, Toolformer with disabled API calls, and GPT3 across LAMA, Math Benchmarks, and QA Benchmarks as model parameters increase. Table 9 details Toolformer's performance on T-REx and WebQS datasets for different values of k (the number of most likely tokens considered), showing metrics for overall performance, API call success, and non-call accuracy. The text discusses how increasing k leads to more API calls and improved performance, particularly on WebQS. A section titled 'Data Quality' qualitatively analyzes API calls generated by the approach, referencing Table 10 (not fully visible) and discussing filtering criteria based on score differences. The page concludes with the beginning of Section 6, 'Related Work,' which introduces approaches to augment language models with textual information during pretraining.",
- "chunk_count": 3,
- "children": [
- {
- "title": "Tool Use",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/6 Related Work/Tool Use",
- "level": 3,
- "summary": "The section discusses approaches to equip language models (LMs) with the ability to use external tools such as search engines, web browsers, calculators, translation systems, and Python interpreters. It categorizes learning methods into those relying on large amounts of human supervision versus few-shot prompting setups. It also mentions TALM as a related self-supervised approach.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Bootstrapping",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/6 Related Work/Bootstrapping",
- "level": 3,
- "summary": "The section titled 'Bootstrapping' discusses the concept of using self-training and bootstrapping techniques to improve models. It lists various contexts where these methods have been investigated, including word sense disambiguation, relation extraction, parsing, sequence generation, and few-shot text classification.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "7 Limitations",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/7 Limitations",
- "level": 2,
- "summary": "The section outlines several limitations of the Toolformer approach. Key issues include the inability to use tools in a chain (using one tool's output as another's input) because API calls are generated independently, and the lack of support for interactive tool usage like search engines. Additionally, models are sensitive to prompt wording, the method is sample-inefficient (requiring large datasets for few examples), and it does not account for the computational cost of making API calls.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "8 Conclusion",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/8 Conclusion",
- "level": 2,
- "summary": "The section introduces Toolformer, a language model designed to learn self-supervised usage of various tools like search engines and calculators through simple API calls. The model is finetuned on sampled API calls filtered by their ability to reduce perplexity on future tokens. It is noted that Toolformer significantly improves zero-shot performance compared to larger models.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "References",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/References",
- "level": 2,
- "summary": "[SAME-AS Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/7 Limitations p11] Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung Won Chung, Charles Sutton, Sebastian Gehrmann, Parker Schuh, Kensen Shi, Sasha Tsvyashchenko, Joshua Maynez, Abhishek Rao, Parker Barnes, Yi Tay, Noam Shazeer, Vin- odkumar Prabhakaran, Emily Reif, Nan Du, Ben Hutchinson, Reiner Pope, James Bradbury, Jacob Austin, Michael Isard, Guy Gur-Ari, Pengcheng Yin, Toju Duke, Anselm Levskaya, Sanjay Ghe- mawat, Sunipa Dev, Henryk Michalewski, Xavier Garcia, Vedant Misra, Kevin Robinson, Liam Fe- dus, Denny Zhou, Daphne Ippolito, David Luan, Hyeontaek Lim, Barret Zoph, Alexander Spiridonov, Ryan Sepassi, David Dohan, Shivani Agrawal, Mark Omernick, Andrew M. Dai, Thanumalayan Sankara- narayana Pillai, Marie Pellat, Aitor Lewkowycz, Erica Moreira, Rewon Child, Oleksandr Polozov, Katherine Lee, Zongwei Zhou, Xuezhi Wang, Bren- nan Saeta, Mark Diaz, Orhan Firat, Michele Catasta, Jason Wei, Kathy Meier-Hellstern, Douglas Eck, Jeff Dean, Slav Petrov, and Noah Fiedel. 2022. Palm: Scaling language modeling with pathways. Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, et al. 2021. Training verifiers to solve math word problems...Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen, Christopher De- wan, Mona Diab, Xian Li, Xi Victoria Lin, Todor Mi- haylov, Myle Ott, Sam Shleifer, Kurt Shuster, Daniel Simig, Punit Singh Koura, Anjali Sridhar, Tianlu Wang, and Luke Zettlemoyer. 2022. Opt: Open pre- trained transformer language models.",
- "chunk_count": 4,
- "children": []
- },
- {
- "title": "A API Details",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details",
- "level": 2,
- "summary": "",
- "chunk_count": 5,
- "children": [
- {
- "title": "A.1 Implementation",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.1 Implementation",
- "level": 3,
- "summary": "Section A.1 Implementation details the specific models and tools used for four tasks: Question Answering (Atlas model), Calculator (Python script with basic operators), Calendar (date extraction from URLs), and Machine Translation (600M parameter NLLB). It also describes data filtering heuristics applied to the CCNet dataset, such as language detection using fastText and token chunking.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "A.2 Prompts",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts",
- "level": 3,
- "summary": "Section A.2 lists the specific prompts used to sample API calls for the Question Answering and Calculator tools. For Question Answering, the prompt instructs the model to add API calls to text to answer questions, providing examples of input/output pairs involving Joe Biden and Coca-Cola. For the Calculator tool, the prompt begins by defining the task as adding calls to a Calculator API.",
- "chunk_count": 4,
- "children": [
- {
- "title": "Wikipedia Search",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts/Wikipedia Search",
- "level": 4,
- "summary": "The section titled 'Wikipedia Search' explains how to use a Wikipedia Search API to look up information. It provides examples of input prompts and corresponding outputs, such as searching for the meaning of colors on the Ghana flag and risks associated with nanomaterial production.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Machine Translation",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts/Machine Translation",
- "level": 4,
- "summary": "The section titled 'Machine Translation' explains how to use a Machine Translation API. It instructs the user to complete text by using the tool with the syntax '[MT(text)]', where 'text' is the input to be translated into English. The section provides three examples: translating a book title from Portuguese, translating a German phrase about Morris de Jonge's work, and translating Chinese text regarding Gaochun district in Nanjing.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Calendar",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/A API Details/A.2 Prompts/Calendar",
- "level": 4,
- "summary": "This section explains the usage of a Calendar API tool designed to add calls for processing text. It provides instructions on how to invoke the API using the syntax \"[Calendar()]\" and includes an example where the input 'Today is the first Friday of the year' is processed.",
- "chunk_count": 1,
- "children": []
- }
- ]
- }
- ]
- },
- {
- "title": "B Toolformer Training",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/B Toolformer Training",
- "level": 2,
- "summary": "The section details the training configuration for Toolformer, specifying a maximum of 25k examples per API, a sequence length of 1,024, and an effective batch size of 128. It notes that models are trained using DeepSpeed's ZeRO-3 on 8 NVIDIA A100 40GB GPUs with BF16 precision. The training runs for up to 2k steps, evaluating perplexity (PPL) every 500 steps on a small development set from CCNet containing 1,000 examples.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C Zero-Shot Prompts",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts",
- "level": 2,
- "summary": "",
- "chunk_count": 4,
- "children": [
- {
- "title": "C.1 LAMA and TEMPLAMA",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts/C.1 LAMA and TEMPLAMA",
- "level": 3,
- "summary": "Section C.1 describes the zero-shot prompt used for both LAMA and TEMPLAMA tasks. The prompt instructs the model to complete an input text 'x' so that it is factually correct.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C.2 Math Benchmarks",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts/C.2 Math Benchmarks",
- "level": 3,
- "summary": "Section C.2 Math Benchmarks describes the prompt format used for math benchmarks, which involves providing a context (x) and a question (q) followed by 'The answer is.'",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C.3 Question Answering",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts/C.3 Question Answering",
- "level": 3,
- "summary": "Section C.3 describes the prompt format used for question answering datasets, including DATESET. The method involves prefixing the question with 'Answer the following question:' and appending a question mark if one is not already present.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "C.4 Multilingual Question Answering",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/C Zero-Shot Prompts/C.4 Multilingual Question Answering",
- "level": 3,
- "summary": "Section C.4 describes the prompt format used for Multilingual Question Answering (MLQA). It specifies that given a context x and a question q, the task is to answer the question in English.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "D DATESET",
- "path": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf/Root/D DATESET",
- "level": 2,
- "summary": "Section D describes the creation of the DATESET dataset. It involves randomly selecting 500 'current dates' and generating relative past/future dates within a four-year range to fill query templates found in Table 11. The section provides an example query ('How many days ago was August 14, 2020?') and explains that a Calendar tool is used to return the current date (e.g., 'Today is Sunday, November 20, 2020').",
- "chunk_count": 1,
- "children": []
- }
- ]
- }
- ],
- "resources": {
- "images": [
- {
- "path": "images/image_page_1_figure_1.png",
- "summary": "Figure 1: Exemplary predictions of Toolformer. [images/image_page_1_figure_1.png]"
- },
- {
- "path": "images/image_page_2_figure_1.png",
- "summary": "Figure 2: Key steps in our approach, illustrated for a question answering tool [images/image_page_2_figure_1.png]"
- },
- {
- "path": "images/image_page_3_figure_1.png",
- "summary": "An exemplary prompt P(x) used to generate API calls for the question answering tool. [images/image_page_3_figure_1.png]"
- },
- {
- "path": "images/image_page_9_figure_1.png",
- "summary": "Figure 4: Average performance on LAMA, our math benchmarks and our QA benchmarks for GPT-2 models of different sizes and GPT-J finetuned with our approach, both with and without API calls. [images/image_page_9_figure_1.png]"
- }
- ],
- "tables": [
- {
- "path": "tables/table_page_9_2.html",
- "summary": "tables/table_page_9_2.html"
- }
- ]
- }
-}
diff --git a/apps/api/app/data/demo_documents/research-toolformer/images/image_page_1_figure_1.png b/apps/api/app/data/demo_documents/research-toolformer/images/image_page_1_figure_1.png
deleted file mode 100644
index 9c9c921ff..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/images/image_page_1_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/images/image_page_2_figure_1.png b/apps/api/app/data/demo_documents/research-toolformer/images/image_page_2_figure_1.png
deleted file mode 100644
index 92dfb8530..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/images/image_page_2_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/images/image_page_3_figure_1.png b/apps/api/app/data/demo_documents/research-toolformer/images/image_page_3_figure_1.png
deleted file mode 100644
index 6e6273dfb..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/images/image_page_3_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/images/image_page_9_figure_1.png b/apps/api/app/data/demo_documents/research-toolformer/images/image_page_9_figure_1.png
deleted file mode 100644
index 39963a746..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/images/image_page_9_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/manifest.json b/apps/api/app/data/demo_documents/research-toolformer/manifest.json
deleted file mode 100644
index 174316f81..000000000
--- a/apps/api/app/data/demo_documents/research-toolformer/manifest.json
+++ /dev/null
@@ -1,295 +0,0 @@
-{
- "version": "2.0",
- "job_id": "job_8e54b45fba36",
- "data_id": "demo-research-toolformer",
- "source_file_name": "Toolformer: Language Models Can Teach Themselves to Use Tools.pdf",
- "processing_date": "2026-08-18T17:55:56.932131Z",
- "processing": {
- "page_count": 17,
- "billing_status": "charged",
- "cost": {
- "micro_dollars": 255000,
- "credits": 0.255
- },
- "cost_estimate": {
- "currency": "USD",
- "pricing_version": "2026-08-18:874ab8b6c9e4",
- "total_input_cost": 0.00305376,
- "total_output_cost": 0.0008233,
- "total_cost": 0.00387706,
- "by_model": {
- "qwen3.5-flash": {
- "input_tokens": 17165,
- "output_tokens": 190,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- },
- "deepseek-chat": {
- "input_tokens": 2134,
- "output_tokens": 535,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.00029876,
- "output_cost": 0.0001498,
- "total_cost": 0.00044856,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- },
- "qwen3.6-flash": {
- "input_tokens": 11020,
- "output_tokens": 449,
- "calls": 4,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.002755,
- "output_cost": 0.0006735,
- "total_cost": 0.0034285,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 17165,
- "output_tokens": 190,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "page_memory.hierarchy": {
- "input_cost": 0.00029876,
- "output_cost": 0.0001498,
- "total_cost": 0.00044856,
- "models": {
- "deepseek-chat": {
- "input_tokens": 2134,
- "output_tokens": 535,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.00029876,
- "output_cost": 0.0001498,
- "total_cost": 0.00044856,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- },
- "page_memory.asset_detect": {
- "input_cost": 0.002755,
- "output_cost": 0.0006735,
- "total_cost": 0.0034285,
- "models": {
- "qwen3.6-flash": {
- "input_tokens": 11020,
- "output_tokens": 449,
- "calls": 4,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.002755,
- "output_cost": 0.0006735,
- "total_cost": 0.0034285,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "timing": {
- "started_at": "2026-08-18T17:52:18.974338+00:00",
- "completed_at": "2026-08-18T17:55:56.929376+00:00",
- "duration_ms": 217955
- },
- "stages": {
- "timing_ms": {
- "page_memory.normalize_to_pdf": 0,
- "page_memory.profile": 70827,
- "page_memory.skeleton": 0,
- "page_memory.title_render": 20813,
- "page_memory.title_detection": 25688,
- "page_memory.fine_hierarchy": 3376,
- "page_memory.render": 3785,
- "page_memory.tag": 33461,
- "page_memory.assets": 14382,
- "page_memory.node_summary": 45459,
- "page_memory.node_rows": 30,
- "page_memory.node_assembly": 45491,
- "worker.parse.document": 217854
- },
- "token_usage": {
- "prompt_tokens": 30319,
- "completion_tokens": 1174,
- "total_tokens": 31493,
- "calls": 6,
- "by_model": {
- "qwen3.5-flash": {
- "prompt_tokens": 17165,
- "completion_tokens": 190,
- "total_tokens": 17355,
- "calls": 1
- },
- "deepseek-chat": {
- "prompt_tokens": 2134,
- "completion_tokens": 535,
- "total_tokens": 2669,
- "calls": 1
- },
- "qwen3.6-flash": {
- "prompt_tokens": 11020,
- "completion_tokens": 449,
- "total_tokens": 11469,
- "calls": 4
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "prompt_tokens": 17165,
- "completion_tokens": 190,
- "total_tokens": 17355,
- "calls": 1,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 17165,
- "completion_tokens": 190,
- "total_tokens": 17355,
- "calls": 1
- }
- }
- },
- "page_memory.hierarchy": {
- "prompt_tokens": 2134,
- "completion_tokens": 535,
- "total_tokens": 2669,
- "calls": 1,
- "models": {
- "deepseek-chat": {
- "prompt_tokens": 2134,
- "completion_tokens": 535,
- "total_tokens": 2669,
- "calls": 1
- }
- }
- },
- "page_memory.asset_detect": {
- "prompt_tokens": 11020,
- "completion_tokens": 449,
- "total_tokens": 11469,
- "calls": 4,
- "models": {
- "qwen3.6-flash": {
- "prompt_tokens": 11020,
- "completion_tokens": 449,
- "total_tokens": 11469,
- "calls": 4
- }
- }
- }
- }
- }
- }
- },
- "statistics": {
- "total_chunks": 41,
- "text_chunks": 0,
- "image_chunks": 4,
- "table_chunks": 1,
- "page_chunks": 36,
- "total_pages": null
- },
- "HIERARCHY": {
- "Root": {
- "Abstract": {},
- "1 Introduction": {},
- "2 Approach": {
- "Sampling API Calls": {},
- "Executing API Calls": {},
- "Filtering API Calls": {},
- "Model Finetuning": {}
- },
- "3 Tools": {},
- "4 Experiments": {},
- "Appendix A": {
- "Baseline Models": {},
- "4.2 Downstream Tasks": {
- "4.2.1 LAMA": {},
- "4.2.2 Math Datasets": {},
- "4.2.3 Question Answering": {},
- "4.2.4 Multilingual Question Answering": {},
- "4.2.5 Temporal Datasets": {}
- },
- "4.3 Language Modeling": {},
- "4.4 Scaling Laws": {}
- },
- "5 Analysis": {},
- "6 Related Work": {
- "Tool Use": {},
- "Bootstrapping": {}
- },
- "7 Limitations": {},
- "8 Conclusion": {},
- "References": {},
- "A API Details": {
- "A.1 Implementation": {},
- "A.2 Prompts": {
- "Wikipedia Search": {},
- "Machine Translation": {},
- "Calendar": {}
- }
- },
- "B Toolformer Training": {},
- "C Zero-Shot Prompts": {
- "C.1 LAMA and TEMPLAMA": {},
- "C.2 Math Benchmarks": {},
- "C.3 Question Answering": {},
- "C.4 Multilingual Question Answering": {}
- },
- "D DATESET": {}
- }
- }
-}
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-1.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-1.png
deleted file mode 100644
index 0ff793be0..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-10.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-10.png
deleted file mode 100644
index 408a5eda3..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-10.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-11.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-11.png
deleted file mode 100644
index 35a970238..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-11.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-12.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-12.png
deleted file mode 100644
index e3eeb2872..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-12.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-13.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-13.png
deleted file mode 100644
index 5ca7a2d8a..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-13.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-14.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-14.png
deleted file mode 100644
index 3ccf773ec..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-14.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-15.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-15.png
deleted file mode 100644
index 5e75403b8..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-15.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-16.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-16.png
deleted file mode 100644
index 4dfb3241b..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-16.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-17.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-17.png
deleted file mode 100644
index 929c8f814..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-17.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-2.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-2.png
deleted file mode 100644
index 1db97a38c..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-3.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-3.png
deleted file mode 100644
index 8ef6ad322..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-3.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-4.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-4.png
deleted file mode 100644
index 9f9ee8318..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-4.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-5.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-5.png
deleted file mode 100644
index fdec4131f..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-5.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-6.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-6.png
deleted file mode 100644
index c7487c478..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-6.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-7.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-7.png
deleted file mode 100644
index 1933fc36a..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-7.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-8.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-8.png
deleted file mode 100644
index 26601981c..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-8.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-9.png b/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-9.png
deleted file mode 100644
index 4ff60258e..000000000
Binary files a/apps/api/app/data/demo_documents/research-toolformer/page_citation_assets/page-9.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/research-toolformer/tables/table_page_9_2.html b/apps/api/app/data/demo_documents/research-toolformer/tables/table_page_9_2.html
deleted file mode 100644
index 6f1fe832c..000000000
--- a/apps/api/app/data/demo_documents/research-toolformer/tables/table_page_9_2.html
+++ /dev/null
@@ -1,58 +0,0 @@
-
mple 7: A total of 500 married couples are poled about salaries:
-
-
-
Wife Husband makes less than 25,000 Husband makes more than 25,000
-
-
-
Less than $25,000 212 198
-
-
-
More than $25,000 36 54
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_74_1.html b/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_74_1.html
deleted file mode 100644
index 80b777648..000000000
--- a/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_74_1.html
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-
-
outcome
-
1
-
2
-
3
-
-
-
1
-
(X = 1, Y = 2) = (1, 2)
-
(2, 3)
-
(3, 4)
-
-
-
2
-
(2, 3)
-
(2, 4)
-
(3, 5)
-
-
-
3
-
(3, 4)
-
(3, 5)
-
(3, 6)
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_74_2.html b/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_74_2.html
deleted file mode 100644
index 4e71ffd2b..000000000
--- a/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_74_2.html
+++ /dev/null
@@ -1,67 +0,0 @@
-
-
-
-
X\Y
-
2
-
-
3.0
-
4.0
-
5.0
-
6.0
-
-
-
1
-
P (X = 1, Y = 2) =
-
1.0
-
0.0
-
0.0
-
0.0
-
0.0
-
-
-
-
-
9.0
-
-
-
-
-
-
-
2
-
0
-
-
2.0
-
1.0
-
0.0
-
0.0
-
-
-
-
-
-
9.0
-
9.0
-
-
-
-
-
3
-
0
-
-
0.0
-
2.0
-
2.0
-
1.0
-
-
-
-
-
-
-
9.0
-
9.0
-
9.0
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_82_1.html b/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_82_1.html
deleted file mode 100644
index fe87270db..000000000
--- a/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_82_1.html
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
-
x\y
-
0.0
-
1.0
-
-
-
0
-
0.4
-
0.2
-
-
-
1
-
0.1
-
0.3
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_87_1.html b/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_87_1.html
deleted file mode 100644
index b499a0f74..000000000
--- a/apps/api/app/data/demo_documents/stem-probability-theory-lecture-notes/tables/table_page_87_1.html
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
-
-
X
-
-
Y
-
0.0
-
2
-
-
-
-
0.0
-
-
0.2
-
.7 .
-
-
-
-
1.0
-
-
0.0
-
.1
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/chunks.json b/apps/api/app/data/demo_documents/stem-statistical-learning/chunks.json
deleted file mode 100644
index b01d4b5cf..000000000
--- a/apps/api/app/data/demo_documents/stem-statistical-learning/chunks.json
+++ /dev/null
@@ -1,1383 +0,0 @@
-{
- "chunks": [
- {
- "chunk_id": "asset_ee275ce4-7db4-5a02-9192-2bea784587ba",
- "type": "image",
- "content": "[images/image_page_21_figure_1.png]",
- "path": "images/image_page_21_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 21
- ],
- "file_path": "images/image_page_21_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_6411b02f-eafc-5539-b1de-aeb1ced89150",
- "type": "image",
- "content": "[images/image_page_21_figure_1.png]",
- "path": "images/image_page_21_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 21
- ],
- "file_path": "images/image_page_21_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_9ab77f2b-3439-5115-8efa-dd51a5588351",
- "type": "image",
- "content": "[images/image_page_45_figure_1.png]",
- "path": "images/image_page_45_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 45
- ],
- "file_path": "images/image_page_45_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_dc0be7e0-62a0-5c03-b3f0-086a59df6fc8",
- "type": "image",
- "content": "[images/image_page_45_figure_2.png]",
- "path": "images/image_page_45_figure_2.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 45
- ],
- "file_path": "images/image_page_45_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_d7e06362-0ee2-50ed-a599-8027893b2d7d",
- "type": "image",
- "content": "[images/image_page_51_figure_1.png]",
- "path": "images/image_page_51_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 51
- ],
- "file_path": "images/image_page_51_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_aa55d178-c0fc-54dc-871c-de6f78473060",
- "type": "image",
- "content": "[images/image_page_51_figure_1.png]",
- "path": "images/image_page_51_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 51
- ],
- "file_path": "images/image_page_51_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "node_c650f3d3-7801-51d9-b6c7-28fad663c392",
- "type": "page",
- "content": "the penalty and therefore the degree of the shrinkage towards 0, is known as a regularisation\nparameter or tuning parameter. We have explicitly included an intercept term which is not\npenalised. The reason for this is that were the variables to have their origins shifted so\ne.g. a variable representing temperature is given in units of Kelvin rather than Celsius, the\nfitted values would not change. However, X ˆβ is not invariant under scale transformations\nof the variables so it is standard practice to centre each column of X (hence making them\northogonal to the intercept term) and then scale them to have ℓ2-norm √n.\nIt is straightforward to show that after this standardisation of X, ˆµR\nλ = ¯Y := Pn\ni=1 Yi/n,\nso we may assume that Pn\ni=1 Yi = 0 by replacing Yi by Yi −¯Y and then we can remove µ\nfrom our objective function. In this case\nˆβR\nλ = (XTX + λI)−1XTY.\nIn this form, we can see how the addition of the λI term helps to stabilise the estimator.\nNote that when X does not have full column rank (such as in high-dimensional situations),\nwe can still compute this estimator. On the other hand, when X does have full column\nrank, we have the following theorem.\nTheorem 1. For λ sufficiently small (depending on β0 and σ2),\nE(ˆβOLS −β0)(ˆβOLS −β0)T −E(ˆβR\nλ −β0)(ˆβR\nλ −β0)T\nis positive definite.\nProof. First we compute the bias of ˆβR\nλ . We drop the subscript λ and superscript R for\nconvenience.\nE(ˆβ) −β0 = (XTX + λI)−1XTXβ0 −β0\n= (XTX + λI)−1(XTX + λI −λI)β0 −β0\n= −λ(XTX + λI)−1β0.\nNow we look at the variance of ˆβ.\nVar(ˆβ) = E{(XTX + λI)−1XTε}{(XTX + λI)−1XTε}T\n= σ2(XTX + λI)−1XTX(XTX + λI)−1.\nThus E(ˆβOLS −β0)(ˆβOLS −β0)T −E(ˆβ −β0)(ˆβ −β0)T is equal to\nσ2(XTX)−1 −σ2(XTX + λI)−1XTX(XTX + λI)−1 −λ2(XTX + λI)−1β0β0T(XTX + λI)−1.\nAfter some simplification, we see that this is equal to\nλ(XTX + λI)−1[σ2{2I + λ(XTX)−1} −λβ0β0T](XTX + λI)−1.\nThus E(ˆβOLS −β0)(ˆβOLS −β0)T −E(ˆβ −β0)(ˆβ −β0)T is positive definite for λ > 0 if and\nonly if\nσ2{2I + λ(XTX)−1} −λβ0β0T\nis positive definite, which is true for λ > 0 sufficiently small (we can take 0 < λ <\n2σ2/∥β0∥2\n2).\n2\n\nThe theorem says that ˆβR\nλ outperforms ˆβOLS provided λ is chosen appropriately. To\nbe able to use ridge regression effectively, we need a way of selecting a good λ—we will\ncome to this very shortly. What the theorem doesn’t really tell us is in what situations\nwe expect ridge regression to perform well. To understand that, we will turn to one of the\nkey matrix decompositions used in statistics, the singular value decomposition (SVD).\n1.1.1\nThe singular value decomposition and principal compo-\nnents analysis\nThe singular value decomposition (SVD) is a generalisation of an eigendecomposition of a\nsquare matrix. We can factorise any X ∈Rn×p into its SVD\nX = UDV T.\nHere the U ∈Rn×n and V ∈Rp×p are orthogonal matrices and D ∈Rn×p has D11 ≥D22 ≥\n· · · ≥Dmm ≥0 where m := min(n, p) and all other entries of D are zero. To compute\nsuch a decomposition requires O(np min(n, p)) operations. The rth columns of U and V\nare known as the rth left and right singular vectors of X respectively, and Drr is the rth\nsingular value.\nWhen n > p, we can replace U by its first p columns and D by its first p rows to produce\nanother version of the SVD (sometimes known as the thin SVD). Then X = UDV T where\nU ∈Rn×p has orthonormal columns (but is no longer square) and D is square and diagonal.\nThere is an equivalent version for when p > n.\nLet us take X ∈Rn×p as our matrix of predictors and suppose n ≥p. Using the (thin)\nSVD we may write the fitted values from ridge regression as follows.\nX ˆβR\nλ = X(XTX + λI)−1XTY\n= UDV T(V D2V T + λI)−1V DU TY\n= UD(D2 + λI)−1DU TY\n=\np\nX\nj=1\nUj\nD2\njj\nD2\njj + λU T\nj Y.\nHere we have used the notation (that we shall use throughout the course) that Uj is the\njth column of U. For comparison, the fitted values from OLS (when X has full column\nrank) are\nX ˆβOLS = X(XTX)−1XTY = UU TY.\nBoth OLS and ridge regression compute the coordinates of Y with respect to the columns\nof U. Ridge regression then shrinks these coordinates by the factors D2\njj/(D2\njj + λ); if Djj\nis small, the amount of shrinkage will be larger.\nTo interpret this further, note that the SVD is intimately connected with Principal\nComponents Analysis (PCA). Consider v ∈Rp with ∥v∥2 = 1. Since the columns of X\n3",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.1 Ridge regression/1.1.1 The singular value decomposition and principal components analysis",
- "metadata": {
- "length": 4288,
- "summary": "The section titled '1.1.1 The singular value decomposition and principal components analysis' introduces the Singular Value Decomposition (SVD) as a generalization of eigendecomposition for any matrix X, defining its factorization into orthogonal matrices U and V and a diagonal matrix D containing singular values. It details the 'thin SVD' variant used when n ≥ p and applies this decomposition to derive the fitted values for ridge regression, demonstrating how it shrinks coordinates relative to Ordinary Least Squares (OLS). The text concludes by establishing the connection between SVD and Principal Components Analysis (PCA).",
- "page_nums": [
- 7,
- 8
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 7,
- "artifact_ref": "page_citation_assets/page-7.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_d04125f2-8d9d-58b3-b09f-384cfdb51fba",
- "type": "page",
- "content": "have had their means subtracted, the sample variance of Xv ∈Rn, is\n1\nnvTXTXv = 1\nnvTV D2V Tv.\nWriting a = V Tv, so ∥a∥2 = 1, we have\n1\nnvTV D2V Tv = 1\nnaTD2a = 1\nn\nX\nj\na2\njD2\njj ≤1\nnD11\nX\nj\na2\nj = 1\nnD2\n11.\nAs ∥XV1∥2\n2/n = D2\n11/n, V1 determines the linear combination of the columns of X which\nhas the largest sample variance, when the coefficients of the linear combination are con-\nstrained to have ℓ2-norm 1. XV1 = D11U1 is known as the first principal component of\nX. Subsequent principal components D22U2, . . . , DppUp have maximum variance D2\njj/n,\nsubject to being orthogonal to all earlier ones—see example sheet 1 for details.\nReturning to ridge regression, we see that it shrinks Y most in the smaller principal\ncomponents of X. Thus it will work well when most of the signal is in the large principal\ncomponents of X. We now turn to the problem of choosing λ.\n1.2\nv-fold cross-validation\nCross-validation is a general technique for selecting a good regression method from among\nseveral competing regression methods. We illustrate the principle with ridge regression,\nwhere we have a family of regression methods given by different λ values.\nSo far, we have considered the matrix of predictors X as fixed and non-random. How-\never, in many cases, it makes sense to think of it as random. Let us assume that our data\nare i.i.d. pairs (xi, Yi), i = 1, . . . , n. Then ideally, we might want to pick a λ value such\nthat\nE{(Y ∗−x∗T ˆβR\nλ (X, Y ))2|X, Y }\n(1.2.1)\nis minimised. Here (x∗, Y ∗) ∈Rp×R is independent of (X, Y ) and has the same distribution\nas (x1, Y1), and we have made the dependence of ˆβR\nλ on the training data (X, Y ) explicit.\nThis λ is such that conditional on the original training data, it minimises the expected\nprediction error on a new observation drawn from the same distribution as the training\ndata.\nA less ambitious goal is to find a λ value to minimise the expected prediction error,\nE[E{(Y ∗−x∗T ˆβR\nλ (X, Y ))2|X, Y }]\n(1.2.2)\nwhere compared with (1.2.1), we have taken a further expectation over the training set.\nWe still have no way of computing (1.2.2) directly, but we can attempt to estimate it.\nThe idea of v-fold cross-validation is to split the data into v groups or folds of roughly equal\nsize: (X(1), Y (1)), . . . , (X(v), Y (v)). Let (X(−k), Y (−k)) be all the data except that in the kth\nfold. For each λ on a grid of values, we compute ˆβR\nλ (X(−k), Y (−k)): the ridge regression\n4",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.2 v-fold cross-validation",
- "metadata": {
- "length": 2431,
- "summary": "This page continues a mathematical derivation regarding ridge regression and principal components, specifically focusing on how the method shrinks Y in smaller principal components. It then introduces section 1.2, 'v-fold cross-validation,' explaining it as a technique for selecting optimal regression parameters (lambda) by splitting data into groups to estimate prediction error.",
- "page_nums": [
- 9
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_3286f89e-b671-52f1-a518-e025cae55eed",
- "type": "page",
- "content": "estimate based on all the data except the kth fold. Writing κ(i) for the fold to which\n(xi, Yi) belongs, we choose the value of λ that minimises\nCV(λ) = 1\nn\nn\nX\ni=1\n{Yi −xT\ni ˆβR\nλ (X(−κ(i)), Y (−κ(i)))}2.\n(1.2.3)\nWriting λCV for the minimiser, our final estimate of β0 can then be ˆβR\nλCV(X, Y ).\nNote that for each i,\nE{Yi −xT\ni ˆβR\nλ (X(−κ(i)), Y (−κ(i)))}2 = E[E{Yi −xT\ni ˆβR\nλ (X(−κ(i)), Y (−κ(i)))}2|X(−κ(i)), Y (−κ(i))].\n(1.2.4)\nThis is precisely the expected prediction error in (1.2.2) but with the training data X, Y\nreplaced with a training data set of smaller size. If all the folds have the same size, then\nCV(λ) is an average of n identically distributed quantities, each with expected value as\nin (1.2.4). However, the quantities being averaged are not independent as they share the\nsame data.\nThus cross-validation gives a biased estimate of the expected prediction error. The\namount of the bias depends on the size of the folds, the case when the v = n giving the\nleast bias—this is known as leave-one-out cross-validation. The quality of the estimate,\nthough, may be worse as the quantities being averaged in (1.2.3) will be highly positively\ncorrelated. Typical choices of v are 5 or 10.\nCross-validation aims to allow us to choose the single best λ (or more generally regres-\nsion procedure); we could instead aim to find the best weighted combination of regression\nprocedures. Returning to our ridge regression example, suppose λ is restricted to a grid of\nvalues λ1 > λ2 > · · · > λL. We can then minimise\n1\nn\nn\nX\ni=1\n\u001a\nYi −\nL\nX\nl=1\nwlxT\ni ˆβR\nλl(X(−κ(i)), Y (−κ(i)))\n\u001b2\nover w ∈RL subject to wl ≥0 for all l. This is a non-negative least-squares optimisation,\nfor which efficient algorithms are available. This is sometimes known as stacking [Wolpert,\n1992, Breiman, 1996] and it can often outperform cross-validation.\n1.3\nThe kernel trick\nThe fitted values from ridge regression are\nX(XTX + λI)−1XTY.\n(1.3.1)\nAn alternative way of writing this is suggested by the following\nXT(XXT + λI) = (XTX + λI)XT\n(XTX + λI)−1XT = XT(XXT + λI)−1\nX(XTX + λI)−1XTY = XXT(XXT + λI)−1Y.\n(1.3.2)\nTwo remarks are in order:\n5\n\n Note while XTX is p × p, XXT is n × n. Computing fitted values using (1.3.1)\nwould require roughly O(np2 + p3) operations. If p ≫n this could be extremely\ncostly. However, our alternative formulation would only require roughly O(n2p + n3)\noperations, which could be substantially smaller.\n We see that the fitted values of ridge regression depend only on inner products\nK = XXT between observations (note Kij = xT\ni xj).\nNow suppose that we believe the signal depends quadratically on the predictors:\nYi = xT\ni β +\nX\nk,l\nxikxilθkl + εi.\nWe can still use ridge regression provided we work with an enlarged set of predictors\nxi1, . . . , xip, xi1xi1, . . . , xi1xip, xi2xi1, . . . , xi2xip, . . . , xipxip.\nThis will give us O(p2) predictors. Our new approach to computing fitted values would\ntherefore have complexity O(n2p2 + n3), which could be rather costly if p is large.\nHowever, rather than first creating all the additional predictors and then computing\nthe new K matrix, we can attempt to directly compute K. To this end consider\n(1 + xT\ni xj)2 =\n\u0012\n1 +\nX\nk\nxikxjk\n\u00132\n= 1 + 2\nX\nk\nxikxjk +\nX\nk,l\nxikxilxjkxjl.\nObserve this amounts to an inner product between vectors of the form\n(1,\n√\n2xi1, . . . ,\n√\n2xip, xi1xi1, . . . , xi1xip, xi2xi1, . . . , xi2xip, . . . , xipxip)T.\n(1.3.3)\nThus if we set\nKij = (1 + xT\ni xj)2\n(1.3.4)\nand plug this into the formula for the fitted values, it is exactly as if we had performed\nridge regression on an enlarged set of variables given by (1.3.3). Now computing K using\n(1.3.4) would require only p operations per entry, so O(n2p) operations in total. It thus\nseems we have improved things by a factor of p using our new approach. This is a nice\ncomputational trick, but more importantly for us it serves to illustrate some general points.\n Since ridge regression only depends on inner products between observations, rather\nthan fitting non-linear models by first mapping the original data xi ∈Rp to φ(xi) ∈Rd\n(say) using some feature map φ (which could, for example introduce quadratic effects),\nwe can instead try to directly compute k(xi, xj) = ⟨φ(xi), φ(xj)⟩.\n In fact rather than thinking in terms of feature maps, we can instead try to think\nabout an appropriate measure of similarity k(xi, xj) between observations. Modelling\nin this fashion is sometimes much easier.\nWe will now formalise and extend what we have learnt with this example.\n6\n\n1.4\nKernels\nWe have seen how a model with quadratic effects can be fitted very efficiently by replacing\nthe inner product matrix (known as the Gram matrix) XXT in (1.3.2) with the matrix\nin (1.3.4). It is then natural to ask what other non-linear models can be fitted efficiently\nusing this sort of approach.\nWe won’t answer this question directly, but instead we will try to understand the sorts\nof similarity measures k that can be represented as inner products between transformations\nof the original data.\nThat is, we will study the similarity measures k : X × X →R from the input space X\nto R for which there exists a feature map φ : X →H where H is some (real) inner product\nspace with\nk(x, x′) = ⟨φ(x), φ(x′)⟩.\n(1.4.1)\nRecall that an inner product space is a real vector space H endowed with a map ⟨·, ·⟩:\nH × H →R that obeys the following properties.\n(i) Symmetry: ⟨u, v⟩= ⟨v, u⟩.\n(ii) Linearity: for a, b ∈R ⟨au + bw, v⟩= a⟨u, v⟩+ b⟨w, v⟩.\n(iii) Positive-definiteness: ⟨u, u⟩≥0 with equality if and only if u = 0.\nDefinition 1. A positive definite kernel or more simply a kernel (for brevity) k is a\nsymmetric map k : X × X →R for which for all n ∈N and all x1, . . . , xn ∈X, the matrix\nK with entries\nKij = k(xi, xj)\nis positive semi-definite.\nA kernel is a little like an inner product, but need not be bilinear in general. However,\na form of the Cauchy–Schwarz inequality does hold for kernels.\nProposition 2.\nk(x, x′)2 ≤k(x, x)k(x′, x′).\nProof. The matrix\n\u0012k(x, x)\nk(x, x′)\nk(x′, x)\nk(x′, x′)\n\u0013\nmust be positive semi-definite so in particular its determinant must be non-negative.\nFirst we show that any inner product of feature maps will give rise to a kernel.\nProposition 3. k defined by k(x, x′) = ⟨φ(x), φ(x′)⟩is a kernel.\n7",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.3 The kernel trick",
- "metadata": {
- "length": 6243,
- "summary": "The section titled '1.3 The kernel trick' explains a computational method to efficiently handle non-linear models, specifically ridge regression with quadratic effects. It demonstrates that by reformulating the problem to depend only on inner products between observations (the Gram matrix), one can avoid explicitly computing a large number of new features. This allows for a significant reduction in computational complexity from O(n^2*p^2 + n^3) to O(n^2*p). The text concludes by introducing the general concept of using a similarity measure k(x_i, x_j) directly, which is formalized in the subsequent section '1.4 Kernels'.",
- "page_nums": [
- 10,
- 11,
- 12
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 12,
- "artifact_ref": "page_citation_assets/page-12.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_27991d01-535a-5983-8992-ca4d0ab30380",
- "type": "page",
- "content": "Proof. Let x1, . . . , xn ∈X, α1, . . . , αn ∈R and consider\nX\ni,j\nαik(xi, xj)αj =\nX\ni,j\nαi⟨φ(xi), φ(xj)⟩αj\n=\n\u001c X\ni\nαiφ(xi),\nX\nj\nαjφ(xj)\n\u001d\n≥0.\nShowing that every kernel admits a representation of the form (1.4.1) is slightly more\ninvolved, and we delay this until after we have studied some examples.\n1.4.1\nExamples of kernels\nProposition 4. Suppose k1, k2, . . . are kernels.\n(i) If α1, α2 ≥0 then α1k1 + α2k2 is a kernel. If limm→∞km(x, x′) =: k(x, x′) exists for\nall x, x′ ∈X, then k is a kernel.\n(ii) The pointwise product k = k1k2 is a kernel.\nLinear kernel.\nk(x, x′) = xTx′.\nPolynomial kernel.\nk(x, x′) = (1+xTx′)d. To show this is a kernel, we can simply note\nthat 1 + xTx′ gives a kernel owing to the fact that 1 is a kernel and (i) of Proposition 4.\nNext (ii) and induction shows that k as defined above is a kernel.\nGaussian kernel.\nThe highly popular Gaussian kernel is defined by\nk(x, x′) = exp\n\u0012\n−∥x −x′∥2\n2\n2σ2\n\u0013\n.\nFor x close to x′ it is large whilst for x far from x′ the kernel quickly decays towards 0.\nThe additional parameter σ2 known as the bandwidth controls the speed of the decay to\nzero. Note it is less clear how one might find a corresponding feature map and indeed any\nfeature map that represents this must be infinite dimensional.\nTo show that it is a kernel first decompose ∥x −x′∥2\n2 = ∥x∥2\n2 + ∥x′∥2\n2 −2xTx′. Note that\nby Proposition 3,\nk1(x, x′) = exp\n\u0012\n−∥x∥2\n2\n2σ2\n\u0013\nexp\n\u0012\n−∥x′∥2\n2\n2σ2\n\u0013\nis a kernel. Next writing\nk2(x, x′) = exp(xTx′/σ2) =\n∞\nX\nr=0\n(xTx′/σ2)r\nr!\nand using (i) of Proposition 4 shows that k2 is a kernel. Finally observing that k = k1k2\nand using (ii) shows that the Gaussian kernel is indeed a kernel.\n8",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.4 Kernels/1.4.1 Examples of kernels",
- "metadata": {
- "length": 1650,
- "summary": "This page presents a mathematical proof regarding the properties of kernels, followed by Section 1.4.1 which provides examples of specific kernel functions. It details Proposition 4 concerning linear combinations and products of kernels, then defines and explains Linear, Polynomial, and Gaussian kernels with their respective formulas and properties.",
- "page_nums": [
- 13
- ],
- "keywords": [
- "Proposition 4",
- "Proposition 3"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 13,
- "artifact_ref": "page_citation_assets/page-13.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_3f525a0b-6813-5175-a495-c88cfac7f649",
- "type": "page",
- "content": "Sobolev kernel.\nTake X to be [0, 1] and let k(x, x′) = min(x, x′).\nNote this is the\ncovariance function of Brownian motion so it must be positive definite.\nJaccard similarity kernel.\nTake X to be the set of all subsets of {1, . . . , p}.\nFor\nx, x′ ∈X with x ∪x′ ̸= ∅define\nk(x, x′) = |x ∩x′|\n|x ∪x′|\nand if x ∪x′ = ∅then set k(x, x′) = 1. Showing that this is a kernel is left to the example\nsheet.\n1.4.2\nReproducing kernel Hilbert spaces\nTheorem 5. For every kernel k there exists a feature map φ taking values in some inner\nproduct space H such that\nk(x, x′) = ⟨φ(x), φ(x′)⟩.\n(1.4.2)\nProof. We will take H to be the vector space of functions of the form\nf(·) =\nn\nX\ni=1\nαik(·, xi),\n(1.4.3)\nwhere n ∈N, xi ∈X and αi ∈R. Our feature map φ : X →H will be\nφ(x) = k(·, x).\n(1.4.4)\nWe now define an inner product on H. If f is given by (1.4.3) and\ng(·) =\nm\nX\nj=1\nβjk(·, x′\nj)\n(1.4.5)\nwe define their inner product to be\n⟨f, g⟩=\nn\nX\ni=1\nm\nX\nj=1\nαiβjk(xi, x′\nj).\n(1.4.6)\nWe need to check this is well-defined as the representations of f and g in (1.4.3) and\n(1.4.5) need not be unique. To this end, note that\nn\nX\ni=1\nm\nX\nj=1\nαiβjk(xi, x′\nj) =\nn\nX\ni=1\nαig(xi) =\nm\nX\nj=1\nβjf(x′\nj).\n(1.4.7)\nThe first equality shows that the inner product does not depend on the particular expansion\nof g whilst the second equality shows that it also does not depend on the expansion of f.\nThus the inner product is well-defined.\n9\n\nFirst we check that with φ defined as in (1.4.4) we do have relationship (1.4.2). Observe\nthat\n⟨k(·, x), f⟩=\nn\nX\ni=1\nαik(xi, x) = f(x),\n(1.4.8)\nso in particular we have\n⟨φ(x), φ(x′)⟩= ⟨k(·, x), k(·, x′)⟩= k(x, x′).\nIt remains to show that it is indeed an inner product. It is clearly symmetric and (1.4.7)\nshows linearity. We now need to show positive definiteness.\nFirst note that\n⟨f, f⟩=\nX\ni,j\nαik(xi, xj)αj ≥0\n(1.4.9)\nby positive definiteness of the kernel. Now from (1.4.8),\nf(x)2 = (⟨k(·, x), f⟩)2.\nIf we could use the Cauchy–Schwarz inequality on the right-hand side, we would have\nf(x)2 ≤⟨k(·, x), k(·, x)⟩⟨f, f⟩,\n(1.4.10)\nwhich would show that if ⟨f, f⟩= 0 then necessarily f = 0; the final property we need\nto show that ⟨·, ·⟩is an inner product. However, in order to use the traditional Cauchy–\nSchwarz inequality we need to first know we’re dealing with an inner product, which is\nprecisely what we’re trying to show!\nAlthough we haven’t shown that ⟨·, ·⟩is an inner product, we do have enough infor-\nmation to show that it is itself a kernel. We may then appeal to Proposition 2 to obtain\n(1.4.10). With this in mind, we argue as follows. Given functions f1, . . . , fm and coefficients\nγ1, . . . , γm ∈R, we have\nX\ni,j\nγi⟨fi, fj⟩γj =\n\u001c X\ni\nγifi,\nX\nj\nγjfj\n\u001d\n≥0\nwhere we have used linearity and (1.4.9), showing that it is a kernel.\nTo further discuss the space H we recall some facts from analysis. Any inner product\nspace B is also a normed space: for f ∈B we may define ∥f∥2\nB := ⟨f, f⟩B. Recall that a\nCauchy sequence (fm)∞\nm=1 in B has ∥fm −fn∥B →0 as n, m →∞. A normed space where\nevery Cauchy sequence has a limit (in the space) is called complete, and a complete inner\nproduct space is called a Hilbert space.\nHilbert spaces may be thought of as the (potentially) infinite-dimensional analogues of\nfinite-dimensional Euclidean spaces. For later use we note that if V is a closed subspace\nof a Hilbert space B, then any f ∈B has a decomposition f = u + v with u ∈V and\nv ∈V ⊥:= {v ∈B : ⟨v, u⟩B = 0 for all u ∈V }.\n10\n\nBy adding the limits of Cauchy sequences to H (from Theorem 5) we can make H a\nHilbert space. Indeed, note that if (fm)∞\nm=1 ∈H is Cauchy then since by (1.4.10) we have\n|fm(x) −fn(x)| ≤\np\nk(x, x)∥fm −fn∥H,\nwe may define function f ∗: X →R by f ∗(x) = limm→∞fm(x). We can check that all such\nf ∗can be added to H to create a Hilbert space.\nIn fact, the completion of H is a special type of Hilbert space known as a reproducing\nkernel Hilbert space (RKHS).\nDefinition 2. A Hilbert space B of functions f : X →R is a reproducing kernel Hilbert\nspace (RKHS) if for all x ∈X, there exists kx ∈B such that\nf(x) = ⟨kx, f⟩\nfor all f ∈B.\nThe function\nk : X × X →R\n(x, x′) 7→⟨kx, kx′⟩= kx′(x)\nis known as the reproducing kernel of B.\nBy Proposition 3 the reproducing kernel of any RKHS is a (positive definite) kernel,\nand one may show (as Theorem 5 indicates) that to any kernel k is associated a (unique)\nRKHS that has reproducing kernel k.\nExamples\nLinear kernel.\nHere H = {f : f(x) = βTx, β ∈Rp} and if f(x) = βTx then ∥f∥2\nH =\n∥β∥2\n2.\nSobolev kernel.\nIt can be shown that H is roughly the space of continuous functions\nf : [0, 1] →R with f(0) = 0 that are differentiable almost everywhere, and for which\nR 1\n0 f ′(x)2dx < ∞. It contains the class of Lipschitz functions (functions f : [0, 1] →R for\nwhich there exists some L with |f(x) −f(y)| ≤L|x −y| for all x, y ∈[0, 1]) that are 0 at\nthe origin. The norm is\n\u0012 Z 1\n0\nf ′(x)2dx\n\u00131/2\n.\nThough the construction of the RKHS from a kernel is explicit, it can be challenging to\nunderstand precisely the space and the form of the norm.\n11",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.4 Kernels/1.4.2 Reproducing kernel Hilbert spaces",
- "metadata": {
- "length": 5000,
- "summary": "This section defines Reproducing Kernel Hilbert Spaces (RKHS). It begins with Theorem 5, which states that for every kernel k, there exists a feature map φ into an inner product space H such that the kernel can be expressed as an inner product of the mapped features. A proof is provided by constructing the space H from functions formed by linear combinations of the kernel and defining an inner product on it. The text then discusses the completion of this space to form a Hilbert space and provides Definition 2 for an RKHS, where evaluation at any point x is a continuous linear functional represented by an inner product with a specific function k_x. Finally, examples of RKHS are given for the Linear kernel and the Sobolev kernel.",
- "page_nums": [
- 14,
- 15,
- 16
- ],
- "keywords": [
- "Theorem 5",
- "Definition 2"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 14,
- "artifact_ref": "page_citation_assets/page-14.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 15,
- "artifact_ref": "page_citation_assets/page-15.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 16,
- "artifact_ref": "page_citation_assets/page-16.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_fc5fac7c-26e0-54ec-abea-d0950155d5c1",
- "type": "page",
- "content": "1.4.3\nThe representer theorem\nTo recap, what we have shown so far is that replacing the matrix XXT in the definition of\nan algorithm by K derived form a positive definite kernel is essentially equivalent to running\nthe same algorithm on some mapping of the original data, though with the modification\nthat instances of xT\ni xj become ⟨φ(xi), φ(xj)⟩.\nBut what exactly is the optimisation problem we are solving when performing kernel\nridge regression? Clearly it is determined by the kernel or equivalently by the RKHS. Note\nwe know that an alternative way of writing the usual ridge regression optimisation is\narg min\nf∈H\n\u001a\nn\nX\ni=1\n{Yi −f(xi)}2 + λ∥f∥2\nH\n\u001b\n(1.4.11)\nwhere H is the RKHS corresponding to the linear kernel. The following theorem shows in\nparticular that kernel ridge regression (i.e. ridge regression replacing XXT with K) with\nkernel k is equivalent to the above with H now being the RKHS corresponding to k.\nTheorem 6 (Representer theorem, [Kimeldorf and Wahba, 1970, Sch¨olkopf et al., 2001]).\nLet c : Rn × X n × Rn →R be an arbitrary loss function, and let J : [0, ∞) →R be strictly\nincreasing. Let x1, . . . , xn ∈X, Y ∈Rn. Finally, let f ∈H where H is an RKHS with\nreproducing kernel k, and let Kij = k(xi, xj) i, j = 1, . . . , n. Then ˆf minimises\nQ1(f) := c(Y, x1, . . . , xn, f(x1), . . . , f(xn)) + J(∥f∥2\nH)\nover f ∈H iff. ˆf(·) = Pn\ni=1 ˆαik(·, xi) and ˆα ∈Rn minimises Q2 over α ∈Rn where\nQ2(α) = c(Y, x1, . . . , xn, Kα) + J(αTKα).\nProof. Suppose ˆf minimises Q1. We may write ˆf = u+v where u ∈V := span{k(·, x1), . . . , k(·, xn)}\nand v ∈V ⊥. Then\nˆf(xi) = ⟨k(·, xi), u + v⟩= ⟨k(·, xi), u⟩= u(xi).\nMeanwhile, by Pythagoras’ theorem we have J(∥ˆf∥2\nH) = J(∥u∥2\nH + ∥v∥2\nH) ≥J(∥u∥2\nH) with\nequality iff. v = 0. Thus by optimality of ˆf, v = 0, so ˆf(·) = Pn\ni=1 αik(·, xi) for α ∈Rn.\nNow observe that if ˆf takes this form, then ∥ˆf∥2\nH = αTKα, so Q1( ˆf) = Q2(α). Then by\noptimality of ˆf, we have that α must minimise Q2.\nNow suppose ˆα minimises Q2 and ˆf(·) = Pn\ni=1 ˆαik(·, xi). Note that Q1( ˆf) = Q2(ˆα).\nIf ˜f ∈H has Q1( ˜f) ≤Q1( ˆf), by the argument above, writing ˜f = u + v with u ∈V ,\nv ∈V ⊥, we know that Q1(u) ≤Q1( ˜f). But by optimality of ˆα we have Q1( ˆf) ≤Q1(u), so\nQ1( ˆf) = Q1( ˜f).\nConsider the result specialised the ridge regression objective. We see that (1.4.11) is\nessentially equivalent to minimising\n∥Y −Kα∥2\n2 + λαTKα,\n12",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.4 Kernels/1.4.3 The representer theorem",
- "metadata": {
- "length": 2376,
- "summary": "This page presents section 1.4.3, titled 'The representer theorem,' which discusses the optimization problem in kernel ridge regression. It defines the objective function for ridge regression and states Theorem 6, attributed to Kimeldorf and Wahba (1970) and Schölkopf et al. (2001). The theorem establishes that minimizing a loss function over a Reproducing Kernel Hilbert Space (RKHS) is equivalent to minimizing a finite-dimensional quadratic form involving the kernel matrix. The page includes a detailed proof of this theorem using Pythagoras' theorem and concludes by applying the result to the specific case of the ridge regression objective.",
- "page_nums": [
- 17
- ],
- "keywords": [
- "Kimeldorf",
- "Wahba",
- "Schölkopf"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c3163aef-6474-592a-b8a6-76c41d67a7f6",
- "type": "page",
- "content": "and you may check (see example sheet 1) that the minimiser ˆα satisfies K ˆα = K(K +\nλI)−1Y . Thus (1.4.11) is indeed an alternative way of expressing kernel ridge regression.\nViewing the result in the opposite direction gives a more “sensational” perspective. If\nyou had set out trying to minimise Q1, it might appear completely hopeless as H could be\ninfinite-dimensional. However, somewhat remarkably we see that this reduces to finding\nthe coefficients ˆαi which solve the simple(r) optimisation problem Q2.\nThe result also tells us how to form predictions: given a new observation x, our predic-\ntion for f(x) is\nˆf(x) =\nn\nX\ni=1\nˆαik(x, xi).\n1.5\nKernel ridge regression\nWe have seen how the kernel trick allows us to solve a potentially infinite-dimensional\nversion of ridge regression. This may seem impressive, but ultimately we should judge\nkernel ridge regression on its statistical properties e.g. predictive performance. Consider a\nsetting where\nYi = f 0(xi) + εi,\nE(ε) = 0, Var(ε) = σ2I.\nWe shall assume that f 0 ∈H where H is an RKHS with reproducing kernel k. By scaling\nσ2, we may assume ∥f 0∥H ≤1. Let K be the kernel matrix Kij = k(xi, xj) with eigenvalues\nd1 ≥d2 ≥· · · ≥dn ≥0. We will see that the predictive performance depends delicately\non these eigenvalues.\nLet ˆfλ be the estimated regression function from kernel ridge regression with kernel k:\nˆfλ = arg min\nf∈H\n\u001a\nn\nX\ni=1\n{Yi −f(xi)}2 + λ∥f∥2\nH\n\u001b\n.\nTheorem 7. The mean squared prediction error (MSPE) may be bounded above in the\nfollowing way:\n1\nnE\n\u001a\nn\nX\ni=1\n{f 0(xi) −ˆfλ(xi)}2\n\u001b\n≤σ2\nn\nn\nX\ni=1\nd2\ni\n(di + λ)2 + λ\n4n\n(1.5.1)\n≤σ2\nn\n1\nλ\nn\nX\ni=1\nmin(di/4, λ) + λ\n4n.\nProof. We know from the representer theorem that\n\u0010\nˆfλ(x1), . . . , ˆfλ(xn)\n\u0011T\n= K(K + λI)−1Y.\nYou will show on the example sheet that\n\u0010\nf 0(x1), . . . , f 0(xn)\n\u0011T\n= Kα,\n13\n\nfor some α ∈Rn, and moreover that ∥f 0∥2\nH ≥αTKα. Let the eigendecomposition of K be\ngiven by K = UDU T with Dii = di and define θ = U TKα. We see that n times the LHS\nof (1.5.1) is\nE∥K(K + λI)−1(Uθ + ε) −Uθ∥2\n2 = E∥DU T(UDU T + λI)−1(Uθ + ε) −θ∥2\n2\n= E∥D(D + λI)−1(θ + U Tε) −θ∥2\n2\n= ∥{D(D + λI)−1 −I}θ∥2\n2 + E∥D(D + λI)−1U Tε∥2\n2.\nTo compute the second term, we use the ‘trace trick’:\nE∥D(D + λI)−1U Tε∥2\n2 = E[{D(D + λI)−1U Tε}TD(D + λI)−1U Tε]\n= E[tr{D(D + λI)−1U TεεTUD(D + λI)−1}]\n= σ2tr{D(D + λI)−1D(D + λI)−1}\n= σ2\nn\nX\ni=1\nd2\ni\n(di + λ)2.\nFor the first term, we have\n∥{D(D + λI)−1 −I}θ∥2\n2 =\nn\nX\ni=1\nλ2θ2\ni\n(di + λ)2.\nNow as θ = DU Tα note that θi = 0 when di = 0. Let D+ be the diagonal matrix with ith\ndiagonal entry equal to D−1\nii\nif Dii > 0 and 0 otherwise. Then\nX\ni:di>0\nθ2\ni\ndi\n= ∥\n√\nD+θ∥2\n2 = αTKUD+U TKα = αTUDD+DU Tα = αTKα ≤1.\nBy H¨older’s inequality we have\nn\nX\ni=1\nθ2\ni\ndi\ndiλ2\n(di + λ)2 ≤max\ni=1,...,n\ndiλ2\n(di + λ)2 ≤λ/4,\nusing the inequality (a + b)2 ≥4ab in the final line. Finally note that\nd2\ni\n(di + λ)2 ≤min{1, d2\ni /(4diλ)} = min(λ, di/4)/λ.\nTo interpret this result further, it will be helpful to express it in terms of ˆµi := di/n\n(the eigenvalues of K/n) and λn := λ/n. We have\n1\nnE\n\u001a\nn\nX\ni=1\n{f 0(xi) −ˆfλ(xi)}2\n\u001b\n≤σ2\nλn\n1\nn\nn\nX\ni=1\nmin(ˆµi/4, λn) + λn/4 =: δn(λn).\n(1.5.2)\n14\n\nHere we have treated the xi as fixed, but we could equally well think of them as\nrandom.\nConsider a setup where the xi are i.i.d. and independent of ε.\nIf we take a\nfurther expectation on the RHS of (1.5.2), our result still holds true (the ˆµi are random in\nthis setting). Ideally we would like to then replace E min(ˆµi/4, λn) with a quantity more\ndirectly related to the kernel k.\nMercer’s theorem is helpful in this regard. This guarantees (under some mild conditions)\nan eigendecomposition for kernels, which are somewhat like infinite-dimensional analogues\nof symmetric positive semi-definite matrices. Under certain technical conditions, we may\nwrite\nk(x, x′) =\n∞\nX\nj=1\nµjej(x)ej(x′)\nwhere given some density p(x) on X, the eigenfunctions ej and corresponding eigenvalues\nµj obey the integral equation\nµjej(x′) =\nZ\nX\nk(x, x′)ej(x)p(x)dx,\nand the ej form an orthonormal basis of H in the sense that\nZ\nX\nei(x)ej(x)p(x)dx = 1{i=j}.\nOne can further show that (ignoring a multiplicative constant)\nE\n\u00121\nn\nn\nX\ni=1\nmin(ˆµi/4, λn)\n\u0013\n≤1\nn\n∞\nX\ni=1\nmin(µi/4, λn).\nWhen k is the Sobolev kernel and p(x) is the uniform density on [0, 1], we find the eigen-\nvalues satisfy\nµj/4 =\n1\nπ2(2j −1)2.\nThus\n∞\nX\ni=1\nmin(µi/4, λn) ≤λn\n2\n\u0012\n1\n√π2λn\n+ 1\n\u0013\n+ 1\nπ2\nZ ∞\n{(π2λn)−1/2+1}/2\n1\n(2x −1)2dx\n=\np\nλn/π + λn/2 = O(\np\nλn)\nas λn →0. Putting things together, we see that\nE(δn(λn)) = O\n\u0012 σ2\nnλ1/2\nn\n+ λn\n\u0013\n.\nThus an optimal λn ∼(σ2/n)2/3 gives an error rate of order (σ2/n)2/3.\n15",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.5 Kernel ridge regression",
- "metadata": {
- "length": 4572,
- "summary": "This section introduces kernel ridge regression, a method that uses the kernel trick to solve potentially infinite-dimensional versions of ridge regression. It establishes a statistical framework for analyzing predictive performance by defining a data-generating process and assuming the true function lies within a Reproducing Kernel Hilbert Space (RKHS). The core of the section is Theorem 7, which provides an upper bound on the Mean Squared Prediction Error (MSPE) in terms of the eigenvalues of the kernel matrix. A detailed proof follows, utilizing techniques like the 'trace trick' and Hölder's inequality. The analysis is then extended to a random design setting using Mercer's theorem to describe the eigendecomposition of the kernel operator. Finally, it derives the optimal convergence rate for the error when using a Sobolev kernel.",
- "page_nums": [
- 18,
- 19,
- 20
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 18,
- "artifact_ref": "page_citation_assets/page-18.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 19,
- "artifact_ref": "page_citation_assets/page-19.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 20,
- "artifact_ref": "page_citation_assets/page-20.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_fc492514-7db9-50d2-8938-fd68fbf71fbd",
- "type": "page",
- "content": "In fact, one can show that this is the best error rate one can achieve with any estimator\nin this problem. More generally Yang et al. [2015] shows that for essentially any RKHS H\nwe have\ninf\nˆf\nsup\nf0:∥f0∥H≤1\nE\n\u001a1\nn\nn\nX\ni=1\n{f 0(xi) −ˆf(xi)}2\n\u001b\n≥c inf\nλn δn(λn)\nwhere c > 0 is a constant and ˆf is allowed to range over all (measurable) functions of\nthe data Y, X. The conclusion is that kernel ridge regression is the optimal regression\nprocedure up to a constant factor in terms of MSPE when the true signal f 0 is from an\nRKHS.\n1.6\nOther kernel machines\nThus far we have we have only considered applying the kernel trick to ridge regression,\nwhich as we have seen has attractive theoretical properties as a regression method. However\nthe kernel trick and the representer theorem are much more generally applicable. In settings\nwhere the Yi are not continuous but are in {−1, 1} (e.g. labels for spam and ham, fraud\nand not fraud etc.), popular approaches include kernel logistic regression and the support\nvector machine (SVM) [Cortes and Vapnik, 1995].\n1.6.1\nThe support vector machine\nConsider first the simple case where the data in the two classes {xi}i:Yi=1 and {xi}i:Yi=−1\nare separable by a hyperplane through the origin, so there exists β ∈Rp with ∥β∥2 = 1\nsuch that YiβTxi > 0 for all i. Note β would then be a unit normal vector to a plane that\nseparates the two classes.\n16\n\nThere may be an infinite number of planes that separate the classes, in which case\nit seems sensible to use the plane that maximises the margin between the two classes.\nConsider therefore the following optimisation problem.\nmax\nβ∈Rp,M≥0 M\nsubject to YixT\ni β/∥β∥2 ≥M, i = 1, . . . , n.\nNote that by normalising β above we need not impose the constraint that ∥β∥2 = 1.\nSuppose now that the classes are not separable. One way to handle this is to replace\nthe constraint YixT\ni β/∥β∥2 ≥M with a penalty for how far over the margin boundary\nxi is. This penalty should be zero if xi is on the correct side of the boundary (i.e. when\nYixT\ni β/∥β∥2 ≥M), and should be equal to the distance over the boundary, M−YixT\ni β/∥β∥2\notherwise. It will in fact be more convenient to penalise according to 1 −YixT\ni β/(∥β∥2M)\nin the latter case, which is the distance measured in units of M. This penalty is invariant\nto β undergoing any positive scaling, so we may set ∥β∥2 = 1/M, thus eliminating M from\nthe objective function. Switching max 1/∥β∥2 with min ∥β∥2\n2 and adding the penalty we\narrive at\narg min\nβ∈Rp\n∥β∥2\n2 + λ\nn\nX\ni=1\n(1 −YixT\ni β)+,\nwhere (·)+ denotes the positive part. Replacing λ with 1/λ we can write the objective in\nthe more familiar-looking form\narg min\nβ∈Rp\nn\nX\ni=1\n(1 −YixT\ni β)+ + λ∥β∥2\n2.\nThus far we have restricted ourselves to hyperplanes through the origin but we would more\ngenerally want to consider any translate of these i.e. any hyperplane. This can be achieved\nby allowing ourselves to translate the xi by an arbitrary vector b, giving\narg min\nβ∈Rp,b∈Rp\nn\nX\ni=1\n(1 −Yi(xi −b)Tβ)+ + λ∥β∥2\n2,\nor equivalently\n(ˆµ, ˆβ) = arg min\n(µ,β)∈R×Rp\nn\nX\ni=1\n{1 −Yi(xT\ni β + µ)}+ + λ∥β∥2\n2.\n(1.6.1)\nThis final objective defines the support vector classifier; given a new observation x predic-\ntions are obtained via sgn(ˆµ + xT ˆβ).\nNote that the objective in (1.6.1) may be re-written as\n(ˆµ, ˆf) = arg min\n(µ,f)∈R×H\nn\nX\ni=1\n[1 −Yi{f(xi) + µ}]+ + λ∥f∥2\nH,\n(1.6.2)\n17",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.6 Other kernel machines/1.6.1 The support vector machine",
- "metadata": {
- "length": 3359,
- "summary": "This section introduces the support vector machine (SVM) as a method for classification when data labels are discrete (e.g., {-1, 1}). It begins by defining the problem of finding a hyperplane that separates two classes. The text then formulates an optimization problem to find the hyperplane that maximizes the margin between these classes. It further extends this formulation to handle cases where classes are not perfectly separable by introducing a penalty term, leading to the standard SVM objective function. Finally, it generalizes the model to include a bias term (or translation), resulting in the final definition of the support vector classifier.",
- "page_nums": [
- 21,
- 22
- ],
- "keywords": [
- "Cortes",
- "Vapnik"
- ],
- "connect_to": [
- {
- "target": "asset_6411b02f-eafc-5539-b1de-aeb1ced89150",
- "relation": "embeds",
- "ref": "[images/image_page_21_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 21,
- "artifact_ref": "page_citation_assets/page-21.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 22,
- "artifact_ref": "page_citation_assets/page-22.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_7ad002ad-6698-52f3-b6a5-ec6a2aadb112",
- "type": "page",
- "content": "where H is the RKHS corresponding to the linear kernel. The representer theorem (more\nspecifically the variant in question 10 of example sheet 1) shows that (1.6.2) for an arbitrary\nRKHS with kernel k and kernel matrix K is equivalent to the support vector machine\n(ˆµ, ˆα) = arg min\n(µ,α)∈R×Rn\nn\nX\ni=1\n[1 −Yi{KT\ni α + µ}]+ + λαTKα.\nPredictions at a new x are given by\nsgn\n\u0012\nˆµ +\nn\nX\ni=1\nˆαik(x, xi)\n\u0013\n.\n1.6.2\nLogistic regression\nRecall that standard logistic regression may be motivated by assuming\nlog\n\u0012 P(Yi = 1)\nP(Yi = −1)\n\u0013\n= xT\ni β0\nand picking ˆβ to maximise the log-likelihood. This leads to (see example sheet) the fol-\nlowing optimisation problem:\narg min\nβ∈Rp\nn\nX\ni=1\nlog{1 + exp(−YixT\ni β)}.\nThe ‘kernelised’ version is given by\narg min\nf∈H\n\u001a\nn\nX\ni=1\nlog[1 + exp{−Yif(xi)}] + λ∥f∥2\nH\n\u001b\n,\nwhere H is an RKHS. As in the case of the SVM, the representer theorem gives a finite-\ndimensional optimisation that is equivalent to the above.\n18",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.6 Other kernel machines/1.6.2 Logistic regression",
- "metadata": {
- "length": 945,
- "summary": "This page continues a discussion on Support Vector Machines (SVM) and introduces Logistic Regression. It presents the optimization problem for an SVM using a kernel matrix K, followed by the prediction formula. Section 1.6.2 details standard logistic regression, deriving its log-likelihood maximization problem and presenting its 'kernelised' version as an optimization over a Reproducing Kernel Hilbert Space (RKHS).",
- "page_nums": [
- 23
- ],
- "keywords": [
- "support vector machine",
- "Logistic regression"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 23,
- "artifact_ref": "page_citation_assets/page-23.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_0399168a-8276-58eb-b56c-a26a52194096",
- "type": "page",
- "content": "1.7\nLarge-scale kernel machines\nWe introduced the kernel trick as a computational device that avoided performing cal-\nculations in a high or infinite dimensional feature space and, in the case of kernel ridge\nregression reduced computation down to forming the n × n matrix K and then inverting\nK + λI. This can be a huge saving, but when n is very large, this can present serious\ncomputational difficulties. Even if p is small, the O(n3) cost of inverting K +λI may cause\nproblems. What’s worse, the fitted regression function is a sum over n terms:\nˆf(·) =\nn\nX\ni=1\nˆαik(xi, ·).\nEven to evaluate a prediction at a single new observation requires O(n) computations\nunless ˆα is sparse.\nIn recent years, there has been great interest in speeding up computations for kernel\nmachines. We will discuss one exciting approach based on random feature expansions.\nGiven a kernel k, the key idea is to develop a random map\nˆφ : X →Rb\nwith b small such that E{ˆφ(x)T ˆφ(x′)} = k(x, x′). In a sense we are trying to reverse the\nkernel trick by approximating the kernel using a random feature map. To increase the\nquality of the approximation of the kernel, we can consider\nx 7→\n1\n√\nL\n(ˆφ1(x), . . . , ˆφL(x)) ∈RLb\nwith each (ˆφl(x))L\nl=1 being i.i.d. for each x. Let Φ be the matrix with ith row given by\n(ˆφ1(xi), . . . , ˆφL(xi))/\n√\nL. We may then run our learning algorithm replacing the initial\nmatrix of predictors X with Φ. For example, when performing ridge regression, we can\ncompute\n(ΦTΦ + λI)−1ΦTY,\nwhich would require O(nL2b2 + L3b3) operations: a cost linear in n. Predicting a new\nobservation would cost O(Lb).\nThe work of Rahimi and Recht [2007] proposes a construction of such a random map-\nping ˆφ for shift-invariant kernels, that is kernels for which there exists a function h with\nk(x, x′) = h(x −x′) for all x, x′ ∈X = Rp. A useful property of such kernels is given by\nBochner’s theorem.\nTheorem 8 (Bochner’s theorem). Let k : Rp × Rp →R be a continuous kernel. Then k\nis shift-invariant if and only if there exists some c > 0 and distribution F on Rp such that\nwhen W ∼F\nk(x, x′) = cEei(x−x′)T W = cE cos((x −x′)TW).\n19\n\nTo make use of this theorem, first observe the following. Let u ∼U[−π, π], x, y ∈R.\nThen\n2E cos(x + u) cos(y + u) = 2E{(cos x cos u −sin x sin u)(cos y cos u −sin y sin u)}.\nNow as u\nd= −u, E cos u sin u = E cos(−u) sin(−u) = −E cos u sin u = 0. Also of course\ncos2 u + sin2 u = 1 so E cos2 u = E sin2 u = 1/2. Thus\n2E cos(x + u) cos(y + u) = cos x cos y + sin x sin y = cos(x −y).\nGiven a shift-invariant kernel k with associated distribution F, suppose W ∼F and\nlet u ∼U[−π, π] independently. Define\nˆφ(x) =\n√\n2c cos(W Tx + u).\nThen\nEˆφ(x)ˆφ(x′) = 2cE[E{cos(W Tx + u) cos(W Tx′ + u)|W}]\n= cE cos((x −x′)TW) = k(x, x′).\nAs a concrete example of this approach, let us take the Gaussian kernel k(x, x′) = exp{−∥x−\nx′∥2\n2/(2σ2)}.\nNote that if W ∼N(0, σ−2I), it has characteristic function E(eitT W) =\ne−∥t∥2\n2/(2σ2) so we may take ˆφ(x) =\n√\n2 cos(W Tx + u).\n20",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.7 Large-scale kernel machines",
- "metadata": {
- "length": 2982,
- "summary": "This section discusses methods for speeding up computations in large-scale kernel machines. It introduces the concept of random feature expansions to approximate kernels, thereby reducing computational complexity from cubic to linear in the number of data points. The text details how shift-invariant kernels can be approximated using Bochner's theorem and provides a concrete example using the Gaussian kernel.",
- "page_nums": [
- 24,
- 25
- ],
- "keywords": [
- "Rahimi",
- "Recht"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 24,
- "artifact_ref": "page_citation_assets/page-24.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 25,
- "artifact_ref": "page_citation_assets/page-25.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_88a2c60b-e79b-5e39-ac38-bbbcf6f8d511",
- "type": "page",
- "content": "Chapter 2\nIntroduction to the Lasso\n2.1\nModel selection\nLet us revisit the linear model Y = Xβ0 + ε where E(ε) = 0, Var(ε) = σ2I. In many\nmodern datasets, there are reasons to believe there are many more variables present than\nare necessary to explain the response. Let S be the set S = {k : β0\nk ̸= 0} and suppose\ns := |S| ≪p.\nThe MSPE of OLS is\n1\nnE∥Xβ0 −X ˆβOLS∥2\n2 = 1\nnE{(β0 −ˆβOLS)TXTX(β0 −ˆβOLS)}\n= 1\nnE[tr{(β0 −ˆβOLS)(β0 −ˆβOLS)TXTX}]\n= 1\nntr[E{(β0 −ˆβOLS)(β0 −ˆβOLS)T}XTX]\n= 1\nntr(Var(ˆβOLS)XTX) = p\nnσ2.\nIf we could identify S and then fit a linear model using just these variables, we’d obtain\nan MSPE of σ2s/n which could be substantially smaller than σ2p/n. Furthermore, it can\nbe shown that parameter estimates from the reduced model are more accurate. The smaller\nmodel would also be easier to interpret.\nWe now briefly review some classical model selection strategies.\nBest subset regression\nA natural approach to finding S is to consider all 2p possible regression procedures each\ninvolving regressing the response on a different sets of explanatory variables XM where\nM is a subset of {1, . . . , p}. We can then pick the best regression procedure using cross-\nvalidation (say). For general design matrices, this involves an exhaustive search over all\nsubsets, so this is not really feasible for p > 50.\n21\n\nForward selection\nThis can be seen as a greedy way of performing best subsets regression. Given a target\nmodel size m (the tuning parameter), this works as follows.\n1. Start by fitting an intercept only model.\n2. Add to the current model the predictor variable that reduces the residual sum of\nsquares the most.\n3. Continue step 2 until m predictor variables have been selected.\n2.2\nThe Lasso estimator\nThe Least absolute shrinkage and selection operator (Lasso) [Tibshirani, 1996] estimates\nβ0 by ˆβL\nλ, where (ˆµL, ˆβL\nλ) minimise\n1\n2n∥Y −µ1 −Xβ∥2\n2 + λ∥β∥1\n(2.2.1)\nover (µ, β) ∈R × Rp. Here ∥β∥1 is the ℓ1-norm of β: ∥β∥1 = Pp\nk=1 |βk|.\nLike ridge regression, ˆβL\nλ shrinks the OLS estimate towards the origin, but there is\nan important difference. The ℓ1 penalty can force some of the estimated coefficients to be\nexactly 0. In this way the Lasso can perform simultaneous variable selection and parameter\nestimation. As we did with ridge regression, we can centre and scale the X matrix, and\nalso centre Y and thus remove µ from the objective. Define\nQλ(β) = 1\n2n∥Y −Xβ∥2\n2 + λ∥β∥1.\n(2.2.2)\nNow the minimiser(s) of Qλ(β) will also be the minimiser(s) of\n∥Y −Xβ∥2\n2 subject to ∥β∥1 ≤∥ˆβL\nλ∥1.\nSimilarly, with the Ridge regression objective, we know that ˆβR\nλ minimises ∥Y −Xβ∥2\n2\nsubject to ∥β∥2 ≤∥ˆβR\nλ ∥2.\nNow the contours of the OLS objective ∥Y −Xβ∥2\n2 are ellipsoids centred at ˆβOLS,\nwhile the contours of ∥β∥2\n2 are spheres centred at the origin, and the contours of ∥β∥1 are\n‘diamonds’ centred at 0.\nThe important point to note is that the ℓ1 ball {β ∈Rp : ∥β∥1 ≤∥ˆβL\nλ∥1} has corners\nwhere some of the components are zero, and it is likely that the OLS contours will intersect\nthe ℓ1 ball at such a corner.\n22",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.1 Model selection",
- "metadata": {
- "length": 3039,
- "summary": "The section '2.1 Model selection' revisits the linear model and discusses the Mean Squared Prediction Error (MSPE) of Ordinary Least Squares (OLS). It explains that if the true set of relevant variables could be identified, a reduced model would yield a smaller MSPE and more accurate estimates. The text then introduces classical model selection strategies, starting with 'Best subset regression', which involves an exhaustive search over all possible subsets of explanatory variables. This method is noted to be computationally infeasible for large numbers of predictors (p > 50).",
- "page_nums": [
- 26,
- 27
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 26,
- "artifact_ref": "page_citation_assets/page-26.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 27,
- "artifact_ref": "page_citation_assets/page-27.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_3802892b-411b-58ba-87d1-81fcba1929b1",
- "type": "page",
- "content": "2.2.1\nConvexity and the Lasso\nOne important difference between the Lasso and Best subsets regression is that the former\ninvolves a convex optimisation, which makes it computationally tractable for large-scale\nproblems, in contrast to the latter which typically only feasible for problems with p < 50\nor so.\nRecall that a set A ⊆Rd is convex if\nx, y ∈A ⇒(1 −t)x + ty ∈A\nfor all t ∈(0, 1).\nA function f : Rd →R is convex if\nf\n\u0000(1 −t)x + ty\n\u0001\n≤(1 −t)f(x) + tf(y)\nfor all x, y ∈Rd and t ∈(0, 1).\nProposition 9.\n(i) Let f1, . . . , fm : Rd →R be convex functions. Then if c1, . . . , cm ≥0,\nc1f1 + · · · cmfm is a convex function.\n(ii) If f : Rd →R is twice continuously differentiable then\n(a) f is convex iff. its Hessian H(x) is positive semi-definite for all x,\n(b) f is strictly convex if H(x) is positive definite for all x.\nLasso solutions need not be unique (e.g. if X has duplicate columns), though for most\nreasonable design matrices, Lasso solutions will be unique. We will often tacitly assume\nthe Lasso solutions is unique in the statement of our theoretical results. It is however\nstraightforward to show that the Lasso fitted values are unique.\nProposition 10. X ˆβL\nλ is unique.\nProof. Fix λ and suppose ˆβ(1) and ˆβ(2) are two Lasso solutions giving an optimal objective\nvalue of c∗. Now for t ∈(0, 1), by strict convexity of ∥· ∥2\n2,\n∥Y −tX ˆβ(1) −(1 −t)X ˆβ(2)∥2\n2 ≤t∥Y −X ˆβ(1)∥2\n2 + (1 −t)∥Y −X ˆβ(2)∥2\n2,\nwith equality if and only if X ˆβ(1) = X ˆβ(2). Since ∥· ∥1 is also convex, we see that\nc∗≤Qλ(tˆβ(1) + (1 −t)ˆβ(2))\n= ∥Y −tX ˆβ(1) −(1 −t)X ˆβ(2)∥2\n2/(2n) + λ∥tˆβ(1) + (1 −t)ˆβ(2)∥1\n≤t∥Y −X ˆβ(1)∥2\n2/(2n) + (1 −t)∥Y −X ˆβ(2)∥2\n2/(2n) + λ∥tˆβ(1) + (1 −t)ˆβ(2)∥1\n≤t{∥Y −X ˆβ(1)∥2\n2/(2n) + λ∥ˆβ(1)∥1} + (1 −t){∥Y −X ˆβ(2)∥2\n2/(2n) + λ∥ˆβ(2)∥1}\n= tQ(ˆβ(1)) + (1 −t)Q(ˆβ(2)) ≤c∗.\nEquality must prevail throughout this chain of inequalities, so X ˆβ(1) = X ˆβ(2).\n23",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.2 The Lasso estimator/2.2.1 Convexity and the Lasso",
- "metadata": {
- "length": 1876,
- "summary": "This page, titled 'Convexity and the Lasso', discusses the computational advantages of Lasso regression over Best subsets regression due to its convex optimization nature. It defines mathematical concepts of convex sets and functions, presents Proposition 9 regarding properties of convex functions (including conditions involving Hessians), and provides a proof for Proposition 10 demonstrating that Lasso fitted values are unique.",
- "page_nums": [
- 28
- ],
- "keywords": [
- "Lasso",
- "Best subsets regression"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 28,
- "artifact_ref": "page_citation_assets/page-28.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c61cfde9-aa37-5c57-98a9-1840330192a8",
- "type": "page",
- "content": "2.2.2\nPrediction error of the Lasso with no assumptions on the\ndesign\nA remarkable property of the Lasso is that even when p ≫n, it can still perform well in\nterms of prediction error. Suppose the columns of X have been centred and scaled (as we\nwill always assume from now on unless stated otherwise) and assume the normal linear\nmodel (where we have already centred Y ),\nY = Xβ0 + ε −¯ε1\n(2.2.3)\nwhere ε ∼Nn(0, σ2I).\nTheorem 11. Let ˆβ be the Lasso solution when\nλ = Aσ\nr\nlog(p)\nn\n.\nWith probability at least 1 −2p−(A2/2−1)\n1\nn∥X(β0 −ˆβ)∥2\n2 ≤4Aσ\nr\nlog(p)\nn\n∥β0∥1.\nProof. From the definition of ˆβ we have\n1\n2n∥Y −X ˆβ∥2\n2 + λ∥ˆβ∥1 ≤1\n2n∥Y −Xβ0∥2\n2 + λ∥β0∥1.\nRearranging,\n1\n2n∥X(β0 −ˆβ)∥2\n2 ≤1\nnεTX(ˆβ −β0) + λ∥β0∥1 −λ∥ˆβ∥1.\nNow |εTX(ˆβ −β0)| ≤∥XTε∥∞∥ˆβ −β0∥1. Let Ω= {∥XTε∥∞/n ≤λ}. Lemma 15 below\nshows that P(Ω) ≥1 −2p−(A2/2−1). Working on the event Ω, we obtain\n1\n2n∥X(β0 −ˆβ)∥2\n2 ≤λ∥β0 −ˆβ∥1 + λ∥β0∥1 −λ∥ˆβ∥1,\n1\nn∥X(β0 −ˆβ)∥2\n2 ≤4λ∥β0∥1,\nby the triangle inequality.\n2.2.3\nConcentration inequalities I\nThe proof of Theorem 11 relies on a lower bound for the probability of the event Ω. A\nunion bound gives\nP(∥XTε∥∞/n > λ) = P(∪p\nj=1|XT\nj ε|/n > λ)\n≤\np\nX\nj=1\nP(|XT\nj ε|/n > λ).\n24",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.2 The Lasso estimator/2.2.2 Prediction error of the Lasso with no assumptions on the design",
- "metadata": {
- "length": 1183,
- "summary": "This section discusses the prediction error of the Lasso estimator when there are no assumptions on the design matrix, specifically in high-dimensional settings where p >> n. It presents Theorem 11, which establishes a bound on the prediction error with high probability under a normal linear model. The proof involves deriving an inequality based on the definition of the Lasso solution and utilizing Lemma 15 to bound the probability of a specific event.",
- "page_nums": [
- 29
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 29,
- "artifact_ref": "page_citation_assets/page-29.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_d6c6eac4-7e8c-586d-9299-943b4116bdd0",
- "type": "page",
- "content": "[SAME-AS Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.2 The Lasso estimator/2.2.2 Prediction error of the Lasso with no assumptions on the design p29]\n\nNow XT\nj ε/n ∼N(0, σ2/n), so if we obtain a bound on the tail probabilities of normal\ndistributions, the argument above will give a bound for P(Ω).\nMotivated by the need to bound normal tail probabilities, we will briefly discuss the\ntopic of concentration inequalities that provide such bounds for much wider classes of\nrandom variables.\nConcentration inequalities are vital for the study of many modern\nalgorithms and in our case here, they will reveal that the attractive properties of the Lasso\npresented in Theorem 11 hold true for a variety of non-normal errors.\nWe begin our discussion with the simplest tail bound, Markov’s inequality, which states\nthat given a non-negative random variable W,\nP(W ≥t) ≤E(W)\nt\n.\nThis immediately implies that given a strictly increasing function ϕ : R →[0, ∞) and any\nrandom variable W,\nP(W ≥t) = P{ϕ(W) ≥ϕ(t)} ≤E(ϕ(W))\nϕ(t)\n.\nApplying this with ϕ(t) = eαt (α > 0) yields the so-called Chernoffbound:\nP(W ≥t) ≤inf\nα>0 e−αtEeαW.\nConsider the case when W ∼N(0, σ2). Recall that\nEeαW = eα2σ2/2.\n(2.2.4)\nThus\nP(W ≥t) ≤inf\nα>0 eα2σ2/2−αt = e−t2/(2σ2).\nNote that to arrive at this bound, all we required was (an upper bound on) the moment\ngenerating function (mgf) of W (2.2.4).\nSub-Gaussian variables\nDefinition 3. We say a random variable W with mean µ = E(W) is sub-Gaussian if there\nexists σ > 0 such that\nEeα(W−µ) ≤eα2σ2/2\nfor all α ∈R. We then say that W is sub-Gaussian with parameter σ.\nProposition 12 (Sub-Gaussian tail bound). If W is sub-Gaussian with parameter σ and\nE(W) = µ then\nP(W −µ ≥t) ≤e−t2/(2σ2).\n25\n\nAs well as Gaussian random variables, the sub-Gaussian class includes bounded random\nvariables.\nLemma 13 (Hoeffding’s lemma). If W is mean-zero and takes values in [a, b], then W is\nsub-Gaussian with parameter (b −a)/2.\nThe following proposition shows that analogously to how a linear combination of jointly\nGaussian random variables is Gaussian, a linear combination of sub-Gaussian random\nvariables is also sub-Gaussian.\nProposition 14. Let (Wi)n\ni=1 be a sequence of independent mean-zero sub-Gaussian ran-\ndom variables with parameters (σi)n\ni=1 and let γ ∈Rn. Then γTW is sub-Gaussian with\nparameter\n\u0010 P\ni γ2\ni σ2\ni\n\u00111/2\n.\nProof.\nE exp\n\u0010\nα\nn\nX\ni=1\nγiWi\n\u0011\n=\nn\nY\ni=1\nE exp(αγiWi)\n≤\nn\nY\ni=1\nexp(α2γ2\ni σ2\ni /2)\n= exp\n\u0010\nα2\nn\nX\ni=1\nγ2\ni σ2\ni /2\n\u0011\n.\nWe can now prove a more general version of the probability bound required for Theo-\nrem 11.\nLemma 15. Suppose (εi)n\ni=1 are independent, mean-zero and sub-Gaussian with common\nparameter σ. Note that this includes ε ∼Nn(0, σ2I). Let λ = Aσ\np\nlog(p)/n. Then\nP(∥XTε∥∞/n ≤λ) ≥1 −2p−(A2/2−1).\nProof.\nP(∥XTε∥∞/n > λ) ≤\np\nX\nj=1\nP(|XT\nj ε|/n > λ).\nBut ±XT\nj ε/n are both sub-Gaussian with parameter (σ2∥Xj∥2\n2/n2)1/2 = σ/√n. Thus the\nRHS is at most\n2p exp(−A2 log(p)/2) = 2p1−A2/2.\n26",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.2 The Lasso estimator/2.2.3 Concentration inequalities I",
- "metadata": {
- "length": 2945,
- "summary": "This section, titled 'Concentration inequalities I', provides the theoretical foundation for bounding tail probabilities of random variables, which is necessary to prove Theorem 11. It begins by introducing Markov's inequality and deriving the Chernoff bound from it. The text then defines sub-Gaussian variables and presents key results: Hoeffding's lemma (showing bounded mean-zero variables are sub-Gaussian), a proposition on linear combinations of sub-Gaussian variables, and Lemma 15, which establishes a probability bound for the infinity norm of a vector of independent sub-Gaussian variables.",
- "page_nums": [
- 29,
- 30,
- 31
- ],
- "keywords": [
- "Theorem 11",
- "Markov's inequality",
- "Chernoff bound",
- "Definition 3",
- "Proposition 12",
- "Lemma 13",
- "Hoeffding's lemma",
- "Proposition 14",
- "Lemma 15"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 29,
- "artifact_ref": "page_citation_assets/page-29.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 30,
- "artifact_ref": "page_citation_assets/page-30.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 31,
- "artifact_ref": "page_citation_assets/page-31.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_f9b7ee8d-4e09-5b58-8bef-0ee67871f49d",
- "type": "page",
- "content": "Chapter 3\nThe Lasso and beyond\nIn this chapter, we will study the Lasso in more depth and present more precise theory\nabout its behaviour. We will first study the variable selection properties of the Lasso,\nwhich will require some basic results in convex analysis that we present below.\n3.1\nSome facts from optimisation theory and convex\nanalysis\n3.1.1\nSubgradients\nDefinition 4. A vector v ∈Rd is a subgradient of a convex function f : Rd →R at x if\nf(y) ≥f(x) + vT(y −x)\nfor all y ∈Rd.\nThe set of subgradients of f at x is called the subdifferential of f at x and denoted ∂f(x).\nIn order to make use of subgradients, we will require the following two facts:\nProposition 16. Let f : Rd →R be convex, and suppose f is differentiable at x. Then\n∂f(x) = {∇f(x)}.\nProposition 17. Let f, g : Rd →R be convex, and let α > 0. Then\n∂(αf)(x) = α∂f(x) = {αv : v ∈∂f(x)},\n∂(f + g)(x) = ∂f(x) + ∂g(x) = {v + w : v ∈∂f(x), w ∈∂g(x)}.\nThe following easy (but key) result is often referred to in the statistical literature as the\nKarush–Kuhn–Tucker (KKT) conditions, though it is actually a much simplified version\nof them.\nProposition 18. x∗∈arg min\nx∈Rd\nf(x) if and only if 0 ∈∂f(x∗).\n27\n\nProof.\nf(y) ≥f(x∗)\nfor all y ∈Rd ⇔f(y) ≥f(x∗) + 0T(y −x)\nfor all y ∈Rd\n⇔0 ∈∂f(x∗).\nLet us now compute the subdifferential of the ℓ1-norm. First note that ∥· ∥1 : Rd →R\nis convex. Indeed it is a norm so the triangle inequality gives ∥tx + (1 −t)y∥1 ≤t∥x∥1 +\n(1 −t)∥y∥1. We introduce some notation that will be helpful here and throughout the rest\nof the course.\nFor x ∈Rd and A = {k1, . . . , km} ⊆{1, . . . , d} with k1 < · · · < km, by xA we will mean\n(xk1, . . . , xkm)T. Similarly if X has d columns we will write XA for the matrix\nXA = (Xk1 · · · Xkm).\nFurther in this context, by Ac, we will mean {1, . . . , d}\\A. Additionally, when in subscripts\nwe will use the shorthand −j = {j}c and −jk = {j, k}c. Note these column and component\nextraction operations will always be considered to have taken place first before any further\noperations on the matrix, so for example XT\nA = (XA)T. Finally, define\nsgn(x1) =\n\n\n\n\n\n−1\nif x1 < 0\n0\nif x1 = 0\n1\nif x1 > 0,\nand\nsgn(x) = (sgn(x1), . . . , sgn(xd))T.\nProposition 19. For x ∈Rd let A = {j : xj ̸= 0}. Then\n∂∥x∥1 = {v ∈Rd : ∥v∥∞≤1 and vA = sgn(xA)}\nProof. For j = 1, . . . , d, let\ngj : Rd →R\nx 7→|xj|.\nThen ∥· ∥= P\nj gj(·) so by Proposition 17, ∂∥x∥1 = P\nj ∂gj(x). When x has xj ̸= 0, gj\nis differentiable at x so by Proposition 16 ∂gj(x) = {sgn(xj)ej} where ej is the jth unit\nvector. When xj = 0, if v ∈∂gj(x) we must have\ngj(y) ≥gj(x) + vT(y −x)\nfor all y ∈Rd,\nso\n|yj| ≥vT(y −x)\nfor all y ∈Rd.\n(3.1.1)\nwe claim that the above holds iff. vj ∈[−1, 1] and v−j = 0. For the ‘if’ direction, note\nthat vT(y −x) = vjyj ≤|yj|. Conversely, set y−j = x−j + v−j and yj = 0 in (3.1.1) to get\n0 ≥∥v−j∥2\n2, so v−j = 0. Then take y with y−j = x−j to get |yj| ≥vjyj for all yj ∈R, so\n|vj| ≤1. Forming the set sum of the subdifferentials then gives the result.\n28",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.1 Some facts from optimisation theory and convex analysis/3.1.1 Subgradients",
- "metadata": {
- "length": 2967,
- "summary": "The section titled '3.1.1 Subgradients' defines the concept of a subgradient for a convex function and introduces the subdifferential as the set of all such subgradients. It presents key propositions regarding subgradients, including their relationship to gradients when a function is differentiable, their behavior under scalar multiplication and addition, and the Karush-Kuhn-Tucker (KKT) conditions for optimality. The section concludes by deriving the subdifferential of the ℓ₁-norm.",
- "page_nums": [
- 32,
- 33
- ],
- "keywords": [
- "Karush-Kuhn-Tucker",
- "KKT"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 32,
- "artifact_ref": "page_citation_assets/page-32.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 33,
- "artifact_ref": "page_citation_assets/page-33.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_f5af8f86-8460-54fd-951a-f2af52f88fc4",
- "type": "page",
- "content": "3.1.2\nThe Lagrangian method\nConsider an optimisation problem of the form\nminimise f(x), subject to g(x) = 0\n(3.1.2)\nwhere g : Rd →Rb. Suppose the optimal value is c∗∈R. The Lagrangian for this problem\nis defined as\nL(x, θ) = f(x) + θTg(x)\nwhere θ ∈Rb. Note that\ninf\nx∈Rd L(x, θ) ≤\ninf\nx∈Rd:g(x)=0 L(x, θ) = c∗\nfor all θ. The Lagrangian method involves finding a θ∗such that the minimising x∗on the\nLHS satisfies g(x∗) = 0. This x∗must then be a minimiser in the original problem (3.1.2).\n3.2\nLasso solutions\nEquipped with these tools from convex analysis, we can now fully characterise the solutions\nto the Lasso. We have that ˆβL\nλ is a Lasso solution if and only if 0 ∈∂Qλ(ˆβL\nλ), which is\nequivalent to\n1\nnXT(Y −X ˆβL\nλ) = λˆν,\nfor ˆν with ∥ˆν∥∞≤1 and writing ˆSλ = {k : ˆβL\nλ,k ̸= 0}, ˆν ˆSλ = sgn(ˆβL\nλ, ˆSλ). These are known\nas the KKT conditions for the Lasso.\n3.3\nVariable selection\nConsider now the “noiseless” version of the high-dimensional linear model (2.2.3), Y =\nXβ0. The case with noise can dealt with by similar arguments to those we will use below\nwhen we work on an event that ∥XTε∥∞/n is small (see example sheet).\nLet S = {k : β0\nk ̸= 0}, N = {1, . . . , p} \\ S and assume wlog that S = {1, . . . , s}, and\nalso that rank(XS) = s.\nTheorem 20. Let λ > 0 and define ∆= XT\nNXS(XT\nS XS)−1sgn(β0\nS). If ∥∆∥∞≤1 and for\nk ∈S,\n|β0\nk| > λ|sgn(β0\nS)T[{ 1\nnXT\nS XS}−1]k|,\n(3.3.1)\nthen there exists a Lasso solution ˆβL\nλ with sgn(ˆβL\nλ) = sgn(β0). As a partial converse, if\nthere exists a Lasso solution ˆβL\nλ with sgn(ˆβL\nλ) = sgn(β0), then ∥∆∥∞≤1.\nRemark 1. We can interpret ∥∆∥∞as the maximum in absolute value over k ∈N of the\ndot product of sgn(β0\nS) and (XT\nS XS)−1XT\nS Xk, the coefficient vector obtained by regressing\nXk on XS. The condition ∥∆∥∞≤1 is known as the irrepresentable condition.\n29",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.1 Some facts from optimisation theory and convex analysis/3.1.2 The Lagrangian method",
- "metadata": {
- "length": 1806,
- "summary": "Section 3.1.2 introduces the Lagrangian method for solving constrained optimization problems of the form minimize f(x) subject to g(x)=0. It defines the Lagrangian function L(x, θ) = f(x) + θ^T g(x) and explains that the method involves finding a parameter θ* such that the minimizer x* of the Lagrangian satisfies the original constraint g(x*)=0.",
- "page_nums": [
- 34
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 34,
- "artifact_ref": "page_citation_assets/page-34.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_041885f6-128d-503b-9bdb-f79178cd8ec4",
- "type": "page",
- "content": "[SAME-AS Statistical Learning Notes.pdf/3 The Lasso and beyond/3.1 Some facts from optimisation theory and convex analysis/3.1.2 The Lagrangian method p34]",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.2 Lasso solutions",
- "metadata": {
- "length": 155,
- "summary": "Section 3.2 characterizes the solutions to the Lasso problem using tools from convex analysis. It states that a vector is a Lasso solution if and only if it satisfies specific Karush-Kuhn-Tucker (KKT) conditions, which are expressed as an equation involving the transpose of the design matrix X, the response vector Y, and a subgradient vector with an infinity norm less than or equal to 1.",
- "page_nums": [
- 34
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 34,
- "artifact_ref": "page_citation_assets/page-34.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_75c37156-8e12-5a2d-b0d3-2b1a59c7de6e",
- "type": "page",
- "content": "[SAME-AS Statistical Learning Notes.pdf/3 The Lasso and beyond/3.1 Some facts from optimisation theory and convex analysis/3.1.2 The Lagrangian method p34]",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.3 Variable selection",
- "metadata": {
- "length": 155,
- "summary": "This section discusses variable selection in the context of a noiseless high-dimensional linear model. It presents Theorem 20, which establishes conditions under which a Lasso solution correctly recovers the sign pattern of the true coefficient vector. A key condition mentioned is the 'irrepresentable condition' (||Δ||_∞ ≤ 1), which ensures that variables not in the true support set do not correlate too strongly with those in the support set.",
- "page_nums": [
- 34
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 34,
- "artifact_ref": "page_citation_assets/page-34.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_edb92fd7-277d-5c79-ba2e-aff3bce4cc3c",
- "type": "page",
- "content": "Proof. Fix λ > 0 and write ˆβ = ˆβL\nλ and ˆS = {k : ˆβk ̸= 0} for convenience. The KKT\nconditions for the Lasso give\n1\nnXTX(β0 −ˆβ) = λˆν\nwhere ∥ˆν∥∞≤1 and ˆν ˆS = sgn(ˆβ ˆS). We can expand this into\n1\nn\n\u0012XT\nS XS\nXT\nS XN\nXT\nNXS\nXT\nNXN\n\u0013 \u0012β0\nS −ˆβS\n−ˆβN\n\u0013\n= λ\n\u0012ˆνS\nˆνN\n\u0013\n.\n(3.3.2)\nWe prove the converse first. If sgn(ˆβ) = sgn(β0) then ˆνS = sgn(β0\nS) and ˆβN = 0. The\ntop block of (3.3.2) gives\nβ0\nS −ˆβS = λ( 1\nnXT\nS XS)−1sgn(β0\nS).\nSubstituting this into the bottom block, we get\nλ 1\nnXT\nNXS( 1\nnXT\nS XS)−1sgn(β0\nS) = λˆνN.\nThus as ∥ˆνN∥∞≤1, we have ∥∆∥∞≤1.\nFor the positive statement, we need to find a ˆβ and ˆν such that sgn(ˆβS) = sgn(β0\nS) and\nˆβN = 0, for which the KKT conditions hold. We claim that taking\n(ˆβS, ˆβN) = (β0\nS −λ( 1\nnXT\nS XS)−1sgn(β0\nS), 0)\n(ˆνS, ˆνN) = (sgn(β0\nS), ∆)\nsatisfies (3.3.2). We only need to check that sgn(β0\nS) = sgn(ˆβS), but this follows from\n(3.3.1).\n3.4\nPrediction and estimation\nConsider again the model Y = Xβ0 + ε −¯ε1 where the components of ε are independent\nmean-zero sub-Gaussian random variables with common parameter σ. Let S, s and N be\ndefined as in the previous section. As we have noted before, in an artificial situation where\nS is known, we could apply OLS on XS and have an MSPE of σ2s/n. Under a so-called\ncompatibility condition on the design matrix, we can obtain a similar MSPE for the Lasso.\nDefinition 5. Given a matrix of predictors X ∈Rn×p and support set S, define\nφ2 =\ninf\nβ∈Rp:βS̸=0, ∥βN∥1≤3∥βS∥1\n1\nn∥Xβ∥2\n2\n1\ns∥βS∥2\n1\n,\nwhere s = |S| and we take φ ≥0. The compatibility condition is that φ2 > 0.\n30\n\nNote that if XTX/n has minimum eigenvalue cmin > 0 (so necessarily p ≤n), then\nφ2 > cmin. Indeed by the Cauchy–Schwarz inequality,\n∥βS∥1 = sgn(βS)TβS ≤√s∥βS∥2 ≤√s∥β∥2.\nThus\nφ2 ≥inf\nβ̸=0\n1\nn∥Xβ∥2\n2\n∥β∥2\n2\n= cmin.\nAlthough in the high-dimensional setting we would have cmin = 0, the fact that the infimum\nin the definition of φ2 is over a restricted set of β can still allow φ2 to be positive even in\nthis case, as we discuss after the presentation of the theorem.\nTheorem 21. Suppose the compatibility condition holds and let ˆβ be the Lasso solution\nwith λ = Aσ\np\nlog(p)/n for A > 0. Then with probability at least 1 −2p−(A2/8−1), we have\n1\nn∥X(β0 −ˆβ)∥2\n2 + λ∥ˆβ −β0∥1 ≤16λ2s\nφ2\n= 16A2 log(p)\nφ2\nσ2s\nn .\nProof. As in Theorem 11 we start with the “basic inequality”:\n1\n2n∥X(ˆβ −β0)∥2\n2 + λ∥ˆβ∥1 ≤1\nnεTX(ˆβ −β0) + λ∥β0∥1.\nWe work on the event Ω= {2∥XTε∥∞/n ≤λ} where after applying H¨older’s inequality,\nwe get\n1\nn∥X(ˆβ −β0)∥2\n2 + 2λ∥ˆβ∥1 ≤λ∥ˆβ −β0∥1 + 2λ∥β0∥1.\n(3.4.1)\nLemma 15 shows that P(Ω) ≥1 −2p−(A2/8−1).\nTo motivate the rest of the proof, consider the following idea. We know\n1\nn∥X(ˆβ −β0)∥2\n2 ≤3λ∥ˆβ −β0∥1.\nIf we could get\n3λ∥ˆβ −β0∥1 ≤cλ\n√n∥X(ˆβ −β0)∥2\nfor some constant c > 0, then we would have that ∥X(ˆβ −β0)∥2\n2/n ≤c2λ2 and also\n3λ∥β0 −ˆβ∥1 ≤c2λ2.\nReturning to the actual proof, write a = ∥X(ˆβ −β0)∥2\n2/(nλ). Then from (3.4.1) we can\nderive the following string of inequalities:\na + 2(∥ˆβN∥1 + ∥ˆβS∥1) ≤∥ˆβS −β0\nS∥1 + ∥ˆβN∥1 + 2∥β0\nS∥1\na + ∥ˆβN∥1 ≤∥ˆβS −β0\nS∥1 + 2∥β0\nS∥1 −2∥ˆβS∥1\na + ∥ˆβN −β0\nN∥1 ≤3∥β0\nS −ˆβS∥1\na + ∥ˆβ −β0∥1 ≤4∥β0\nS −ˆβS∥1,\n31\n\nthe final inequality coming from adding ∥β0\nS −ˆβS∥1 to both sides.\nNow using the compatibility condition with β = ˆβ −β0 we have\n1\nn∥X(ˆβ −β0)∥2\n2 + λ∥β0 −ˆβ∥1 ≤4λ∥β0\nS −ˆβS∥1\n≤4λ\nφ\nr s\nn∥X(ˆβ −β0)∥2.\n(3.4.2)\nFrom this we get\n1\n√n∥X(ˆβ −β0)∥2 ≤4λ√s\nφ\n,\nand substituting this into the RHS of (3.4.2) gives the result.\n3.5\nThe compatibility condition\nHow strong is the compatibility condition? In order to answer this question, we shall think\nof X as random and try to understand what conditions on the population covariance matrix\nΣ0 := E(XTX/n) imply that X satisfies a compatibility condition with high probability.\nTo this end let us define\nφ2\nΣ(S) =\ninf\nβ:∥βS∥1̸=0,∥βN∥1≤3∥βS∥1\nβTΣβ\n∥βS∥2\n1/|S|\nwhere Σ ∈Rp×p. Note then our φ2 = φ2\nˆΣ(S) where ˆΣ := XTX/n and S is the support set\nof β0. The following result shows that if ˆΣ is close to a matrix ˇΣ for which φ2\nˇΣ(S) > 0,\nthen also φ2\nˆΣ(S) > 0.\nLemma 22. Suppose φ2\nˇΣ(S) > 0 and maxjk |ˆΣjk −ˇΣjk| ≤φ2\nˇΣ(S)/(32|S|). Then φ2\nˆΣ(S) ≥\nφ2\nˇΣ(S)/2.\nProof. In the following we suppress dependence on S. Let s = |S| and let t = φ2\nˇΣ/(32s).\nWe have\n|βT(ˇΣ −ˆΣ)β| ≤∥β∥1∥(ˇΣ −ˆΣ)β∥∞(H¨older)\n≤t∥β∥2\n1 (H¨older again).\nIf ∥βN∥1 ≤3∥βS∥1 then\n∥β∥1 = ∥βN∥1 + ∥βS∥1 ≤4∥βS∥1 ≤4\np\nβT ˇΣβ\nφˇΣ/√s .\nThus if ∥βN∥1 ≤3∥βS∥1,\nβT ˇΣβ −φ2\nˇΣ\n32s\n16βT ˇΣβ\nφ2\nˇΣ/s\n= 1\n2βT ˇΣβ ≤βT ˆΣβ.\n32",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.4 Prediction and estimation",
- "metadata": {
- "length": 4442,
- "summary": "This section, titled 'Prediction and estimation', introduces the compatibility condition as a requirement for obtaining Mean Squared Prediction Error (MSPE) bounds for the Lasso estimator. It defines the compatibility constant φ² via an infimum over a restricted set of vectors. Theorem 21 is then presented, which provides a high-probability bound on the prediction error and estimation error of the Lasso solution under this condition. The proof involves deriving a series of inequalities based on the basic inequality and Hölder's inequality. The section concludes by beginning to discuss the strength of the compatibility condition in relation to the population covariance matrix.",
- "page_nums": [
- 35,
- 36,
- 37
- ],
- "keywords": [
- "Lasso",
- "OLS",
- "Theorem 21",
- "Lemma 22"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 35,
- "artifact_ref": "page_citation_assets/page-35.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 36,
- "artifact_ref": "page_citation_assets/page-36.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 37,
- "artifact_ref": "page_citation_assets/page-37.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_477f140a-e16a-5b66-b3e4-f403b2bce1b7",
- "type": "page",
- "content": "We would like to apply the result above with ˇΣ = Σ0, and use to argue that if Σ0\nsatisfies the compatibility condition, then so will ˆΣ with high probability. In order to do\nthis, we need to argue that the event that maxjk |ˆΣjk −Σ0\njk| is small occurs with high\nprobability. We can obtain such a result with the aid of concentration inequalities.\n3.5.1\nConcentration inequalities II\nWhen trying to understand the concentration properties of ˆΣjk, it will be helpful to have a\ntail bound for a product of sub-Gaussian random variables. Bernstein’s inequality, which\napplies to random variables satisfying the condition below, is helpful in this regard.\nDefinition 6 (Bernstein’s condition). We say that the random variable W with EW = µ\nsatisfies Bernstein’s condition with parameter (σ, b) where σ, b > 0 if\nE(|W −µ|k) ≤1\n2k!σ2bk−2\nfor k = 2, 3, . . . .\nProposition 23 (Bernstein’s inequality). Let W1, W2, . . . be independent random variables\nwith E(Wi) = µ. Suppose each Wi satisfies Bernstein’s condition with parameter (σ, b).\nThen\nE(eα(Wi−µ)) ≤exp\n\u0012 α2σ2/2\n1 −b|α|\n\u0013\nfor all |α| < 1/b\nP\n\u00121\nn\nn\nX\ni=1\nWi −µ ≥t\n\u0013\n≤exp\n\u0012\n−\nnt2\n2(σ2 + bt)\n\u0013\nfor all t ≥0.\nProof. Fix i and let W = Wi. We have\nE(eα(W−µ)) = 1 + αE(W −µ) +\n∞\nX\nk=2\nαk E{(W −µ)k}\nk!\n≤1 + σ2α2\n2\n∞\nX\nk=2\n|α|k−2bk−2\n= 1 + σ2α2\n2\n1\n1 −|α|b ≤exp\n\u0012 α2σ2/2\n1 −b|α|\n\u0013\n,\nprovided |α| < 1/b and using the inequality eu ≥1+u in the final line. For the probability\nbound, first note that\nE exp\n\u0012\nn\nX\ni=1\nα(Wi −µ)/n\n\u0013\n=\nn\nY\ni=1\nE exp{α(Wi −µ)/n}\n≤exp\n\u0012\nn(α/n)2σ2/2\n1 −b|α/n|\n\u0013\nfor |α|/n < 1/b. Then we use the Chernoffmethod and set α/n = t/(bt+σ2) ∈[0, 1/b).\n33",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.5 The compatibility condition/3.5.1 Concentration inequalities II",
- "metadata": {
- "length": 1609,
- "summary": "This page presents a mathematical discussion on concentration inequalities, specifically focusing on Bernstein's inequality. It begins by defining Bernstein's condition for a random variable and then states Proposition 23, which provides tail bounds for the sum of independent random variables satisfying this condition. The page concludes with a detailed proof of the proposition using moment generating functions and the Chernoff method.",
- "page_nums": [
- 38
- ],
- "keywords": [
- "Bernstein"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 38,
- "artifact_ref": "page_citation_assets/page-38.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_61e1c3d8-f2d0-5fff-85f0-ef4bbe74e4bd",
- "type": "page",
- "content": "Lemma 24. Let W, Z be mean-zero and sub-Gaussian with parameters σW and σZ respec-\ntively. Then the product WZ satisfies Bernstein’s condition with parameter (8σWσZ, 4σWσZ).\nProof. In order to use Bernstein’s inequality (Proposition 23) we first obtain bounds on\nthe moments of W and Z. Note that W 2k =\nR ∞\n0 1{x x)dx\n= 2k\nZ ∞\n0\nt2k−1P(|W| > t)dt\nsubstituting t2k = x\n≤4k\nZ ∞\n0\nt2k−1 exp{−t2/(2σ2\nW)}dt\nby Proposition 12\n= 4kσ2\nW\nZ ∞\n0\n(2σ2\nWx)k−1e−xdx\nsubstituting t2/(2σ2\nW) = x\n= 2k+1σ2k\nW k!.\nNext note that for any random variable Y ,\n|E(Y −EY )k| ≤E|Y −EY |k\n= 2kE|Y/2 −EY/2|k\n≤2k−1(E|Y |k + |EY |k)\nby Jensen’s inequality applied to t 7→|t|k,\n≤2kE|Y |k.\nTherefore\nE(|WZ −EWZ|k) ≤2kE|WZ|k\n≤2k(EW 2k)1/2(EZ2k)1/2\nby Cauchy–Schwarz\n≤2k2k+1σk\nWσk\nZk!\n= k!\n2 (8σWσZ)2(4σWσZ)k−2.\n3.5.2\nThe compatibility condition and random design\nWe may now apply this with ˇΣ = Σ0. To make the result more readily interpretable, we\nshall state it in an asymptotic framework. Imagine a sequence of design matrices with n\nand p growing, each with their own compatibility condition. We will however suppress the\nasymptotic regime in the notation.\nTheorem 25. Suppose the rows of X are i.i.d. and each entry of X is mean-zero sub-\nGaussian with parameter v. Suppose s\np\nlog(p)/n →0 (and s, p, n > 1) as n →∞. Let\nφ2\nˆΣ,s = min\nS:|S|=s φ2\nˆΣ(S)\nφ2\nΣ0,s = min\nS:|S|=s φ2\nΣ0(S),\nand suppose the latter is bounded away from 0. Then P(φ2\nˆΣ,s ≥φ2\nΣ0,s/2) →1 as n →∞.\n34",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.5 The compatibility condition/3.5.2 The compatibility condition and random design",
- "metadata": {
- "length": 1505,
- "summary": "This page contains a mathematical proof for Lemma 24 regarding the product of mean-zero sub-Gaussian random variables satisfying Bernstein's condition. It includes detailed derivations using Fubini's theorem, Jensen's inequality, and Cauchy-Schwarz. The page also introduces section 3.5.2 on 'The compatibility condition and random design' and presents Theorem 25 concerning asymptotic frameworks for design matrices.",
- "page_nums": [
- 39
- ],
- "keywords": [
- "Lemma 24",
- "Proposition 23",
- "Proposition 12",
- "Theorem 25"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 39,
- "artifact_ref": "page_citation_assets/page-39.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_6b0b4560-7957-520f-bd34-4deb7816e802",
- "type": "page",
- "content": "Proof. In view of Lemma 22, we need only show that\nP(max\njk |ˆΣjk −Σ0\njk| ≥φ2\nΣ0,s/(32s)) →0.\nLet t = φ2\nΣ0,s/(32s). By a union bound and then Lemma 24 we have\nP(max\njk |ˆΣjk −Σ0\njk| ≥φ2\nΣ0,s/(32s)) < p2 max\njk P\n\u0010\f\f\f\nn\nX\ni=1\nXijXik/n −Σ0\njk\n\f\f\f ≥t\n\u0011\n≤2 exp\n\u0012\n−\nnt2\n2(64v4 + 4v2t)) + 2 log(p)\n\u0013\n≤c1 exp(−c2n/s2 + c3 log(p)) →0.\nCorollary 26. Suppose the rows of X are independent with distribution Np(0, Σ0). Suppose\nthe diagonal entries of Σ0 are bounded above and the minimum eigenvalue of Σ0, cmin is\nbounded away from 0. Then P(φ2\nˆΣ,s ≥cmin/2) →1 provided s\np\nlog(p)/n →0.\n3.6\nComputation\nOne of the most efficient ways of computing Lasso solutions is to use a optimisation tech-\nnique called coordinate descent.\nThis is a quite general way of minimising a function\nf : Rd →R and works particularly well for functions of the form\nf(x) = g(x) +\nd\nX\nj=1\nhj(xj)\nwhere g is convex and differentiable and each hj : R →R is convex (and so continuous). We\nstart with an initial guess of the minimiser x(0) (e.g. x(0) = 0) and repeat for m = 1, 2, . . .\nx(m)\n1\n= arg min\nx1∈R\nf(x1, x(m−1)\n2\n, . . . , x(m−1)\nd\n)\nx(m)\n2\n= arg min\nx2∈R\nf(x(m)\n1\n, x2, x(m−1)\n3\n, . . . , x(m−1)\nd\n)\n...\nx(m)\nd\n= arg min\nxd∈R\nf(x(m)\n1\n, x(m)\n2\n, . . . , x(m)\nd−1, xd).\nTseng [2001] proves that provided A0 = {x : f(x) ≤f(x(0))} is compact, then every\nconverging subsequence of x(m) will converge to a minimiser of f.\nCorollary 27. Suppose A0 is compact. Then\n(i) There exists a minimiser of f, x∗and f(x(m)) →f(x∗).\n35",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.6 Computation",
- "metadata": {
- "length": 1491,
- "summary": "This page contains a mathematical proof concluding with Corollary 26, followed by Section 3.6 titled 'Computation'. The section discusses the coordinate descent optimization technique for computing Lasso solutions, presenting the general function form and the iterative update equations used in the algorithm. It references Tseng [2001] regarding convergence properties and concludes with Corollary 27.",
- "page_nums": [
- 40
- ],
- "keywords": [
- "Lemma 22",
- "Lemma 24",
- "Corollary 26",
- "Tseng",
- "Corollary 27"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 40,
- "artifact_ref": "page_citation_assets/page-40.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_073a9681-0199-5f84-8297-69863769b17d",
- "type": "page",
- "content": "(ii) If x∗is the unique minimiser of f then x(m) →x∗.\nProof. f is a continuous function so it attains its infimum on the compact set A0. Sup-\npose f(x(m)) ↛f(x∗). Then there exists ϵ > 0 and a subsequence (x(mj))∞\nj=0 such that\nf(x(mj)) ≥f(x∗)+ϵ for all j. Note that since f(x(m)) ≤f(x(m−1)), we know that x(m) ∈A0\nfor all m. Thus if A0 is compact then any subsequence of (x(m))∞\nm=0 has a further sub-\nsequence that converges by the Bolzano–Weierstrass theorem. Let ˜x be the limit of the\nconverging subsequence of (x(mj))∞\nj=0. Then f(˜x) ≥f(x∗) + ϵ, contradicting the result of\nTseng [2001]. Thus (i) holds. The proof of (ii) is similar.\nWe often want to solve the Lasso on a grid of λ values λ0 > · · · > λL (for the purposes\nof cross-validation for example). To do this, we can first solve for λ0, and then solve at\nsubsequent grid points by using the solution at the previous grid points as an initial guess\n(known as a warm start). An active set strategy can further speed up computation. This\nworks as follows: For l = 1, . . . , L\n1. Initialise A = {k : ˆβL\nλl−1,k ̸= 0}.\n2. Perform coordinate descent only on coordinates in A obtaining a solution ˆβ (all\ncomponents ˆβk with k /∈A are set to zero).\n3. Let V = {k : |XT\nk (Y −X ˆβ)|/n > λl}, the set of coordinates which violate the KKT\nconditions when ˆβ is taken as a candidate solution.\n4. If V is empty, we set ˆβL\nλl = ˆβ. Else we update A ←A ∪V and return to 2.\n3.7\nExtensions of the Lasso\nWe can add an ℓ1 penalty to many other log-likelihoods, or more generally other loss\nfunctions besides the squared-error loss that arises from the normal linear model. For\nLasso-penalised generalised linear models, such as logistic regression, similar theoretical\nresults to those we have obtained are available and computations can proceed in a similar\nfashion to above.\n3.7.1\nStructural penalties\nThe Lasso penalty encourages the estimated coefficients to be shrunk towards 0 and some-\ntimes exactly to 0. Other penalty functions can be constructed to encourage different types\nof sparsity.\n36",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.7 Extensions of the Lasso/3.7.1 Structural penalties",
- "metadata": {
- "length": 2045,
- "summary": "This page contains a mathematical proof regarding the convergence of a minimizer for a continuous function, referencing the Bolzano-Weierstrass theorem and Tseng [2001]. It then details an algorithmic strategy using coordinate descent with a 'warm start' to solve the Lasso on a grid of lambda values. The text concludes with section 3.7, discussing extensions of the Lasso to other loss functions like logistic regression, and section 3.7.1, which introduces structural penalties.",
- "page_nums": [
- 41
- ],
- "keywords": [
- "Tseng",
- "Lasso"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 41,
- "artifact_ref": "page_citation_assets/page-41.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_6fb03822-c033-5d90-b5e2-6f30ac456667",
- "type": "page",
- "content": "Group Lasso\nSuppose we have a partition G1, . . . , Gq of {1, . . . , p} (so ∪q\nk=1Gk = {1, . . . , p}, Gj ∩Gk = ∅\nfor j ̸= k). The group Lasso penalty [Yuan and Lin, 2006] is given by\nλ\nq\nX\nj=1\nmj∥βGj∥2.\nThe multipliers mj > 0 serve to balance cases where the groups are of very different sizes;\ntypically we choose mj =\np\n|Gj|. This penalty encourages either an entire group G to\nhave ˆβG = 0 or ˆβk ̸= 0 for all k ∈G. Such a property is useful when groups occur through\ncoding for categorical predictors or when expanding predictors using basis functions.\nFused Lasso\nIf there is a sense in which the coefficients are ordered, so β0\nj is expected to be close to\nβ0\nj+1, a fused Lasso penalty [Tibshirani et al., 2005] may be appropriate. This takes the\nform\nλ1\np−1\nX\nj=1\n|βj −βj+1| + λ2∥β∥1,\nwhere the second term may be omitted depending on whether shrinkage towards 0 is\ndesired. As an example, consider the simple setting where Yi = µ0\ni + εi, and it is thought\nthat the (µ0\ni )n\ni=1 form a piecewise constant sequence. Then one option is to minimise over\nµ ∈Rn, the following objective\n1\nn∥Y −µ∥2\n2 + λ\nn−1\nX\ni=1\n|µi −µi+1|.\n3.7.2\nReducing the bias of the Lasso\nOne potential drawback of the Lasso is that the same shrinkage effect that sets many\nestimated coefficients exactly to zero also shrinks all non-zero estimated coefficients towards\nzero. One possible solution is to take ˆSλ = {k : ˆβL\nλ,k ̸= 0} and then re-estimate β0\nˆSλ by OLS\nregression on X ˆSλ.\nAnother option is to re-estimate using the Lasso on X ˆSλ; this procedure is known as\nthe relaxed Lasso [Meinshausen, 2007]. The adaptive Lasso [Zou, 2006] takes an initial\nestimate of β0, ˆβinit (e.g. from the Lasso) and then performs weighted Lasso regression:\nˆβadapt\nλ\n=\narg min\nβ∈Rp:β ˆ\nSc\ninit=0\n\u001a 1\n2n∥Y −Xβ∥2\n2 + λ\nX\nk∈ˆSinit\n|βk|\n|ˆβinit\nk |\n\u001b\n,\nwhere ˆSinit = {k : ˆβinit\nk\n̸= 0}.\n37\n\nYet another approach involves using a family of non-convex penalty functions pλ,γ :\n[0, ∞) →[0, ∞) and attempting to minimise\n1\n2n∥Y −Xβ∥2\n2 +\np\nX\nk=1\npλ,γ(|βk|).\nA prominent example is the minimax concave penalty (MCP) [Zhang, 2010] which takes\np′\nλ,γ(u) =\n\u0012\nλ −u\nγ\n\u0013\n+\n.\nOne disadvantage of using a non-convex penalty is that there may be multiple local minima\nwhich can make optimisation problematic. However, typically if the non-convexity is not\ntoo severe, coordinate descent can produce reasonable results.\n38",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.7 Extensions of the Lasso/3.7.2 Reducing the bias of the Lasso",
- "metadata": {
- "length": 2373,
- "summary": "This section discusses methods to reduce the bias inherent in the Lasso estimator, which shrinks non-zero coefficients towards zero. It introduces two primary approaches: re-estimation techniques like the 'relaxed Lasso' (Meinshausen, 2007) and the 'adaptive Lasso' (Zou, 2006), which uses weighted regression based on an initial estimate. Additionally, it covers the use of non-convex penalty functions, such as the 'minimax concave penalty' (MCP) by Zhang (2010), noting that while these can mitigate bias, they may introduce multiple local minima that complicate optimization.",
- "page_nums": [
- 42,
- 43
- ],
- "keywords": [
- "Meinshausen",
- "Zou",
- "Zhang"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 42,
- "artifact_ref": "page_citation_assets/page-42.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 43,
- "artifact_ref": "page_citation_assets/page-43.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_e9272750-0d84-5a09-ad0e-92d4a5d2102c",
- "type": "page",
- "content": "Chapter 4\nGraphical modelling and causal\ninference\nSo far we have considered the problem of relating a particular response to a large collection\nof explanatory variables.\nIn some settings however, we do not have a distinguished response variable and instead\nwe would like to better understand relationships between all the variables. In other sit-\nuations, rather than being able to predict variables, we would like to understand causal\nrelationships between them. Representing relationships between random variables through\ngraphs will be an important tool in tackling these problems.\n4.1\nGraphs\nDefinition 7. A graph is a pair G = (V, E) where V is a set of vertices or nodes and\nE ⊆V × V with (v, v) /∈E for any v ∈V is a set of edges.\nLet Z = (Z1, . . . , Zp)T be a collection of random variables. The graphs we will consider\nwill always have V = {1, . . . , p} so V indexes the random variables.\nLet j, k ∈V .\n We say there is an edge between j and k and that j and k are adjacent if either\n(j, k) ∈E or (k, j) ∈E.\n An edge (j, k) is undirected if also (k, j) ∈E. We then sometimes write {j, k} ∈E\nand j −k to represent this. Otherwise if only (j, k) ∈E the edge is directed and we\nmay write j →k to represent this.\n If all edges in the graph are (un)directed we call it an (un)directed graph. We can\nrepresent graphs as pictures: for example, we can draw the graph when p = 4 and\nE = {(2, 1), (3, 4), (2, 3)} as\n39\n\nZ1\nZ3\nZ2\nZ4\nIf instead we have E = {{1, 2}, {2, 4}} we get the undirected graph\nZ1\nZ3\nZ2\nZ4\n A graph G1 = (V1, E1) is a subgraph of G = (V, E) if V1 ⊆V and E1 ⊆E and a\nproper subgraph if either of these are proper inclusions.\n Say j is a parent of k and k is a child of j if j →k. The sets of parents and children\nof k will be denoted pa(k) and ch(k) respectively.\n A set of three nodes is called a v-structure if one node is a child of the two other\nnodes, and these two nodes are not adjacent.\n The skeleton of G is a copy of G with every edge replaced by an undirected edge.\n A path from j to k is a sequence j = j1, j2, . . . , jm = k of (at least two) distinct\nvertices such that jl and jl+1 are adjacent. Such a path is a directed path if jl →jl+1\nfor all l. We then call k a descendant of j. The set of descendants of j will be denoted\nde(j). If jl−1 →jl ←jl+1, jl is called a collider (relative to the path).\n A directed cycle is (almost) a directed path but with the start and end points the\nsame. A partially directed acyclic graph (PDAG) is a graph containing no directed\ncycles. A directed acyclic graph (DAG) is a directed graph containing no directed\ncycles.\n In a DAG, a path between j1 and jm (j1, j2, . . . , jm) is blocked by a set S with neither j1\nnor jm in S whenever there is a node jl such that one of the following two possibilities\nhold:\n1. jl ∈S and we don’t have jl−1 →jl ←jl+1\n2. jl−1 →jl ←jl+1 and neither jl nor any of its descendants are in S.\n If G is a DAG, given a triple of subsets of nodes A, B, S, we say S d-separates A from\nB if S blocks every path from A to B.\n40",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.1 Graphs",
- "metadata": {
- "length": 3035,
- "summary": "Section 4.1 introduces the fundamental concepts of graphs used in graphical modelling and causal inference. It defines a graph as a pair of vertices (nodes) and edges, specifically indexing them by random variables. The section details types of edges (directed vs. undirected), adjacency, subgraphs, parent-child relationships, v-structures, skeletons, paths, descendants, colliders, directed cycles, and Directed Acyclic Graphs (DAGs). It concludes with definitions for blocked paths and d-separation within DAGs.",
- "page_nums": [
- 44,
- 45
- ],
- "keywords": [
- "Chapter 4",
- "Definition 7"
- ],
- "connect_to": [
- {
- "target": "asset_9ab77f2b-3439-5115-8efa-dd51a5588351",
- "relation": "embeds",
- "ref": "[images/image_page_45_figure_1.png]"
- },
- {
- "target": "asset_dc0be7e0-62a0-5c03-b3f0-086a59df6fc8",
- "relation": "embeds",
- "ref": "[images/image_page_45_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 44,
- "artifact_ref": "page_citation_assets/page-44.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 45,
- "artifact_ref": "page_citation_assets/page-45.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_795c86a8-b750-5bf6-bce0-87709ba2e34f",
- "type": "page",
- "content": " The moralised graph of a DAG G is the undirected graph obtained by adding edges\nbetween (marrying) the parents of each node and removing all edge directions.\nDefinition 8. Given a DAG G with V = {1, . . . , p}, we say that a permutation π of V is\na topological (or causal) ordering of the variables if it satisfies\nπ(j) < π(k)\nwhenever k ∈de(j).\nProposition 28. Every DAG has a topological ordering.\nProof. We use induction on the number of nodes p. Clearly the result is true when p = 1.\nNow we show that in any DAG, we can find a node with no parents. Pick any node\nand move to one of its parents, if possible. Then move to one of the new node’s parents,\nand continue in this fashion. This procedure must terminate since no node can be visited\ntwice, or we would have found a cycle. The final node we visit must therefore have no\nparents, which we call a source node.\nSuppose then that p ≥2, and we know that all DAGs with p−1 nodes have a topological\nordering. Find a source s (wlog s = p) and form a new DAG ˜G with p−1 nodes by removing\nthe source (and all edges emanating from it). Note we keep the labelling of the nodes in\nthis new DAG the same. This smaller DAG must have a topological order ˜π. A topological\nordering π for our original DAG is then given by π(s) = 1 and π(k) = ˜π(k)+1 for k ̸= s.\n4.2\nConditional independence graphs\nWe would like to understand which variables may be ‘related’ to each other. Trying to find\npairs of variables that are independent and so unlikely to be related to each other is not\nnecessarily a good way to proceed as each variable may be correlated with a large number\nof variables without being directly related to them. A better approach is to use conditional\nindependence.\nDefinition 9. If X, Y and Z are random vectors with a joint density fXY Z (w.r.t. a\nproduct measure µ) then we say X is conditionally independent of Y given Z, and write\nX ⊥⊥Y |Z\nif\nfXY |Z(x, y|z) = fX|Z(x|z)fY |Z(y|z).\nEquivalently\nX ⊥⊥Y |Z ⇐⇒fX|Y Z(x|y, z) = fX|Z(x|z).\nWe will first look at how undirected graphs can be used to visualise conditional inde-\npendencies between random variables; thus in the next few subsections by graph we will\nmean undirected graph.\nDefinition 10. Let Z = (Z1, . . . , Zp)T be a collection of random variables with joint law\nP. The conditional independence graph (CIG) for P is the graph with V = {1, . . . , p} and\nan edge {j, k} between j and k if and only if\nZj ̸⊥⊥Zk | Z−jk.\n41",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.2 Conditional independence graphs",
- "metadata": {
- "length": 2433,
- "summary": "This page contains mathematical definitions and proofs related to Directed Acyclic Graphs (DAGs) and conditional independence. It begins with the definition of a moralised graph and Definition 8 regarding topological orderings, followed by Proposition 28 which states that every DAG has a topological ordering along with its proof. The page then introduces Section 4.2 on Conditional Independence Graphs, defining conditional independence (Definition 9) and the conditional independence graph (CIG) (Definition 10).",
- "page_nums": [
- 46
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 46,
- "artifact_ref": "page_citation_assets/page-46.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_59f1456c-972e-5cc9-9292-0cb4b5d0f3c4",
- "type": "page",
- "content": "4.3\nGaussian graphical models\nEstimating the CIG given samples from P is a difficult task in general. However, in the\ncase where P is multivariate Gaussian, things simplify considerably as we shall see. We\nbegin with some notation. For a matrix M ∈Rp×p, and sets A, B ⊆{1, . . . , p}, let MA,B\nbe the |A| × |B| submatrix of M consisting of those rows and columns of M indexed by\nthe sets A and B respectively. The submatrix extraction operation is always performed\nfirst (so e.g. M T\nk,−k = (Mk,−k)T).\n4.3.1\nNormal conditionals\nNow let Z ∼Np(µ, Σ) with Σ positive definite. Note ΣA,A is also positive definite for any\nA.\nProposition 29.\nZA|ZB = zB ∼N|A|(µA + ΣA,BΣ−1\nB,B(zB −µB), ΣA,A −ΣA,BΣ−1\nB,BΣB,A)\nProof. Idea: write ZA = MZB+(ZA−MZB) with matrix M ∈R|A|×|B| such that ZA−MZB\nand ZB are independent, i.e. such that\nCov(ZB, ZA −MZB) = ΣB,A −ΣB,BM T = 0.\nThis occurs when we take M T = Σ−1\nB,BΣB,A. Because ZA −MZB and ZB are indepen-\ndent, the distribution of ZA −MZB conditional on ZB = zB is equal to its unconditional\ndistribution. Now\nE(ZA −MZB) = µA −ΣA,BΣ−1\nB,BµB\nVar(ZA −MZB) = ΣA,A + ΣA,BΣ−1\nB,BΣB,BΣ−1\nB,BΣB,A −2ΣA,BΣ−1\nB,BΣB,A\n= ΣA,A −ΣA,BΣ−1\nB,BΣB,A.\nSince MZB is a function of ZB and ZA −MZB is normally distributed, we have the\nresult.\n4.3.2\nNodewise regression\nSpecialising to the case where A = {k} and B = Ac we see that when conditioning on\nZ−k = z−k, we may write\nZk = mk + zT\n−kΣ−1\n−k,−kΣ−k,k + εk,\nwhere\nmk = µk −Σk,−kΣ−1\n−k,−kµ−k\nεk|Z−k = z−k ∼N(0, Σk,k −Σk,−kΣ−1\n−k,−kΣ−k,k).\n42",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.1 Normal conditionals",
- "metadata": {
- "length": 1499,
- "summary": "Section 4.3.1 Normal conditionals presents Proposition 29, which defines the conditional distribution of a subset of variables $Z_A$ given another subset $Z_B = z_B$ for a multivariate Gaussian vector $Z$. The section includes a proof demonstrating that by constructing an independent component $Z_A - MZ_B$, one can derive the mean and variance of the resulting normal distribution.",
- "page_nums": [
- 47
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 47,
- "artifact_ref": "page_citation_assets/page-47.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_290a4c11-1b31-5773-b626-efc4ed774082",
- "type": "page",
- "content": "[SAME-AS Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.1 Normal conditionals p47]",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.2 Nodewise regression",
- "metadata": {
- "length": 143,
- "summary": "Section 4.3.2, titled 'Nodewise regression', specializes the previously discussed normal conditionals to the specific case where a single variable $Z_k$ is conditioned on all other variables $Z_{-k}$. It derives the resulting linear regression equation for $Z_k$, defining the intercept term $m_k$ and the conditional distribution of the error term $\\varepsilon_k$.",
- "page_nums": [
- 47
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 47,
- "artifact_ref": "page_citation_assets/page-47.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_04a92779-c505-54d6-af44-0bf93a9dc0bc",
- "type": "page",
- "content": "Note that if the jth element of the vector of coefficients Σ−1\n−k,−kΣ−k,k is zero, then the\ndistribution of Zk conditional on Z−k will not depend at all on the jth component of Z−k.\nThen if that jth component was Zj′, we would have that Zk|Z−k = z−k has the same\ndistribution as Zk|Z−j′k = z−j′k, so Zk ⊥⊥Zj|Z−j′k.\nThus given x1, . . . , xn\ni.i.d.\n∼Z and writing\nX =\n\n\n\nxT\n1...\nxT\nn\n\n\n,\nwe may estimate the coefficient vector Σ−1\n−k,−kΣ−k,k by regressing Xk on X{k}c and including\nan intercept term.\nThe technique of neighbourhood selection [Meinshausen and B¨uhlmann, 2006] involves\nperforming such a regression for each variable, using the Lasso. There are two options for\npopulating our estimate of the CIG with edges based on the Lasso estimates. Writing ˆSk\nfor the selected set of variables when regressing Xk on X{k}c, we can use the “OR” rule\nand put an edge between vertices j and k if and only if k ∈ˆSj or j ∈ˆSk. An alternative is\nthe “AND” rule where we put an edge between j and k if and only if k ∈ˆSj and j ∈ˆSk.\nAnother popular approach to estimating the CIG works by first directly estimating Ω,\nas we’ll now see.\n4.3.3\nThe precision matrix and conditional independence\nThe following facts about blockwise inversion of matrices will help us to interpret the mean\nand variance in Proposition 29.\nProposition 30. Let M ∈Rp×p be a symmetric positive definite matrix and suppose\nM =\n\u0012P\nQT\nQ\nR\n\u0013\nwith P and R square matrices. The Schur complement of R is P −QTR−1Q =: S. We\nhave that S is positive definite and\nM −1 =\n\u0012\nS−1\n−S−1QTR−1\n−R−1QS−1\nR−1 + R−1QS−1QTR−1\n\u0013\n.\nFurthermore det(M) = det(S)det(R).\nLet Ω= Σ−1 be the precision matrix. Note that Σk,k −Σk,−kΣ−1\n−k,−kΣ−k,k = Ω−1\nkk , and\nmore generally that Var(ZA|ZAc) = Ω−1\nA,A. Also, we see that Σ−1\n−k,−kΣ−k,k = −Ω−1\nkk Ω−k,k, so\n(Σ−1\n−k,−kΣ−k,k)j = 0 ⇔\n(\nΩj,k = 0 for j < k\nΩj+1,k = 0 for j ≥k.\nThus\nZk ⊥⊥Zj|Z−jk ⇔Ωjk = 0.\nThis motivates another approach to estimating the CIG.\n43",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.3 The precision matrix and conditional independence",
- "metadata": {
- "length": 1947,
- "summary": "This page discusses statistical methods for estimating Conditional Independence Graphs (CIG), specifically focusing on neighborhood selection using the Lasso and the relationship between precision matrices and conditional independence. It details the 'OR' and 'AND' rules for edge selection in CIG estimation and presents Proposition 30 regarding blockwise inversion of symmetric positive definite matrices to interpret mean and variance.",
- "page_nums": [
- 48
- ],
- "keywords": [
- "Meinshausen",
- "Bühlmann"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 48,
- "artifact_ref": "page_citation_assets/page-48.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_9b0ff888-28a3-5aef-ab2c-35b756fa88f3",
- "type": "page",
- "content": "4.3.4\nThe Graphical Lasso\nRecall that the density of Np(µ, Σ) is\nf(z) =\n1\n(2π)p/2det(Σ)1/2 exp\n\u0012\n−1\n2(z −µ)TΣ−1(z −µ)\n\u0013\n.\nThe log-likelihood of (µ, Σ) based on an i.i.d. sample x1, . . . , xn is\nℓ(µ, Ω) = n\n2 log det(Ω) −1\n2\nn\nX\ni=1\n(xi −µ)TΩ(xi −µ).\nWrite\n¯X = 1\nn\nn\nX\ni=1\nxi,\nS = 1\nn\nn\nX\ni=1\n(xi −¯X)(xi −¯X)T.\nThen\nn\nX\ni=1\n(xi −µ)TΩ(xi −µ) =\nn\nX\ni=1\n(xi −¯X + ¯X −µ)TΩ(xi −¯X + ¯X −µ)\n=\nn\nX\ni=1\n(xi −¯X)TΩ(xi −¯X) + n( ¯X −µ)TΩ( ¯X −µ)\n+ 2\nn\nX\ni=1\n(xi −¯X)TΩ( ¯X −µ).\nAlso,\nn\nX\ni=1\n(xi −¯X)TΩ(xi −¯X) =\nn\nX\ni=1\ntr{(xi −¯X)TΩ(xi −¯X)}\n=\nn\nX\ni=1\ntr{(xi −¯X)(xi −¯X)TΩ}\n= ntr(SΩ).\nThus\nℓ(µ, Ω) = −n\n2{tr(SΩ) −log det(Ω) + ( ¯X −µ)TΩ( ¯X −µ)}\nand\nmax\nµ∈Rp ℓ(µ, Ω) = −n\n2{tr(SΩ) −log det(Ω)}.\nHence the maximum likelihood estimate of Ω, ˆΩML can be obtained by solving\nmin\nΩ:Ω≻0{−log det(Ω) + tr(SΩ)},\n44",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.4 The Graphical Lasso",
- "metadata": {
- "length": 802,
- "summary": "This page presents a mathematical derivation for the Graphical Lasso method, starting with the density function of a multivariate normal distribution. It defines the log-likelihood and sample statistics (mean and covariance), then algebraically manipulates these terms to derive the maximum likelihood estimate for the precision matrix Omega.",
- "page_nums": [
- 49
- ],
- "keywords": [
- "Graphical Lasso",
- "N_p(mu, Sigma)"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 49,
- "artifact_ref": "page_citation_assets/page-49.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_4e7b2530-097e-5117-a312-4f7235656ca3",
- "type": "page",
- "content": "where Ω≻0 means Ωis positive definite. One can show that the objective is convex and\nwe are minimising over a convex set. As\n∂\n∂Ωjk\nlog det(Ω) = (Ω−1)kj = (Ω−1)jk,\n∂\n∂Ωjk\ntr(SΩ) = Skj = Sjk,\nif X has full column rank so S is positive definite, ˆΩML = S−1.\nThe graphical Lasso [Yuan and Lin, 2007] penalises the log-likelihood for Ωand solves\nmin\nΩ:Ω≻0{−log det(Ω) + tr(SΩ) + λ∥Ω∥1},\nwhere ∥Ω∥1 = P\nj,k |Ωjk|; this results in a sparse estimate of the precision matrix from\nwhich an estimate of the CIG can be constructed. Often the ∥Ω∥1 is modified such that\nthe diagonal elements are not penalised.\n4.4\nStructural equation models\nConditional independence graphs give us some understanding of the relationships between\nvariables. However they do not tell us how, if we were to set the kth variable to a particular\nvalue, say 0.5, then how the distribution of the other values would be altered. Yet this is\noften the sort of question that we would like to answer.\nIn order to reach this more ambitious goal, we introduce the notion of structural equation\nmodels (SEMs). These give a way of representing the data generating process. We will now\nhave to make use of not just undirected graphs but other sorts of graphs (and particularly\nDAGs), so by graph we will now mean any sort of graph satisfying definition 7.\nDefinition 11. A structural equation model S for a random vector Z ∈Rp is a collection\nof p equations\nZk = hk(ZPk, εk),\nk = 1, . . . , p\nwhere\n ε1, . . . , εp are all independent random variables;\n Pk ⊆{1, . . . , p} \\ {k} are such that the graph with edges given by Pk being pa(k) is\na DAG.\nExample 4.4.1. Consider the following (totally artificial) SEM which has whether you\nare taking this course (Z1 = 1) depending on whether you went to the statistical modelling\ncourse (Z2 = 1) and whether you have heard about machine learning (Z3 = 1). Suppose\nZ3 = ε3 ∼Bern(1/4)\nZ2 = 1{ε2(1+Z3)>1/2}\nε2 ∼U[0, 1]\nZ1 = 1{ε1(Z2+Z3)>1/2}\nε1 ∼U[0, 1].\nThe corresponding DAG is\n45",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.4 Structural equation models",
- "metadata": {
- "length": 1974,
- "summary": "This page continues a discussion on statistical modeling, specifically focusing on the graphical Lasso method for estimating precision matrices and sparse conditional independence graphs. It then introduces Section 4.4, 'Structural equation models' (SEMs), defining them as collections of equations representing data-generating processes using directed acyclic graphs (DAGs). The section concludes with Example 4.4.1, which presents an artificial SEM involving variables related to taking a course, attending a statistical modeling class, and hearing about machine learning.",
- "page_nums": [
- 50
- ],
- "keywords": [
- "Yuan",
- "Lin"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 50,
- "artifact_ref": "page_citation_assets/page-50.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_31d81c7a-3a02-5c2d-9147-4b1a2c2a8116",
- "type": "page",
- "content": "Z1\nZ3\nZ2\nNote that an SEM for Z determines its law. Indeed using a topological ordering π for\nthe associated DAG, we can write each Zk as a function of επ−1(1), επ−1(2), . . . , επ−1(π(k)).\nImportantly, though, we can use it to tell us much more than simply the law of Z: for\nexample we can query properties of the distribution of Z after having set a particular\ncomponent to any given value. This is what we study next.\n4.5\nInterventions\nGiven an SEM S, we can replace one (or more) of the structural equations by a new\nstructural equation, for example for a chosen variable k we could replace the structural\nequation Zk = hk(ZPk, εk) by Zk = ˜hk( ˜Z ˜Pk, ˜εk). This gives us a new structural equation\nmodel ˜S which in turn determines a new joint law for Z.\nWhen we have ˜hk( ˜Z ˜Pk, ˜εk) = a for some a ∈R, so we are setting the value of Zk to\nbe a, we call this a (perfect) intervention. Expectations and probabilities under this new\nlaw for Z are written by adding |do(Zk = a) e.g. E(Zj|do(Zk = a)). Note that this will in\ngeneral be different from the conditional expectation E(Zj|Zk = a).\nExample 4.4.1 continued.\nAfter the intervention do(Z2 = 1) (everyone is forced to go\nto statistical modelling), we have a new SEM ˜S:\nZ3 = ε3 ∼Bern(1/4)\nZ2 = 1\nZ1 = 1{ε1(1+Z3)>1/2}\nε1 ∼U[0, 1].\nThus P(Z1 = 1|do(Z2 = 1)) = 1\n4\n3\n4 + 3\n4\n1\n2 =\n9\n16. On the other hand,\nP(Z1 = 1|Z2 = 1) =\nX\nj∈{0,1}\nP(Z1 = 1|Z2 = 1, Z3 = j)P(Z3 = j|Z2 = 1)\n=\n1\nP(Z2 = 1)\nX\nj∈{0,1}\nP(Z1 = 1|Z2 = 1, Z3 = j)P(Z2 = 1|Z3 = j)P(Z3 = j)\n=\n1\n1\n4\n3\n4 + 3\n4\n1\n2\n\u00123\n4\n3\n4\n1\n4 + 1\n2\n1\n2\n3\n4\n\u0013\n= 7\n12 ̸= 9\n16.\n46",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.5 Interventions",
- "metadata": {
- "length": 1575,
- "summary": "This page discusses interventions in Structural Equation Models (SEM), specifically focusing on how replacing structural equations affects the joint law of variables. It defines a 'perfect intervention' using the do-operator notation and provides a continued example (4.4.1) where variable Z2 is set to 1. The text includes mathematical derivations comparing the probability of Z1=1 under an intervention versus conditional expectation, showing they are not equal.",
- "page_nums": [
- 51
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_aa55d178-c0fc-54dc-871c-de6f78473060",
- "relation": "embeds",
- "ref": "[images/image_page_51_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 51,
- "artifact_ref": "page_citation_assets/page-51.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_68ab802f-5317-50da-8c99-c60c27cbfa04",
- "type": "page",
- "content": "4.6\nThe Markov properties on DAGs\nThe DAG of an SEM can encode a number of conditional independencies present in the\nlaw of the random vector Z. To understand this, we first introduce some so-called Markov\nproperties on DAGs.\nLet P be the joint law of Z and suppose it has a density f.\nDefinition 12. Given a DAG G, we say P satisfies the\n(i) Markov factorisation property w.r.t. the DAG G if\nf(z1, . . . , zp) =\np\nY\nk=1\nf(zk|zpa(k)).\n(ii) global Markov property w.r.t. the DAG G if for all disjoint A, B, S ⊆{1, . . . , p},\nA, B d-separated by S ⇒ZA ⊥⊥ZB|ZS.\nTheorem 31. If P has a density f (with respect to a product measure), then all Markov\nproperties in definition 12 are equivalent.\nIn view of this, we will henceforth use the term Markov to mean global Markov.\nProposition 32. Let P be the law of an SEM with DAG G. Then P obeys the Markov\nfactorisation property w.r.t. G.\nThus we can read offfrom the DAG of an SEM a great deal of information concerning the\ndistribution it generates. We can use this to help us calculate the effects of interventions.\nWe have seen now how an SEM can be used to not only query properties of the joint\ndistribution, but also to determine the effects of certain perturbations to the system. In\nmany settings, we may not have a prespecified SEM to work with, but instead we would\nlike to learn the DAG from observational data. This is the problem we turn to next.\n4.7\nCausal structure learning\nGiven a sample of observations from P, we would like to determine the DAG which gener-\nated it. We can think of this task in terms of two subtasks: firstly we need to understand\nhow to extract information concerning P from a sample, which is a traditional statistical\nquestion of the sort we are used to; secondly, given P itself, we need to relate this to the\nDAG which generated it. The latter problem is unique to casual inference and we discuss\nthis first.\n4.7.1\nThree obstacles\nThere are three obstacles to causal structure learning. The first two are more immediate\nbut the last is somewhat subtle.\n47",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.6 The Markov properties on DAGs",
- "metadata": {
- "length": 2029,
- "summary": "Section 4.6 introduces the Markov properties on Directed Acyclic Graphs (DAGs) in the context of Structural Equation Models (SEMs). It defines two key properties: the Markov factorisation property, which relates the joint density to a product of conditional densities based on parent nodes, and the global Markov property, which links d-separation in the graph to conditional independence in the probability distribution. The section states that these properties are equivalent for distributions with a density (Theorem 31) and asserts that the law of an SEM obeys the Markov factorisation property (Proposition 32). It concludes by noting that these properties allow one to read information about the distribution from the DAG and determine the effects of interventions.",
- "page_nums": [
- 52
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 52,
- "artifact_ref": "page_citation_assets/page-52.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_e9d5b832-6e26-51ae-bf93-895c8e206ddc",
- "type": "page",
- "content": "[SAME-AS Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.6 The Markov properties on DAGs p52]\n\nCausal minimality\nWe know that if P is generated by an SEM with DAG G, then P will be Markov w.r.t. G.\nConversely, one can show that if P is Markov w.r.t. a DAG G, then there is also an SEM\nwith DAG G that could have generated P. But P will be Markov w.r.t. a great number of\nDAGs, e.g. Z1 and Z2 being independent can be represented by\nZ1 = 0 × Z2 + ε1 = ε1,\nZ2 = ε2.\nThis motivates the following definition.\nDefinition 13. P satisfies causal minimality with respect to G if it is (global) Markov\nw.r.t. G but not to a proper subgraph of G with the same nodes.\nMarkov equivalent DAGs\nIt is possible for two different DAGs to satisfy the same collection of d-separations e.g.\nZ1\nZ2\nZ1\nZ2\nFor a DAG G, let\nM(G) = {distributions P : P satisfies the global Markov property w.r.t. G}.\nDefinition 14. We say two DAGs G1 and G2 are Markov equivalent if M(G1) = M(G2).\nProposition 33. Two DAGs are Markov equivalent if and only if they have the same\nskeleton and v-structures.\nThe set of all DAGs that are Markov equivalent to a DAG can be represented by a\ncompleted PDAG (CPDAG) which contains an edge (j, k) if and only if one member of the\nMarkov equivalence class does. We can only ever hope to obtain the Markov equivalence\nclass i.e. the CPDAG of a DAG with which P satisfies causal minimality (unless we place\nrestrictions on the functional forms of the SEM equations).\nFaithfulness\nConsider the following SEM.\nZ1 = ε1\nZ2 = αZ1 + ε2\nZ3 = βZ1 + γZ2 + ε3,\nwhere ε ∼N3(0, I). Then (Z1, Z2, Z3) ∼N3(0, Σ) = P 0 with\nΣ =\n\n\n1\nα\nβ + αγ\nα\nα2 + 1\nαβ + γ(α2 + 1)\nβ + αγ\nαβ + γ(α2 + 1)\nβ2 + γ2(α2 + 1) + 2βγα + 1\n\n.\n48\n\nIf β + αγ = 0 e.g. if β = −1, α, γ = 1, then Z1 ⊥⊥Z3. We claim that in this case P 0 can\nalso be generated by the SEM\n˜Z1 = ˜ε1\n˜Z2 = ˜Z1 + ˜α ˜Z3 + ˜ε2\n˜Z3 = ˜ε3.\nHere the ˜εj are independent with ˜ε1 ∼N(0, 1), ˜ε3 ∼N(0, 2), ˜α = 1/2 and ˜ε3 ∼N(0, 1/2).\nWriting the DAGs for the two SEMs above as G and ˜G, note that P 0 satisfies causal\nminimality w.r.t. both G and ˜G.\nDefinition 15. We say P is faithful to the DAG G if it is Markov w.r.t. G and for all\ndisjoint A, B, S ⊆{1, . . . , p},\nA, B d-separated by S ⇐ZA ⊥⊥ZB|ZS.\nFaithfulness demands that all conditional independencies in P are represented in the\nDAG. In our example P 0 is not faithful to G, but it is faithful to ˜G.\n4.7.2\nThe PC algorithm\nProposition 34. If nodes j and k in a DAG G are adjacent, then no set can d-separate\nthem. If they are not adjacent and π is a topological order with π(j) < π(k), then they are\nd-separated by pa(k).\nProof. Consider a path j = j1, . . . , jm = k. We may assume we don’t have jm−1 →k\nas otherwise the path would be blocked since jm−1 ∈pa(k). Let l be the largest l′ with\njl′−1 →jl′ ←jl′+1; this must exist as otherwise we would have a directed path from k to j\ncontradicting the topological ordering. In order for the path to be active, jl′ must have a\ndescendant in pa(k), but this would introduce a cycle.\nThis shows in particular that any non-adjacent nodes must have a d-separating set. If\nwe assume that P is faithful w.r.t. a DAG G, we can check whether nodes j and k are\nadjacent in G by testing whether there is a set S with Zj ⊥⊥Zk|ZS. If there is no such set\nS, j and k must be adjacent. This allows us to recover the skeleton of G.\nProposition 35. Suppose we have a triple of nodes j, k, l in a DAG and the only non-\nadjacent pair is j, k (i.e. in the skeleton j −l −k).\n(i) If the nodes are in a v-structure (j →l ←k) then no S that d-separates j and k can\ncontain l.\n(ii) If there exists an S that d-separates j and k and l /∈S, then we must have j →l ←k.\nProof. For (i) note that any set containing l cannot block the path j, l, k. For (ii) note we\nknow that the path j, l, k is blocked by S, so we must have j →l ←k.\n49",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.1 Three obstacles",
- "metadata": {
- "length": 3866,
- "summary": "The section titled '4.7.1 Three obstacles' introduces the challenges in causal structure learning. It states that there are three specific obstacles to this process, noting that the first two are more immediate while the third is somewhat subtle.",
- "page_nums": [
- 52,
- 53,
- 54
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 52,
- "artifact_ref": "page_citation_assets/page-52.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 53,
- "artifact_ref": "page_citation_assets/page-53.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 54,
- "artifact_ref": "page_citation_assets/page-54.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_3765ca36-6264-5451-aeeb-a4a17c67454b",
- "type": "page",
- "content": "This last result then allows us to find the v-structures given the skeleton and a d-\nseparating set S(j, k) corresponding to each absent edge. Given a skeleton and v-structures,\nit may be possible to orient further edges by making use of the acyclicity of DAGs; we do\nnot cover this here.\nPopulation version\nThe PC-algorithm, named after its inventors Peter Spirtes and Clarke Glymour [Spirtes\net al., 2000], exploits the fact that we need not search over all sets S but only subsets of\neither pa(j) or pa(k) for efficiency. The version assumes P is known and so conditional\nindependencies can be queried directly. A sample version that is applicable in practice is\ngiven in the following subsection. We denote the set of nodes that are adjacent to a node\nj in graph G by adj(G, j).\nAlgorithm 1 First part of the PC algorithm: finding the skeleton.\nSet ˆG to be the complete undirected graph. Set ℓ= −1.\nrepeat\nIncrement ℓ→ℓ+ 1.\nrepeat\nSelect a (new) ordered pair of nodes j, k that are adjacent in ˆG and such that\n|adj( ˆG, j) \\ {k}| ≥ℓ.\nrepeat\nChoose new S ⊆adj( ˆG, j) \\ {k} with |S| = ℓ.\nIf Zj ⊥⊥Zk|ZS then delete edges (j, k) and (k, j) and set S(j, k) = S(k, j) = S.\nuntil edges (j, k), (k, j) are deleted or all relevant subsets have been chosen.\nuntil all relevant ordered pairs have been chosen.\nuntil for every ordered pair j, k that are adjacent in ˆG we have |adj( ˆG, j) \\ {k}| < ℓ.\nSuppose P is faithful to DAG G0. At each stage of the Algorithm 1 we must have that\nthe skeleton is a subgraph of ˆG. By the end of the algorithm, for each pair j, k adjacent in\nˆG, we would have searched through adj( ˆG, j) and adj( ˆG, k) for sets S such that Zj ⊥⊥Zk|ZS.\nIf P were faithful to G0 then, we would know that j and k must be adjacent in G0. That\nis the output of Algorithm 1 would be the skeleton of G0.\nAlgorithm 2 Second part of the PC algorithm: finding the v-structures\nfor all pairs of non-adjacent variables j, k (in skeleton ˆG) with common neighbour l do\nIf l /∈S(j, k) then orient j →l ←k.\nend for\n50",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm/Population version",
- "metadata": {
- "length": 2016,
- "summary": "This page details the 'Population version' of the PC algorithm, a method for learning causal structures from data. It introduces Algorithm 1, which finds the skeleton of a Directed Acyclic Graph (DAG) by iteratively removing edges based on conditional independence tests. The text also presents Algorithm 2, which identifies v-structures to orient the remaining edges in the graph.",
- "page_nums": [
- 55
- ],
- "keywords": [
- "Peter Spirtes",
- "Clarke Glymour"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 55,
- "artifact_ref": "page_citation_assets/page-55.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_39fb99f8-e688-5082-8fa5-8c9f837c5e47",
- "type": "page",
- "content": "Sample version\nThe sample version of the PC algorithm replaces the querying of conditional independence\nwith a conditional independence test applied to data x1, . . . , xn.\nThe level of the test\nα will be a tuning parameter of the method. If the data are assumed to be multivariate\nnormal, the (sample) partial correlation can be used to test conditional independence since\nif Zj ⊥⊥Zk|ZS then\nCorr(Zj, Zk|ZS) := ρjk·S = 0.\nTo compute the sample partial correlation, we regress Xj and Xk on XS and compute the\ncorrelation between the resulting residuals.\n51",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm/Sample version",
- "metadata": {
- "length": 556,
- "summary": "This page describes the sample version of the PC algorithm, which uses conditional independence tests on data rather than direct queries. It explains that if data is multivariate normal, sample partial correlation can be used to test for conditional independence, providing a mathematical formula for this condition. The text concludes by outlining the computational method: regressing variables X_j and X_k on X_S and computing the correlation between the resulting residuals.",
- "page_nums": [
- 56
- ],
- "keywords": [
- "PC algorithm"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 56,
- "artifact_ref": "page_citation_assets/page-56.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_c0859016-7a1e-540c-9827-b86a681d792c",
- "type": "page",
- "content": "Chapter 5\nHigh-dimensional inference\nConsider the normal linear model Y = Xβ0 + ε where ε ∼Nn(0, σ2I).\nIn the low-\ndimensional setting, the fact that ˆβOLS −β0 ∼Np(0, σ2(XTX)−1) allows us to form con-\nfidence intervals for components of β0\nj and perform hypothesis tests with H0 : β0\nj = 0, for\nexample.\nOne might hope that studying the distribution of ˆβL\nλ −β0 would enable us to perform\nthese tasks in the high-dimensional setting when p ≫n. However, the distribution of\nˆβL\nλ −β0 is intractable and depends delicately on the unknown β0, making it unsuitable as\na basis for establishing confidence intervals.\nWhilst several methods have been proposed over the years, typically they have involved\nplacing conditions on the unknown β0, other than the usual assumption of sparsity. Given\nthat the task is to perform inference for β0, such conditions are undesirable. In the last\ncouple of years, there has been a breakthrough on this front [Zhang and Zhang, 2014,\nVan de Geer et al., 2014], and here we will aim to cover the main ideas in this exciting\ndevelopment. Our treatment follows Van de Geer et al. [2014].\nWe begin our investigation by considering the KKT conditions of the Lasso. Fix λ > 0\nand let ˆβ be the Lasso estimator with tuning parameter λ. Recall that the KKT conditions\ngive\n1\nnXT(Y −X ˆβ) = λˆν\nwhere ∥ˆν∥∞≤1 and writing ˆS = {k : ˆβk ̸= 0}, ˆν ˆS = sgn(ˆβ ˆS). Setting ˆΣ = XTX/n and\nrearranging we have\nˆΣ(ˆβ −β0) + λˆν = 1\nnXTε.\nThe key idea is now to form an approximate inverse ˆΘ of ˆΣ. Then we have\nˆβ + λˆΘˆν −β0 = 1\nn\nˆΘXTε + 1\n√n∆\nwhere ∆= √n(ˆΘˆΣ −I)(β0 −ˆβ). Define\nˆb = ˆβ + λˆΘˆν = ˆβ + ˆΘXT(Y −X ˆβ)/n,\n52\n\nwhich we shall refer to as the debiased Lasso. If we choose ˆΘ such that ∆is small, we will\nhave ˆb −β0 ≈ˆΘXTε/n, which can be used as a basis for performing inference.\nWe already know that under a compatibility condition on the design matrix X, ∥ˆβ−β0∥1\nis small (Theorem 21) with high probability. If we can also show that the ℓ∞-norms of\nrows of ˆΘˆΣ −I are small, we can leverage this fact using H¨older’s inequality to show that\n∥∆∥∞is small. Let ˆθj be the jth row of ˆΘ. Then ∥(ˆΣˆΘT −I)j∥∞≤η is equivalent to\n1\nn∥XT\n−jX ˆθj∥∞≤η and |XT\nj X ˆθj/n −1| ≤η.\nThe first of these inequalities is somewhat reminiscent of the KKT conditions for the Lasso.\nLet\nˆγ(j) = arg min\nγ∈Rp−1\n\u001a 1\n2n∥Xj −X−jγ∥2\n2 + λj∥γ∥1\n\u001b\n.\n(5.0.1)\nFurther let\nˆτ 2\nj = XT\nj (Xj −X−jˆγ(j))/n = 1\nn∥Xj −X−jˆγ(j)∥2\n2 + λj∥ˆγ(j)∥1;\nsee the example sheet for the final equality. Then set\nˆθj = −1\nˆτ 2\nj\n(ˆγ(j)\n1 , . . . , ˆγ(j)\nj−1, −1, ˆγ(j)\nj , . . . , ˆγ(j)\np−1)T.\n↑\njth position\nNote that by construction,\nX ˆθj =\nXj −X−jˆγ(j)\nXT\nj (X −X−jˆγ(j))/n.\nThus XT\nj X ˆθj/n = 1 and by the KKT conditions of the Lasso optimisation (5.0.1), we have\nˆτ 2\nj ∥XT\n−jX ˆθj∥∞/n ≤λj.\nThus with the choice of ˆΘ defined as above, we have\n∥∆∥∞≤√n∥ˆβ −β0∥1 max\nj\nλj\nˆτ 2\nj\nWhen can we expect λj/ˆτ 2\nj to be small? One way of answering this is to consider a random\ndesign setting. Let us assume that each row of X is independent and distributed as Np(0, Σ)\nwhere Σ is positive definite. Write Ω= Σ−1. From Proposition 29 and our study of the\nneighbourhood selection procedure (see also Section 4.3.3), we know that for each j, we\ncan write\nXj = X−jγ(j) + ε(j),\n(5.0.2)\n53",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm/Chapter 5",
- "metadata": {
- "length": 3263,
- "summary": "This section introduces high-dimensional inference, focusing on the limitations of standard estimators like the Lasso in settings where the number of predictors (p) exceeds the sample size (n). It discusses the development of the 'debiased Lasso' as a method to perform valid statistical inference. The text details the mathematical derivation of this estimator, starting from the Karush-Kuhn-Tucker (KKT) conditions of the Lasso and constructing an approximate inverse of the covariance matrix to correct for bias.",
- "page_nums": [
- 57,
- 58
- ],
- "keywords": [
- "Zhang and Zhang",
- "Van de Geer et al."
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 57,
- "artifact_ref": "page_citation_assets/page-57.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 58,
- "artifact_ref": "page_citation_assets/page-58.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_3004ca97-d460-546a-bdee-70859bfc1cb6",
- "type": "page",
- "content": "where ε(j)\ni |X−j\ni.i.d.\n∼N(0, Ω−1\njj ) and γ(j) = −Ω−1\njj Ω−j,j. Theorem 21 can therefore be used to\nunderstand properties of ˆγ(j) and hence the ˆτ 2\nj . In order to apply this result however, we\nneed γ(j) to be sparse. Let us therefore define\nsj =\nX\nk̸=j\n1{Ωkj̸=0}\nand smax = max(maxj sj, s). In order to make the following result more easily interpretable,\nwe will consider an asymptotic regime where X, s, smax etc. are all allowed to change as\nn →∞, though we suppress this in the notation. We will consider σ as constant.\nTheorem 36. Suppose the minimum eigenvalue of Σ is always at least cmin > 0 and\nmaxj Σjj ≤1. Suppose further that smax\np\nlog(p)/n →0. Then there exists constants\nA1, A2 such that setting λ = λj = A1\np\nlog(p)/n, we have\n√n(ˆb −β0) = W + ∆\nW|X ∼Np(0, σ2 ˆΘˆΣˆΘT),\nand as n, p →∞,\nP(∥∆∥∞> A2s log(p)/√n) →0.\nProof. Consider the sequence of events Λn described by the following properties:\n φ2\nˆΣ,s ≥cmin/2 and φ2\nˆΣ−j,−j,sj ≥cmin/2 for all j,\n 2∥XTε∥∞/n ≤λ and 2∥XT\n−jε(j)∥∞/n ≤λ for all j,\n ∥ε(j)∥2\n2/n ≥Ω−1\njj (1 −4\np\nlog(p)/n) for all j.\nYou will show on the example sheet that then P(Λn) →1 for A1 sufficiently large. In the\nfollowing we work on Λn, and c1, c2, . . . will be constants.\nBy Theorem 21, we have\n∥ˆβ −β0∥1 ≤c1s\np\nlog(p)/n.\nWe now seek a lower bound for the ˆτ 2\nj . Consider the linear models in (5.0.2). Note that\nthe maximum eigenvalue of Ωis at most c−1\nmin so Ωjj ≤c−1\nmin. Also, Ω−1\njj = Var(Xij|Xi,−j) ≤\nVar(Xij) = Σjj ≤1. Thus applying Theorem 21 to the linear models (5.0.2), we know that\n∥γ(j) −ˆγ(j)∥1 ≤c2sj\np\nlog(p)/n.\nThen\nˆτ 2\nj ≥1\nn∥Xj −X−jˆγ(j)∥2\n2 ≥1\nn∥ε(j)∥2\n2 −2\nn∥XT\n−jε(j)∥∞∥γ(j) −ˆγ(j)∥1\n≥Ω−1\njj (1 −4\np\nlog(p)/n) −c4smax log(p)\nn\n≥cmin/2\n54",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm/Theorem 36.",
- "metadata": {
- "length": 1706,
- "summary": "This page presents a mathematical proof for Theorem 36, which establishes asymptotic properties of an estimator in a high-dimensional statistical setting. It defines sparsity measures and sets up an asymptotic regime where the number of variables $p$ and sample size $n$ grow to infinity. The proof relies on Theorem 21 and derives bounds for the estimation error of coefficients ($\\hat{\\beta} - \\beta^0$) and variance estimates ($\\hat{\\tau}_j^2$).",
- "page_nums": [
- 59
- ],
- "keywords": [
- "Theorem 21",
- "Theorem 36"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 59,
- "artifact_ref": "page_citation_assets/page-59.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- },
- {
- "chunk_id": "node_695b1c13-5b5b-5b87-a7f3-feacf2c91528",
- "type": "page",
- "content": "for all j when n is sufficiently large. Putting things together we see that on Λn,\n∥∆∥∞≤λ√n∥ˆβ −β0∥1 max\nj\nˆτ −2\nj\n≤2A1\np\nlog(p)(c1s\np\nlog(p)/n)/cmin ≤A2s log(p)/√n\nwhere A2 = 2c1A1/cmin. Thus\nP(∥∆∥∞> A2s log(p)/√n) ≤P(Λc\nn) →0.\n5.1\nUsing the debiased Lasso in practice\nTheorem 36 shows in particular that\n√n(ˆbj −β0\nj ) ≈Wj\nwhere Wj ∼N(0, σ2(ˆΘˆΣˆΘT)jj). Let (ˆΘˆΣˆΘT)jj = dj. The approximate equality above\nsuggests constructing (1-α)-level confidence intervals of the form\nh\nˆbj −zα/2σ\np\ndj/√n, ˆbj + zα/2σ\np\ndj/√n\ni\n,\nwhere zα is the upper α point of a standard normal. The only unknown quantity in the\nconfidence interval above is σ: this can be estimated using e.g. the square-root / scaled\nLasso [Sun and Zhang, 2012].\n55\n\nBibliography\nL. Breiman. Stacked regressions. Machine Learning, 24:49–64, 1996.\nC. Cortes and V. Vapnik.\nSupport-vector networks. Machine learning, 20(3):273–297,\n1995.\nA. Hoerl and R. Kennard. Ridge regression: Biased estimation for nonorthogonal problems.\nTechnometrics, pages 55–67, 1970.\nG. S. Kimeldorf and G. Wahba. A correspondence between bayesian estimation on stochas-\ntic processes and smoothing by splines. The Annals of Mathematical Statistics, 41(2):\n495–502, 1970.\nN. Meinshausen. Relaxed lasso. Computational Statistics and Data Analysis, 52:374–393,\n2007.\nN. Meinshausen and P. B¨uhlmann. High dimensional graphs and variable selection with\nthe lasso. Annals of Statistics, 34:1436–1462, 2006.\nA. Rahimi and B. Recht. Random features for large-scale kernel machines. In Advances in\nNeural Information Processing Systems, pages 1177–1184, 2007.\nB. Sch¨olkopf, R. Herbrich, and A. J. Smola. A generalized representer theorem. In Inter-\nnational Conference on Computational Learning Theory, pages 416–426. Springer, 2001.\nP. Spirtes, C. N. Glymour, and R. Scheines. Causation, prediction, and search. MIT press,\n2000.\nT. Sun and C.-H. Zhang. Scaled sparse linear regression. Biometrika, page ass043, 2012.\nR. Tibshirani.\nRegression shrinkage and selection via the lasso.\nJournal of the Royal\nStatistical Society, Series B, 58:267–288, 1996.\nR. Tibshirani, M. Saunders, S. Rosset, J. Zhu, and K. Knight.\nSparsity and smooth-\nness via the fused lasso. Journal of the Royal Statistical Society: Series B (Statistical\nMethodology), 67(1):91–108, 2005.\nP. Tseng. Convergence of a block coordinate descent method for nondifferentiable mini-\nmization. Journal of optimization theory and applications, 109(3):475–494, 2001.\n56\n\nS. Van de Geer, P. B¨uhlmann, Y. Ritov, R. Dezeure, et al. On asymptotically optimal\nconfidence regions and tests for high-dimensional models. The Annals of Statistics, 42\n(3):1166–1202, 2014.\nD. Wolpert. Stacked generalization. Neural Networks, 5:241–259, 1992.\nY. Yang, M. Pilanci, and M. J. Wainwright. Randomized sketches for kernels: Fast and\noptimal non-parametric regression. arXiv preprint arXiv:1501.06195, 2015.\nM. Yuan and Y. Lin. Model selection and estimation in regression with grouped variables.\nJournal of the Royal Statistical Society: Series B, 68:49–67, 2006.\nM. Yuan and Y. Lin. Model selection and estimation in the gaussian graphical model.\nBiometrika, 94(1):19–35, 2007.\nC.-H. Zhang. Nearly unbiased variable selection under minimax concave penalty. The\nAnnals of statistics, pages 894–942, 2010.\nC.-H. Zhang and S. S. Zhang. Confidence intervals for low dimensional parameters in high\ndimensional linear models. Journal of the Royal Statistical Society: Series B (Statistical\nMethodology), 76(1):217–242, 2014.\nH. Zou. The adaptive lasso and its oracle properties. Journal of the American Statistical\nAssociation, 101:1418–1429, 2006.\n57",
- "path": "Statistical Learning Notes.pdf/5 High-dimensional inference/5.1 Using the debiased Lasso in practice",
- "metadata": {
- "length": 3618,
- "summary": "This section explains the practical application of the debiased Lasso method. It derives an approximate normal distribution for the estimator, which is then used to construct (1-α)-level confidence intervals. The text notes that the only unknown parameter in these intervals, σ, can be estimated using methods like the square-root or scaled Lasso.",
- "page_nums": [
- 60,
- 61,
- 62
- ],
- "keywords": [
- "Sun and Zhang",
- "The Annals of Statistics",
- "Journal of the Royal Statistical Society: Series B"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 60,
- "artifact_ref": "page_citation_assets/page-60.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 61,
- "artifact_ref": "page_citation_assets/page-61.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- },
- {
- "page_num": 62,
- "artifact_ref": "page_citation_assets/page-62.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 1191,
- "height": 1684
- }
- ]
- }
- }
- ]
-}
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/doc_nav.json b/apps/api/app/data/demo_documents/stem-statistical-learning/doc_nav.json
deleted file mode 100644
index 86052bdda..000000000
--- a/apps/api/app/data/demo_documents/stem-statistical-learning/doc_nav.json
+++ /dev/null
@@ -1,490 +0,0 @@
-{
- "version": "1.0",
- "file_name": "Statistical Learning Notes.pdf",
- "stats": {
- "total_chunks": 45,
- "text_chunks": 0,
- "image_chunks": 6,
- "table_chunks": 0,
- "page_chunks": 39,
- "max_depth": 4
- },
- "sections": [
- {
- "title": "1 Kernel machines",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines",
- "level": 1,
- "summary": "",
- "chunk_count": 19,
- "children": [
- {
- "title": "1.1 Ridge regression",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.1 Ridge regression",
- "level": 2,
- "summary": "",
- "chunk_count": 2,
- "children": [
- {
- "title": "1.1.1 The singular value decomposition and principal components analysis",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.1 Ridge regression/1.1.1 The singular value decomposition and principal components analysis",
- "level": 3,
- "summary": "The section titled '1.1.1 The singular value decomposition and principal components analysis' introduces the Singular Value Decomposition (SVD) as a generalization of eigendecomposition for any matrix X, defining its factorization into orthogonal matrices U and V and a diagonal matrix D containing singular values. It details the 'thin SVD' variant used when n ≥ p and applies this decomposition to derive the fitted values for ridge regression, demonstrating how it shrinks coordinates relative to Ordinary Least Squares (OLS). The text concludes by establishing the connection between SVD and Principal Components Analysis (PCA).",
- "chunk_count": 2,
- "children": []
- }
- ]
- },
- {
- "title": "1.2 v-fold cross-validation",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.2 v-fold cross-validation",
- "level": 2,
- "summary": "This page continues a mathematical derivation regarding ridge regression and principal components, specifically focusing on how the method shrinks Y in smaller principal components. It then introduces section 1.2, 'v-fold cross-validation,' explaining it as a technique for selecting optimal regression parameters (lambda) by splitting data into groups to estimate prediction error.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "1.3 The kernel trick",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.3 The kernel trick",
- "level": 2,
- "summary": "The section titled '1.3 The kernel trick' explains a computational method to efficiently handle non-linear models, specifically ridge regression with quadratic effects. It demonstrates that by reformulating the problem to depend only on inner products between observations (the Gram matrix), one can avoid explicitly computing a large number of new features. This allows for a significant reduction in computational complexity from O(n^2*p^2 + n^3) to O(n^2*p). The text concludes by introducing the general concept of using a similarity measure k(x_i, x_j) directly, which is formalized in the subsequent section '1.4 Kernels'.",
- "chunk_count": 3,
- "children": []
- },
- {
- "title": "1.4 Kernels",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.4 Kernels",
- "level": 2,
- "summary": "",
- "chunk_count": 5,
- "children": [
- {
- "title": "1.4.1 Examples of kernels",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.4 Kernels/1.4.1 Examples of kernels",
- "level": 3,
- "summary": "This page presents a mathematical proof regarding the properties of kernels, followed by Section 1.4.1 which provides examples of specific kernel functions. It details Proposition 4 concerning linear combinations and products of kernels, then defines and explains Linear, Polynomial, and Gaussian kernels with their respective formulas and properties.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "1.4.2 Reproducing kernel Hilbert spaces",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.4 Kernels/1.4.2 Reproducing kernel Hilbert spaces",
- "level": 3,
- "summary": "This section defines Reproducing Kernel Hilbert Spaces (RKHS). It begins with Theorem 5, which states that for every kernel k, there exists a feature map φ into an inner product space H such that the kernel can be expressed as an inner product of the mapped features. A proof is provided by constructing the space H from functions formed by linear combinations of the kernel and defining an inner product on it. The text then discusses the completion of this space to form a Hilbert space and provides Definition 2 for an RKHS, where evaluation at any point x is a continuous linear functional represented by an inner product with a specific function k_x. Finally, examples of RKHS are given for the Linear kernel and the Sobolev kernel.",
- "chunk_count": 3,
- "children": []
- },
- {
- "title": "1.4.3 The representer theorem",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.4 Kernels/1.4.3 The representer theorem",
- "level": 3,
- "summary": "This page presents section 1.4.3, titled 'The representer theorem,' which discusses the optimization problem in kernel ridge regression. It defines the objective function for ridge regression and states Theorem 6, attributed to Kimeldorf and Wahba (1970) and Schölkopf et al. (2001). The theorem establishes that minimizing a loss function over a Reproducing Kernel Hilbert Space (RKHS) is equivalent to minimizing a finite-dimensional quadratic form involving the kernel matrix. The page includes a detailed proof of this theorem using Pythagoras' theorem and concludes by applying the result to the specific case of the ridge regression objective.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "1.5 Kernel ridge regression",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.5 Kernel ridge regression",
- "level": 2,
- "summary": "This section introduces kernel ridge regression, a method that uses the kernel trick to solve potentially infinite-dimensional versions of ridge regression. It establishes a statistical framework for analyzing predictive performance by defining a data-generating process and assuming the true function lies within a Reproducing Kernel Hilbert Space (RKHS). The core of the section is Theorem 7, which provides an upper bound on the Mean Squared Prediction Error (MSPE) in terms of the eigenvalues of the kernel matrix. A detailed proof follows, utilizing techniques like the 'trace trick' and Hölder's inequality. The analysis is then extended to a random design setting using Mercer's theorem to describe the eigendecomposition of the kernel operator. Finally, it derives the optimal convergence rate for the error when using a Sobolev kernel.",
- "chunk_count": 3,
- "children": []
- },
- {
- "title": "1.6 Other kernel machines",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.6 Other kernel machines",
- "level": 2,
- "summary": "",
- "chunk_count": 3,
- "children": [
- {
- "title": "1.6.1 The support vector machine",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.6 Other kernel machines/1.6.1 The support vector machine",
- "level": 3,
- "summary": "This section introduces the support vector machine (SVM) as a method for classification when data labels are discrete (e.g., {-1, 1}). It begins by defining the problem of finding a hyperplane that separates two classes. The text then formulates an optimization problem to find the hyperplane that maximizes the margin between these classes. It further extends this formulation to handle cases where classes are not perfectly separable by introducing a penalty term, leading to the standard SVM objective function. Finally, it generalizes the model to include a bias term (or translation), resulting in the final definition of the support vector classifier.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "1.6.2 Logistic regression",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.6 Other kernel machines/1.6.2 Logistic regression",
- "level": 3,
- "summary": "This page continues a discussion on Support Vector Machines (SVM) and introduces Logistic Regression. It presents the optimization problem for an SVM using a kernel matrix K, followed by the prediction formula. Section 1.6.2 details standard logistic regression, deriving its log-likelihood maximization problem and presenting its 'kernelised' version as an optimization over a Reproducing Kernel Hilbert Space (RKHS).",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "1.7 Large-scale kernel machines",
- "path": "Statistical Learning Notes.pdf/1 Kernel machines/1.7 Large-scale kernel machines",
- "level": 2,
- "summary": "This section discusses methods for speeding up computations in large-scale kernel machines. It introduces the concept of random feature expansions to approximate kernels, thereby reducing computational complexity from cubic to linear in the number of data points. The text details how shift-invariant kernels can be approximated using Bochner's theorem and provides a concrete example using the Gaussian kernel.",
- "chunk_count": 2,
- "children": []
- }
- ]
- },
- {
- "title": "2 Introduction to the Lasso",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso",
- "level": 1,
- "summary": "",
- "chunk_count": 7,
- "children": [
- {
- "title": "2.1 Model selection",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.1 Model selection",
- "level": 2,
- "summary": "The section '2.1 Model selection' revisits the linear model and discusses the Mean Squared Prediction Error (MSPE) of Ordinary Least Squares (OLS). It explains that if the true set of relevant variables could be identified, a reduced model would yield a smaller MSPE and more accurate estimates. The text then introduces classical model selection strategies, starting with 'Best subset regression', which involves an exhaustive search over all possible subsets of explanatory variables. This method is noted to be computationally infeasible for large numbers of predictors (p > 50).",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "2.2 The Lasso estimator",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.2 The Lasso estimator",
- "level": 2,
- "summary": "",
- "chunk_count": 5,
- "children": [
- {
- "title": "2.2.1 Convexity and the Lasso",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.2 The Lasso estimator/2.2.1 Convexity and the Lasso",
- "level": 3,
- "summary": "This page, titled 'Convexity and the Lasso', discusses the computational advantages of Lasso regression over Best subsets regression due to its convex optimization nature. It defines mathematical concepts of convex sets and functions, presents Proposition 9 regarding properties of convex functions (including conditions involving Hessians), and provides a proof for Proposition 10 demonstrating that Lasso fitted values are unique.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "2.2.2 Prediction error of the Lasso with no assumptions on the design",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.2 The Lasso estimator/2.2.2 Prediction error of the Lasso with no assumptions on the design",
- "level": 3,
- "summary": "This section discusses the prediction error of the Lasso estimator when there are no assumptions on the design matrix, specifically in high-dimensional settings where p >> n. It presents Theorem 11, which establishes a bound on the prediction error with high probability under a normal linear model. The proof involves deriving an inequality based on the definition of the Lasso solution and utilizing Lemma 15 to bound the probability of a specific event.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "2.2.3 Concentration inequalities I",
- "path": "Statistical Learning Notes.pdf/2 Introduction to the Lasso/2.2 The Lasso estimator/2.2.3 Concentration inequalities I",
- "level": 3,
- "summary": "This section, titled 'Concentration inequalities I', provides the theoretical foundation for bounding tail probabilities of random variables, which is necessary to prove Theorem 11. It begins by introducing Markov's inequality and deriving the Chernoff bound from it. The text then defines sub-Gaussian variables and presents key results: Hoeffding's lemma (showing bounded mean-zero variables are sub-Gaussian), a proposition on linear combinations of sub-Gaussian variables, and Lemma 15, which establishes a probability bound for the infinity norm of a vector of independent sub-Gaussian variables.",
- "chunk_count": 3,
- "children": []
- }
- ]
- }
- ]
- },
- {
- "title": "3 The Lasso and beyond",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond",
- "level": 1,
- "summary": "",
- "chunk_count": 14,
- "children": [
- {
- "title": "3.1 Some facts from optimisation theory and convex analysis",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.1 Some facts from optimisation theory and convex analysis",
- "level": 2,
- "summary": "",
- "chunk_count": 3,
- "children": [
- {
- "title": "3.1.1 Subgradients",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.1 Some facts from optimisation theory and convex analysis/3.1.1 Subgradients",
- "level": 3,
- "summary": "The section titled '3.1.1 Subgradients' defines the concept of a subgradient for a convex function and introduces the subdifferential as the set of all such subgradients. It presents key propositions regarding subgradients, including their relationship to gradients when a function is differentiable, their behavior under scalar multiplication and addition, and the Karush-Kuhn-Tucker (KKT) conditions for optimality. The section concludes by deriving the subdifferential of the ℓ₁-norm.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "3.1.2 The Lagrangian method",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.1 Some facts from optimisation theory and convex analysis/3.1.2 The Lagrangian method",
- "level": 3,
- "summary": "Section 3.1.2 introduces the Lagrangian method for solving constrained optimization problems of the form minimize f(x) subject to g(x)=0. It defines the Lagrangian function L(x, θ) = f(x) + θ^T g(x) and explains that the method involves finding a parameter θ* such that the minimizer x* of the Lagrangian satisfies the original constraint g(x*)=0.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "3.2 Lasso solutions",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.2 Lasso solutions",
- "level": 2,
- "summary": "Section 3.2 characterizes the solutions to the Lasso problem using tools from convex analysis. It states that a vector is a Lasso solution if and only if it satisfies specific Karush-Kuhn-Tucker (KKT) conditions, which are expressed as an equation involving the transpose of the design matrix X, the response vector Y, and a subgradient vector with an infinity norm less than or equal to 1.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "3.3 Variable selection",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.3 Variable selection",
- "level": 2,
- "summary": "This section discusses variable selection in the context of a noiseless high-dimensional linear model. It presents Theorem 20, which establishes conditions under which a Lasso solution correctly recovers the sign pattern of the true coefficient vector. A key condition mentioned is the 'irrepresentable condition' (||Δ||_∞ ≤ 1), which ensures that variables not in the true support set do not correlate too strongly with those in the support set.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "3.4 Prediction and estimation",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.4 Prediction and estimation",
- "level": 2,
- "summary": "This section, titled 'Prediction and estimation', introduces the compatibility condition as a requirement for obtaining Mean Squared Prediction Error (MSPE) bounds for the Lasso estimator. It defines the compatibility constant φ² via an infimum over a restricted set of vectors. Theorem 21 is then presented, which provides a high-probability bound on the prediction error and estimation error of the Lasso solution under this condition. The proof involves deriving a series of inequalities based on the basic inequality and Hölder's inequality. The section concludes by beginning to discuss the strength of the compatibility condition in relation to the population covariance matrix.",
- "chunk_count": 3,
- "children": []
- },
- {
- "title": "3.5 The compatibility condition",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.5 The compatibility condition",
- "level": 2,
- "summary": "",
- "chunk_count": 2,
- "children": [
- {
- "title": "3.5.1 Concentration inequalities II",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.5 The compatibility condition/3.5.1 Concentration inequalities II",
- "level": 3,
- "summary": "This page presents a mathematical discussion on concentration inequalities, specifically focusing on Bernstein's inequality. It begins by defining Bernstein's condition for a random variable and then states Proposition 23, which provides tail bounds for the sum of independent random variables satisfying this condition. The page concludes with a detailed proof of the proposition using moment generating functions and the Chernoff method.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "3.5.2 The compatibility condition and random design",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.5 The compatibility condition/3.5.2 The compatibility condition and random design",
- "level": 3,
- "summary": "This page contains a mathematical proof for Lemma 24 regarding the product of mean-zero sub-Gaussian random variables satisfying Bernstein's condition. It includes detailed derivations using Fubini's theorem, Jensen's inequality, and Cauchy-Schwarz. The page also introduces section 3.5.2 on 'The compatibility condition and random design' and presents Theorem 25 concerning asymptotic frameworks for design matrices.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "3.6 Computation",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.6 Computation",
- "level": 2,
- "summary": "This page contains a mathematical proof concluding with Corollary 26, followed by Section 3.6 titled 'Computation'. The section discusses the coordinate descent optimization technique for computing Lasso solutions, presenting the general function form and the iterative update equations used in the algorithm. It references Tseng [2001] regarding convergence properties and concludes with Corollary 27.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "3.7 Extensions of the Lasso",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.7 Extensions of the Lasso",
- "level": 2,
- "summary": "",
- "chunk_count": 3,
- "children": [
- {
- "title": "3.7.1 Structural penalties",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.7 Extensions of the Lasso/3.7.1 Structural penalties",
- "level": 3,
- "summary": "This page contains a mathematical proof regarding the convergence of a minimizer for a continuous function, referencing the Bolzano-Weierstrass theorem and Tseng [2001]. It then details an algorithmic strategy using coordinate descent with a 'warm start' to solve the Lasso on a grid of lambda values. The text concludes with section 3.7, discussing extensions of the Lasso to other loss functions like logistic regression, and section 3.7.1, which introduces structural penalties.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "3.7.2 Reducing the bias of the Lasso",
- "path": "Statistical Learning Notes.pdf/3 The Lasso and beyond/3.7 Extensions of the Lasso/3.7.2 Reducing the bias of the Lasso",
- "level": 3,
- "summary": "This section discusses methods to reduce the bias inherent in the Lasso estimator, which shrinks non-zero coefficients towards zero. It introduces two primary approaches: re-estimation techniques like the 'relaxed Lasso' (Meinshausen, 2007) and the 'adaptive Lasso' (Zou, 2006), which uses weighted regression based on an initial estimate. Additionally, it covers the use of non-convex penalty functions, such as the 'minimax concave penalty' (MCP) by Zhang (2010), noting that while these can mitigate bias, they may introduce multiple local minima that complicate optimization.",
- "chunk_count": 2,
- "children": []
- }
- ]
- }
- ]
- },
- {
- "title": "4 Graphical modelling and causal inference",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference",
- "level": 1,
- "summary": "",
- "chunk_count": 18,
- "children": [
- {
- "title": "4.1 Graphs",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.1 Graphs",
- "level": 2,
- "summary": "Section 4.1 introduces the fundamental concepts of graphs used in graphical modelling and causal inference. It defines a graph as a pair of vertices (nodes) and edges, specifically indexing them by random variables. The section details types of edges (directed vs. undirected), adjacency, subgraphs, parent-child relationships, v-structures, skeletons, paths, descendants, colliders, directed cycles, and Directed Acyclic Graphs (DAGs). It concludes with definitions for blocked paths and d-separation within DAGs.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "4.2 Conditional independence graphs",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.2 Conditional independence graphs",
- "level": 2,
- "summary": "This page contains mathematical definitions and proofs related to Directed Acyclic Graphs (DAGs) and conditional independence. It begins with the definition of a moralised graph and Definition 8 regarding topological orderings, followed by Proposition 28 which states that every DAG has a topological ordering along with its proof. The page then introduces Section 4.2 on Conditional Independence Graphs, defining conditional independence (Definition 9) and the conditional independence graph (CIG) (Definition 10).",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.3 Gaussian graphical models",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models",
- "level": 2,
- "summary": "",
- "chunk_count": 4,
- "children": [
- {
- "title": "4.3.1 Normal conditionals",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.1 Normal conditionals",
- "level": 3,
- "summary": "Section 4.3.1 Normal conditionals presents Proposition 29, which defines the conditional distribution of a subset of variables $Z_A$ given another subset $Z_B = z_B$ for a multivariate Gaussian vector $Z$. The section includes a proof demonstrating that by constructing an independent component $Z_A - MZ_B$, one can derive the mean and variance of the resulting normal distribution.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.3.2 Nodewise regression",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.2 Nodewise regression",
- "level": 3,
- "summary": "Section 4.3.2, titled 'Nodewise regression', specializes the previously discussed normal conditionals to the specific case where a single variable $Z_k$ is conditioned on all other variables $Z_{-k}$. It derives the resulting linear regression equation for $Z_k$, defining the intercept term $m_k$ and the conditional distribution of the error term $\\varepsilon_k$.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.3.3 The precision matrix and conditional independence",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.3 The precision matrix and conditional independence",
- "level": 3,
- "summary": "This page discusses statistical methods for estimating Conditional Independence Graphs (CIG), specifically focusing on neighborhood selection using the Lasso and the relationship between precision matrices and conditional independence. It details the 'OR' and 'AND' rules for edge selection in CIG estimation and presents Proposition 30 regarding blockwise inversion of symmetric positive definite matrices to interpret mean and variance.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.3.4 The Graphical Lasso",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.3 Gaussian graphical models/4.3.4 The Graphical Lasso",
- "level": 3,
- "summary": "This page presents a mathematical derivation for the Graphical Lasso method, starting with the density function of a multivariate normal distribution. It defines the log-likelihood and sample statistics (mean and covariance), then algebraically manipulates these terms to derive the maximum likelihood estimate for the precision matrix Omega.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "4.4 Structural equation models",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.4 Structural equation models",
- "level": 2,
- "summary": "This page continues a discussion on statistical modeling, specifically focusing on the graphical Lasso method for estimating precision matrices and sparse conditional independence graphs. It then introduces Section 4.4, 'Structural equation models' (SEMs), defining them as collections of equations representing data-generating processes using directed acyclic graphs (DAGs). The section concludes with Example 4.4.1, which presents an artificial SEM involving variables related to taking a course, attending a statistical modeling class, and hearing about machine learning.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.5 Interventions",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.5 Interventions",
- "level": 2,
- "summary": "This page discusses interventions in Structural Equation Models (SEM), specifically focusing on how replacing structural equations affects the joint law of variables. It defines a 'perfect intervention' using the do-operator notation and provides a continued example (4.4.1) where variable Z2 is set to 1. The text includes mathematical derivations comparing the probability of Z1=1 under an intervention versus conditional expectation, showing they are not equal.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.6 The Markov properties on DAGs",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.6 The Markov properties on DAGs",
- "level": 2,
- "summary": "Section 4.6 introduces the Markov properties on Directed Acyclic Graphs (DAGs) in the context of Structural Equation Models (SEMs). It defines two key properties: the Markov factorisation property, which relates the joint density to a product of conditional densities based on parent nodes, and the global Markov property, which links d-separation in the graph to conditional independence in the probability distribution. The section states that these properties are equivalent for distributions with a density (Theorem 31) and asserts that the law of an SEM obeys the Markov factorisation property (Proposition 32). It concludes by noting that these properties allow one to read information about the distribution from the DAG and determine the effects of interventions.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "4.7 Causal structure learning",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning",
- "level": 2,
- "summary": "",
- "chunk_count": 8,
- "children": [
- {
- "title": "4.7.1 Three obstacles",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.1 Three obstacles",
- "level": 3,
- "summary": "The section titled '4.7.1 Three obstacles' introduces the challenges in causal structure learning. It states that there are three specific obstacles to this process, noting that the first two are more immediate while the third is somewhat subtle.",
- "chunk_count": 3,
- "children": []
- },
- {
- "title": "4.7.2 The PC algorithm",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm",
- "level": 3,
- "summary": "",
- "chunk_count": 5,
- "children": [
- {
- "title": "Population version",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm/Population version",
- "level": 4,
- "summary": "This page details the 'Population version' of the PC algorithm, a method for learning causal structures from data. It introduces Algorithm 1, which finds the skeleton of a Directed Acyclic Graph (DAG) by iteratively removing edges based on conditional independence tests. The text also presents Algorithm 2, which identifies v-structures to orient the remaining edges in the graph.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Sample version",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm/Sample version",
- "level": 4,
- "summary": "This page describes the sample version of the PC algorithm, which uses conditional independence tests on data rather than direct queries. It explains that if data is multivariate normal, sample partial correlation can be used to test for conditional independence, providing a mathematical formula for this condition. The text concludes by outlining the computational method: regressing variables X_j and X_k on X_S and computing the correlation between the resulting residuals.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Chapter 5",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm/Chapter 5",
- "level": 4,
- "summary": "This section introduces high-dimensional inference, focusing on the limitations of standard estimators like the Lasso in settings where the number of predictors (p) exceeds the sample size (n). It discusses the development of the 'debiased Lasso' as a method to perform valid statistical inference. The text details the mathematical derivation of this estimator, starting from the Karush-Kuhn-Tucker (KKT) conditions of the Lasso and constructing an approximate inverse of the covariance matrix to correct for bias.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "Theorem 36.",
- "path": "Statistical Learning Notes.pdf/4 Graphical modelling and causal inference/4.7 Causal structure learning/4.7.2 The PC algorithm/Theorem 36.",
- "level": 4,
- "summary": "This page presents a mathematical proof for Theorem 36, which establishes asymptotic properties of an estimator in a high-dimensional statistical setting. It defines sparsity measures and sets up an asymptotic regime where the number of variables $p$ and sample size $n$ grow to infinity. The proof relies on Theorem 21 and derives bounds for the estimation error of coefficients ($\\hat{\\beta} - \\beta^0$) and variance estimates ($\\hat{\\tau}_j^2$).",
- "chunk_count": 1,
- "children": []
- }
- ]
- }
- ]
- }
- ]
- },
- {
- "title": "5 High-dimensional inference",
- "path": "Statistical Learning Notes.pdf/5 High-dimensional inference",
- "level": 1,
- "summary": "",
- "chunk_count": 3,
- "children": [
- {
- "title": "5.1 Using the debiased Lasso in practice",
- "path": "Statistical Learning Notes.pdf/5 High-dimensional inference/5.1 Using the debiased Lasso in practice",
- "level": 2,
- "summary": "This section explains the practical application of the debiased Lasso method. It derives an approximate normal distribution for the estimator, which is then used to construct (1-α)-level confidence intervals. The text notes that the only unknown parameter in these intervals, σ, can be estimated using methods like the square-root or scaled Lasso.",
- "chunk_count": 3,
- "children": []
- }
- ]
- }
- ],
- "resources": {
- "images": [
- {
- "path": "images/image_page_21_figure_1.png",
- "summary": "[images/image_page_21_figure_1.png]"
- },
- {
- "path": "images/image_page_21_figure_1.png",
- "summary": "[images/image_page_21_figure_1.png]"
- },
- {
- "path": "images/image_page_45_figure_1.png",
- "summary": "[images/image_page_45_figure_1.png]"
- },
- {
- "path": "images/image_page_45_figure_2.png",
- "summary": "[images/image_page_45_figure_2.png]"
- },
- {
- "path": "images/image_page_51_figure_1.png",
- "summary": "[images/image_page_51_figure_1.png]"
- },
- {
- "path": "images/image_page_51_figure_1.png",
- "summary": "[images/image_page_51_figure_1.png]"
- }
- ],
- "tables": []
- }
-}
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_21_figure_1.png b/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_21_figure_1.png
deleted file mode 100644
index 72a91f45e..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_21_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_45_figure_1.png b/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_45_figure_1.png
deleted file mode 100644
index 47f3319ec..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_45_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_45_figure_2.png b/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_45_figure_2.png
deleted file mode 100644
index 14cc4eee4..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_45_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_51_figure_1.png b/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_51_figure_1.png
deleted file mode 100644
index 3e601460d..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/images/image_page_51_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/manifest.json b/apps/api/app/data/demo_documents/stem-statistical-learning/manifest.json
deleted file mode 100644
index ec213adda..000000000
--- a/apps/api/app/data/demo_documents/stem-statistical-learning/manifest.json
+++ /dev/null
@@ -1,281 +0,0 @@
-{
- "version": "2.0",
- "job_id": "job_d5b4e9750b05",
- "data_id": "demo-stem-statistical-learning",
- "source_file_name": "Statistical Learning Notes.pdf",
- "processing_date": "2026-08-19T04:18:05.907034Z",
- "processing": {
- "page_count": 62,
- "billing_status": "charged",
- "cost": {
- "micro_dollars": 930000,
- "credits": 0.93
- },
- "cost_estimate": {
- "currency": "USD",
- "pricing_version": "2026-08-19:874ab8b6c9e4",
- "total_input_cost": 0.0,
- "total_output_cost": 0.0,
- "total_cost": 0.0,
- "by_model": {
- "qwen3.5-flash": {
- "input_tokens": 57500,
- "output_tokens": 1547,
- "calls": 14,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 17142,
- "output_tokens": 153,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "calibration.react_loop": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 11664,
- "output_tokens": 658,
- "calls": 3,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "calibration.inspect_pages": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 12000,
- "output_tokens": 151,
- "calls": 2,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "page_memory.page_locate": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 16694,
- "output_tokens": 585,
- "calls": 8,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "timing": {
- "started_at": "2026-08-19T04:10:50.975742+00:00",
- "completed_at": "2026-08-19T04:18:05.877765+00:00",
- "duration_ms": 434902
- },
- "stages": {
- "timing_ms": {
- "page_memory.normalize_to_pdf": 0,
- "page_memory.profile": 232624,
- "page_memory.skeleton": 2,
- "page_memory.node_summary": 30326,
- "page_memory.node_rows": 109,
- "page_memory.node_assembly": 30451,
- "worker.parse.document": 434743
- },
- "token_usage": {
- "prompt_tokens": 57500,
- "completion_tokens": 1547,
- "total_tokens": 59047,
- "calls": 14,
- "by_model": {
- "qwen3.5-flash": {
- "prompt_tokens": 57500,
- "completion_tokens": 1547,
- "total_tokens": 59047,
- "calls": 14
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "prompt_tokens": 17142,
- "completion_tokens": 153,
- "total_tokens": 17295,
- "calls": 1,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 17142,
- "completion_tokens": 153,
- "total_tokens": 17295,
- "calls": 1
- }
- }
- },
- "calibration.react_loop": {
- "prompt_tokens": 11664,
- "completion_tokens": 658,
- "total_tokens": 12322,
- "calls": 3,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 11664,
- "completion_tokens": 658,
- "total_tokens": 12322,
- "calls": 3
- }
- }
- },
- "calibration.inspect_pages": {
- "prompt_tokens": 12000,
- "completion_tokens": 151,
- "total_tokens": 12151,
- "calls": 2,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 12000,
- "completion_tokens": 151,
- "total_tokens": 12151,
- "calls": 2
- }
- }
- },
- "page_memory.page_locate": {
- "prompt_tokens": 16694,
- "completion_tokens": 585,
- "total_tokens": 17279,
- "calls": 8,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 16694,
- "completion_tokens": 585,
- "total_tokens": 17279,
- "calls": 8
- }
- }
- }
- }
- }
- }
- },
- "statistics": {
- "total_chunks": 45,
- "text_chunks": 0,
- "image_chunks": 6,
- "table_chunks": 0,
- "page_chunks": 39,
- "total_pages": null
- },
- "HIERARCHY": {
- "1 Kernel machines": {
- "1.1 Ridge regression": {
- "1.1.1 The singular value decomposition and principal components analysis": {}
- },
- "1.2 v-fold cross-validation": {},
- "1.3 The kernel trick": {},
- "1.4 Kernels": {
- "1.4.1 Examples of kernels": {},
- "1.4.2 Reproducing kernel Hilbert spaces": {},
- "1.4.3 The representer theorem": {}
- },
- "1.5 Kernel ridge regression": {},
- "1.6 Other kernel machines": {
- "1.6.1 The support vector machine": {},
- "1.6.2 Logistic regression": {}
- },
- "1.7 Large-scale kernel machines": {}
- },
- "2 Introduction to the Lasso": {
- "2.1 Model selection": {},
- "2.2 The Lasso estimator": {
- "2.2.1 Convexity and the Lasso": {},
- "2.2.2 Prediction error of the Lasso with no assumptions on the design": {},
- "2.2.3 Concentration inequalities I": {}
- }
- },
- "3 The Lasso and beyond": {
- "3.1 Some facts from optimisation theory and convex analysis": {
- "3.1.1 Subgradients": {},
- "3.1.2 The Lagrangian method": {}
- },
- "3.2 Lasso solutions": {},
- "3.3 Variable selection": {},
- "3.4 Prediction and estimation": {},
- "3.5 The compatibility condition": {
- "3.5.1 Concentration inequalities II": {},
- "3.5.2 The compatibility condition and random design": {}
- },
- "3.6 Computation": {},
- "3.7 Extensions of the Lasso": {
- "3.7.1 Structural penalties": {},
- "3.7.2 Reducing the bias of the Lasso": {}
- }
- },
- "4 Graphical modelling and causal inference": {
- "4.1 Graphs": {},
- "4.2 Conditional independence graphs": {},
- "4.3 Gaussian graphical models": {
- "4.3.1 Normal conditionals": {},
- "4.3.2 Nodewise regression": {},
- "4.3.3 The precision matrix and conditional independence": {},
- "4.3.4 The Graphical Lasso": {}
- },
- "4.4 Structural equation models": {},
- "4.5 Interventions": {},
- "4.6 The Markov properties on DAGs": {},
- "4.7 Causal structure learning": {
- "4.7.1 Three obstacles": {},
- "4.7.2 The PC algorithm": {
- "Population version": {},
- "Sample version": {},
- "Chapter 5": {},
- "Theorem 36.": {}
- }
- }
- },
- "5 High-dimensional inference": {
- "5.1 Using the debiased Lasso in practice": {}
- }
- }
-}
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-10.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-10.png
deleted file mode 100644
index 7bd9659e7..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-10.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-11.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-11.png
deleted file mode 100644
index 435d8a04c..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-11.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-12.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-12.png
deleted file mode 100644
index ad3ac7ce3..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-12.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-13.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-13.png
deleted file mode 100644
index 3d8ab64b4..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-13.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-14.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-14.png
deleted file mode 100644
index 9d559b65a..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-14.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-15.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-15.png
deleted file mode 100644
index 8dd06751e..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-15.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-16.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-16.png
deleted file mode 100644
index 9e3512eef..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-16.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-17.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-17.png
deleted file mode 100644
index 3cb066077..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-17.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-18.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-18.png
deleted file mode 100644
index fd6512cfb..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-18.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-19.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-19.png
deleted file mode 100644
index 48eecb4f2..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-19.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-20.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-20.png
deleted file mode 100644
index bb6426613..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-20.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-21.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-21.png
deleted file mode 100644
index af436f1bf..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-21.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-22.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-22.png
deleted file mode 100644
index 5f6c4e0d6..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-22.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-23.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-23.png
deleted file mode 100644
index 17c5167c7..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-23.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-24.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-24.png
deleted file mode 100644
index c41629b09..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-24.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-25.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-25.png
deleted file mode 100644
index dbf8bba3e..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-25.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-26.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-26.png
deleted file mode 100644
index 517aa4060..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-26.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-27.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-27.png
deleted file mode 100644
index 32265a3bb..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-27.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-28.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-28.png
deleted file mode 100644
index ca52a68ba..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-28.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-29.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-29.png
deleted file mode 100644
index b8bb1989c..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-29.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-30.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-30.png
deleted file mode 100644
index 701f4b4f3..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-30.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-31.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-31.png
deleted file mode 100644
index 1a5fdbd85..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-31.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-32.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-32.png
deleted file mode 100644
index 3d30430d2..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-32.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-33.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-33.png
deleted file mode 100644
index f92cd1ecb..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-33.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-34.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-34.png
deleted file mode 100644
index e5c639619..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-34.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-35.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-35.png
deleted file mode 100644
index f6ea99deb..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-35.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-36.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-36.png
deleted file mode 100644
index 8ac855f53..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-36.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-37.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-37.png
deleted file mode 100644
index c41e64d4c..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-37.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-38.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-38.png
deleted file mode 100644
index 6fdce26b6..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-38.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-39.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-39.png
deleted file mode 100644
index 4bb7a795e..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-39.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-40.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-40.png
deleted file mode 100644
index bc596ceef..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-40.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-41.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-41.png
deleted file mode 100644
index 5b8307a3c..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-41.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-42.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-42.png
deleted file mode 100644
index 894cf1636..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-42.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-43.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-43.png
deleted file mode 100644
index 63fc70336..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-43.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-44.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-44.png
deleted file mode 100644
index 85e8af57f..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-44.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-45.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-45.png
deleted file mode 100644
index 2d9166729..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-45.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-46.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-46.png
deleted file mode 100644
index 14401c403..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-46.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-47.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-47.png
deleted file mode 100644
index 825d9993d..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-47.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-48.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-48.png
deleted file mode 100644
index dce28c3db..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-48.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-49.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-49.png
deleted file mode 100644
index d88e30241..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-49.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-50.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-50.png
deleted file mode 100644
index cb07dee69..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-50.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-51.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-51.png
deleted file mode 100644
index 7adf227a6..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-51.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-52.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-52.png
deleted file mode 100644
index ee7cfb756..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-52.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-53.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-53.png
deleted file mode 100644
index 403e9f6b9..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-53.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-54.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-54.png
deleted file mode 100644
index dd748adcd..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-54.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-55.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-55.png
deleted file mode 100644
index 44cf23b77..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-55.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-56.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-56.png
deleted file mode 100644
index 22ebab631..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-56.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-57.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-57.png
deleted file mode 100644
index 8d42d2639..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-57.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-58.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-58.png
deleted file mode 100644
index ead1f89b7..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-58.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-59.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-59.png
deleted file mode 100644
index 1c4c0399a..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-59.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-60.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-60.png
deleted file mode 100644
index 739a1d77d..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-60.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-61.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-61.png
deleted file mode 100644
index 646513c2a..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-61.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-62.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-62.png
deleted file mode 100644
index 501c7563c..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-62.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-7.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-7.png
deleted file mode 100644
index c38e9d32c..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-7.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-8.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-8.png
deleted file mode 100644
index 18fdfb2ec..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-8.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-9.png b/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-9.png
deleted file mode 100644
index 7fdb36e52..000000000
Binary files a/apps/api/app/data/demo_documents/stem-statistical-learning/page_citation_assets/page-9.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/chunks.json b/apps/api/app/data/demo_documents/tsla-q4-2025/chunks.json
deleted file mode 100644
index d3f8332a8..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/chunks.json
+++ /dev/null
@@ -1,2008 +0,0 @@
-{
- "chunks": [
- {
- "chunk_id": "asset_85f035bb-05a0-5e36-9bc4-8d44f13e042e",
- "type": "table",
- "content": "tables/table_page_2_1.html",
- "path": "tables/table_page_2_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 2
- ],
- "file_path": "tables/table_page_2_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_03a68271-090e-5345-8773-68c974244d44",
- "type": "table",
- "content": "tables/table_page_4_1.html",
- "path": "tables/table_page_4_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 4
- ],
- "file_path": "tables/table_page_4_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_1d12f8a0-f279-5e5b-a23e-2bbc39167342",
- "type": "table",
- "content": "tables/table_page_5_1.html",
- "path": "tables/table_page_5_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 5
- ],
- "file_path": "tables/table_page_5_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_4ff45cac-1440-522e-8f7e-9e7742d5f203",
- "type": "table",
- "content": "tables/table_page_6_1.html",
- "path": "tables/table_page_6_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 6
- ],
- "file_path": "tables/table_page_6_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_12a65600-46f3-5d1c-b2b7-f083a8593149",
- "type": "table",
- "content": "tables/table_page_7_1.html",
- "path": "tables/table_page_7_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 7
- ],
- "file_path": "tables/table_page_7_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_052c1e7a-29f1-5fee-92cd-0b74b320dfc4",
- "type": "table",
- "content": "tables/table_page_8_1.html",
- "path": "tables/table_page_8_1.html",
- "metadata": {
- "length": 26,
- "summary": "",
- "page_nums": [
- 8
- ],
- "file_path": "tables/table_page_8_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_0b4cbe9b-7731-5df7-8668-77dcc85e3e2e",
- "type": "image",
- "content": "Tesla AI Training Capacity Ramp (H100 equivalent GPUs)\n[images/image_page_9_figure_2.png]",
- "path": "images/image_page_9_figure_2.png",
- "metadata": {
- "length": 89,
- "summary": "",
- "page_nums": [
- 9
- ],
- "file_path": "images/image_page_9_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_509e8dfa-6ebc-56a6-8060-caf683f4e521",
- "type": "image",
- "content": "Cumulative Miles Driven with FSD (Supervised) (billions)\n[images/image_page_10_figure_1.png]",
- "path": "images/image_page_10_figure_1.png",
- "metadata": {
- "length": 92,
- "summary": "",
- "page_nums": [
- 10
- ],
- "file_path": "images/image_page_10_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_3614d9d2-de2a-56c6-9837-d3c869f461a2",
- "type": "image",
- "content": "Targeting Step-Function Improvement for our Next-Generation Inference Chip, AI5\n[images/image_page_10_figure_2.png]",
- "path": "images/image_page_10_figure_2.png",
- "metadata": {
- "length": 115,
- "summary": "",
- "page_nums": [
- 10
- ],
- "file_path": "images/image_page_10_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_e5e3668a-29f3-560a-81b4-8777871db0df",
- "type": "image",
- "content": "Cumulative Paid Robotaxi Miles\n[images/image_page_11_figure_1.png]",
- "path": "images/image_page_11_figure_1.png",
- "metadata": {
- "length": 66,
- "summary": "",
- "page_nums": [
- 11
- ],
- "file_path": "images/image_page_11_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_237b9b1a-cb17-5080-ba09-5e5ceb5a3bb6",
- "type": "table",
- "content": "tables/table_page_11_2.html",
- "path": "tables/table_page_11_2.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 11
- ],
- "file_path": "tables/table_page_11_2.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_70d54e04-aae7-5be8-ab6d-e83d599dc636",
- "type": "image",
- "content": "MODEL Y - 2025 BEST IN CLASS EURO NCAP SMALL SUV\n[images/image_page_15_figure_1.png]",
- "path": "images/image_page_15_figure_1.png",
- "metadata": {
- "length": 84,
- "summary": "",
- "page_nums": [
- 15
- ],
- "file_path": "images/image_page_15_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_c8366c98-3281-5e5c-9420-4fd22da01b8c",
- "type": "image",
- "content": "MODEL 3 - 2025 BEST IN CLASS EURO NCAP LARGE FAMILY CAR\n[images/image_page_16_figure_1.png]",
- "path": "images/image_page_16_figure_1.png",
- "metadata": {
- "length": 91,
- "summary": "",
- "page_nums": [
- 16
- ],
- "file_path": "images/image_page_16_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_92efada6-9ac0-5fd1-b5a0-c16f41d3bc94",
- "type": "image",
- "content": "[images/image_page_17_figure_1.png]",
- "path": "images/image_page_17_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 17
- ],
- "file_path": "images/image_page_17_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_01432e7f-9816-5765-aa22-09df1164d996",
- "type": "image",
- "content": "DRIVERLESS ROBOTAXI - TESTING IN AUSTIN\n[images/image_page_18_figure_1.png]",
- "path": "images/image_page_18_figure_1.png",
- "metadata": {
- "length": 75,
- "summary": "",
- "page_nums": [
- 18
- ],
- "file_path": "images/image_page_18_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_10a13f13-e347-5787-b65a-24d0419df947",
- "type": "image",
- "content": "CYBERCAB – COLD WEATHER TESTING IN ALASKA\n[images/image_page_19_figure_1.png]",
- "path": "images/image_page_19_figure_1.png",
- "metadata": {
- "length": 77,
- "summary": "",
- "page_nums": [
- 19
- ],
- "file_path": "images/image_page_19_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_9036f2b6-99af-5adb-810b-66877dc00820",
- "type": "image",
- "content": "TESLA SEMI - PRODUCTION BEGINNING IN 2026\n[images/image_page_20_figure_1.png]",
- "path": "images/image_page_20_figure_1.png",
- "metadata": {
- "length": 77,
- "summary": "",
- "page_nums": [
- 20
- ],
- "file_path": "images/image_page_20_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_71a6700a-6ba2-532d-8dd3-74cd5a840d83",
- "type": "image",
- "content": "TESLA SEMI - MEGACHARGER NETWORK PLANNED SITES FOR 2026\n[images/image_page_21_figure_1.png]",
- "path": "images/image_page_21_figure_1.png",
- "metadata": {
- "length": 91,
- "summary": "",
- "page_nums": [
- 21
- ],
- "file_path": "images/image_page_21_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_e9a56942-8c8d-5d1c-bca1-43581a0266b5",
- "type": "image",
- "content": "GIGAFACTORY SHANGHAI – 9 MILLIONTH VEHICLE PRODUCED (GLOBALLY)\n[images/image_page_22_figure_1.png]",
- "path": "images/image_page_22_figure_1.png",
- "metadata": {
- "length": 98,
- "summary": "",
- "page_nums": [
- 22
- ],
- "file_path": "images/image_page_22_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_27ec6411-6eed-5c56-b989-f06f827ebc48",
- "type": "image",
- "content": "GIGAFACTORY NEVADA - 6 MILLIONTH DRIVE UNIT PRODUCED\n[images/image_page_23_figure_1.png]",
- "path": "images/image_page_23_figure_1.png",
- "metadata": {
- "length": 88,
- "summary": "",
- "page_nums": [
- 23
- ],
- "file_path": "images/image_page_23_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_acd3b2a7-87cc-5480-8edc-4560f6bf9581",
- "type": "image",
- "content": "KEY METRICS QUARTERLY\n[images/image_page_24_figure_1.png]",
- "path": "images/image_page_24_figure_1.png",
- "metadata": {
- "length": 57,
- "summary": "",
- "page_nums": [
- 24
- ],
- "file_path": "images/image_page_24_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_65df6a5c-1947-5912-9de0-9ceb53c28524",
- "type": "image",
- "content": "Vehicle Deliveries (millions of units)\n[images/image_page_25_figure_1.png]",
- "path": "images/image_page_25_figure_1.png",
- "metadata": {
- "length": 74,
- "summary": "",
- "page_nums": [
- 25
- ],
- "file_path": "images/image_page_25_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_f103abcb-b20a-50d0-b29c-ca62a69a3fb7",
- "type": "image",
- "content": "Operating Cash Flow ($B) and Free Cash Flow ($B)\n[images/image_page_25_figure_2.png]",
- "path": "images/image_page_25_figure_2.png",
- "metadata": {
- "length": 84,
- "summary": "",
- "page_nums": [
- 25
- ],
- "file_path": "images/image_page_25_figure_2.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_2e016cd6-faa5-54d4-ab58-5ffd594e0ab9",
- "type": "image",
- "content": "Net Income ($B) and Adjusted EBITDA ($B)\n[images/image_page_25_figure_3.png]",
- "path": "images/image_page_25_figure_3.png",
- "metadata": {
- "length": 76,
- "summary": "",
- "page_nums": [
- 25
- ],
- "file_path": "images/image_page_25_figure_3.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "asset_fb8bc966-dde3-51cd-ba9e-73ada5aeb41a",
- "type": "table",
- "content": "tables/table_page_28_1.html",
- "path": "tables/table_page_28_1.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 28
- ],
- "file_path": "tables/table_page_28_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_1c5afc0e-5600-579a-8657-da973eba36c6",
- "type": "table",
- "content": "tables/table_page_29_1.html",
- "path": "tables/table_page_29_1.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 29
- ],
- "file_path": "tables/table_page_29_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_b0f611a6-9c75-52b0-ad2f-762b56694a4f",
- "type": "table",
- "content": "tables/table_page_30_1.html",
- "path": "tables/table_page_30_1.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 30
- ],
- "file_path": "tables/table_page_30_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_ba33916c-76ab-58fd-9d67-2b5da73878cf",
- "type": "table",
- "content": "tables/table_page_31_1.html",
- "path": "tables/table_page_31_1.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 31
- ],
- "file_path": "tables/table_page_31_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_07f3203e-41aa-5eab-a843-d13b61d02751",
- "type": "table",
- "content": "tables/table_page_32_1.html",
- "path": "tables/table_page_32_1.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 32
- ],
- "file_path": "tables/table_page_32_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_be11d68c-c437-5e3f-b88e-a26fce773560",
- "type": "table",
- "content": "tables/table_page_33_1.html",
- "path": "tables/table_page_33_1.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 33
- ],
- "file_path": "tables/table_page_33_1.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_0b9e3b34-bfbe-5a15-89fa-b65f82c795ae",
- "type": "table",
- "content": "tables/table_page_33_2.html",
- "path": "tables/table_page_33_2.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 33
- ],
- "file_path": "tables/table_page_33_2.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_3aff0fc7-a9a4-5b72-8176-d26482a26a91",
- "type": "table",
- "content": "tables/table_page_33_3.html",
- "path": "tables/table_page_33_3.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 33
- ],
- "file_path": "tables/table_page_33_3.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_9538f471-c622-52dc-8b7e-2d45f59cfe34",
- "type": "table",
- "content": "tables/table_page_33_4.html",
- "path": "tables/table_page_33_4.html",
- "metadata": {
- "length": 27,
- "summary": "",
- "page_nums": [
- 33
- ],
- "file_path": "tables/table_page_33_4.html",
- "keywords": [],
- "tokens": [],
- "connect_to": []
- }
- },
- {
- "chunk_id": "asset_e0461421-3c85-5cd0-90d8-be7ad00ad97b",
- "type": "image",
- "content": "[images/image_page_35_figure_1.png]",
- "path": "images/image_page_35_figure_1.png",
- "metadata": {
- "length": 35,
- "summary": "",
- "page_nums": [
- 35
- ],
- "file_path": "images/image_page_35_figure_1.png",
- "keywords": [],
- "tokens": []
- }
- },
- {
- "chunk_id": "node_c8fd0bf4-d2df-5106-8e78-4a62d663f503",
- "type": "page",
- "content": "Q4 and FY 2025 Update\n1\n\nHighlights\n03\nFinancial Summary\n04\nOperational Summary\n06\nManufacturing & Hardware\n08\nSupporting Infrastructure\n09\nAI & Software\n10\nServices\n11\nOther Updates\n12\nOutlook\n13\nPhotos & Charts\n14\nKey Metrics\n24\nFinancial Statements\n27\nAdditional Information\n34\n2",
- "path": "TSLA-Q4-2025-Update.pdf/Q4 and FY 2025 Update",
- "metadata": {
- "length": 282,
- "summary": "This section serves as the title page and table of contents for the 'Q4 and FY 2025 Update' presentation. It outlines the document's structure, listing key sections such as Highlights, Financial Summary, Operational Summary, Manufacturing & Hardware, AI & Software, Services, Outlook, and Key Metrics, along with their corresponding page numbers.",
- "page_nums": [
- 1,
- 2
- ],
- "keywords": [
- "Tesla"
- ],
- "connect_to": [
- {
- "target": "asset_85f035bb-05a0-5e36-9bc4-8d44f13e042e",
- "relation": "embeds",
- "ref": "[tables/table_page_2_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 1,
- "artifact_ref": "page_citation_assets/page-1.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 2,
- "artifact_ref": "page_citation_assets/page-2.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_fd965122-2ecf-518d-a9f3-15e04ee8fdae",
- "type": "page",
- "content": "S U M M A R Y\nH I G H L I G H T S\n(1) Excludes SBC (stock-based compensation) & Digital assets gains and losses, net of tax; (2) Free cash flow = operating cash flow less capex; (3) Includes cash, cash equivalents and investments; (4)Active driver supervision required; does not make the vehicle autonomous; Note: all \ninformation herein refers to the current quarter unless otherwise noted. \nProfitability\n$4.4B GAAP operating income in 2025; $1.4B in Q4\n$3.8B GAAP net income in 2025; $0.8B in Q4\n$5.9B non-GAAP net income1 in 2025; $1.8B in Q4\n2025 marked a critical year for Tesla as we further expanded our mission and continued our \ntransition from a hardware-centric business to a physical AI company. We laid the foundation \nfor the future of Tesla as we further advanced FSD (Supervised)4, launched our Robotaxi service, \nbegan installing production lines for Cybercab and fine-tuned our production-primed Optimus \ndesign while expanding our AI training infrastructure. \nOur approach to autonomous vehicles and humanoid robots mirrors the way we approached \nelectric vehicles and energy storage – at the system level where we identify the limiting factor \nand develop bespoke and scalable solutions (batteries, power electronics, inverters, software, \nAI silicon, etc.) to optimize for cost, functionality, efficiency and safety. Our vertical integration \nhas enabled us to achieve economies of scale in a profitable manner, quickly troubleshoot \nbottlenecks in production and iteratively optimize our technologies more rapidly than others. \nIn 2025, we completed the refresh of our vehicle lineup with the launch of the new Model Y, \nincluding additional variants. We believe that maintaining an optimized and efficient product \nportfolio, with a continued focus on high-value features such as long range, best-in-class \nsoftware and autonomy, is the correct strategy to win the autos market of the future. Similarly, \nwe continued to evolve our energy offerings for commercial, utility and retail customers, as we \nposition ourselves as a supplier of choice for clean, affordable and rapidly deployable energy \ncapacity ahead of expected sustained demand growth for electricity. \nIn 2026, we will further invest in the infrastructure needed to support clean energy and \ntransport and autonomous robots, including the ramp of six new production lines across \nvehicle, robots, energy storage and battery manufacturing, while further leveraging our existing \nfactory, charging and service center footprint to support future growth.\nCash\nOperating cash flow of $14.7B in 2025; $3.8B in Q4\nFree cash flow2 of $6.2B in 2025; $1.4B in Q4\n$7.5B increase in our cash and investments3 in 2025 to $44.1B\nOperations\nBegan removing safety monitor from our Robotaxis in Austin in January\nRecord Q4 & FY’25 energy storage deployments\nRecord vehicle deliveries in APAC\n3",
- "path": "TSLA-Q4-2025-Update.pdf/HIGHLIGHTS",
- "metadata": {
- "length": 2865,
- "summary": "The 'HIGHLIGHTS' section outlines key financial and operational achievements for 2025. Under Profitability, it lists GAAP operating income of $4.4B, net income of $3.8B, and non-GAAP net income of $5.9B. The Cash section details an operating cash flow of $14.7B, free cash flow of $6.2B, and a total cash and investments balance of $44.1B. Operational highlights include the removal of safety monitors from Robotaxis in Austin, record energy storage deployments, and record vehicle deliveries in APAC.",
- "page_nums": [
- 3
- ],
- "keywords": [
- "Tesla",
- "Austin",
- "APAC"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_d2e97eeb-b861-5636-a69b-aff0f27e5610",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/HIGHLIGHTS p3]",
- "path": "TSLA-Q4-2025-Update.pdf/SUMMARY",
- "metadata": {
- "length": 41,
- "summary": "The 'SUMMARY' section outlines Tesla's strategic transition in 2025 from a hardware-centric business to a physical AI company. Key achievements include advancing Full Self-Driving (FSD) technology, launching the Robotaxi service, and initiating production lines for Cybercab and Optimus robots. The text details the company's system-level approach to autonomous vehicles and humanoid robots, emphasizing vertical integration and cost optimization. It also highlights the refresh of the vehicle lineup with the new Model Y and the evolution of energy offerings. Looking ahead to 2026, the summary notes plans to invest in infrastructure for clean energy and transport, including the ramp-up of six new production lines across various manufacturing sectors.",
- "page_nums": [
- 3
- ],
- "keywords": [
- "Tesla",
- "FSD",
- "Robotaxi",
- "Cybercab",
- "Optimus",
- "Model Y"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 3,
- "artifact_ref": "page_citation_assets/page-3.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_e25f6d84-6f97-5010-839c-b99fe8c3c196",
- "type": "page",
- "content": "F I N A N C I A L S U M M A R Y\n(Unaudited)\n($ in millions, except percentages and per share data)\nQ4-2024\nQ1-2025\nQ2-2025\nQ3-2025\nQ4-2025\nYoY\nTotal automotive revenues\n19,798\n13,967\n16,661\n21,205\n17,693\n-11%\nEnergy generation and storage revenue\n3,061\n2,730\n2,789\n3,415\n3,837\n25%\nServices and other revenue\n2,848\n2,638\n3,046\n3,475\n3,371\n18%\nTotal revenues\n25,707\n19,335\n22,496\n28,095\n24,901\n-3%\nTotal gross profit\n4,179\n3,153\n3,878\n5,054\n5,009\n20%\nTotal GAAP gross margin\n16.3%\n16.3%\n17.2%\n18.0%\n20.1%\n386 bp\nOperating expenses\n2,596\n2,754\n2,955\n3,430\n3,600\n39%\nIncome from operations\n1,583\n399\n923\n1,624\n1,409\n-11%\nOperating margin\n6.2%\n2.1%\n4.1%\n5.8%\n5.7%\n-50 bp\nAdjusted EBITDA (1) (2)\n4,333\n2,814\n3,401\n4,227\n4,154\n-4%\nAdjusted EBITDA margin (1) (2)\n16.9%\n14.6%\n15.1%\n15.0%\n16.7%\n-17 bp\nNet income attributable to common stockholders (GAAP) (1)\n2,128\n409\n1,172\n1,373\n840\n-61%\nNet income attributable to common stockholders (non-GAAP) (1) (3)\n2,107\n934\n1,393\n1,770\n1,761\n-16%\nEPS attributable to common stockholders, diluted (GAAP) (1)\n0.60\n0.12\n0.33\n0.39\n0.24\n-60%\nEPS attributable to common stockholders, diluted (non-GAAP) (1) (3)\n0.60\n0.27\n0.40\n0.50\n0.50\n-17%\nNet cash provided by operating activities\n4,814\n2,156\n2,540\n6,238\n3,813\n-21%\nCapital expenditures (4)\n(2,780)\n(1,492)\n(2,394)\n(2,248)\n(2,393)\n-14%\nFree cash flow (4)\n2,034\n664\n146\n3,990\n1,420\n-30%\nCash, cash equivalents and investments\n36,563\n36,996\n36,782\n41,647\n44,059\n21%\n4\n(1) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.\n(2) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(3) Beginning in Q1'25, Net income attributable to common stockholders (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(4) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted.\n\nF I N A N C I A L S U M M A R Y\n(Unaudited)\n($ in millions, except percentages and per share data)\n2021\n2022\n2023\n2024\n2025\nYoY\nTotal automotive revenues\n47,232\n71,462\n82,419\n77,070\n69,526\n-10%\nEnergy generation and storage revenue\n2,789\n3,909\n6,035\n10,086\n12,771\n27%\nServices and other revenue\n3,802\n6,091\n8,319\n10,534\n12,530\n19%\nTotal revenues\n53,823\n81,462\n96,773\n97,690\n94,827\n-3%\nTotal gross profit\n13,606\n20,853\n17,660\n17,450\n17,094\n-2%\nTotal GAAP gross margin\n25.3%\n25.6%\n18.2%\n17.9%\n18.0%\n16 bp\nOperating expenses\n7,083\n7,197\n8,769\n10,374\n12,739\n23%\nIncome from operations\n6,523\n13,656\n8,891\n7,076\n4,355\n-38%\nOperating margin\n12.1%\n16.8%\n9.2%\n7.2%\n4.6%\n-265 bp\nAdjusted EBITDA (1)\n11,722\n19,390\n16,631\n16,056\n14,596\n-9%\nAdjusted EBITDA margin (1)\n21.8%\n23.8%\n17.2%\n16.4%\n15.4%\n-104 bp\nNet income attributable to common stockholders (GAAP)\n5,519\n12,556\n14,997\n7,091\n3,794\n-46%\nNet income attributable to common stockholders (non-GAAP) (2)\n7,719\n14,276\n10,882\n7,960\n5,858\n-26%\nEPS attributable to common stockholders, diluted (GAAP)\n1.63\n3.62\n4.30\n2.04\n1.08\n-47%\nEPS attributable to common stockholders, diluted (non-GAAP) (2)\n2.28\n4.12\n3.12\n2.29\n1.66\n-28%\nNet cash provided by operating activities\n11,497\n14,724\n13,256\n14,923\n14,747\n-1%\nCapital expenditures (3)\n(6,514)\n(7,163)\n(8,899)\n(11,342)\n(8,527)\n-25%\nFree cash flow (3)\n4,983\n7,561\n4,357\n3,581\n6,220\n74%\nCash, cash equivalents and investments\n17,707\n22,185\n29,094\n36,563\n44,059\n21%\n5\n(1) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(2) Beginning in Q1'25, Net income attributable to common stockholders (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(3) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted.",
- "path": "TSLA-Q4-2025-Update.pdf/FINANCIAL SUMMARY",
- "metadata": {
- "length": 4002,
- "summary": "The section titled 'FINANCIAL SUMMARY' presents two comprehensive tables of unaudited financial data. The first table details quarterly performance metrics for the fiscal year 2024 and 2025 (Q4-2024 through Q4-2025), while the second table provides annual historical data from 2021 to 2025. Both tables report figures in millions of dollars, excluding percentages and per share data. Key metrics tracked include revenue breakdowns by segment (automotive, energy generation/storage, services), total revenues, gross profit, operating expenses, income from operations, adjusted EBITDA, net income (GAAP and non-GAAP), earnings per share (EPS), and cash flow indicators such as capital expenditures and free cash flow.",
- "page_nums": [
- 4,
- 5
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_03a68271-090e-5345-8773-68c974244d44",
- "relation": "embeds",
- "ref": "[tables/table_page_4_1.html]"
- },
- {
- "target": "asset_1d12f8a0-f279-5e5b-a23e-2bbc39167342",
- "relation": "embeds",
- "ref": "[tables/table_page_5_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 4,
- "artifact_ref": "page_citation_assets/page-4.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 5,
- "artifact_ref": "page_citation_assets/page-5.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_8e667dc0-e0af-500b-bc83-306df392d9a7",
- "type": "page",
- "content": "Q4-2024\nQ1-2025\nQ2-2025\nQ3-2025\nQ4-2025\nYoY\nModel 3/Y production\n436,718\n345,454\n396,835\n435,826\n422,652\n-3%\nOther models production\n22,727\n17,161\n13,409\n11,624\n11,706\n-48%\nTotal production\n459,445\n362,615\n410,244\n447,450\n434,358\n-5%\nModel 3/Y deliveries\n471,930\n323,800\n373,728\n481,166\n406,585\n-14%\nOther models deliveries\n23,640\n12,881\n10,394\n15,933\n11,642\n-51%\nTotal deliveries\n495,570\n336,681\n384,122\n497,099\n418,227\n-16%\n of which subject to operating lease accounting\n26,962\n13,721\n6,670\n10,230\n10,996\n-59%\nCumulative deliveries(1) (all-time; mil)\n7.3\n7.6\n8.0\n8.5\n8.9\n22%\nActive FSD Subscriptions(2) (mil)\n0.8\n0.8\n0.9\n1.0\n1.1\n38%\nTotal end of quarter operating lease (new vehicle) count(3)\n180,523\n179,930\n172,882\n167,163\n163,075\n-10%\nGlobal vehicle inventory (days of supply)(4)\n12\n22\n24\n10\n15\n25%\nStorage deployed (GWh)\n11.0\n10.4\n9.6\n12.5\n14.2\n29%\nTesla locations\n1,359\n1,390\n1,454\n1,498\n1,553\n14%\nSupercharger stations\n6,975\n7,131\n7,377\n7,753\n8,182\n17%\nSupercharger connectors\n65,495\n67,316\n70,228\n73,817\n77,682\n19%\nO P E R A T I O N A L S U M M A R Y\n(Unaudited)\n6\n(1) In accordance with our 2025 CEO Performance Award, metric includes all new Tesla vehicles delivered to customers plus any unsupervised Robotaxis placed into commercial operation. For further detail see our 2025 Proxy Statement.\n(2) In accordance with our 2025 CEO Performance Award, metric includes both up-front payment and monthly subscriptions and excludes free trials. For further detail see our 2025 Proxy Statement.\n(3) Beginning in Q4’25, end of quarter operating lease count pertains only to new vehicles. Q3’25 has been adjusted to exclude used vehicles.\n(4) Days of supply is calculated by dividing new vehicle ending inventory by the relevant quarter’s deliveries and using 75 trading days (aligned with Automotive News definition).\n\n2021\n2022\n2023\n2024\n2025\nYoY\nModel 3/Y production\n906,032\n1,298,434\n1,775,159\n1,679,338\n1,600,767\n-5%\nOther models production\n24,390\n71,177\n70,826\n94,105\n53,900\n-43%\nTotal production\n930,422\n1,369,611\n1,845,985\n1,773,443\n1,654,667\n-7%\nModel 3/Y deliveries\n911,242\n1,247,146\n1,739,707\n1,704,093\n1,585,279\n-7%\nOther models deliveries\n24,980\n66,705\n68,874\n85,133\n50,850\n-40%\nTotal deliveries\n936,222\n1,313,851\n1,808,581\n1,789,226\n1,636,129\n-9%\n of which subject to operating lease accounting\n60,912\n47,582\n72,226\n60,003\n41,617\n-31%\nCumulative deliveries(1) (all-time; mil)\n2.3\n3.7\n5.5\n7.3\n8.9\n22%\nActive FSD Subscriptions(2) (mil)\n0.4\n0.5\n0.6\n0.8\n1.1\n38%\nTotal end of year operating lease (new vehicle) count\n120,342\n140,667\n176,564\n180,523\n163,075\n-10%\nGlobal vehicle inventory (days of supply)(3)\n6\n16\n16\n13\n15\n15%\nStorage deployed (GWh)\n4.0\n6.5\n14.7\n31.4\n46.7\n49%\nTesla locations\n644\n963\n1,208\n1,359\n1,553\n14%\nSupercharger stations\n3,476\n4,678\n5,952\n6,975\n8,182\n17%\nSupercharger connectors\n31,498\n42,419\n54,892\n65,495\n77,682\n19%\nO P E R A T I O N A L S U M M A R Y\n(Unaudited)\n7\n(1) In accordance with our 2025 CEO Performance Award, metric includes all new Tesla vehicles delivered to customers plus any unsupervised Robotaxis placed into commercial operation. For further detail see our 2025 Proxy Statement.\n(2) In accordance with our 2025 CEO Performance Award, metric includes both up-front payment and monthly subscriptions and excludes free trials. For further detail see our 2025 Proxy Statement.\n(3) Days of supply is calculated by dividing new vehicle ending inventory by the relevant year’s deliveries and using 300 trading days (aligned with Automotive News definition).",
- "path": "TSLA-Q4-2025-Update.pdf/OPERATIONAL SUMMARY",
- "metadata": {
- "length": 3508,
- "summary": "The 'OPERATIONAL SUMMARY' section presents two tables detailing Tesla's operational metrics. The first table provides quarterly data from Q4 2024 through Q4 2025, while the second table offers annual historical data from 2021 to 2025. Key metrics tracked include vehicle production (broken down by Model 3/Y and other models), deliveries (including total, subject to operating lease accounting, and cumulative all-time figures), Active FSD Subscriptions, end-of-period operating lease counts, global vehicle inventory days of supply, storage deployed in GWh, Tesla locations, and Supercharger infrastructure (stations and connectors).",
- "page_nums": [
- 6,
- 7
- ],
- "keywords": [
- "Tesla",
- "Model 3/Y",
- "FSD",
- "Robotaxis",
- "Automotive News"
- ],
- "connect_to": [
- {
- "target": "asset_4ff45cac-1440-522e-8f7e-9e7742d5f203",
- "relation": "embeds",
- "ref": "[tables/table_page_6_1.html]"
- },
- {
- "target": "asset_12a65600-46f3-5d1c-b2b7-f083a8593149",
- "relation": "embeds",
- "ref": "[tables/table_page_7_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 6,
- "artifact_ref": "page_citation_assets/page-6.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 7,
- "artifact_ref": "page_citation_assets/page-7.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_8d1e6a15-928b-52d8-ac84-cd6fbc171097",
- "type": "page",
- "content": "M A N U F A C T U R I N G & H A R D W A R E\nInstalled Annual Manufacturing Capacity\nRegion\nProduct\nCapacity\nStatus\nAutomotive\nCalifornia\nModel 3 / Model Y\n>550,000\nProduction\nModel S / Model X\n100,000\nProduction\nShanghai\nModel 3 / Model Y\n>950,000\nProduction\nBerlin\nModel Y\n>375,000\nProduction\nTexas\nModel Y\n>250,000\nProduction\nCybertruck\n>125,000\nProduction\nCybercab\n-\nTooling\nNevada\nTesla Semi\n-\nTooling\nTBD\nRoadster\n-\nDesign development\nEnergy Generation and Storage\nCalifornia\nMegapack\n40 GWh\nProduction\nNevada\nPowerwall\n>6 GWh\nProduction\nShanghai\nMegapack\n40 GWh\nProduction\nTexas\nMegapack\n-\nConstruction\nRobotics\nCalifornia\nOptimus\n-\nConstruction\nInstalled capacity ≠ current production rate and there may be limitations discovered as production rates approach \ncapacity. Production rates depend on a variety of factors, including equipment uptime, component supply, \ndowntime related to factory upgrades, regulatory considerations and other factors. Construction includes factory \nand infrastructure buildout as well as tool installation.\nAutomotive\nWhile automotive sales declined sequentially, gross margin (even when excluding the impact of \nregulatory credits) improved. The APAC region continued to show strength across multiple \nmarkets and set a record for deliveries in the quarter. We continued the rollout of Model Y \nvariants across markets in Q4, including the standard and performance versions.\nPreparations continue in North America for the production ramps of Tesla Semi and Cybercab, \nboth commencing 1H26, and production of the next-generation Roadster.\nEnergy generation and storage\nWe achieved our highest quarterly energy storage deployments, driven by record Megapack \ndeployments. Total gross profit rose, both sequentially and year-over-year, to a record $1.1 \nbillion, marking the fifth consecutive record quarter. We plan to begin Megapack 3 and \nMegablock production at Megafactory Houston in 2026. In 2025, our global Powerwall network \nsupported more than 89,000 Virtual Power Plant events across over 1 million installed units, \nallowing homeowners to save over $1 billion in electricity bills as Virtual Power Plant \nparticipation continues to scale rapidly.\nRobotics\nWe made further progress on the Optimus program in 2025. In Q1 of this year, we plan to unveil \nthe Gen 3 version of Optimus, which will include major upgrades from version 2.5, including our \nlatest hand design. The Gen 3 is our first design meant for mass production. Preparations are \nunderway for the first production line, including supply chain readiness, with start of production \nplanned before the end of 2026 and eventual planned capacity of 1 million robots per year.\n8",
- "path": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Automotive",
- "metadata": {
- "length": 2688,
- "summary": "The section discusses automotive sales performance, noting a sequential decline but an improvement in gross margin and strong results in the APAC region. It details the rollout of Model Y variants and preparations for future production ramps of Tesla Semi, Cybercab, and the next-generation Roadster. The accompanying table outlines installed annual manufacturing capacity across various regions (California, Shanghai, Berlin, Texas, Nevada) for products including Model 3, Model Y, Model S, Model X, Cybertruck, Cybercab, and Tesla Semi.",
- "page_nums": [
- 8
- ],
- "keywords": [
- "APAC",
- "Tesla Semi",
- "Cybercab",
- "Roadster"
- ],
- "connect_to": [
- {
- "target": "asset_052c1e7a-29f1-5fee-92cd-0b74b320dfc4",
- "relation": "embeds",
- "ref": "[tables/table_page_8_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_1d86e7c7-e9ac-50c0-aa1f-020eaa97a545",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Automotive p8]",
- "path": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Energy generation and storage",
- "metadata": {
- "length": 66,
- "summary": "The section details record quarterly energy storage deployments driven by Megapack, resulting in a record gross profit of $1.1 billion. It outlines plans to begin production of Megapack 3 and Megablock at the Megafactory Houston in 2026. Additionally, it notes that the global Powerwall network supported over 89,000 Virtual Power Plant events across more than 1 million units in 2025. The accompanying table lists installed annual manufacturing capacity for Energy Generation and Storage products (Megapack, Powerwall) across regions including California, Nevada, Shanghai, and Texas.",
- "page_nums": [
- 8
- ],
- "keywords": [
- "Megapack",
- "Megafactory Houston",
- "Powerwall",
- "Virtual Power Plant"
- ],
- "connect_to": [
- {
- "target": "asset_052c1e7a-29f1-5fee-92cd-0b74b320dfc4",
- "relation": "related",
- "ref": "[tables/table_page_8_1.html]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Automotive"
- }
- ],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_eebd73e8-7cf3-5513-be1e-566c4e55c567",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Automotive p8]",
- "path": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Robotics",
- "metadata": {
- "length": 66,
- "summary": "The Robotics section details progress on the Optimus program in 2025, specifically the unveiling of the Gen 3 version with major upgrades and a new hand design intended for mass production. It outlines plans to start production before the end of 2026 with an eventual capacity of 1 million robots per year. The accompanying table indicates that the Optimus robot is currently under construction in California.",
- "page_nums": [
- 8
- ],
- "keywords": [
- "Optimus",
- "California"
- ],
- "connect_to": [
- {
- "target": "asset_052c1e7a-29f1-5fee-92cd-0b74b320dfc4",
- "relation": "related",
- "ref": "[tables/table_page_8_1.html]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Automotive"
- }
- ],
- "page_assets": [
- {
- "page_num": 8,
- "artifact_ref": "page_citation_assets/page-8.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_2f0af9ef-5199-5f1d-8b48-a9837b84e56a",
- "type": "page",
- "content": "S U P P O R T I N G I N F R A S T R U C T U R E\nInstalled Annual Capacity\nRegion\nProduct\nCapacity\nStatus\nAI Training Compute\nTexas\nCortex 1\n>100k H100e\nProduction\nCortex 2\n-\nConstruction\nBattery Manufacturing\nNevada\nLFP\n7 GWh\nEarly Ramp\nTexas\n4680\n40 GWh\nProduction\nCathode Materials\n10 GWh\nEarly Ramp\nLithium Refining\n30 GWh\nEarly Ramp\nInstalled capacity ≠ current production rate and there may be limitations discovered as production rates approach \ncapacity. Production rates depend on a variety of factors, including equipment uptime, component supply, \ndowntime related to factory upgrades, regulatory considerations and other factors. Construction includes factory \nand infrastructure buildout as well as tool installation.\nAI Training Compute\nWe are currently building Cortex 2 at Gigafactory Texas to further increase our AI training \ncompute capacity. In the first half of 2026, we plan to more than double the size of onsite \ncompute in Texas (in terms of H100 equivalents). We aim to maximize capital efficiency by \nscaling training compute judiciously, including when the training backlog gets too long or in \nanticipation of greater demand from our engineers to support our AI-related offerings.\nBattery\nOur lithium refinery commenced pilot production and is the first spodumene to lithium \nhydroxide refinery in North America, leveraging a simpler, cheaper and more environmentally \nfriendly process. This refinery enables us to domestically produce critical minerals in support of \nenergy storage, battery manufacturing and ultimately for EV growth. \nWe have begun to produce battery packs for certain Model Ys with our 4680 cells, unlocking an \nadditional vector of supply to help navigate increasingly complex supply chain challenges caused \nby trade barriers and tariff risks. We now produce dry-electrode for 4680 cells with both anode \nand cathode made in Austin. We expect both domestic cathode material in Texas and LFP lines \nin Nevada to begin production in 2026.\nOther Supporting Infrastructure\nWe continue to efficiently utilize our existing physical footprint in North America, with targeted \naugmentation to support the rollout of Robotaxi. While in the short-term, operational \nworkstreams such as charging, cleaning and maintenance can be managed through our existing \ncharging network, service centers and sales and delivery locations, we will have to add more \ncapacity as the service expands. We added over 3,800 net new Supercharging stalls, growing the \nnetwork 19% year-over-year.\n9\nTesla AI Training Capacity Ramp (H100 equivalent GPUs)\n0\n50,000\n100,000\n150,000\n200,000\n250,000\n300,000\nExisting Capacity\nFuture Planned Capacity",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Training Compute",
- "metadata": {
- "length": 2666,
- "summary": "The section details the construction of Cortex 2 at Gigafactory Texas to increase AI training compute capacity, with plans to more than double onsite compute in terms of H100 equivalents by the first half of 2026. The accompanying table outlines installed annual capacity for AI Training Compute, listing 'Cortex 1' in Texas as having over 100k H100e capacity in production and 'Cortex 2' under construction.",
- "page_nums": [
- 9
- ],
- "keywords": [
- "Cortex 2",
- "Gigafactory Texas",
- "Texas",
- "H100",
- "Cortex 1"
- ],
- "connect_to": [
- {
- "target": "asset_0b4cbe9b-7731-5df7-8668-77dcc85e3e2e",
- "relation": "embeds",
- "ref": "[images/image_page_9_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_2bde273d-c2d3-54fe-b1e5-064308d80f16",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Training Compute p9]",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/Battery",
- "metadata": {
- "length": 76,
- "summary": "The section details the company's battery manufacturing progress, including the commencement of pilot production at its lithium refinery in North America (the first spodumene-to-lithium hydroxide facility). It also covers the production of 4680 cells and battery packs for Model Ys using dry-electrode technology made in Austin. The text notes expectations for domestic cathode material production in Texas and LFP lines in Nevada to begin in 2026.",
- "page_nums": [
- 9
- ],
- "keywords": [
- "North America",
- "Texas",
- "Nevada",
- "Austin",
- "Model Ys"
- ],
- "connect_to": [
- {
- "target": "asset_0b4cbe9b-7731-5df7-8668-77dcc85e3e2e",
- "relation": "related",
- "ref": "[images/image_page_9_figure_2.png]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Training Compute"
- }
- ],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_6d3cdb81-2669-54ed-a567-39213cc0c996",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Training Compute p9]",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/Other Supporting Infrastructure",
- "metadata": {
- "length": 76,
- "summary": "This section outlines the company's strategy for supporting infrastructure, focusing on the efficient utilization of its North American physical footprint to support the Robotaxi rollout. It details plans to augment existing charging networks, service centers, and sales locations as demand grows, specifically noting the addition of over 3,800 new Supercharging stalls.",
- "page_nums": [
- 9
- ],
- "keywords": [
- "North America",
- "Robotaxi",
- "Supercharging"
- ],
- "connect_to": [
- {
- "target": "asset_0b4cbe9b-7731-5df7-8668-77dcc85e3e2e",
- "relation": "related",
- "ref": "[images/image_page_9_figure_2.png]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Training Compute"
- }
- ],
- "page_assets": [
- {
- "page_num": 9,
- "artifact_ref": "page_citation_assets/page-9.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_031d8b49-63c7-54e2-ac73-11c39f8f4b4e",
- "type": "page",
- "content": "Targeting Step-Function Improvement for our Next-Generation Inference Chip, AI5\nA I & S O F T W A R E\nCumulative Miles Driven with FSD (Supervised)1 (billions)\nAI Software\nWe continue to enhance FSD (Supervised)1 via our end-to-end foundation model trained on both \ncustomer and Robotaxi real-world data with our latest version, v14. FSD (Supervised)1 increasingly \nprovides safety and convenience functionality that can relieve drivers of many tedious and \npotentially dangerous aspects of road travel, including giving access to personal transport for \nthose who otherwise have difficulty driving. V14 offers unparalleled driver assistance to safely \ndrive to the customer’s destination, find a free parking spot and park at that location. Our global \nfleet can collect the equivalent of over 500 years of continuous driving data per day2, allowing us \nto safely deploy and scale capabilities that can handle the long and fat tail of corner cases across \ndiverse geographies and driving environments. \nAI Inference Compute\nDevelopment of our in-house, custom designed AI5 and AI6 inference chips for autonomy \nprogressed during the quarter, with production planned for 2027 and 2028, respectively. We are \ntargeting a 50x improvement in performance for AI5 relative to AI4 thanks to 10x raw compute, 9x \nmemory capacity and 5x hardened block quantization and softmax function (the latter enabling \nefficient low-precision computing without sacrificing model accuracy).\nAutomotive and Other Software\nThe Robotaxi iOS app no longer has a waitlist in the areas we serve. Our vehicles keep getting \nbetter with our over-the-air updates, including: Grok (an AI companion) which now supports \nnavigation commands (allowing users to find, add and edit navigation destinations hands-free); \nTesla Photobooth which enables users to take photos in their car and download or share via the \nTesla mobile app; Supercharger Site Maps which displays Supercharger layouts, nearby businesses \nand live availability details; Automatic HOV Lane Routing based on interior camera occupancy \ndetection; Phone Left Behind Chime and SpaceX ISS Docking Simulator Game.\n10\n(1) Active driver supervision required; does not make the vehicle autonomous\n(2) Calculated based on continuous hours of driving at an average of 30 miles per hour\n0.0\n1.0\n2.0\n3.0\n4.0\n5.0\n6.0\n7.0\n8.0\nFSD Miles on V12 and Beyond\nFSD Miles on V11 and Before",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Software",
- "metadata": {
- "length": 2408,
- "summary": "The section details the enhancement of FSD (Supervised) version v14, an end-to-end foundation model trained on customer and Robotaxi data. It highlights safety features like autonomous parking and accessibility for those with difficulty driving. The text also mentions the global fleet's ability to collect over 500 years of continuous driving data daily to handle corner cases.",
- "page_nums": [
- 10
- ],
- "keywords": [
- "FSD",
- "Robotaxi",
- "Tesla"
- ],
- "connect_to": [
- {
- "target": "asset_509e8dfa-6ebc-56a6-8060-caf683f4e521",
- "relation": "embeds",
- "ref": "[images/image_page_10_figure_1.png]"
- },
- {
- "target": "asset_3614d9d2-de2a-56c6-9837-d3c869f461a2",
- "relation": "embeds",
- "ref": "[images/image_page_10_figure_2.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_0b3d2022-92be-52a6-9ce0-57073c65aaea",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Software p10]",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Inference Compute",
- "metadata": {
- "length": 69,
- "summary": "The section details the development of in-house AI5 and AI6 inference chips for autonomy, with production planned for 2027 and 2028. It highlights a targeted 50x performance improvement for the AI5 chip over the AI4, driven by a 10x increase in raw compute, 9x memory capacity, and 5x hardened block quantization and softmax function.",
- "page_nums": [
- 10
- ],
- "keywords": [
- "AI5",
- "AI6",
- "AI4"
- ],
- "connect_to": [
- {
- "target": "asset_509e8dfa-6ebc-56a6-8060-caf683f4e521",
- "relation": "related",
- "ref": "[images/image_page_10_figure_1.png]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Software"
- },
- {
- "target": "asset_3614d9d2-de2a-56c6-9837-d3c869f461a2",
- "relation": "related",
- "ref": "[images/image_page_10_figure_2.png]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Software"
- }
- ],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_b091cc3b-cafb-590c-9e50-c6e7c858551d",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Software p10]",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/Automotive and Other Software",
- "metadata": {
- "length": 69,
- "summary": "This section details recent software updates and features for Tesla vehicles, including the removal of waitlists for the Robotaxi iOS app. It highlights over-the-air enhancements such as Grok AI companion support for navigation commands, Tesla Photobooth for in-car photography, Supercharger Site Maps with live availability, Automatic HOV Lane Routing based on camera occupancy detection, a Phone Left Behind Chime, and the SpaceX ISS Docking Simulator Game.",
- "page_nums": [
- 10
- ],
- "keywords": [
- "Robotaxi",
- "Grok",
- "Tesla",
- "SpaceX"
- ],
- "connect_to": [
- {
- "target": "asset_509e8dfa-6ebc-56a6-8060-caf683f4e521",
- "relation": "related",
- "ref": "[images/image_page_10_figure_1.png]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Software"
- },
- {
- "target": "asset_3614d9d2-de2a-56c6-9837-d3c869f461a2",
- "relation": "related",
- "ref": "[images/image_page_10_figure_2.png]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Software"
- }
- ],
- "page_assets": [
- {
- "page_num": 10,
- "artifact_ref": "page_citation_assets/page-10.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_cb7c79b3-402b-5ecb-b902-1b48baf55e64",
- "type": "page",
- "content": "S E R V I C E S\nPlanned Robotaxi Coverage\nCumulative Paid Robotaxi Miles\n11\nState\nMetro\nStatus\nCalifornia\nSF Bay Area\nSafety Driver\nTexas\nAustin\nRamping Unsupervised\nDallas\n1H 2026\nHouston\n1H 2026\nArizona\nPhoenix\n1H 2026\nFlorida\nMiami\n1H 2026\nOrlando\n1H 2026\nTampa\n1H 2026\nNevada\nLas Vegas\n1H 2026\n0\n100,000\n200,000\n300,000\n400,000\n500,000\n600,000\n700,000\n(1) Active driver supervision required; does not make the vehicle autonomous.\nRobotaxi\nWe began testing driverless Robotaxis in Austin in December and began removing the safety \nmonitor from customer rides in January on a limited basis, which will unlock further expansion of \nour Robotaxi fleet and coverage area in the Austin-metro. Our Bay Area ride-hailing service began \nserving the San Jose Airport in October, with plans to expand to other major airports in the Bay \nArea upon receiving required permitting.\nFSD (Supervised)1\nWe launched FSD (Supervised)1 in South Korea, where customers drove over 1 million kilometers \nusing the software in just one month. While we continue to pursue regulatory approval in China \nand Europe, we began offering ride-along experiences to consumers in Italy, Germany, France and \nSwitzerland. \nMonthly subscriptions to FSD (Supervised)1 continued to grow sequentially and more than \ndoubled in 2025. Starting this quarter, we are transitioning access to FSD (Supervised)1 to monthly \nsubscriptions only as we begin to sunset the up-front payment option.\nAutomotive Services\nServices and Other gross profit of approximately $300 million was partly driven by Part Sales and \nSupercharging. We now offer Tesla Insurance in Florida, as we continue to expand our insurance \nproduct to new states. In certain states, customers receive a discount on their insurance \npremiums when using FSD (Supervised)1. The more you drive with FSD (Supervised)1 enabled, the \nbigger the discount is on your insurance premium – helping, in certain cases, to completely offset \nthe monthly subscription cost for FSD (Supervised)1.",
- "path": "TSLA-Q4-2025-Update.pdf/SERVICES/Robotaxi",
- "metadata": {
- "length": 2004,
- "summary": "The section details the progress of driverless Robotaxi services, including testing in Austin starting in December and removing safety monitors in January. It also mentions the launch of Bay Area ride-hailing service at San Jose Airport in October. The section includes a chart showing cumulative paid Robotaxi miles projected through December 2025 and a table outlining planned coverage for various US states (California, Texas, Arizona, Florida, Nevada) with specific metro areas and expected status or launch dates.",
- "page_nums": [
- 11
- ],
- "keywords": [
- "Austin",
- "Bay Area",
- "San Jose Airport"
- ],
- "connect_to": [
- {
- "target": "asset_e5e3668a-29f3-560a-81b4-8777871db0df",
- "relation": "embeds",
- "ref": "[images/image_page_11_figure_1.png]"
- },
- {
- "target": "asset_237b9b1a-cb17-5080-ba09-5e5ceb5a3bb6",
- "relation": "embeds",
- "ref": "[tables/table_page_11_2.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_1cd98fe2-b2b2-5e9b-a3b3-1fb3ba2291d3",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/SERVICES/Robotaxi p11]",
- "path": "TSLA-Q4-2025-Update.pdf/SERVICES/FSD (Supervised)",
- "metadata": {
- "length": 49,
- "summary": "The section details the launch and growth of FSD (Supervised) in South Korea, where customers drove over 1 million kilometers in one month. It mentions ongoing regulatory pursuits in China and Europe and ride-along experiences in Italy, Germany, France, and Switzerland. Additionally, it notes that monthly subscriptions for the service more than doubled in 2025 and that access is transitioning to a monthly subscription model only.",
- "page_nums": [
- 11
- ],
- "keywords": [
- "South Korea",
- "China",
- "Europe",
- "Italy",
- "Germany",
- "France",
- "Switzerland"
- ],
- "connect_to": [
- {
- "target": "asset_e5e3668a-29f3-560a-81b4-8777871db0df",
- "relation": "related",
- "ref": "[images/image_page_11_figure_1.png]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SERVICES/Robotaxi"
- },
- {
- "target": "asset_237b9b1a-cb17-5080-ba09-5e5ceb5a3bb6",
- "relation": "related",
- "ref": "[tables/table_page_11_2.html]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SERVICES/Robotaxi"
- }
- ],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_dfe08665-4806-5bec-b3ce-bcc2d9ed6993",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/SERVICES/Robotaxi p11]",
- "path": "TSLA-Q4-2025-Update.pdf/SERVICES/Automotive Services",
- "metadata": {
- "length": 49,
- "summary": "The section details the performance of 'Services and Other' gross profit, which was approximately $300 million, driven by Part Sales and Supercharging. It highlights the expansion of Tesla Insurance to Florida and explains a discount program where customers using FSD (Supervised) receive reduced insurance premiums.",
- "page_nums": [
- 11
- ],
- "keywords": [
- "Tesla",
- "Florida"
- ],
- "connect_to": [
- {
- "target": "asset_e5e3668a-29f3-560a-81b4-8777871db0df",
- "relation": "related",
- "ref": "[images/image_page_11_figure_1.png]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SERVICES/Robotaxi"
- },
- {
- "target": "asset_237b9b1a-cb17-5080-ba09-5e5ceb5a3bb6",
- "relation": "related",
- "ref": "[tables/table_page_11_2.html]",
- "same_as_owner": "TSLA-Q4-2025-Update.pdf/SERVICES/Robotaxi"
- }
- ],
- "page_assets": [
- {
- "page_num": 11,
- "artifact_ref": "page_citation_assets/page-11.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_0ce2f147-0fb9-5c16-8731-b0c6ae772e7e",
- "type": "page",
- "content": "O T H E R U P D A T E S\nOn January 16, 2026, Tesla entered into an agreement to invest approximately $2 billion to acquire shares of Series E Preferred Stock of xAI as part of their recent\npublicly-disclosed financing round. Tesla’s investment was made on market terms consistent with those previously agreed to by other investors in the financing round. As\nset forth in Master Plan Part IV, Tesla is building products and services that bring AI into the physical world. Meanwhile, xAI is developing leading digital AI products and\nservices, such as its large language model (Grok).\n \nIn that context, and as part of Tesla’s broader strategy under Master Plan Part IV, Tesla and xAI also entered into a framework agreement in connection with the\ninvestment. Among other things, the framework agreement builds upon the existing relationship between Tesla and xAI by providing a framework for evaluating\npotential AI collaborations between the companies. Together, the investment and the related framework agreement are intended to enhance Tesla’s ability to develop\nand deploy AI products and services into the physical world at scale. This investment is subject to customary regulatory conditions with the expectation to close in\nQ1'2026.\n12",
- "path": "TSLA-Q4-2025-Update.pdf/OTHER UPDATES",
- "metadata": {
- "length": 1243,
- "summary": "This page details an agreement reached on January 16, 2026, where Tesla invested approximately $2 billion to acquire Series E Preferred Stock of xAI. The text explains that this investment is part of Tesla's Master Plan Part IV and includes a framework agreement to evaluate potential AI collaborations between the two companies. The transaction is subject to regulatory conditions with an expected closing in Q1'2026.",
- "page_nums": [
- 12
- ],
- "keywords": [
- "Tesla",
- "xAI",
- "Grok"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 12,
- "artifact_ref": "page_citation_assets/page-12.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_9f6c6c83-7175-55f8-b9e4-07c527c35a8f",
- "type": "page",
- "content": "O U T L O O K\nVolume\nWe are focused on maximum capacity utilization at our factories. Deliveries and deployments will be impacted by aggregate demand for \nour products, supply chain readiness and allocation decisions between sale to customers or use for our owned and operated fleet.\nCash\nWe will manage the businesses such that we ensure a strong balance sheet, maintaining sufficient liquidity to fund our product roadmap, \nlong-term capacity expansion plans – including further vertical integration – and other expenses.\nProfit\nWhile we continue to execute on innovations to reduce the cost of manufacturing and operations, over time, we expect our hardware-\nrelated profits to be accompanied by an acceleration of AI, software and fleet-based profits.\nProduct\nWe continue to evolve and augment our product lineup with a focus on cost, scale and future monetization opportunities via services \npowered by our AI software. We remain focused on growing our sales volumes through a differentiated and efficiently managed product \nportfolio, which includes leveraging and optimizing our existing production capacity before building new factories and production lines. \nCybercab, Tesla Semi and Megapack 3 are on schedule for volume production starting in 2026. First generation production lines for \nOptimus are being installed in anticipation of volume production.\n13",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK/Volume",
- "metadata": {
- "length": 1367,
- "summary": "The section outlines the company's focus on maximizing factory capacity utilization. It notes that deliveries and deployments depend on aggregate demand, supply chain readiness, and allocation decisions between customer sales and use in the company's own fleet.",
- "page_nums": [
- 13
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 13,
- "artifact_ref": "page_citation_assets/page-13.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_41a85c59-b580-58ff-8827-f895c0320559",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/OUTLOOK/Volume p13]",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK/Cash",
- "metadata": {
- "length": 46,
- "summary": "The section outlines the company's strategy to maintain a strong balance sheet and sufficient liquidity. This financial management approach is intended to fund the product roadmap, long-term capacity expansion plans (including vertical integration), and other expenses.",
- "page_nums": [
- 13
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 13,
- "artifact_ref": "page_citation_assets/page-13.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_87d5e45d-05ec-51cd-a3b4-fac482b1ed72",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/OUTLOOK/Volume p13]",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK/Profit",
- "metadata": {
- "length": 46,
- "summary": "The section outlines the company's strategy to reduce manufacturing and operational costs through innovation. It states that over time, hardware-related profits are expected to be supplemented by an acceleration of profits from AI, software, and fleet-based services.",
- "page_nums": [
- 13
- ],
- "keywords": [],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 13,
- "artifact_ref": "page_citation_assets/page-13.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_70368561-bd63-5676-84d5-6f12198fdafe",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/OUTLOOK/Volume p13]",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK/Product",
- "metadata": {
- "length": 46,
- "summary": "The section outlines the company's strategy to evolve its product lineup with a focus on cost, scale, and monetization via AI software. It emphasizes growing sales volumes by optimizing existing production capacity before building new facilities. Additionally, it provides specific timelines for volume production: Cybercab, Tesla Semi, and Megapack 3 are scheduled to start in 2026, while first-generation production lines for Optimus are currently being installed.",
- "page_nums": [
- 13
- ],
- "keywords": [
- "Cybercab",
- "Tesla Semi",
- "Megapack 3",
- "Optimus"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 13,
- "artifact_ref": "page_citation_assets/page-13.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_8cba5227-e547-5aaf-be0d-fa36a7456f91",
- "type": "page",
- "content": "P H O T O S & C H A R T S\n\nM O D E L Y – 2 0 2 5 B E S T I N C L A S S E U R O N C A P (1) S M A L L S U V \n15\n(1) EURO NCAP: European New Car Assessment Programme \n.\n\nM O D E L 3 – 2 0 2 5 B E S T I N C L A S S E U R O N C A P (1) L A R G E F A M I L Y C A R\n16\n(1) EURO NCAP: European New Car Assessment Programme\n\nF S D ( S U P E R V I S E D ) 1 – V 1 4 O F F E R S U N P A R A L L E L E D D R I V E R A S S I S T A N C E\n17\n(1) Active driver supervision required; does not make the vehicle autonomous.\n\nD R I V E R L E S S R O B O T A X I – T E S T I N G I N A U S T I N\n18\n\nC Y B E R C A B – C O L D W E A T H E R T E S T I N G I N A L A S K A\n19\n\nT E S L A S E M I –\nP R O D U C T I O N B E G I N N I N G I N 2 0 2 6\n20",
- "path": "TSLA-Q4-2025-Update.pdf/PHOTOS & CHARTS",
- "metadata": {
- "length": 800,
- "summary": "This section presents a series of images highlighting Tesla's vehicle safety ratings and autonomous driving capabilities. It features the Model Y, recognized as the 2025 best-in-class small SUV by Euro NCAP, and the Model 3, recognized as the 2025 best-in-class large family car by Euro NCAP. The section also showcases Tesla's Full Self-Driving (Supervised) V14 software, which offers driver assistance, and documents the testing phase of the Driverless Robotaxi in Austin.",
- "page_nums": [
- 14,
- 15,
- 16,
- 17,
- 18,
- 19,
- 20
- ],
- "keywords": [
- "Tesla",
- "Model Y",
- "Euro NCAP",
- "Model 3",
- "FSD (Supervised)",
- "V14",
- "Driverless Robotaxi",
- "Austin"
- ],
- "connect_to": [
- {
- "target": "asset_70d54e04-aae7-5be8-ab6d-e83d599dc636",
- "relation": "embeds",
- "ref": "[images/image_page_15_figure_1.png]"
- },
- {
- "target": "asset_c8366c98-3281-5e5c-9420-4fd22da01b8c",
- "relation": "embeds",
- "ref": "[images/image_page_16_figure_1.png]"
- },
- {
- "target": "asset_92efada6-9ac0-5fd1-b5a0-c16f41d3bc94",
- "relation": "embeds",
- "ref": "[images/image_page_17_figure_1.png]"
- },
- {
- "target": "asset_01432e7f-9816-5765-aa22-09df1164d996",
- "relation": "embeds",
- "ref": "[images/image_page_18_figure_1.png]"
- },
- {
- "target": "asset_10a13f13-e347-5787-b65a-24d0419df947",
- "relation": "embeds",
- "ref": "[images/image_page_19_figure_1.png]"
- },
- {
- "target": "asset_9036f2b6-99af-5adb-810b-66877dc00820",
- "relation": "embeds",
- "ref": "[images/image_page_20_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 14,
- "artifact_ref": "page_citation_assets/page-14.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 15,
- "artifact_ref": "page_citation_assets/page-15.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 16,
- "artifact_ref": "page_citation_assets/page-16.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 17,
- "artifact_ref": "page_citation_assets/page-17.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 18,
- "artifact_ref": "page_citation_assets/page-18.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 19,
- "artifact_ref": "page_citation_assets/page-19.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 20,
- "artifact_ref": "page_citation_assets/page-20.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_ff3e0ea8-9de8-5bbc-9551-5b0028c8b874",
- "type": "page",
- "content": "T E S L A S E M I –\nM E G A C H A R G E R N E T W O R K P L A N N E D S I T E S F O R 2 0 2 6\n21\n\nG I G A F A C T O R Y S H A N G H A I – 9 M I L L I O N T H V E H I C L E P R O D U C E D ( G L O B A L L Y )\n22\n\nG I G A F A C T O R Y N E V A D A – 6 M I L L I O N T H D R I V E U N I T P R O D U C E D\n23",
- "path": "TSLA-Q4-2025-Update.pdf/TESLA SEMI - MEGACHARGER NETWORK PLANNED SITES FOR 2026",
- "metadata": {
- "length": 342,
- "summary": "The section displays a map of the United States illustrating the planned locations for Tesla Semi Megacharger network sites in 2026. Red markers indicate the specific cities and regions where these charging stations are scheduled to be built.",
- "page_nums": [
- 21,
- 22,
- 23
- ],
- "keywords": [
- "TESLA SEMI",
- "MEGACHARGER NETWORK",
- "UNITED STATES"
- ],
- "connect_to": [
- {
- "target": "asset_71a6700a-6ba2-532d-8dd3-74cd5a840d83",
- "relation": "embeds",
- "ref": "[images/image_page_21_figure_1.png]"
- },
- {
- "target": "asset_e9a56942-8c8d-5d1c-bca1-43581a0266b5",
- "relation": "embeds",
- "ref": "[images/image_page_22_figure_1.png]"
- },
- {
- "target": "asset_27ec6411-6eed-5c56-b989-f06f827ebc48",
- "relation": "embeds",
- "ref": "[images/image_page_23_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 21,
- "artifact_ref": "page_citation_assets/page-21.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 22,
- "artifact_ref": "page_citation_assets/page-22.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 23,
- "artifact_ref": "page_citation_assets/page-23.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_2a2d2437-af4a-5494-be3c-4554fb04b176",
- "type": "page",
- "content": "Vehicle Deliveries \n(millions of units)\nK E Y M E T R I C S Q U A R T E R L Y\n(Unaudited)\nOperating Cash Flow ($B)\nFree Cash Flow ($B) (1)\nNet Income ($B) (2)\nAdjusted EBITDA ($B) (2) (3)\n24\n(1) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted. \n(2) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.\n(3) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n0.0\n0.1\n0.2\n0.3\n0.4\n0.5\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025\n-3\n-2\n-1\n0\n1\n2\n3\n4\n5\n6\n7\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025\n0\n1\n2\n3\n4\n5\n6\n7\n8\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS QUARTERLY",
- "metadata": {
- "length": 990,
- "summary": "This page displays quarterly key metrics for a company, presented as three bar charts covering the period from 1Q-2023 to 4Q-2025. The first chart shows Vehicle Deliveries in millions of units, while the second and third charts display financial performance including Operating Cash Flow, Free Cash Flow, Net Income, and Adjusted EBITDA in billions of dollars. The data is marked as Unaudited.",
- "page_nums": [
- 24
- ],
- "keywords": [
- "Vehicle Deliveries",
- "Operating Cash Flow",
- "Free Cash Flow",
- "Net Income",
- "Adjusted EBITDA"
- ],
- "connect_to": [
- {
- "target": "asset_acd3b2a7-87cc-5480-8edc-4560f6bf9581",
- "relation": "embeds",
- "ref": "[images/image_page_24_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 24,
- "artifact_ref": "page_citation_assets/page-24.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_78b0206b-26b3-524f-84f0-0f2eb21885ff",
- "type": "page",
- "content": "Operating Cash Flow ($B)\nFree Cash Flow ($B) (1)\nK E Y M E T R I C S T R A I L I N G 1 2 M O N T H S ( T T M )\n(Unaudited)\nNet Income ($B) (2)\nAdjusted EBITDA ($B) (2) (3)\nVehicle Deliveries \n(millions of units)\n25\n(1) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted. \n(2) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.\n(3) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n0.0\n0.2\n0.4\n0.6\n0.8\n1.0\n1.2\n1.4\n1.6\n1.8\n2.0\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025\n0\n2\n4\n6\n8\n10\n12\n14\n16\n18\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025\n0\n2\n4\n6\n8\n10\n12\n14\n16\n18\n20\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS TRAILING 12 MONTHS (TTM)",
- "metadata": {
- "length": 1044,
- "summary": "The page displays a set of three bar charts titled 'KEY METRICS TRAILING 12 MONTHS (TTM)' showing financial and operational data from Q1 2023 through Q4 2025. The first chart tracks Vehicle Deliveries in millions of units, the second compares Operating Cash Flow and Free Cash Flow in billions of dollars, and the third contrasts Net Income with Adjusted EBITDA, also in billions.",
- "page_nums": [
- 25
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_65df6a5c-1947-5912-9de0-9ceb53c28524",
- "relation": "embeds",
- "ref": "[images/image_page_25_figure_1.png]"
- },
- {
- "target": "asset_f103abcb-b20a-50d0-b29c-ca62a69a3fb7",
- "relation": "embeds",
- "ref": "[images/image_page_25_figure_2.png]"
- },
- {
- "target": "asset_2e016cd6-faa5-54d4-ab58-5ffd594e0ab9",
- "relation": "embeds",
- "ref": "[images/image_page_25_figure_3.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 25,
- "artifact_ref": "page_citation_assets/page-25.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_78eeb59d-fe56-5abf-a8db-f2b22087f16f",
- "type": "page",
- "content": "Revenue\nTotal quarterly revenue decreased 3% YoY to $24.9B. YoY, revenue was impacted by the following items(1):\n- decrease in vehicle deliveries\n- lower regulatory credit revenue\n+ growth in Energy Generation and Storage\n+ growth in Services and Other\n+ positive FX impact of $0.3B1\n+ growth in other automotive ancillary sales, partly driven by an increase in FSD subscriptions\n+ higher vehicle average selling price (ASP) (excl. FX impact1), inclusive of mix impact\nProfitability\nOur quarterly operating income decreased 11% YoY to $1.4B, resulting in a 5.7% operating margin. YoY, operating income was primarily \nimpacted by the following items(1):\n- increase in SBC and Restructuring and Other charges\n- increase in operating expenses (excl. SBC and Restructuring and Other) driven by AI and other R&D projects and SG&A\n- higher average cost per vehicle due to lower fixed cost absorption for certain models and an increase in tariffs\n- decrease in vehicle deliveries\n- lower regulatory credit revenue\n+ higher vehicle average gross profit due to mix and pricing impacts \n+ growth in Energy Generation and Storage gross profit\n+ growth in Services and Other gross profit\n+ growth in other automotive ancillary sales, partly driven by an increase in FSD subscriptions\nCash\nQuarter-end cash, cash equivalents and investments was $44.1B. The sequential increase of $2.4B was primarily the result of positive free cash \nflow.\n26\n(1) Impact is calculated on a constant currency basis. Actuals are compared against current results converted into USD using average exchange rates from Q4’24.\nK E Y M E T R I C S Y E A R – O V E R – Y E A R F I N A N C I A L S U M M A R Y",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS YEAR-OVER-YEAR FINANCIAL SUMMARY/Revenue",
- "metadata": {
- "length": 1673,
- "summary": "The section details that total quarterly revenue decreased 3% year-over-year to $24.9 billion. It lists the factors impacting this change, including decreases in vehicle deliveries and regulatory credit revenue, offset by growth in Energy Generation and Storage, Services and Other, positive foreign exchange impact, and higher average selling prices.",
- "page_nums": [
- 26
- ],
- "keywords": [
- "$24.9B",
- "Energy Generation and Storage",
- "Services and Other",
- "FSD"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 26,
- "artifact_ref": "page_citation_assets/page-26.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_a25cf04f-09bd-5cc1-beb4-79dbdeb154f8",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/KEY METRICS YEAR-OVER-YEAR FINANCIAL SUMMARY/Revenue p26]",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS YEAR-OVER-YEAR FINANCIAL SUMMARY/Profitability",
- "metadata": {
- "length": 84,
- "summary": "The Profitability section details a 11% year-over-year decrease in quarterly operating income to $1.4 billion, resulting in a 5.7% operating margin. The decline was primarily driven by increased SBC and restructuring charges, higher operating expenses related to AI and R&D projects, and an increase in average cost per vehicle due to lower fixed cost absorption and tariffs. These negative factors were partially offset by growth in gross profit from Energy Generation and Storage, Services and Other, and other automotive ancillary sales.",
- "page_nums": [
- 26
- ],
- "keywords": [
- "SBC",
- "AI"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 26,
- "artifact_ref": "page_citation_assets/page-26.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_6c377d7f-e831-58ad-b176-b052bf497330",
- "type": "page",
- "content": "F I N A N C I A L S T A T E M E N T S",
- "path": "TSLA-Q4-2025-Update.pdf/FINANCIAL STATEMENTS",
- "metadata": {
- "length": 41,
- "summary": "This page serves as a section divider or title page for the financial statements. It features a minimalist black background with the text 'FINANCIAL STATEMENTS' centered in white capital letters. The Tesla logo is visible in the bottom right corner.",
- "page_nums": [
- 27
- ],
- "keywords": [
- "Tesla"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 27,
- "artifact_ref": "page_citation_assets/page-27.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_38e58b09-ea3d-5400-8eed-7bfce3225d39",
- "type": "page",
- "content": "In millions of USD or shares as applicable, except per share data\nQ4-2024\nQ1-2025\nQ2-2025\nQ3-2025\nQ4-2025\nREVENUES\nAutomotive sales\n18,659\n12,925\n15,787\n20,359\n16,750\nAutomotive regulatory credits\n692\n595\n439\n417\n542\nAutomotive leasing\n447\n447\n435\n429\n401\nTotal automotive revenues\n19,798\n13,967\n16,661\n21,205\n17,693\nEnergy generation and storage\n3,061\n2,730\n2,789\n3,415\n3,837\nServices and other\n2,848\n2,638\n3,046\n3,475\n3,371\nTotal revenues\n25,707\n19,335\n22,496\n28,095\n24,901\nCOST OF REVENUES\nAutomotive sales\n16,268\n11,461\n13,567\n17,365\n13,874\nAutomotive leasing\n242\n239\n228\n225\n206\nTotal automotive cost of revenues\n16,510\n11,700\n13,795\n17,590\n14,080\nEnergy generation and storage\n2,289\n1,945\n1,943\n2,342\n2,739\nServices and other\n2,729\n2,537\n2,880\n3,109\n3,073\nTotal cost of revenues\n21,528\n16,182\n18,618\n23,041\n19,892\nGross profit\n4,179\n3,153\n3,878\n5,054\n5,009\nOPERATING EXPENSES\nResearch and development\n1,276\n1,409\n1,589\n1,630\n1,783\nSelling, general and administrative\n1,313\n1,251\n1,366\n1,562\n1,655\nRestructuring and other\n7\n94\n—\n238\n162\nTotal operating expenses\n2,596\n2,754\n2,955\n3,430\n3,600\nINCOME FROM OPERATIONS\n1,583\n399\n923\n1,624\n1,409\nInterest income\n442\n400\n392\n439\n449\nInterest expense\n(96)\n(91)\n(86)\n(76)\n(85)\nOther income (expense), net (1)\n595\n(119)\n320\n(28)\n(592)\nINCOME BEFORE INCOME TAXES (1)\n2,524\n589\n1,549\n1,959\n1,181\nProvision for income taxes (1)\n381\n169\n359\n570\n325\nNET INCOME (1)\n2,143\n420\n1,190\n1,389\n856\nNet income attributable to noncontrolling interests and redeemable noncontrolling interests in subsidiaries\n15\n11\n18\n16\n16\nNET INCOME ATTRIBUTABLE TO COMMON STOCKHOLDERS (1)\n2,128\n409\n1,172\n1,373\n840\nLess: Buy-out of noncontrolling interest\n3\n—\n—\n—\n—\nNET INCOME USED IN COMPUTING NET INCOME PER SHARE OF COMMON STOCK (1)\n2,125\n409\n1,172\n1,373\n840\nNet income per share of common stock attributable to common stockholders\nBasic (1)\n$ 0.66 \n$ 0.13 \n$ 0.36 \n$ 0.43 \n$ 0.26 \nDiluted (1)\n$ 0.60 \n$ 0.12 \n$ 0.33 \n$ 0.39 \n$ 0.24 \nWeighted average shares used in computing net income per share of common stock\nBasic\n3,213\n3,218\n3,223\n3,227\n3,231\nDiluted\n3,517\n3,521\n3,519\n3,526\n3,539\nS T A T E M E N T O F O P E R A T I O N S\n(Unaudited)\n28\n(1) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.",
- "path": "TSLA-Q4-2025-Update.pdf/STATEMENT OF OPERATIONS",
- "metadata": {
- "length": 2371,
- "summary": "The page displays an Unaudited Statement of Operations for a company, detailing financial performance across five quarters from Q4-2024 to Q4-2025. The table is structured into sections covering Revenues (Automotive sales, Energy generation, Services), Cost of Revenues, Operating Expenses, and Net Income. Key columns include Total revenues, Gross profit, Income from operations, and Net income attributable to common stockholders, with figures presented in millions of USD.",
- "page_nums": [
- 28
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_fb8bc966-dde3-51cd-ba9e-73ada5aeb41a",
- "relation": "embeds",
- "ref": "[tables/table_page_28_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 28,
- "artifact_ref": "page_citation_assets/page-28.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_9e2478cc-f365-51c9-b7f4-be54ce6cd49b",
- "type": "page",
- "content": "B A L A N C E S H E E T\n(Unaudited)\nIn millions of USD\n31-Dec-24\n31-Mar-25\n30-Jun-25\n30-Sep-25\n31-Dec-25\nASSETS\nCurrent assets\n Cash, cash equivalents and investments\n36,563\n36,996\n36,782\n41,647\n44,059\n Accounts receivable, net\n4,418\n3,782\n3,838\n4,703\n4,576\n Inventory\n12,017\n13,706\n14,570\n12,276\n12,392\n Prepaid expenses and other current assets\n5,362\n4,905\n5,943\n6,027\n7,615\n Total current assets\n58,360\n59,389\n61,133\n64,653\n68,642\nOperating lease vehicles, net\n5,581\n5,477\n5,230\n5,019\n4,912\nEnergy generation and storage systems, net\n4,924\n4,855\n4,788\n4,673\n4,604\nProperty, plant and equipment, net\n35,836\n37,088\n38,574\n39,407\n40,643\nOperating lease right-of-use assets\n5,160\n5,330\n5,633\n5,783\n6,027\nDigital assets (2)\n1,076\n951\n1,235\n1,315\n1,008\nDeferred tax assets (2)\n6,524\n6,687\n6,721\n6,637\n6,925\nOther non-current assets\n4,609\n5,334\n5,253\n6,248\n5,045\n Total assets (2)\n122,070\n125,111\n128,567\n133,735\n137,806\nLIABILITIES AND EQUITY\nCurrent liabilities\n Accounts payable\n12,474\n13,471\n13,212\n12,819\n13,371\n Accrued liabilities and other\n10,723\n10,802\n11,519\n12,791\n13,279\n Deferred revenue\n3,168\n3,243\n3,237\n3,756\n3,424\n Current portion of debt and finance leases (1)\n2,456\n2,237\n2,040\n1,924\n1,640\n Total current liabilities\n28,821\n29,753\n30,008\n31,290\n31,714\nDebt and finance leases, net of current portion (1)\n5,757\n5,292\n5,180\n5,778\n6,736\nDeferred revenue, net of current portion\n3,317\n3,610\n3,764\n3,746\n3,631\nOther long-term liabilities\n10,495\n11,038\n11,543\n12,205\n12,860\n Total liabilities\n48,390\n49,693\n50,495\n53,019\n54,941\nRedeemable noncontrolling interests in subsidiaries\n63\n62\n61\n59\n58\nTotal stockholders' equity (2)\n72,913\n74,653\n77,314\n79,970\n82,137\nNoncontrolling interests in subsidiaries\n704\n703\n697\n687\n670\n Total liabilities and equity (2)\n122,070\n125,111\n128,567\n133,735\n137,806\n(1) Breakdown of our debt is as follows:\nNon-recourse debt\n7,871\n7,238\n6,953\n7,458\n8,150\n Recourse debt\n7\n6\n3\n3\n3\n Days sales outstanding\n14\n19\n15\n14\n17\n Days payable outstanding\n58\n72\n65\n52\n61\n29\n(2) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.",
- "path": "TSLA-Q4-2025-Update.pdf/BALANCE SHEET",
- "metadata": {
- "length": 2137,
- "summary": "The page displays an Unaudited Balance Sheet for a company, presenting financial data in millions of USD across five quarterly periods from December 31, 2024, to December 31, 2025. The table details Assets (including current assets like cash and inventory, and non-current assets such as property and digital assets), Liabilities (current and long-term debt, accounts payable), and Equity. Key totals show Total Assets increasing from 122,070 million to 137,806 million over the period.",
- "page_nums": [
- 29
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_1c5afc0e-5600-579a-8657-da973eba36c6",
- "relation": "embeds",
- "ref": "[tables/table_page_29_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 29,
- "artifact_ref": "page_citation_assets/page-29.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_06182fff-d260-57c0-93e6-bb2b43a09e9f",
- "type": "page",
- "content": "In millions of USD\nQ4-2024\nQ1-2025\nQ2-2025\nQ3-2025\nQ4-2025\nCASH FLOWS FROM OPERATING ACTIVITIES\nNet income (1)\n2,143\n420\n1,190\n1,389\n856\nAdjustments to reconcile net income to net cash provided by operating activities:\nDepreciation, amortization and impairment\n1,496\n1,447\n1,433\n1,625\n1,643\nStock-based compensation\n579\n573\n635\n663\n954\nDeferred income taxes (1)\n6\n(43)\n52\n225\n(111)\nDigital assets (gain) loss, net (1)\n(347)\n125\n(284)\n(80)\n307\nOther\n(93)\n188\n187\n333\n378\nChanges in operating assets and liabilities\n1030\n(554)\n(673)\n2,083\n(214)\nNet cash provided by operating activities\n4,814\n2,156\n2,540\n6,238\n3,813\nCASH FLOWS FROM INVESTING ACTIVITIES\nCapital expenditures (2)\n(2,780)\n(1,492)\n(2,394)\n(2,248)\n(2,393)\nPurchases of investments\n(15,158)\n(6,015)\n(7,485)\n(11,402)\n(12,207)\nProceeds from maturities of investments\n10,335\n5,856\n6,935\n9,295\n8,072\nNet cash used in investing activities\n(7,603)\n(1,651)\n(2,944)\n(4,355)\n(6,528)\nCASH FLOWS FROM FINANCING ACTIVITIES\nNet cash flows from other debt activities\n(108)\n(50)\n(23)\n410\n963\nNet borrowings (repayments) under vehicle and energy product financing\n677\n(674)\n(400)\n81\n(377)\nNet cash flows from noncontrolling interests – Solar\n(37)\n(22)\n(14)\n(20)\n(22)\nOther\n453\n414\n215\n512\n146\nNet cash provided by (used in) financing activities\n985\n(332)\n(222)\n983\n710\nEffect of exchange rate changes on cash and cash equivalents and restricted cash\n(133)\n40\n111\n(17)\n37\nNet (decrease) increase in cash and cash equivalents and restricted cash\n(1,937)\n213\n(515)\n2,849\n(1,968)\nCash and cash equivalents and restricted cash at beginning of period\n18,974\n17,037\n17,250\n16,735\n19,584\nCash and cash equivalents and restricted cash at end of period\n17,037\n17,250\n16,735\n19,584\n17,616\nS T A T E M E N T O F C A S H F L O W S\n(Unaudited)\n30\n(1) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.\n(2) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted.",
- "path": "TSLA-Q4-2025-Update.pdf/STATEMENT OF CASH FLOWS",
- "metadata": {
- "length": 2078,
- "summary": "The page displays an Unaudited Statement of Cash Flows, presenting financial data in millions of USD across five quarters from Q4-2024 to Q4-2025. The table details cash flows categorized into operating activities (including net income and adjustments like depreciation), investing activities (such as capital expenditures and investment purchases), and financing activities (including debt activities and vehicle financing). It concludes with the net increase or decrease in cash and the ending cash balance for each period.",
- "page_nums": [
- 30
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_b0f611a6-9c75-52b0-ad2f-762b56694a4f",
- "relation": "embeds",
- "ref": "[tables/table_page_30_1.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 30,
- "artifact_ref": "page_citation_assets/page-30.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_1cbddb6d-e352-52bc-8b20-8809c0d4bf06",
- "type": "page",
- "content": "In millions of USD or shares as applicable, except per share data\nQ4-2024\nQ1-2025\nQ2-2025\nQ3-2025\nQ4-2025\nNet income attributable to common stockholders (GAAP) (1)\n2,128\n409\n1,172\n1,373\n840\nStock-based compensation expense, net of tax\n249\n428\n443\n459\n682\nDigital assets (gain) loss, net of tax (1)\n(270)\n97\n(222)\n(62)\n239\nNet income attributable to common stockholders (non-GAAP) (1) (2)\n2,107\n934\n1,393\n1,770\n1,761\nLess: Buy-outs of noncontrolling interests\n3\n—\n—\n—\n—\nNet income used in computing diluted EPS attributable to common stockholders (non-GAAP) (1) (2)\n2,104\n934\n1,393\n1,770\n1,761\nEPS attributable to common stockholders, diluted (GAAP) (1)\n0.60\n0.12\n0.33\n0.39\n0.24\nStock-based compensation expense, net of tax, per share\n0.08\n0.12\n0.13\n0.13\n0.19\nDigital assets (gain) loss, net of tax, per share (1)\n(0.08)\n0.03\n(0.06)\n(0.02)\n0.07\nEPS attributable to common stockholders, diluted (non-GAAP) (1) (2)\n0.60\n0.27\n0.40\n0.50\n0.50\nShares used in EPS calculation, diluted (GAAP and non-GAAP)\n3,517\n3,521\n3,519\n3,526\n3,539\nNet income attributable to common stockholders (GAAP) (1)\n2,128\n409\n1,172\n1,373\n840\nInterest expense\n96\n91\n86\n76\n85\nProvision for income taxes (1)\n381\n169\n359\n570\n325\nDepreciation, amortization and impairment\n1,496\n1,447\n1,433\n1,625\n1,643\nStock-based compensation expense\n579\n573\n635\n663\n954\nDigital assets (gain) loss, net (1)\n(347)\n125\n(284)\n(80)\n307\nAdjusted EBITDA (non-GAAP) (1) (3)\n4,333\n2,814\n3,401\n4,227\n4,154\nTotal revenues\n25,707\n19,335\n22,496\n28,095\n24,901\nAdjusted EBITDA margin (non-GAAP) (1) (3)\n16.9%\n14.6%\n15.1%\n15.0%\n16.7%\nAutomotive gross margin (GAAP)\n16.6%\n16.2%\n17.2%\n17.0%\n20.4%\nLess: Total regulatory credit revenue recognized\n3.0%\n3.7%\n2.2%\n1.6%\n2.5%\nAutomotive gross margin excluding regulatory credit sales (non-GAAP)\n13.6%\n12.5%\n15.0%\n15.4%\n17.9%\nR E C O N C I L I A T I O N O F G A A P T O N O N – G A A P F I N A N C I A L I N F O R M A T I O N \n(Unaudited)\n31\n(1) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.\n(2) Beginning in Q1'25, Net income attributable to common stockholders (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(3) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n\nIn millions of USD or shares as applicable, except per share data\n2021\n2022\n2023\n2024\n2025\nNet income attributable to common stockholders (GAAP)\n5,519\n12,556\n14,997\n7,091\n3,794\nStock-based compensation expense, net of tax\n2,121\n1,560\n1,812\n1,328\n2,012\nDigital assets loss (gain), net of tax\n79\n160\n—\n(459)\n52\nRelease of valuation allowance on deferred tax assets\n—\n—\n(5,927)\n—\n—\nNet income attributable to common stockholders (non-GAAP) (1)\n7,719\n14,276\n10,882\n7,960\n5,858\nLess: Buy-outs of noncontrolling interests\n(5)\n(27)\n(2)\n(39)\n—\nLess: Dilutive convertible debt\n(9)\n(1)\n—\n—\n—\nNet income used in computing diluted EPS attributable to common stockholders (non-GAAP) (1)\n7,733\n14,304\n10,884\n7,999\n5,858\nEPS attributable to common stockholders, diluted (GAAP)\n1.63\n3.62\n4.30\n2.04\n1.08\nStock-based compensation expense, net of tax, per share\n0.63\n0.45\n0.52\n0.38\n0.57\nDigital assets loss (gain), net of tax, per share\n0.02\n0.05\n—\n(0.13)\n0.01\nRelease of valuation allowance on deferred tax assets\n—\n—\n(1.70)\n—\n—\nEPS attributable to common stockholders, diluted (non-GAAP) (1)\n2.28\n4.12\n3.12\n2.29\n1.66\nShares used in EPS calculation, diluted (GAAP and non-GAAP)\n3,386\n3,475\n3,485\n3,498\n3,528\nNet income attributable to common stockholders (GAAP)\n5,519\n12,556\n14,997\n7,091\n3,794\nInterest expense\n371\n191\n156\n350\n338\nProvision for (benefit from) income taxes\n699\n1132\n(5,001)\n1,837\n1,423\nDepreciation, amortization and impairment\n2,911\n3,747\n4,667\n5,368\n6,148\nStock-based compensation expense\n2,121\n1,560\n1,812\n1,999\n2,825\nDigital assets loss (gain), net\n101\n204\n—\n(589)\n68\nAdjusted EBITDA (non-GAAP) (2)\n11,722\n19,390\n16,631\n16,056\n14,596\nTotal revenues\n53,823\n81,462\n96,773\n97,690\n94,827\nAdjusted EBITDA margin (non-GAAP) (2)\n21.8%\n23.8%\n17.2%\n16.4%\n15.4%\nAutomotive gross margin (GAAP)\n29.3%\n28.5%\n19.4%\n18.4%\n17.8%\nLess: Total regulatory credit revenue recognized\n2.3%\n1.8%\n1.7%\n3.0%\n2.4%\nAutomotive gross margin excluding regulatory credit sales (non-GAAP)\n27.0%\n26.7%\n17.7%\n15.4%\n15.4%\nR E C O N C I L I A T I O N O F G A A P T O N O N – G A A P F I N A N C I A L I N F O R M A T I O N \n(Unaudited)\n32\n(1) Beginning in Q1'25, Net income attributable to common stockholders (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n(2) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.\n\nR E C O N C I L I A T I O N O F G A A P T O N O N – G A A P F I N A N C I A L I N F O R M A T I O N \n(Unaudited)\nIn millions of USD\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025\nNet cash provided by operating activities – TTM (GAAP)\n13,242 \n13,956 \n12,164 \n13,256 \n10,985\n11,532\n14,479\n14,923\n16,837\n15,765\n15,748\n14,747\nCapital expenditures – TTM (1)\n(7,464)\n(7,794)\n(8,450)\n(8,899)\n(9,603)\n(9,815)\n(10,869)\n(11,342)\n(10,057)\n(10,179)\n(8,914)\n(8,527)\nFree cash flow – TTM (non-GAAP) (1)\n5,778 \n6,162 \n3,714 \n4,357 \n1,382\n1,717\n3,610\n3,581\n6,780\n5,586\n6,834\n6,220\nIn millions of USD\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025\nNet income attributable to common stockholders – TTM (GAAP) (2)\n11,751 \n12,195 \n10,756 \n14,997 \n13,874\n12,571\n12,891\n7,091\n6,110\n5,882\n5,082\n3,794\nInterest expense – TTM\n159 \n143 \n128 \n156 \n203\n261\n315\n350\n365\n365\n349\n338\nProvision for (benefit from) income taxes – TTM (2)\n1,047 \n1,165 \n1,027 \n(5,001)\n(4,779)\n(4,731)\n(4,296)\n1,837\n1,523\n1,511\n1,479\n1,423\nDepreciation, amortization and impairment – TTM\n3,913 \n4,145 \n4,424 \n4,667 \n4,867\n4,991\n5,104\n5,368\n5,569\n5,724\n6,001\n6,148\nStock-based compensation expense – TTM\n1,560 \n1,644 \n1,747 \n1,812 \n1,918\n1,912\n1,904\n1,999\n2,048\n2,244\n2,450\n2,825\nDigital assets loss (gain), net – TTM (2)\n204\n34\n34\n—\n(335)\n(235)\n(242)\n(589)\n(129)\n(513)\n(586)\n68\nAdjusted EBITDA – TTM (non-GAAP) (2) (3)\n18,634 \n19,326 \n18,116 \n16,631 \n15,748\n14,769\n15,676\n16,056\n15,486\n15,213\n14,775\n14,596\nIn millions of USD\n2Q-2022\n3Q-2022\n4Q-2022\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025\nNet cash provided by operating activities (GAAP)\n2,351 \n5,100 \n3,278 \n2,513 \n3,065 \n3,308 \n4,370 \n242\n3,612\n6,255\n4,814\n2,156\n2,540\n6,238\n3,813\nCapital expenditures (1)\n(1,730)\n(1,803)\n(1,858)\n(2,073)\n(2,060)\n(2,459)\n(2,307)\n(2,777)\n(2,272)\n(3,513)\n(2,780)\n(1,492)\n(2,394)\n(2,248)\n(2,393)\nFree cash flow (non-GAAP) (1)\n621 \n3,297 \n1,420 \n440 \n1,005 \n849 \n2,063 \n(2,535)\n1,340\n2,742\n2,034\n664\n146\n3,990\n1,420\nIn millions of USD\n2Q-2022\n3Q-2022\n4Q-2022\n1Q-2023\n2Q-2023\n3Q-2023\n4Q-2023\n1Q-2024\n2Q-2024\n3Q-2024\n4Q-2024\n1Q-2025\n2Q-2025\n3Q-2025\n4Q-2025\nNet income attributable to common stockholders (GAAP) (2)\n2,259 \n3,292 \n3,687 \n2,513 \n2,703 \n1,853 \n7,928 \n1,390\n1,400\n2,173\n2,128\n409\n1,172\n1,373\n840\nInterest expense\n44 \n53 \n33 \n29 \n28 \n38 \n61 \n76\n86\n92\n96\n91\n86\n76\n85\nProvision for (benefit from) income taxes (2)\n205 \n305 \n276 \n261 \n323 \n167 \n(5,752)\n483\n371\n602\n381\n169\n359\n570\n325\nDepreciation, amortization and impairment\n922 \n956 \n989 \n1,046 \n1,154 \n1,235 \n1,232 \n1,246\n1,278\n1,348\n1,496\n1,447\n1,433\n1,625\n1,643\nStock-based compensation expense\n361 \n362 \n419 \n418 \n445 \n465 \n484 \n524\n439\n457\n579\n573\n635\n663\n954\nDigital assets loss (gain), net (2)\n170\n—\n34\n—\n—\n—\n—\n(335)\n100\n(7)\n(347)\n125\n(284)\n(80)\n307\nAdjusted EBITDA (non-GAAP) (2) (3)\n3,961 \n4,968 \n5,438 \n4,267 \n4,653 \n3,758 \n3,953 \n3,384\n3,674\n4,665\n4,333\n2,814\n3,401\n4,227\n4,154\n33\nTTM = Trailing twelve months\n(1) Beginning in Q1'25, Capital expenditures is presented inclusive of purchases of energy generation and storage systems and all prior periods have been adjusted. \n(2) As a result of the adoption of the new crypto assets standard, the previously reported quarterly periods in 2024 have been recast.\n(3) Beginning in Q1'25, Adjusted EBITDA (non-GAAP) is presented net of digital assets gains and losses and all prior periods have been adjusted.",
- "path": "TSLA-Q4-2025-Update.pdf/RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION",
- "metadata": {
- "length": 8326,
- "summary": "The section titled 'RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION' presents a series of financial tables detailing the reconciliation between Generally Accepted Accounting Principles (GAAP) and non-GAAP metrics. The data is provided in millions of USD, with per-share data where applicable, covering quarterly periods from Q2 2022 through Q4 2025 and annual periods from 2021 to 2025. Key metrics reconciled include Net Income attributable to common stockholders, Earnings Per Share (EPS), Adjusted EBITDA, Free Cash Flow, and Automotive Gross Margin. The tables show adjustments for items such as stock-based compensation, digital assets gains/losses, interest expense, taxes, and regulatory credit revenue.",
- "page_nums": [
- 31,
- 32,
- 33
- ],
- "keywords": [],
- "connect_to": [
- {
- "target": "asset_ba33916c-76ab-58fd-9d67-2b5da73878cf",
- "relation": "embeds",
- "ref": "[tables/table_page_31_1.html]"
- },
- {
- "target": "asset_07f3203e-41aa-5eab-a843-d13b61d02751",
- "relation": "embeds",
- "ref": "[tables/table_page_32_1.html]"
- },
- {
- "target": "asset_be11d68c-c437-5e3f-b88e-a26fce773560",
- "relation": "embeds",
- "ref": "[tables/table_page_33_1.html]"
- },
- {
- "target": "asset_0b9e3b34-bfbe-5a15-89fa-b65f82c795ae",
- "relation": "embeds",
- "ref": "[tables/table_page_33_2.html]"
- },
- {
- "target": "asset_3aff0fc7-a9a4-5b72-8176-d26482a26a91",
- "relation": "embeds",
- "ref": "[tables/table_page_33_3.html]"
- },
- {
- "target": "asset_9538f471-c622-52dc-8b7e-2d45f59cfe34",
- "relation": "embeds",
- "ref": "[tables/table_page_33_4.html]"
- }
- ],
- "page_assets": [
- {
- "page_num": 31,
- "artifact_ref": "page_citation_assets/page-31.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 32,
- "artifact_ref": "page_citation_assets/page-32.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 33,
- "artifact_ref": "page_citation_assets/page-33.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_87a37514-5b8d-5fa1-bd73-9e13c6fd6624",
- "type": "page",
- "content": "A D D I T I O N A L I N F O R M A T I O N\nWEBCAST INFORMATION\nTesla will provide a live webcast of its fourth quarter 2025 financial results conference call beginning at 4:30 p.m. CT on January 28, 2026 at ir.tesla.com. This webcast will also be available for replay for approximately one year thereafter.\nCERTAIN TERMS\nWhen used in this update, certain terms have the following meanings. Our vehicle deliveries include only vehicles that have been transferred to end customers with all paperwork correctly completed. Our energy product deployment volume includes \nboth customer units when installed and equipment sales at time of delivery. “Net income attributable to common stockholders (non-GAAP)\" is equal to (i) net income attributable to common stockholders before (ii)(a) stock-based compensation expense, \nnet of tax, (b) digital assets (gain) loss, net of tax and (c) release of valuation allowance on deferred tax assets. \"Adjusted EBITDA (non-GAAP)\" is equal to (i) net income attributable to common stockholders before (ii)(a) interest expense, (b) provision for \n(benefit from) income taxes, (c) depreciation, amortization and impairment, (d) stock-based compensation expense and (e) digital assets loss (gain), net. \"Free cash flow\" is operating cash flow less capital expenditures. Average cost per vehicle is cost of \nautomotive sales divided by new vehicle deliveries (excluding operating leases). “Days sales outstanding” is equal to (i) average accounts receivable, net for the period divided by (ii) total revenues and multiplied by (iii) the number of days in the period. \n“Days payable outstanding” is equal to (i) average accounts payable for the period divided by (ii) total cost of revenues and multiplied by (iii) the number of days in the period. “Days of supply” is calculated by dividing new car ending inventory by the \nrelevant period's deliveries and using trading days. Constant currency impacts are calculated by comparing actuals against current results converted into USD using average exchange rates from the prior period.\nNON-GAAP FINANCIAL INFORMATION\nConsolidated financial information has been presented in accordance with GAAP as well as on a non-GAAP basis to supplement our consolidated financial results. Our non-GAAP financial measures include non-GAAP net income (loss) attributable to \ncommon stockholders, non-GAAP net income (loss) attributable to common stockholders on a diluted per share basis (calculated using weighted average shares for GAAP diluted net income (loss) attributable to common stockholders), Adjusted EBITDA, \nAdjusted EBITDA margin, non-GAAP automotive gross margin and free cash flow. These non-GAAP financial measures also facilitate management’s internal comparisons to Tesla’s historical performance as well as comparisons to the operating results of \nother companies. Management believes that it is useful to supplement its GAAP financial statements with this non-GAAP information because management uses such information internally for its operating, budgeting and financial planning purposes. \nManagement also believes that presentation of the non-GAAP financial measures provides useful information to our investors regarding our financial condition and results of operations, so that investors can see through the eyes of Tesla management \nregarding important financial metrics that Tesla uses to run the business and allowing investors to better understand Tesla’s performance. Non-GAAP information is not prepared under a comprehensive set of accounting rules and therefore, should only \nbe read in conjunction with financial information reported under U.S. GAAP when understanding Tesla’s operating performance. A reconciliation between GAAP and non-GAAP financial information is provided above.\nFORWARD-LOOKING STATEMENTS\nCertain statements in this update, including, but not limited to, statements in the “Outlook” section; statements relating to the development, strategy, ramp, production and capacity, demand and market growth, cost, pricing and profitability, investment, \ndeliveries, deployment, availability and other features and improvements and timing of existing and future Tesla products and services and supporting infrastructure; statements regarding operating margin, operating profits, spending and liquidity; and \nstatements regarding expansion, improvements and/or ramp and related timing at our facilities are “forward-looking statements” within the meaning of the Private Securities Litigation Reform Act of 1995. Forward-looking statements are based on \nassumptions and management’s current expectations, involve certain risks and uncertainties, and are not guarantees. Future results may differ materially from those expressed in any forward-looking statement. The following important factors, without \nlimitation, could cause actual results to differ materially from those in the forward-looking statements: the risk of delays in launching and/or manufacturing our products, services and features cost-effectively; our ability to build and/or grow our products \nand services, sales, delivery, installation, servicing and charging capabilities and effectively manage this growth; our ability to successfully and timely develop, introduce and scale, as well as our consumers’ demand for, products and services based on \nartificial intelligence, robotics and automation, electric vehicles, advanced driver assistance systems, and ride-hailing services generally and our vehicles and services specifically; the ability of suppliers to deliver components according to schedules, \nprices, quality and volumes acceptable to us, and our ability to manage such components effectively; any issues with lithium-ion cells or other components manufactured at our factories; our ability to ramp our factories in accordance with our plans; our \nability to procure supply of battery cells, including through our own manufacturing; risks relating to international operations and expansion, including unfavorable and uncertain regulatory, political, economic, tax, tariff, export controls and labor \nconditions; any failures by Tesla products to perform as expected or if product recalls occur; the risk of product liability claims; competition in the automotive, transportation and energy product and services and robotics markets; our ability to maintain \npublic credibility and confidence in our long-term business prospects; our ability to manage risks relating to our various product financing programs; the status of government and economic incentives for electric vehicles and energy products; our ability \nto attract, hire and retain key employees and qualified personnel; our ability to maintain the security of our information and production and product systems; our compliance with various regulations and laws applicable to our operations and products, \nwhich may evolve from time to time; risks relating to our indebtedness and financing strategies; and adverse foreign exchange movements. More information on potential factors that could affect our financial results is included from time to time in our \nSecurities and Exchange Commission filings and reports, including the risks identified under the section captioned “Risk Factors” in our annual report on Form 10-K filed with the SEC on January 30, 2025 and subsequent quarterly reports on Form 10-Q. \nTesla disclaims any obligation to update information contained in these forward-looking statements whether as a result of new information, future events or otherwise.\n34",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/WEBCAST INFORMATION",
- "metadata": {
- "length": 7529,
- "summary": "The section provides details regarding a live webcast of Tesla's fourth quarter 2025 financial results conference call. It specifies the start time as 4:30 p.m. CT on January 28, 2026, and notes that the event will be available at ir.tesla.com. Additionally, it mentions that a replay of the webcast will be accessible for approximately one year.",
- "page_nums": [
- 34
- ],
- "keywords": [
- "Tesla",
- "ir.tesla.com"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 34,
- "artifact_ref": "page_citation_assets/page-34.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_b7614083-6642-5aa3-8918-48613b9fe066",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/WEBCAST INFORMATION p34]",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/CERTAIN TERMS",
- "metadata": {
- "length": 74,
- "summary": "This section defines specific terms used in the document, including vehicle deliveries, energy product deployment volume, Net income attributable to common stockholders (non-GAAP), Adjusted EBITDA (non-GAAP), Free cash flow, Average cost per vehicle, Days sales outstanding, Days payable outstanding, and Days of supply.",
- "page_nums": [
- 34
- ],
- "keywords": [
- "Tesla"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 34,
- "artifact_ref": "page_citation_assets/page-34.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_bb98469a-1c28-5a4c-aa86-1346c1adb57f",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/WEBCAST INFORMATION p34]",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/NON-GAAP FINANCIAL INFORMATION",
- "metadata": {
- "length": 74,
- "summary": "This section explains that Tesla presents consolidated financial information on both a GAAP and non-GAAP basis. It lists specific non-GAAP measures used by management, including non-GAAP net income (loss), Adjusted EBITDA, Adjusted EBITDA margin, non-GAAP automotive gross margin, and free cash flow. The text states these metrics are used for internal planning and to help investors understand the company's performance, noting that they should be read in conjunction with U.S. GAAP financial information.",
- "page_nums": [
- 34
- ],
- "keywords": [
- "Tesla",
- "U.S. GAAP"
- ],
- "connect_to": [],
- "page_assets": [
- {
- "page_num": 34,
- "artifact_ref": "page_citation_assets/page-34.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- },
- {
- "chunk_id": "node_e60b3f11-70be-5d4c-87bb-bf4fb73cff1b",
- "type": "page",
- "content": "[SAME-AS TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/WEBCAST INFORMATION p34]",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/FORWARD-LOOKING STATEMENTS",
- "metadata": {
- "length": 74,
- "summary": "This section contains a legal disclaimer regarding forward-looking statements made by Tesla. It defines these statements as those related to future expectations, strategies, and performance metrics (such as production, demand, and profitability) which are based on assumptions and involve risks and uncertainties. The text lists numerous potential factors that could cause actual results to differ materially from these projections, including manufacturing delays, supply chain issues, regulatory changes, competition, and product safety. It references the Private Securities Litigation Reform Act of 1995 and directs readers to SEC filings for more detailed risk information.",
- "page_nums": [
- 34,
- 35
- ],
- "keywords": [
- "Tesla",
- "Private Securities Litigation Reform Act of 1995",
- "Securities and Exchange Commission",
- "SEC"
- ],
- "connect_to": [
- {
- "target": "asset_e0461421-3c85-5cd0-90d8-be7ad00ad97b",
- "relation": "embeds",
- "ref": "[images/image_page_35_figure_1.png]"
- }
- ],
- "page_assets": [
- {
- "page_num": 34,
- "artifact_ref": "page_citation_assets/page-34.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- },
- {
- "page_num": 35,
- "artifact_ref": "page_citation_assets/page-35.png",
- "content_type": "image/png",
- "source": "knowhere-rendered-page-citation-source",
- "width": 3167,
- "height": 1782
- }
- ]
- }
- }
- ]
-}
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/doc_nav.json b/apps/api/app/data/demo_documents/tsla-q4-2025/doc_nav.json
deleted file mode 100644
index b3fda644f..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/doc_nav.json
+++ /dev/null
@@ -1,506 +0,0 @@
-{
- "version": "1.0",
- "file_name": "TSLA-Q4-2025-Update.pdf",
- "stats": {
- "total_chunks": 71,
- "text_chunks": 0,
- "image_chunks": 18,
- "table_chunks": 16,
- "page_chunks": 37,
- "max_depth": 2
- },
- "sections": [
- {
- "title": "Q4 and FY 2025 Update",
- "path": "TSLA-Q4-2025-Update.pdf/Q4 and FY 2025 Update",
- "level": 1,
- "summary": "This section serves as the title page and table of contents for the 'Q4 and FY 2025 Update' presentation. It outlines the document's structure, listing key sections such as Highlights, Financial Summary, Operational Summary, Manufacturing & Hardware, AI & Software, Services, Outlook, and Key Metrics, along with their corresponding page numbers.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "HIGHLIGHTS",
- "path": "TSLA-Q4-2025-Update.pdf/HIGHLIGHTS",
- "level": 1,
- "summary": "The 'HIGHLIGHTS' section outlines key financial and operational achievements for 2025. Under Profitability, it lists GAAP operating income of $4.4B, net income of $3.8B, and non-GAAP net income of $5.9B. The Cash section details an operating cash flow of $14.7B, free cash flow of $6.2B, and a total cash and investments balance of $44.1B. Operational highlights include the removal of safety monitors from Robotaxis in Austin, record energy storage deployments, and record vehicle deliveries in APAC.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "SUMMARY",
- "path": "TSLA-Q4-2025-Update.pdf/SUMMARY",
- "level": 1,
- "summary": "The 'SUMMARY' section outlines Tesla's strategic transition in 2025 from a hardware-centric business to a physical AI company. Key achievements include advancing Full Self-Driving (FSD) technology, launching the Robotaxi service, and initiating production lines for Cybercab and Optimus robots. The text details the company's system-level approach to autonomous vehicles and humanoid robots, emphasizing vertical integration and cost optimization. It also highlights the refresh of the vehicle lineup with the new Model Y and the evolution of energy offerings. Looking ahead to 2026, the summary notes plans to invest in infrastructure for clean energy and transport, including the ramp-up of six new production lines across various manufacturing sectors.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "FINANCIAL SUMMARY",
- "path": "TSLA-Q4-2025-Update.pdf/FINANCIAL SUMMARY",
- "level": 1,
- "summary": "The section titled 'FINANCIAL SUMMARY' presents two comprehensive tables of unaudited financial data. The first table details quarterly performance metrics for the fiscal year 2024 and 2025 (Q4-2024 through Q4-2025), while the second table provides annual historical data from 2021 to 2025. Both tables report figures in millions of dollars, excluding percentages and per share data. Key metrics tracked include revenue breakdowns by segment (automotive, energy generation/storage, services), total revenues, gross profit, operating expenses, income from operations, adjusted EBITDA, net income (GAAP and non-GAAP), earnings per share (EPS), and cash flow indicators such as capital expenditures and free cash flow.",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "OPERATIONAL SUMMARY",
- "path": "TSLA-Q4-2025-Update.pdf/OPERATIONAL SUMMARY",
- "level": 1,
- "summary": "The 'OPERATIONAL SUMMARY' section presents two tables detailing Tesla's operational metrics. The first table provides quarterly data from Q4 2024 through Q4 2025, while the second table offers annual historical data from 2021 to 2025. Key metrics tracked include vehicle production (broken down by Model 3/Y and other models), deliveries (including total, subject to operating lease accounting, and cumulative all-time figures), Active FSD Subscriptions, end-of-period operating lease counts, global vehicle inventory days of supply, storage deployed in GWh, Tesla locations, and Supercharger infrastructure (stations and connectors).",
- "chunk_count": 2,
- "children": []
- },
- {
- "title": "MANUFACTURING & HARDWARE",
- "path": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE",
- "level": 1,
- "summary": "",
- "chunk_count": 3,
- "children": [
- {
- "title": "Automotive",
- "path": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Automotive",
- "level": 2,
- "summary": "The section discusses automotive sales performance, noting a sequential decline but an improvement in gross margin and strong results in the APAC region. It details the rollout of Model Y variants and preparations for future production ramps of Tesla Semi, Cybercab, and the next-generation Roadster. The accompanying table outlines installed annual manufacturing capacity across various regions (California, Shanghai, Berlin, Texas, Nevada) for products including Model 3, Model Y, Model S, Model X, Cybertruck, Cybercab, and Tesla Semi.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Energy generation and storage",
- "path": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Energy generation and storage",
- "level": 2,
- "summary": "The section details record quarterly energy storage deployments driven by Megapack, resulting in a record gross profit of $1.1 billion. It outlines plans to begin production of Megapack 3 and Megablock at the Megafactory Houston in 2026. Additionally, it notes that the global Powerwall network supported over 89,000 Virtual Power Plant events across more than 1 million units in 2025. The accompanying table lists installed annual manufacturing capacity for Energy Generation and Storage products (Megapack, Powerwall) across regions including California, Nevada, Shanghai, and Texas.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Robotics",
- "path": "TSLA-Q4-2025-Update.pdf/MANUFACTURING & HARDWARE/Robotics",
- "level": 2,
- "summary": "The Robotics section details progress on the Optimus program in 2025, specifically the unveiling of the Gen 3 version with major upgrades and a new hand design intended for mass production. It outlines plans to start production before the end of 2026 with an eventual capacity of 1 million robots per year. The accompanying table indicates that the Optimus robot is currently under construction in California.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "SUPPORTING INFRASTRUCTURE",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE",
- "level": 1,
- "summary": "",
- "chunk_count": 6,
- "children": [
- {
- "title": "AI Training Compute",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Training Compute",
- "level": 2,
- "summary": "The section details the construction of Cortex 2 at Gigafactory Texas to increase AI training compute capacity, with plans to more than double onsite compute in terms of H100 equivalents by the first half of 2026. The accompanying table outlines installed annual capacity for AI Training Compute, listing 'Cortex 1' in Texas as having over 100k H100e capacity in production and 'Cortex 2' under construction.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Battery",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/Battery",
- "level": 2,
- "summary": "The section details the company's battery manufacturing progress, including the commencement of pilot production at its lithium refinery in North America (the first spodumene-to-lithium hydroxide facility). It also covers the production of 4680 cells and battery packs for Model Ys using dry-electrode technology made in Austin. The text notes expectations for domestic cathode material production in Texas and LFP lines in Nevada to begin in 2026.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Other Supporting Infrastructure",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/Other Supporting Infrastructure",
- "level": 2,
- "summary": "This section outlines the company's strategy for supporting infrastructure, focusing on the efficient utilization of its North American physical footprint to support the Robotaxi rollout. It details plans to augment existing charging networks, service centers, and sales locations as demand grows, specifically noting the addition of over 3,800 new Supercharging stalls.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "AI Software",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Software",
- "level": 2,
- "summary": "The section details the enhancement of FSD (Supervised) version v14, an end-to-end foundation model trained on customer and Robotaxi data. It highlights safety features like autonomous parking and accessibility for those with difficulty driving. The text also mentions the global fleet's ability to collect over 500 years of continuous driving data daily to handle corner cases.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "AI Inference Compute",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/AI Inference Compute",
- "level": 2,
- "summary": "The section details the development of in-house AI5 and AI6 inference chips for autonomy, with production planned for 2027 and 2028. It highlights a targeted 50x performance improvement for the AI5 chip over the AI4, driven by a 10x increase in raw compute, 9x memory capacity, and 5x hardened block quantization and softmax function.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Automotive and Other Software",
- "path": "TSLA-Q4-2025-Update.pdf/SUPPORTING INFRASTRUCTURE/Automotive and Other Software",
- "level": 2,
- "summary": "This section details recent software updates and features for Tesla vehicles, including the removal of waitlists for the Robotaxi iOS app. It highlights over-the-air enhancements such as Grok AI companion support for navigation commands, Tesla Photobooth for in-car photography, Supercharger Site Maps with live availability, Automatic HOV Lane Routing based on camera occupancy detection, a Phone Left Behind Chime, and the SpaceX ISS Docking Simulator Game.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "SERVICES",
- "path": "TSLA-Q4-2025-Update.pdf/SERVICES",
- "level": 1,
- "summary": "",
- "chunk_count": 3,
- "children": [
- {
- "title": "Robotaxi",
- "path": "TSLA-Q4-2025-Update.pdf/SERVICES/Robotaxi",
- "level": 2,
- "summary": "The section details the progress of driverless Robotaxi services, including testing in Austin starting in December and removing safety monitors in January. It also mentions the launch of Bay Area ride-hailing service at San Jose Airport in October. The section includes a chart showing cumulative paid Robotaxi miles projected through December 2025 and a table outlining planned coverage for various US states (California, Texas, Arizona, Florida, Nevada) with specific metro areas and expected status or launch dates.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "FSD (Supervised)",
- "path": "TSLA-Q4-2025-Update.pdf/SERVICES/FSD (Supervised)",
- "level": 2,
- "summary": "The section details the launch and growth of FSD (Supervised) in South Korea, where customers drove over 1 million kilometers in one month. It mentions ongoing regulatory pursuits in China and Europe and ride-along experiences in Italy, Germany, France, and Switzerland. Additionally, it notes that monthly subscriptions for the service more than doubled in 2025 and that access is transitioning to a monthly subscription model only.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Automotive Services",
- "path": "TSLA-Q4-2025-Update.pdf/SERVICES/Automotive Services",
- "level": 2,
- "summary": "The section details the performance of 'Services and Other' gross profit, which was approximately $300 million, driven by Part Sales and Supercharging. It highlights the expansion of Tesla Insurance to Florida and explains a discount program where customers using FSD (Supervised) receive reduced insurance premiums.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "OTHER UPDATES",
- "path": "TSLA-Q4-2025-Update.pdf/OTHER UPDATES",
- "level": 1,
- "summary": "This page details an agreement reached on January 16, 2026, where Tesla invested approximately $2 billion to acquire Series E Preferred Stock of xAI. The text explains that this investment is part of Tesla's Master Plan Part IV and includes a framework agreement to evaluate potential AI collaborations between the two companies. The transaction is subject to regulatory conditions with an expected closing in Q1'2026.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "OUTLOOK",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK",
- "level": 1,
- "summary": "",
- "chunk_count": 4,
- "children": [
- {
- "title": "Volume",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK/Volume",
- "level": 2,
- "summary": "The section outlines the company's focus on maximizing factory capacity utilization. It notes that deliveries and deployments depend on aggregate demand, supply chain readiness, and allocation decisions between customer sales and use in the company's own fleet.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Cash",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK/Cash",
- "level": 2,
- "summary": "The section outlines the company's strategy to maintain a strong balance sheet and sufficient liquidity. This financial management approach is intended to fund the product roadmap, long-term capacity expansion plans (including vertical integration), and other expenses.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Profit",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK/Profit",
- "level": 2,
- "summary": "The section outlines the company's strategy to reduce manufacturing and operational costs through innovation. It states that over time, hardware-related profits are expected to be supplemented by an acceleration of profits from AI, software, and fleet-based services.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Product",
- "path": "TSLA-Q4-2025-Update.pdf/OUTLOOK/Product",
- "level": 2,
- "summary": "The section outlines the company's strategy to evolve its product lineup with a focus on cost, scale, and monetization via AI software. It emphasizes growing sales volumes by optimizing existing production capacity before building new facilities. Additionally, it provides specific timelines for volume production: Cybercab, Tesla Semi, and Megapack 3 are scheduled to start in 2026, while first-generation production lines for Optimus are currently being installed.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "PHOTOS & CHARTS",
- "path": "TSLA-Q4-2025-Update.pdf/PHOTOS & CHARTS",
- "level": 1,
- "summary": "This section presents a series of images highlighting Tesla's vehicle safety ratings and autonomous driving capabilities. It features the Model Y, recognized as the 2025 best-in-class small SUV by Euro NCAP, and the Model 3, recognized as the 2025 best-in-class large family car by Euro NCAP. The section also showcases Tesla's Full Self-Driving (Supervised) V14 software, which offers driver assistance, and documents the testing phase of the Driverless Robotaxi in Austin.",
- "chunk_count": 7,
- "children": []
- },
- {
- "title": "TESLA SEMI - MEGACHARGER NETWORK PLANNED SITES FOR 2026",
- "path": "TSLA-Q4-2025-Update.pdf/TESLA SEMI - MEGACHARGER NETWORK PLANNED SITES FOR 2026",
- "level": 1,
- "summary": "The section displays a map of the United States illustrating the planned locations for Tesla Semi Megacharger network sites in 2026. Red markers indicate the specific cities and regions where these charging stations are scheduled to be built.",
- "chunk_count": 3,
- "children": []
- },
- {
- "title": "KEY METRICS QUARTERLY",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS QUARTERLY",
- "level": 1,
- "summary": "This page displays quarterly key metrics for a company, presented as three bar charts covering the period from 1Q-2023 to 4Q-2025. The first chart shows Vehicle Deliveries in millions of units, while the second and third charts display financial performance including Operating Cash Flow, Free Cash Flow, Net Income, and Adjusted EBITDA in billions of dollars. The data is marked as Unaudited.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "KEY METRICS TRAILING 12 MONTHS (TTM)",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS TRAILING 12 MONTHS (TTM)",
- "level": 1,
- "summary": "The page displays a set of three bar charts titled 'KEY METRICS TRAILING 12 MONTHS (TTM)' showing financial and operational data from Q1 2023 through Q4 2025. The first chart tracks Vehicle Deliveries in millions of units, the second compares Operating Cash Flow and Free Cash Flow in billions of dollars, and the third contrasts Net Income with Adjusted EBITDA, also in billions.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "KEY METRICS YEAR-OVER-YEAR FINANCIAL SUMMARY",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS YEAR-OVER-YEAR FINANCIAL SUMMARY",
- "level": 1,
- "summary": "",
- "chunk_count": 2,
- "children": [
- {
- "title": "Revenue",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS YEAR-OVER-YEAR FINANCIAL SUMMARY/Revenue",
- "level": 2,
- "summary": "The section details that total quarterly revenue decreased 3% year-over-year to $24.9 billion. It lists the factors impacting this change, including decreases in vehicle deliveries and regulatory credit revenue, offset by growth in Energy Generation and Storage, Services and Other, positive foreign exchange impact, and higher average selling prices.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "Profitability",
- "path": "TSLA-Q4-2025-Update.pdf/KEY METRICS YEAR-OVER-YEAR FINANCIAL SUMMARY/Profitability",
- "level": 2,
- "summary": "The Profitability section details a 11% year-over-year decrease in quarterly operating income to $1.4 billion, resulting in a 5.7% operating margin. The decline was primarily driven by increased SBC and restructuring charges, higher operating expenses related to AI and R&D projects, and an increase in average cost per vehicle due to lower fixed cost absorption and tariffs. These negative factors were partially offset by growth in gross profit from Energy Generation and Storage, Services and Other, and other automotive ancillary sales.",
- "chunk_count": 1,
- "children": []
- }
- ]
- },
- {
- "title": "FINANCIAL STATEMENTS",
- "path": "TSLA-Q4-2025-Update.pdf/FINANCIAL STATEMENTS",
- "level": 1,
- "summary": "This page serves as a section divider or title page for the financial statements. It features a minimalist black background with the text 'FINANCIAL STATEMENTS' centered in white capital letters. The Tesla logo is visible in the bottom right corner.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "STATEMENT OF OPERATIONS",
- "path": "TSLA-Q4-2025-Update.pdf/STATEMENT OF OPERATIONS",
- "level": 1,
- "summary": "The page displays an Unaudited Statement of Operations for a company, detailing financial performance across five quarters from Q4-2024 to Q4-2025. The table is structured into sections covering Revenues (Automotive sales, Energy generation, Services), Cost of Revenues, Operating Expenses, and Net Income. Key columns include Total revenues, Gross profit, Income from operations, and Net income attributable to common stockholders, with figures presented in millions of USD.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "BALANCE SHEET",
- "path": "TSLA-Q4-2025-Update.pdf/BALANCE SHEET",
- "level": 1,
- "summary": "The page displays an Unaudited Balance Sheet for a company, presenting financial data in millions of USD across five quarterly periods from December 31, 2024, to December 31, 2025. The table details Assets (including current assets like cash and inventory, and non-current assets such as property and digital assets), Liabilities (current and long-term debt, accounts payable), and Equity. Key totals show Total Assets increasing from 122,070 million to 137,806 million over the period.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "STATEMENT OF CASH FLOWS",
- "path": "TSLA-Q4-2025-Update.pdf/STATEMENT OF CASH FLOWS",
- "level": 1,
- "summary": "The page displays an Unaudited Statement of Cash Flows, presenting financial data in millions of USD across five quarters from Q4-2024 to Q4-2025. The table details cash flows categorized into operating activities (including net income and adjustments like depreciation), investing activities (such as capital expenditures and investment purchases), and financing activities (including debt activities and vehicle financing). It concludes with the net increase or decrease in cash and the ending cash balance for each period.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION",
- "path": "TSLA-Q4-2025-Update.pdf/RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION",
- "level": 1,
- "summary": "The section titled 'RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION' presents a series of financial tables detailing the reconciliation between Generally Accepted Accounting Principles (GAAP) and non-GAAP metrics. The data is provided in millions of USD, with per-share data where applicable, covering quarterly periods from Q2 2022 through Q4 2025 and annual periods from 2021 to 2025. Key metrics reconciled include Net Income attributable to common stockholders, Earnings Per Share (EPS), Adjusted EBITDA, Free Cash Flow, and Automotive Gross Margin. The tables show adjustments for items such as stock-based compensation, digital assets gains/losses, interest expense, taxes, and regulatory credit revenue.",
- "chunk_count": 3,
- "children": []
- },
- {
- "title": "ADDITIONAL INFORMATION",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION",
- "level": 1,
- "summary": "",
- "chunk_count": 5,
- "children": [
- {
- "title": "WEBCAST INFORMATION",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/WEBCAST INFORMATION",
- "level": 2,
- "summary": "The section provides details regarding a live webcast of Tesla's fourth quarter 2025 financial results conference call. It specifies the start time as 4:30 p.m. CT on January 28, 2026, and notes that the event will be available at ir.tesla.com. Additionally, it mentions that a replay of the webcast will be accessible for approximately one year.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "CERTAIN TERMS",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/CERTAIN TERMS",
- "level": 2,
- "summary": "This section defines specific terms used in the document, including vehicle deliveries, energy product deployment volume, Net income attributable to common stockholders (non-GAAP), Adjusted EBITDA (non-GAAP), Free cash flow, Average cost per vehicle, Days sales outstanding, Days payable outstanding, and Days of supply.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "NON-GAAP FINANCIAL INFORMATION",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/NON-GAAP FINANCIAL INFORMATION",
- "level": 2,
- "summary": "This section explains that Tesla presents consolidated financial information on both a GAAP and non-GAAP basis. It lists specific non-GAAP measures used by management, including non-GAAP net income (loss), Adjusted EBITDA, Adjusted EBITDA margin, non-GAAP automotive gross margin, and free cash flow. The text states these metrics are used for internal planning and to help investors understand the company's performance, noting that they should be read in conjunction with U.S. GAAP financial information.",
- "chunk_count": 1,
- "children": []
- },
- {
- "title": "FORWARD-LOOKING STATEMENTS",
- "path": "TSLA-Q4-2025-Update.pdf/ADDITIONAL INFORMATION/FORWARD-LOOKING STATEMENTS",
- "level": 2,
- "summary": "This section contains a legal disclaimer regarding forward-looking statements made by Tesla. It defines these statements as those related to future expectations, strategies, and performance metrics (such as production, demand, and profitability) which are based on assumptions and involve risks and uncertainties. The text lists numerous potential factors that could cause actual results to differ materially from these projections, including manufacturing delays, supply chain issues, regulatory changes, competition, and product safety. It references the Private Securities Litigation Reform Act of 1995 and directs readers to SEC filings for more detailed risk information.",
- "chunk_count": 2,
- "children": []
- }
- ]
- }
- ],
- "resources": {
- "images": [
- {
- "path": "images/image_page_9_figure_2.png",
- "summary": "Tesla AI Training Capacity Ramp (H100 equivalent GPUs) [images/image_page_9_figure_2.png]"
- },
- {
- "path": "images/image_page_10_figure_1.png",
- "summary": "Cumulative Miles Driven with FSD (Supervised) (billions) [images/image_page_10_figure_1.png]"
- },
- {
- "path": "images/image_page_10_figure_2.png",
- "summary": "Targeting Step-Function Improvement for our Next-Generation Inference Chip, AI5 [images/image_page_10_figure_2.png]"
- },
- {
- "path": "images/image_page_11_figure_1.png",
- "summary": "Cumulative Paid Robotaxi Miles [images/image_page_11_figure_1.png]"
- },
- {
- "path": "images/image_page_15_figure_1.png",
- "summary": "MODEL Y - 2025 BEST IN CLASS EURO NCAP SMALL SUV [images/image_page_15_figure_1.png]"
- },
- {
- "path": "images/image_page_16_figure_1.png",
- "summary": "MODEL 3 - 2025 BEST IN CLASS EURO NCAP LARGE FAMILY CAR [images/image_page_16_figure_1.png]"
- },
- {
- "path": "images/image_page_17_figure_1.png",
- "summary": "[images/image_page_17_figure_1.png]"
- },
- {
- "path": "images/image_page_18_figure_1.png",
- "summary": "DRIVERLESS ROBOTAXI - TESTING IN AUSTIN [images/image_page_18_figure_1.png]"
- },
- {
- "path": "images/image_page_19_figure_1.png",
- "summary": "CYBERCAB – COLD WEATHER TESTING IN ALASKA [images/image_page_19_figure_1.png]"
- },
- {
- "path": "images/image_page_20_figure_1.png",
- "summary": "TESLA SEMI - PRODUCTION BEGINNING IN 2026 [images/image_page_20_figure_1.png]"
- },
- {
- "path": "images/image_page_21_figure_1.png",
- "summary": "TESLA SEMI - MEGACHARGER NETWORK PLANNED SITES FOR 2026 [images/image_page_21_figure_1.png]"
- },
- {
- "path": "images/image_page_22_figure_1.png",
- "summary": "GIGAFACTORY SHANGHAI – 9 MILLIONTH VEHICLE PRODUCED (GLOBALLY) [images/image_page_22_figure_1.png]"
- },
- {
- "path": "images/image_page_23_figure_1.png",
- "summary": "GIGAFACTORY NEVADA - 6 MILLIONTH DRIVE UNIT PRODUCED [images/image_page_23_figure_1.png]"
- },
- {
- "path": "images/image_page_24_figure_1.png",
- "summary": "KEY METRICS QUARTERLY [images/image_page_24_figure_1.png]"
- },
- {
- "path": "images/image_page_25_figure_1.png",
- "summary": "Vehicle Deliveries (millions of units) [images/image_page_25_figure_1.png]"
- },
- {
- "path": "images/image_page_25_figure_2.png",
- "summary": "Operating Cash Flow ($B) and Free Cash Flow ($B) [images/image_page_25_figure_2.png]"
- },
- {
- "path": "images/image_page_25_figure_3.png",
- "summary": "Net Income ($B) and Adjusted EBITDA ($B) [images/image_page_25_figure_3.png]"
- },
- {
- "path": "images/image_page_35_figure_1.png",
- "summary": "[images/image_page_35_figure_1.png]"
- }
- ],
- "tables": [
- {
- "path": "tables/table_page_2_1.html",
- "summary": "tables/table_page_2_1.html"
- },
- {
- "path": "tables/table_page_4_1.html",
- "summary": "tables/table_page_4_1.html"
- },
- {
- "path": "tables/table_page_5_1.html",
- "summary": "tables/table_page_5_1.html"
- },
- {
- "path": "tables/table_page_6_1.html",
- "summary": "tables/table_page_6_1.html"
- },
- {
- "path": "tables/table_page_7_1.html",
- "summary": "tables/table_page_7_1.html"
- },
- {
- "path": "tables/table_page_8_1.html",
- "summary": "tables/table_page_8_1.html"
- },
- {
- "path": "tables/table_page_11_2.html",
- "summary": "tables/table_page_11_2.html"
- },
- {
- "path": "tables/table_page_28_1.html",
- "summary": "tables/table_page_28_1.html"
- },
- {
- "path": "tables/table_page_29_1.html",
- "summary": "tables/table_page_29_1.html"
- },
- {
- "path": "tables/table_page_30_1.html",
- "summary": "tables/table_page_30_1.html"
- },
- {
- "path": "tables/table_page_31_1.html",
- "summary": "tables/table_page_31_1.html"
- },
- {
- "path": "tables/table_page_32_1.html",
- "summary": "tables/table_page_32_1.html"
- },
- {
- "path": "tables/table_page_33_1.html",
- "summary": "tables/table_page_33_1.html"
- },
- {
- "path": "tables/table_page_33_2.html",
- "summary": "tables/table_page_33_2.html"
- },
- {
- "path": "tables/table_page_33_3.html",
- "summary": "tables/table_page_33_3.html"
- },
- {
- "path": "tables/table_page_33_4.html",
- "summary": "tables/table_page_33_4.html"
- }
- ]
- }
-}
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_10_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_10_figure_1.png
deleted file mode 100644
index b326a7e2d..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_10_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_10_figure_2.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_10_figure_2.png
deleted file mode 100644
index 02fac07b6..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_10_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_11_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_11_figure_1.png
deleted file mode 100644
index f921e1fdd..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_11_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_15_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_15_figure_1.png
deleted file mode 100644
index b7c28f6c6..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_15_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_16_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_16_figure_1.png
deleted file mode 100644
index 05342236f..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_16_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_17_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_17_figure_1.png
deleted file mode 100644
index e00063225..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_17_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_18_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_18_figure_1.png
deleted file mode 100644
index a20aa71f6..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_18_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_19_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_19_figure_1.png
deleted file mode 100644
index beb7ca822..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_19_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_20_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_20_figure_1.png
deleted file mode 100644
index f554a6fa1..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_20_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_21_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_21_figure_1.png
deleted file mode 100644
index 9c0d99691..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_21_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_22_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_22_figure_1.png
deleted file mode 100644
index 0db28028b..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_22_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_23_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_23_figure_1.png
deleted file mode 100644
index 0e67978e5..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_23_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_24_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_24_figure_1.png
deleted file mode 100644
index 68c5ac030..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_24_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_1.png
deleted file mode 100644
index 347a74f48..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_2.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_2.png
deleted file mode 100644
index c8e7e6d09..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_3.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_3.png
deleted file mode 100644
index 7ea5362a6..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_25_figure_3.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_35_figure_1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_35_figure_1.png
deleted file mode 100644
index 12a6f8265..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_35_figure_1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_9_figure_2.png b/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_9_figure_2.png
deleted file mode 100644
index 841eb3842..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/images/image_page_9_figure_2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/manifest.json b/apps/api/app/data/demo_documents/tsla-q4-2025/manifest.json
deleted file mode 100644
index fce279b14..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/manifest.json
+++ /dev/null
@@ -1,338 +0,0 @@
-{
- "version": "2.0",
- "job_id": "job_424aa26c4f99",
- "data_id": null,
- "source_file_name": "TSLA-Q4-2025-Update.pdf",
- "processing_date": "2026-08-18T09:54:35.950352Z",
- "processing": {
- "page_count": 35,
- "billing_status": "charged",
- "cost": {
- "micro_dollars": 525000,
- "credits": 0.525
- },
- "cost_estimate": {
- "currency": "USD",
- "pricing_version": "2026-08-18:874ab8b6c9e4",
- "total_input_cost": 0.03063332,
- "total_output_cost": 0.00541008,
- "total_cost": 0.0360434,
- "by_model": {
- "qwen3.5-flash": {
- "input_tokens": 21291,
- "output_tokens": 200,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- },
- "deepseek-chat": {
- "input_tokens": 2238,
- "output_tokens": 561,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.00031332,
- "output_cost": 0.00015708,
- "total_cost": 0.0004704,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- },
- "qwen3.6-flash": {
- "input_tokens": 121280,
- "output_tokens": 3502,
- "calls": 41,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.03032,
- "output_cost": 0.005253,
- "total_cost": 0.035573,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "models": {
- "qwen3.5-flash": {
- "input_tokens": 21291,
- "output_tokens": 200,
- "calls": 1,
- "input_cost": 0.0,
- "output_cost": 0.0,
- "total_cost": 0.0,
- "pricing_missing": true
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "page_memory.hierarchy": {
- "input_cost": 0.00031332,
- "output_cost": 0.00015708,
- "total_cost": 0.0004704,
- "models": {
- "deepseek-chat": {
- "input_tokens": 2238,
- "output_tokens": 561,
- "calls": 1,
- "input_rate_per_1m": 0.14,
- "output_rate_per_1m": 0.28,
- "input_cost": 0.00031332,
- "output_cost": 0.00015708,
- "total_cost": 0.0004704,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.14,
- "output_per_1m": 0.28,
- "source": "DeepSeek official pricing",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- },
- "page_memory.asset_detect": {
- "input_cost": 0.02863875,
- "output_cost": 0.0046335,
- "total_cost": 0.03327225,
- "models": {
- "qwen3.6-flash": {
- "input_tokens": 114555,
- "output_tokens": 3089,
- "calls": 35,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.02863875,
- "output_cost": 0.0046335,
- "total_cost": 0.03327225,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- },
- "page_memory.table_continuity": {
- "input_cost": 0.00168125,
- "output_cost": 0.0006195,
- "total_cost": 0.00230075,
- "models": {
- "qwen3.6-flash": {
- "input_tokens": 6725,
- "output_tokens": 413,
- "calls": 6,
- "input_rate_per_1m": 0.25,
- "output_rate_per_1m": 1.5,
- "input_cost": 0.00168125,
- "output_cost": 0.0006195,
- "total_cost": 0.00230075,
- "pricing": {
- "currency": "USD",
- "unit": "per_1m_tokens",
- "input_per_1m": 0.25,
- "output_per_1m": 1.5,
- "source": "Qwen Cloud official pricing, <=256K input tier",
- "effective_date": "2026-06-11"
- }
- }
- },
- "missing_pricing_models": []
- }
- },
- "missing_pricing_models": [
- "qwen3.5-flash"
- ]
- },
- "timing": {
- "started_at": "2026-08-18T09:47:33.118517+00:00",
- "completed_at": "2026-08-18T09:54:35.946390+00:00",
- "duration_ms": 422827
- },
- "stages": {
- "timing_ms": {
- "page_memory.normalize_to_pdf": 0,
- "page_memory.profile": 24090,
- "page_memory.skeleton": 0,
- "page_memory.title_render": 8467,
- "page_memory.title_detection": 61266,
- "page_memory.fine_hierarchy": 3800,
- "page_memory.render": 9305,
- "page_memory.tag": 48637,
- "page_memory.assets": 217531,
- "page_memory.node_summary": 49523,
- "page_memory.node_rows": 51,
- "page_memory.node_assembly": 49577,
- "worker.parse.document": 422703
- },
- "token_usage": {
- "prompt_tokens": 144809,
- "completion_tokens": 4263,
- "total_tokens": 149072,
- "calls": 43,
- "by_model": {
- "qwen3.5-flash": {
- "prompt_tokens": 21291,
- "completion_tokens": 200,
- "total_tokens": 21491,
- "calls": 1
- },
- "deepseek-chat": {
- "prompt_tokens": 2238,
- "completion_tokens": 561,
- "total_tokens": 2799,
- "calls": 1
- },
- "qwen3.6-flash": {
- "prompt_tokens": 121280,
- "completion_tokens": 3502,
- "total_tokens": 124782,
- "calls": 41
- }
- },
- "by_task": {
- "document_agent.coarse_profile": {
- "prompt_tokens": 21291,
- "completion_tokens": 200,
- "total_tokens": 21491,
- "calls": 1,
- "models": {
- "qwen3.5-flash": {
- "prompt_tokens": 21291,
- "completion_tokens": 200,
- "total_tokens": 21491,
- "calls": 1
- }
- }
- },
- "page_memory.hierarchy": {
- "prompt_tokens": 2238,
- "completion_tokens": 561,
- "total_tokens": 2799,
- "calls": 1,
- "models": {
- "deepseek-chat": {
- "prompt_tokens": 2238,
- "completion_tokens": 561,
- "total_tokens": 2799,
- "calls": 1
- }
- }
- },
- "page_memory.asset_detect": {
- "prompt_tokens": 114555,
- "completion_tokens": 3089,
- "total_tokens": 117644,
- "calls": 35,
- "models": {
- "qwen3.6-flash": {
- "prompt_tokens": 114555,
- "completion_tokens": 3089,
- "total_tokens": 117644,
- "calls": 35
- }
- }
- },
- "page_memory.table_continuity": {
- "prompt_tokens": 6725,
- "completion_tokens": 413,
- "total_tokens": 7138,
- "calls": 6,
- "models": {
- "qwen3.6-flash": {
- "prompt_tokens": 6725,
- "completion_tokens": 413,
- "total_tokens": 7138,
- "calls": 6
- }
- }
- }
- }
- }
- }
- },
- "statistics": {
- "total_chunks": 71,
- "text_chunks": 0,
- "image_chunks": 18,
- "table_chunks": 16,
- "page_chunks": 37,
- "total_pages": null
- },
- "HIERARCHY": {
- "Root": {
- "Q4 and FY 2025 Update": {},
- "HIGHLIGHTS": {},
- "SUMMARY": {},
- "FINANCIAL SUMMARY": {},
- "OPERATIONAL SUMMARY": {},
- "MANUFACTURING & HARDWARE": {
- "Automotive": {},
- "Energy generation and storage": {},
- "Robotics": {}
- },
- "SUPPORTING INFRASTRUCTURE": {
- "AI Training Compute": {},
- "Battery": {},
- "Other Supporting Infrastructure": {},
- "AI Software": {},
- "AI Inference Compute": {},
- "Automotive and Other Software": {}
- },
- "SERVICES": {
- "Robotaxi": {},
- "FSD (Supervised)": {},
- "Automotive Services": {}
- },
- "OTHER UPDATES": {},
- "OUTLOOK": {
- "Volume": {},
- "Cash": {},
- "Profit": {},
- "Product": {}
- },
- "PHOTOS & CHARTS": {},
- "TESLA SEMI - MEGACHARGER NETWORK PLANNED SITES FOR 2026": {},
- "KEY METRICS QUARTERLY": {},
- "KEY METRICS TRAILING 12 MONTHS (TTM)": {},
- "KEY METRICS YEAR-OVER-YEAR FINANCIAL SUMMARY": {
- "Revenue": {},
- "Profitability": {}
- },
- "FINANCIAL STATEMENTS": {},
- "STATEMENT OF OPERATIONS": {},
- "BALANCE SHEET": {},
- "STATEMENT OF CASH FLOWS": {},
- "RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL INFORMATION": {},
- "ADDITIONAL INFORMATION": {
- "WEBCAST INFORMATION": {},
- "CERTAIN TERMS": {},
- "NON-GAAP FINANCIAL INFORMATION": {},
- "FORWARD-LOOKING STATEMENTS": {}
- }
- }
- }
-}
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/original.pdf b/apps/api/app/data/demo_documents/tsla-q4-2025/original.pdf
deleted file mode 100644
index 87cdb1be5..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/original.pdf and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-1.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-1.png
deleted file mode 100644
index dbedc79a8..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-1.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-10.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-10.png
deleted file mode 100644
index f79576332..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-10.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-11.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-11.png
deleted file mode 100644
index 425ccb514..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-11.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-12.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-12.png
deleted file mode 100644
index acb2a8829..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-12.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-13.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-13.png
deleted file mode 100644
index 729764438..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-13.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-14.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-14.png
deleted file mode 100644
index 20df0747f..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-14.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-15.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-15.png
deleted file mode 100644
index 32c2ff15c..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-15.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-16.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-16.png
deleted file mode 100644
index 24769d6ed..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-16.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-17.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-17.png
deleted file mode 100644
index 390ae68f9..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-17.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-18.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-18.png
deleted file mode 100644
index ad3017cf7..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-18.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-19.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-19.png
deleted file mode 100644
index f77b7c4a8..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-19.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-2.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-2.png
deleted file mode 100644
index 1c4b7b20a..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-2.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-20.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-20.png
deleted file mode 100644
index 85ba16ef2..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-20.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-21.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-21.png
deleted file mode 100644
index d914873b5..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-21.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-22.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-22.png
deleted file mode 100644
index 1eb2800c4..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-22.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-23.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-23.png
deleted file mode 100644
index 7f47336ed..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-23.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-24.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-24.png
deleted file mode 100644
index 19bb5be1c..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-24.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-25.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-25.png
deleted file mode 100644
index bc887a89f..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-25.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-26.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-26.png
deleted file mode 100644
index ed493ef37..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-26.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-27.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-27.png
deleted file mode 100644
index fec021612..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-27.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-28.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-28.png
deleted file mode 100644
index ebbf2f8cd..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-28.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-29.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-29.png
deleted file mode 100644
index 1e2d426ff..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-29.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-3.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-3.png
deleted file mode 100644
index c5e5e3443..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-3.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-30.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-30.png
deleted file mode 100644
index 1ce7eb561..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-30.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-31.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-31.png
deleted file mode 100644
index 83f68778e..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-31.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-32.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-32.png
deleted file mode 100644
index a240148a8..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-32.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-33.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-33.png
deleted file mode 100644
index 11db16bd3..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-33.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-34.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-34.png
deleted file mode 100644
index 91a4ee0f6..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-34.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-35.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-35.png
deleted file mode 100644
index 4215ed44b..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-35.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-4.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-4.png
deleted file mode 100644
index e712453f2..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-4.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-5.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-5.png
deleted file mode 100644
index 2d1419fde..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-5.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-6.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-6.png
deleted file mode 100644
index 047f42354..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-6.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-7.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-7.png
deleted file mode 100644
index ef5d980bb..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-7.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-8.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-8.png
deleted file mode 100644
index 89a0ce1f8..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-8.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-9.png b/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-9.png
deleted file mode 100644
index da32e3daf..000000000
Binary files a/apps/api/app/data/demo_documents/tsla-q4-2025/page_citation_assets/page-9.png and /dev/null differ
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_11_2.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_11_2.html
deleted file mode 100644
index 35101ba3a..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_11_2.html
+++ /dev/null
@@ -1,54 +0,0 @@
-
-
-
-
State
-
Metro
-
Status
-
-
-
California
-
SF Bay Area
-
Safety Driver
-
-
-
Texas
-
Austin
-
Ramping Unsupervised
-
-
-
-
Dallas
-
1H 2026
-
-
-
-
Houston
-
1H 2026
-
-
-
Arizona
-
Phoenix
-
1H 2026
-
-
-
Florida
-
Miami
-
1H 2026
-
-
-
-
Orlando
-
1H 2026
-
-
-
-
Tampa
-
1H 2026
-
-
-
Nevada
-
Las Vegas
-
1H 2026
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_28_1.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_28_1.html
deleted file mode 100644
index 9737feb32..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_28_1.html
+++ /dev/null
@@ -1,511 +0,0 @@
-
-
-
-
In millions of USD or shares as applicable, except per share data
-
-
Q4-2024
-
-
Q1-2025
-
-
Q2-2025
-
-
Q3-2025
-
-
Q4-2025
-
-
-
REVENUES
-
-
-
-
-
-
-
-
-
-
-
-
-
Automotive sales
-
-
18,659
-
-
12,925
-
-
15,787
-
-
20,359
-
-
16,750
-
-
-
Automotive regulatory credits
-
-
692
-
-
595
-
-
439
-
-
417
-
-
542
-
-
-
Automotive leasing
-
-
447
-
-
447
-
-
435
-
-
429
-
-
401
-
-
-
Total automotive revenues
-
-
19,798
-
-
13,967
-
-
16,661
-
-
21,205
-
-
17,693
-
-
-
Energy generation and storage
-
-
3,061
-
-
2,730
-
-
2,789
-
-
3,415
-
-
3,837
-
-
-
Services and other
-
-
2,848
-
-
2,638
-
-
3,046
-
-
3,475
-
-
3,371
-
-
-
Total revenues
-
-
25,707
-
-
19,335
-
-
22,496
-
-
28,095
-
-
24,901
-
-
-
COST OF REVENUES
-
-
-
-
-
-
-
-
-
-
-
-
-
Automotive sales
-
-
16,268
-
-
11,461
-
-
13,567
-
-
17,365
-
-
13,874
-
-
-
Automotive leasing
-
-
242
-
-
239
-
-
228
-
-
225
-
-
206
-
-
-
Total automotive cost of revenues
-
-
16,510
-
-
11,700
-
-
13,795
-
-
17,590
-
-
14,080
-
-
-
Energy generation and storage
-
-
2,289
-
-
1,945
-
-
1,943
-
-
2,342
-
-
2,739
-
-
-
Services and other
-
-
2,729
-
-
2,537
-
-
2,880
-
-
3,109
-
-
3,073
-
-
-
Total cost of revenues
-
-
21,528
-
-
16,182
-
-
18,618
-
-
23,041
-
-
19,892
-
-
-
Gross profit
-
-
4,179
-
-
3,153
-
-
3,878
-
-
5,054
-
-
5,009
-
-
-
OPERATING EXPENSES
-
-
-
-
-
-
-
-
-
-
-
-
-
Research and development
-
-
1,276
-
-
1,409
-
-
1,589
-
-
1,630
-
-
1,783
-
-
-
Selling, general and administrative
-
-
1,313
-
-
1,251
-
-
1,366
-
-
1,562
-
-
1,655
-
-
-
Restructuring and other
-
-
7
-
-
94
-
-
—
-
-
238
-
-
162
-
-
-
Total operating expenses
-
-
2,596
-
-
2,754
-
-
2,955
-
-
3,430
-
-
3,600
-
-
-
INCOME FROM OPERATIONS
-
-
1,583
-
-
399
-
-
923
-
-
1,624
-
-
1,409
-
-
-
Interest income
-
-
442
-
-
400
-
-
392
-
-
439
-
-
449
-
-
-
Interest expense
-
-
(96)
-
-
(91)
-
-
(86)
-
-
(76)
-
-
(85)
-
-
-
Other income (expense), net (1)
-
-
595
-
-
(119)
-
-
320
-
-
(28)
-
-
(592)
-
-
-
INCOME BEFORE INCOME TAXES (1)
-
-
2,524
-
-
589
-
-
1,549
-
-
1,959
-
-
1,181
-
-
-
Provision for income taxes (1)
-
-
381
-
-
169
-
-
359
-
-
570
-
-
325
-
-
-
NET INCOME (1)
-
-
2,143
-
-
420
-
-
1,190
-
-
1,389
-
-
856
-
-
-
Net income attributable to noncontrolling interests and redeemable noncontrolling interests in subsidiaries
-
-
15
-
-
11
-
-
18
-
-
16
-
-
16
-
-
-
NET INCOME ATTRIBUTABLE TO COMMON STOCKHOLDERS (1)
-
-
2,128
-
-
409
-
-
1,172
-
-
1,373
-
-
840
-
-
-
Less: Buy-out of noncontrolling interest
-
-
3
-
-
—
-
-
—
-
-
—
-
-
—
-
-
-
NET INCOME USED IN COMPUTING NET INCOME PER SHARE OF COMMON STOCK (1)
-
-
2,125
-
-
409
-
-
1,172
-
-
1,373
-
-
840
-
-
-
Net income per share of common stock attributable to common stockholders
-
-
-
-
-
-
-
-
-
-
-
-
-
Basic (1)
-
$
-
0.66
-
$
-
0.13
-
$
-
0.36
-
$
-
0.43
-
$
-
0.26
-
-
-
Diluted (1)
-
$
-
0.60
-
$
-
0.12
-
$
-
0.33
-
$
-
0.39
-
$
-
0.24
-
-
-
Weighted average shares used in computing net income per share of common stock
-
-
-
-
-
-
-
-
-
-
-
-
-
Basic
-
-
3,213
-
-
3,218
-
-
3,223
-
-
3,227
-
-
3,231
-
-
-
Diluted
-
-
3,517
-
-
3,521
-
-
3,519
-
-
3,526
-
-
3,539
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_29_1.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_29_1.html
deleted file mode 100644
index c8fd1140e..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_29_1.html
+++ /dev/null
@@ -1,292 +0,0 @@
-
-
-
-
In millions of USD
-
31-Dec-24
-
31-Mar-25
-
30-Jun-25
-
30-Sep-25
-
31-Dec-25
-
-
-
ASSETS
-
-
-
-
-
-
-
-
Current assets
-
-
-
-
-
-
-
-
Cash, cash equivalents and investments
-
36,563
-
36,996
-
36,782
-
41,647
-
44,059
-
-
-
Accounts receivable, net
-
4,418
-
3,782
-
3,838
-
4,703
-
4,576
-
-
-
Inventory
-
12,017
-
13,706
-
14,570
-
12,276
-
12,392
-
-
-
Prepaid expenses and other current assets
-
5,362
-
4,905
-
5,943
-
6,027
-
7,615
-
-
-
Total current assets
-
58,360
-
59,389
-
61,133
-
64,653
-
68,642
-
-
-
Operating lease vehicles, net
-
5,581
-
5,477
-
5,230
-
5,019
-
4,912
-
-
-
Energy generation and storage systems, net
-
4,924
-
4,855
-
4,788
-
4,673
-
4,604
-
-
-
Property, plant and equipment, net
-
35,836
-
37,088
-
38,574
-
39,407
-
40,643
-
-
-
Operating lease right-of-use assets
-
5,160
-
5,330
-
5,633
-
5,783
-
6,027
-
-
-
Digital assets (2)
-
1,076
-
951
-
1,235
-
1,315
-
1,008
-
-
-
Deferred tax assets (2)
-
6,524
-
6,687
-
6,721
-
6,637
-
6,925
-
-
-
Other non-current assets
-
4,609
-
5,334
-
5,253
-
6,248
-
5,045
-
-
-
Total assets (2)
-
122,070
-
125,111
-
128,567
-
133,735
-
137,806
-
-
-
LIABILITIES AND EQUITY
-
-
-
-
-
-
-
-
Current liabilities
-
-
-
-
-
-
-
-
Accounts payable
-
12,474
-
13,471
-
13,212
-
12,819
-
13,371
-
-
-
Accrued liabilities and other
-
10,723
-
10,802
-
11,519
-
12,791
-
13,279
-
-
-
Deferred revenue
-
3,168
-
3,243
-
3,237
-
3,756
-
3,424
-
-
-
Current portion of debt and finance leases (1)
-
2,456
-
2,237
-
2,040
-
1,924
-
1,640
-
-
-
Total current liabilities
-
28,821
-
29,753
-
30,008
-
31,290
-
31,714
-
-
-
Debt and finance leases, net of current portion (1)
-
5,757
-
5,292
-
5,180
-
5,778
-
6,736
-
-
-
Deferred revenue, net of current portion
-
3,317
-
3,610
-
3,764
-
3,746
-
3,631
-
-
-
Other long-term liabilities
-
10,495
-
11,038
-
11,543
-
12,205
-
12,860
-
-
-
Total liabilities
-
48,390
-
49,693
-
50,495
-
53,019
-
54,941
-
-
-
Redeemable noncontrolling interests in subsidiaries
-
63
-
62
-
61
-
59
-
58
-
-
-
Total stockholders' equity (2)
-
72,913
-
74,653
-
77,314
-
79,970
-
82,137
-
-
-
Noncontrolling interests in subsidiaries
-
704
-
703
-
697
-
687
-
670
-
-
-
Total liabilities and equity (2)
-
122,070
-
125,111
-
128,567
-
133,735
-
137,806
-
-
-
(1) Breakdown of our debt is as follows:
-
-
-
-
-
-
-
-
Non-recourse debt
-
7,871
-
7,238
-
6,953
-
7,458
-
8,150
-
-
-
Recourse debt
-
7
-
6
-
3
-
3
-
3
-
-
-
Days sales outstanding
-
14
-
19
-
15
-
14
-
17
-
-
-
Days payable outstanding
-
58
-
72
-
65
-
52
-
61
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_2_1.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_2_1.html
deleted file mode 100644
index 29c7b9590..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_2_1.html
+++ /dev/null
@@ -1,56 +0,0 @@
-
-
-
-
Highlights
-
3
-
-
-
Financial Summary
-
4
-
-
-
Operational Summary
-
6
-
-
-
Manufacturing & Hardware
-
8
-
-
-
Supporting Infrastructure
-
9
-
-
-
AI & Software
-
10
-
-
-
Services
-
11
-
-
-
Other Updates
-
12
-
-
-
Outlook
-
13
-
-
-
Photos & Charts
-
14
-
-
-
Key Metrics
-
24
-
-
-
Financial Statements
-
27
-
-
-
Additional Information
-
34
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_30_1.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_30_1.html
deleted file mode 100644
index ac4681863..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_30_1.html
+++ /dev/null
@@ -1,212 +0,0 @@
-
-
-
-
In millions of USD
-
Q4-2024
-
Q1-2025
-
Q2-2025
-
Q3-2025
-
Q4-2025
-
-
-
CASH FLOWS FROM OPERATING ACTIVITIES
-
-
-
-
-
-
-
-
Net income (1)
-
2,143
-
420
-
1,190
-
1,389
-
856
-
-
-
Adjustments to reconcile net income to net cash provided by operating activities:
-
-
-
-
-
-
-
-
Depreciation, amortization and impairment
-
1,496
-
1,447
-
1,433
-
1,625
-
1,643
-
-
-
Stock-based compensation
-
579
-
573
-
635
-
663
-
954
-
-
-
Deferred income taxes (1)
-
6
-
(43)
-
52
-
225
-
(111)
-
-
-
Digital assets (gain) loss, net (1)
-
(347)
-
125
-
(284)
-
(80)
-
307
-
-
-
Other
-
(93)
-
188
-
187
-
333
-
378
-
-
-
Changes in operating assets and liabilities
-
1030
-
(554)
-
(673)
-
2,083
-
(214)
-
-
-
Net cash provided by operating activities
-
4,814
-
2,156
-
2,540
-
6,238
-
3,813
-
-
-
CASH FLOWS FROM INVESTING ACTIVITIES
-
-
-
-
-
-
-
-
Capital expenditures (2)
-
(2,780)
-
(1,492)
-
(2,394)
-
(2,248)
-
(2,393)
-
-
-
Purchases of investments
-
(15,158)
-
(6,015)
-
(7,485)
-
(11,402)
-
(12,207)
-
-
-
Proceeds from maturities of investments
-
10,335
-
5,856
-
6,935
-
9,295
-
8,072
-
-
-
Net cash used in investing activities
-
(7,603)
-
(1,651)
-
(2,944)
-
(4,355)
-
(6,528)
-
-
-
CASH FLOWS FROM FINANCING ACTIVITIES
-
-
-
-
-
-
-
-
Net cash flows from other debt activities
-
(108)
-
(50)
-
(23)
-
410
-
963
-
-
-
Net borrowings (repayments) under vehicle and energy product financing
-
677
-
(674)
-
(400)
-
81
-
(377)
-
-
-
Net cash flows from noncontrolling interests – Solar
-
(37)
-
(22)
-
(14)
-
(20)
-
(22)
-
-
-
Other
-
453
-
414
-
215
-
512
-
146
-
-
-
Net cash provided by (used in) financing activities
-
985
-
(332)
-
(222)
-
983
-
710
-
-
-
Effect of exchange rate changes on cash and cash equivalents and restricted cash
-
(133)
-
40
-
111
-
(17)
-
37
-
-
-
Net (decrease) increase in cash and cash equivalents and restricted cash
-
(1,937)
-
213
-
(515)
-
2,849
-
(1,968)
-
-
-
Cash and cash equivalents and restricted cash at beginning of period
-
18,974
-
17,037
-
17,250
-
16,735
-
19,584
-
-
-
Cash and cash equivalents and restricted cash at end of period
-
17,037
-
17,250
-
16,735
-
19,584
-
17,616
-
-
-
\ No newline at end of file
diff --git a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_31_1.html b/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_31_1.html
deleted file mode 100644
index 4ac566467..000000000
--- a/apps/api/app/data/demo_documents/tsla-q4-2025/tables/table_page_31_1.html
+++ /dev/null
@@ -1,196 +0,0 @@
-
-
-
-
In millions of USD or shares as applicable, except per share data
-
Q4-2024
-
Q1-2025
-
Q2-2025
-
Q3-2025
-
Q4-2025
-
-
-
Net income attributable to common stockholders (GAAP) (1)
-
2,128
-
409
-
1,172
-
1,373
-
840
-
-
-
Stock-based compensation expense, net of tax
-
249
-
428
-
443
-
459
-
682
-
-
-
Digital assets (gain) loss, net of tax (1)
-
(270)
-
97
-
(222)
-
(62)
-
239
-
-
-
Net income attributable to common stockholders (non-GAAP) (1) (2)
-
2,107
-
934
-
1,393
-
1,770
-
1,761
-
-
-
Less: Buy-outs of noncontrolling interests
-
3
-
—
-
—
-
—
-
—
-
-
-
Net income used in computing diluted EPS attributable to common stockholders (non-GAAP) (1) (2)
-
2,104
-
934
-
1,393
-
1,770
-
1,761
-
-
-
EPS attributable to common stockholders, diluted (GAAP) (1)
-
0.60
-
0.12
-
0.33
-
0.39
-
0.24
-
-
-
Stock-based compensation expense, net of tax, per share
-
0.08
-
0.12
-
0.13
-
0.13
-
0.19
-
-
-
Digital assets (gain) loss, net of tax, per share (1)
-
(0.08)
-
0.03
-
(0.06)
-
(0.02)
-
0.07
-
-
-
EPS attributable to common stockholders, diluted (non-GAAP) (1) (2)
-
0.60
-
0.27
-
0.40
-
0.50
-
0.50
-
-
-
Shares used in EPS calculation, diluted (GAAP and non-GAAP)
-
3,517
-
3,521
-
3,519
-
3,526
-
3,539
-
-
-
Net income attributable to common stockholders (GAAP) (1)
-
-
-
-
diff --git a/docs/shared-demo-rollout.md b/docs/shared-demo-rollout.md
index 6985ddca9..97a6cbfa8 100644
--- a/docs/shared-demo-rollout.md
+++ b/docs/shared-demo-rollout.md
@@ -37,6 +37,48 @@ archive. API job deletion requests for these revisions return 409. This checkout
has no general job deletion/cancellation API; ordinary jobs remain retained
processing records, and DELETE remains unsupported for them.
+## Public revision structure
+
+`GET /api/v1/demo/sources/{demo_source_id}/structure` returns the complete
+published section outline without loading chunk bodies or result archives.
+It requires no authentication and follows the same active-source and complete
+revision checks as the other public demo reads.
+
+Pass `job_result_id` to pin the outline to the same revision as subsequent chunk
+and asset requests. Omitting it selects the current complete revision. Responses
+use `Cache-Control: no-store`.
+
+```json
+{
+ "demo_source_id": "example-source",
+ "canonical_document_id": "ddoc_example",
+ "job_result_id": "published-revision-id",
+ "sections": [
+ {
+ "section_id": "sec_parent",
+ "parent_section_id": null,
+ "section_path": "Operations",
+ "section_title": "Operations",
+ "section_level": 1,
+ "sort_order": 0
+ }
+ ]
+}
+```
+
+Sections follow persisted `sort_order`, with `section_id` as a deterministic tie
+breaker. Build the tree from `parent_section_id`; do not infer parentage from
+title splitting or alphabetically reorder siblings. Structural parents are
+included even when they own no chunk. This is the published section tree;
+TOC-only entries never persisted during parsing/publication are not restored by
+this read endpoint. Unknown sources or unavailable pinned revisions return 404;
+known sources without a complete revision retain the existing preparing error.
+
+The HTTP/PostgreSQL shared-demo contract suite passes all ten tests, including
+revision-pinned structure reads, structural parents, persisted sibling order,
+preparing/error behavior, and guards against chunk or artifact downloads.
+Ruff and Pyright pass for the changed API modules.
+
## Backup gate
Use the authoritative originals in `knowhere-storage-staging`, not public URLs.
diff --git a/docs/source-content-deduplication-review.md b/docs/source-content-deduplication-review.md
new file mode 100644
index 000000000..9aff23b4b
--- /dev/null
+++ b/docs/source-content-deduplication-review.md
@@ -0,0 +1,58 @@
+# Source content deduplication review and ablations
+
+Reviewed on 2026-10-10 against `5264697481567b76abf8abcf463573018847701c`, including the uncommitted implementation and new files. The specification is the agreed conversation: compare original bytes within one user and namespace, reject unchanged updates, use current active revisions and in-progress reservations, and perform no historical backfill.
+
+## Standards
+
+The review found one concrete migration defect: a valid, ready index with the expected name but incorrect columns or predicate was retained. Two real PostgreSQL contracts reproduced the defect under ordinary and caller-owned transactions. The migration now validates the table, nonunique B-tree definition, column order and expression, predicate, validity, and readiness before retaining an index. Twelve cases pass, including retention of the correct index without rebuilding it and replacement of indexes whose JSON keys differ in case or contain a cast-like suffix. Definition comparison preserves JSON string literals rather than normalizing their case or stripping cast-like text from them.
+
+No remaining blocker was found. Optional improvements include a direct interrupted-index recovery contract, concurrent index removal during ordinary downgrade, and a precise fingerprint return type. Canonical index-definition formatting is verified on the local PostgreSQL version; a future formatting change could cause an unnecessary rebuild.
+
+## Spec
+
+No blocking behavior mismatch was found. Publication and the terminal job transition commit together, accepted fingerprints survive ordinary metadata updates, and completed task redelivery exits before admission.
+
+The review identified a lost export contract. It is restored for two same-named XLSX files differing only in ZIP comments: both succeed, retain full document and ZIP chunks, and produce identical parsed chunk identifiers. The duplicate worker contract now enables billing and verifies that the duplicate leaves account balance unchanged, has no result, and creates an `ALREADY_EXISTS` webhook outbox event and dispatch request. The dispatch is captured locally; external webhook delivery is not exercised.
+
+Exact byte equality remains the agreed boundary. Re-exported files with changed container metadata can pass even when their visible content is identical. An unchanged source is also rejected when only parsing options change. Existing unhashed sources remain outside admission.
+
+## Component ablations
+
+Every variant ran the same 14 admission contracts against an isolated real PostgreSQL fixture and local source files. Mutations were test-scoped patches in separate processes; production code was not replaced with ablated implementations. All runs below had zero setup errors. Failures in the altered variants are intentional evidence that the contracts detect the removed behavior.
+
+| Variant | Passed | Failed | Observed failure |
+| --- | ---: | ---: | --- |
+| Unchanged implementation | 14 | 0 | Agreed behavior holds |
+| Use file size instead of hashing bytes | 12 | 2 | Different bytes of equal length are falsely rejected |
+| Omit fingerprint persistence | 5 | 9 | Later duplicates are accepted and durable fingerprints are unavailable |
+| Remove advisory transaction lock | 13 | 1 | Both concurrent claims are admitted |
+| Check published documents only | 12 | 2 | Unpublished reservations and the publication transition are missed |
+| Split the union into two separate reads | 13 | 1 | Publication between the snapshots allows a duplicate |
+
+The concurrency contract pauses the first transaction after its empty lookup and starts the second claim before releasing it. With the lock present, PostgreSQL reports the second transaction waiting for its advisory lock. Without the lock, the second claim can finish against the empty state. The publication contract commits publication after the first lookup executes, exposing the gap in separate statement snapshots.
+
+## Index ablation and skew experiment
+
+The experiment used the actual ORM union query and the ordinary runtime database role, with planner defaults enabled. The first dataset contained 5,000 active current document revisions with distinct hashes and one incoming job. The last dataset additionally contained 2,000 running jobs with the requested hash in other namespaces. Every query correctly returned no duplicate in the incoming namespace.
+
+| Local workload | SQL execution time | Shared buffer hits | Query-plan evidence |
+| --- | ---: | ---: | --- |
+| Hash index, absent hash | 0.093 ms | 6 | Published branch uses user/hash index; no revision joins execute |
+| Same dataset, hash index removed | 37.883 ms | 30,210 | Visits 5,000 documents and performs 5,000 result/job lookups |
+| Hash index, same hash in 2,000 other namespaces | 6.948 ms | 4,177 | Published branch probes 2,000 candidate results; processing branch filters 2,001 jobs |
+
+These are single local `EXPLAIN (ANALYZE, BUFFERS)` probes with zero shared disk reads, not production latency measurements or a five-million-document benchmark. SQL execution time excludes download, hashing, pool acquisition, network round trips, and lock waits. Buffer counts and visited rows provide the stronger evidence.
+
+The index is necessary, but matching hash multiplicity still matters because namespace and lifecycle state are later filters. The earlier 5–20 ms planning estimate assumes few matching user/hash rows and is not a worst-case guarantee. We keep the current design for this change; a workload dominated by repeated content across namespaces would justify revisiting index scope and the published-candidate query. Terminal and archived history were not measured by this skew experiment.
+
+## Reproduction and validation
+
+Local experiment artifacts are in `/tmp/knowhere-source-content-ablation/`: `source_content_ablation.py`, `run_matrix.py`, per-variant logs and JUnit reports, `results.json`, `test_query_plan_experiment.py`, and the complete `query-plans.json`.
+
+The matrix can be repeated with `python3 /tmp/knowhere-source-content-ablation/run_matrix.py`; it uses the existing uv environment, applies one mutation per process, and runs the admission contract file. Local PostgreSQL sockets must be permitted. The index experiment loads `contract.conftest` as a pytest plugin, executes its temporary experiment file, and drops/recreates the hash index only in the isolated fixture database.
+
+The strengthened worker suite passes all 33 contracts, and the API and migration suite passes all 61 contracts. The final expanded index migration check passes all 12 cases; this check overlaps the broader migration suite and is not an additional 12 distinct contracts. Ruff, Pyright (production paths, changed API contracts, and the new migration), and `git diff --check` pass. URL transport and real external webhook delivery remain outside these local contracts.
+
+Before PR publication, the change was rebased onto `main` at `65c0445a` and revalidated: all 33 worker contracts and all 65 API/migration contracts pass, including the expanded migration cases. Ruff, Pyright, and the diff whitespace check also pass on that base.
+
+Standards: one confirmed defect, fixed; optional operational and typing suggestions remain. Spec: no blocking mismatch; export, billing, webhook, and concurrency coverage strengthened.
diff --git a/docs/spacex-hierarchy-fix.md b/docs/spacex-hierarchy-fix.md
index 62eca2ad0..3ba737dbe 100644
--- a/docs/spacex-hierarchy-fix.md
+++ b/docs/spacex-hierarchy-fix.md
@@ -50,3 +50,86 @@ reproduced the loop before this change and finishes in four turns afterward.
Both real reader launch-comparison queries finished with precise page ranges.
See [the validation report](shared-demo-validation-20261003.md) for timings and
the exact verification boundary.
+
+## Preserve the document prefix before anchored chapters
+
+A separate production result omitted the pages before the first anchored
+chapter: Meta's nine-page release contained only pages 5–9, and SpaceX's
+407-page filing contained only pages 20–407. Resolving a TOC correctly did not
+guarantee complete document coverage: only the resolved section ranges were
+sent to rendering, tagging, and node assembly.
+
+The document prefix is every physical page before the first recognized chapter,
+including printed TOC pages. It belongs directly to the existing document root.
+Its chunk path is the filename itself; no `filename/Root` chapter is inserted,
+and existing chapter paths and ranges stay unchanged. Publication already maps
+a filename-only chunk to its canonical `Root` section. Navigation retains its
+existing root-body projection at the filename path.
+
+The prefix has explicit `document_prefix` evidence so an isolated prefix scope
+does not expand into the full-document Root fallback. It receives ordinary
+rendering, page tagging, and full-page citation assets, but does not undergo
+chapter-title refinement or single-child collapse. Prefix TOCs are retained
+through production and staged replay; TOC policy inside chapters is unchanged.
+The source evidence and level zero survive hierarchy artifact round trips.
+
+This repair covers opening pages only. It does not fill arbitrary middle or
+trailing gaps, regenerate existing production results, or change no-TOC
+full-document fallback behavior. The historical full-provider validation above
+is separate from the prefix-repair validation.
+
+### Prefix-repair verification on 2026-10-22
+
+Validation ran in an isolated Linux ARM64 environment, based on `main` commit
+`2ed1889626f63240bd0894ec0c24b7a7c5217dfb`.
+
+The new output contract failed for both missing-prefix examples before the
+repair; the first-page chapter and no-TOC fallback controls already passed.
+A separate staged-artifact contract reproduced the loss of level zero on
+reload before its repair. The final focused run passed **102 contracts**:
+
+```sh
+.venv/bin/pytest apps/worker/tests/contract/test_page_memory* \
+ apps/worker/tests/contract/test_profile_skeleton_anchor_wiring_contract.py -q
+```
+
+The full Worker contract suite also passed **405 contracts**, including the
+database contracts that reload application modules between environments:
+
+```sh
+.venv/bin/pytest apps/worker/tests/contract -q
+```
+
+This includes direct-root text and full-page citation assets, TOC retention in
+the prefix, unchanged chapter TOC policy, first-page chapters, no-TOC fallback,
+filename-only publication/navigation projection, and staged artifact reload.
+Ruff passed on the page-memory modules, changed stage scripts, and contracts;
+Pyright passed on the five changed application modules with zero errors and
+warnings.
+
+Captured production anatomy was also replayed against the actual originals:
+
+| Document | Real root-body output | Citation assets | Coverage evidence |
+| --- | --- | --- | --- |
+| Meta Q1 2026 Exhibit 99.1 | Filename path, physical pages 1–4, 11,265 text characters | Four real full-page PNGs | All nine pages assembled; five existing chapter paths retained |
+| SpaceX S-1 | Filename path, physical pages 1–19, 24,086 text characters; printed TOC page 17 retained | Nineteen real full-page PNGs | New prefix plus 683 unchanged original body chunks covers all 407 pages |
+
+The replay used real PDF rasterization, node assembly, and ZIP chunk-schema
+conversion. It reused captured chapter anatomy and substituted page-tag
+responses; it did not call model providers or publish a document revision.
+SpaceX rendered only the missing prefix and reused the original body results,
+rather than rerunning all 407 pages. Thus this is captured-profile seam
+verification, not a fresh full-provider parse or production deployment.
+
+### Existing demo revisions after deployment
+
+Deploying the Worker repair does not regenerate existing parsed revisions.
+After confirming the deployed Worker includes this change, reparse the known
+affected Meta release and SpaceX filing through the maintainer demo ingestion
+flow. Audit the remaining READY demos for missing opening pages before
+selecting additional sources. The restoration helper supports source selection
+and `--reparse-ready`; keep the previous revision available while an update runs.
+
+Validate physical-page coverage, prefix TOC retention, filename-only root-body
+paths, and page-citation assets on each new revision. Confirm that downstream
+readers consume the new revision after publication.
diff --git a/docs/vlm-usage-points.md b/docs/vlm-usage-points.md
new file mode 100644
index 000000000..157565ed5
--- /dev/null
+++ b/docs/vlm-usage-points.md
@@ -0,0 +1,77 @@
+# VLM Usage Points
+
+Inventory of live places that send **page or image pixels** to a vision model.
+Text-only LLM calls are listed at the bottom so they are not mistaken for VLM.
+
+Checked against the worker parse path and retrieval attach path. Re-trace before treating a new call as compliant.
+
+## Config
+
+| Knob | Role |
+|---|---|
+| `IMAGE_MODEL` | One vision model for parse. Official id: `deepseek-flash` (DeepSeek-V4.1-Flash, native multimodal). Routes through `DS_KEY` / `DS_URL`. |
+| `ASSET_MODEL` | Optional override for page-memory chart/table bbox only. Empty follows `IMAGE_MODEL`. Job metadata `page_memory_config.asset_model` still wins when set. |
+| Request `llm_config.vision` | Per-request override at call time. Not a server env switch. |
+
+`IMAGE_MODEL_MAX` is gone. Staging worker env may still pin an older name; local worker `.env` is what a local parse reads. Restart the worker after changing the env.
+
+Already-created jobs that baked a concrete `asset_model` keep that pin.
+
+## Shared PROFILE (PDF / PPTX, both parse tracks)
+
+Runs inside document profile before page-memory or chunk parse.
+
+| # | What it does | When it fires | Model |
+|---|---|---|---|
+| 1 | Coarse classify (scan / atlas / slides / generic) | Sample up to 10 pages | `IMAGE_MODEL` |
+| 2 | Confirm TOC start pages | Text scan found candidates | `IMAGE_MODEL` |
+| 3 | Extract TOC entries | After confirm | `IMAGE_MODEL` |
+| 4 | Calibrate: walk forward to find where a TOC title first starts | Printed TOC beat the outline; used to compute printed→physical offset | `IMAGE_MODEL` |
+| 5 | Calibrate: confirm the guessed page (usually one page) | Offset already computed; yes/no. If no, bisect then run 4 again | `IMAGE_MODEL` |
+| 6 | Null-page locate: grep first, then confirm the start page | TOC entry has no printed page | `IMAGE_MODEL` |
+
+If the PDF outline wins coverage, 4–6 are skipped.
+
+### PROFILE 4–6 in one example
+
+Cover + TOC occupy physical pages 1–3. Printed TOC says `1. Overview … 1`, `2. Method … 5`, `Appendix A` (no page).
+
+- **4** asks “Does `1. Overview` start on this page?” from after the TOC. Finds physical page 4 → offset +3.
+- **5** asks “Does `2. Method` start on page 8?” If yes, apply +3 to the rest. If no, find where the offset breaks.
+- **6** searches text for `Appendix A`, then asks which of those pages is the real section start. A text hit is not enough.
+
+## Page-memory track (API v2 `.pdf` / `.pptx`)
+
+| # | What it does | When it fires | Model |
+|---|---|---|---|
+| 7 | Detect outline titles on fat leaves | Default: leaf ≥ 4 pages | `IMAGE_MODEL` |
+| 8 | Per-page summary / entities | Almost every in-scope page; skip-tagged pages do not call | `IMAGE_MODEL` |
+| 9 | Node summary | Section spans pages, or one page hosts several nodes. Single exclusive page reuses 8 | `IMAGE_MODEL` |
+| 10 | Box figures/tables | Default on | `ASSET_MODEL` or `IMAGE_MODEL` |
+| 11 | Summarize cropped assets | Default **off** | same as 10 when enabled |
+
+## Text / chunk track (API v1, and non PDF/PPTX)
+
+| # | What it does | When it fires | Model |
+|---|---|---|---|
+| 12 | Atlas page info | Coarse routing is atlas **and** chunk track. Page-memory ignores atlas routing | `IMAGE_MODEL` |
+| 13 | Embedded image description | DOCX / Markdown (including PDF→Markdown) when image summary is on (default on) | `IMAGE_MODEL` |
+| 14 | Standalone image upload | Classify type; text-like images also transcribe | `IMAGE_MODEL` |
+
+Table / body summaries on this track are text LLM, not VLM.
+
+## Retrieval (after parse)
+
+| # | What it does | When it fires | Model |
+|---|---|---|---|
+| 15 | Attach HTTPS image URLs to the agent | Agentic retrieval on, and a tool returned a public image | Retrieval harness model, **not** `IMAGE_MODEL`. Cursor attaches HTTPS images. Own harness only attaches when the model name contains `vision`. |
+
+## Not VLM
+
+- Scanned-page text: local RapidOCR
+- TOC source choose, cross-page table continuity, title-tree refine, Excel/DOCX table summary: text LLM
+- Archived map-nav image filter: not on the live path
+
+## Debug
+
+Staged page-memory / text-track scripts reuse the same production calls. They are not extra VLM products.
diff --git a/packages/shared-python/shared/core/exceptions/webhook_exceptions.py b/packages/shared-python/shared/core/exceptions/webhook_exceptions.py
index 9e90c9224..2ca14dfbf 100644
--- a/packages/shared-python/shared/core/exceptions/webhook_exceptions.py
+++ b/packages/shared-python/shared/core/exceptions/webhook_exceptions.py
@@ -38,8 +38,8 @@ def __init__(
internal_message=internal_message,
user_message=user_message,
http_status_code=400,
+ details=details,
)
- self.details = details
class WebhookDeliveryException(WebhookException):
diff --git a/packages/shared-python/shared/models/database/job.py b/packages/shared-python/shared/models/database/job.py
index 8a4fb6bf3..d4d4831bc 100644
--- a/packages/shared-python/shared/models/database/job.py
+++ b/packages/shared-python/shared/models/database/job.py
@@ -126,6 +126,12 @@ class Job(Base):
# Indexes
__table_args__ = (
+ Index(
+ "idx_jobs_user_source_content_hash",
+ "user_id",
+ text("(job_metadata ->> 'source_content_sha256')"),
+ postgresql_where=text("job_metadata ->> 'source_content_sha256' IS NOT NULL"),
+ ),
Index("idx_job_status", "status"),
Index("idx_job_type", "job_type"),
Index("idx_job_created_at", "created_at"),
diff --git a/packages/shared-python/shared/models/schemas/job_metadata.py b/packages/shared-python/shared/models/schemas/job_metadata.py
index ec7d23454..9b5c7d930 100644
--- a/packages/shared-python/shared/models/schemas/job_metadata.py
+++ b/packages/shared-python/shared/models/schemas/job_metadata.py
@@ -49,6 +49,12 @@ class JobMetadataBase(BaseModel):
# Source-file fields.
source_type: Optional[str] = Field(None, description="Source type")
source_file_name: Optional[str] = Field(None, description="Source file name")
+ source_content_sha256: Optional[str] = Field(
+ None, pattern=r"^[0-9a-f]{64}$", description="SHA-256 of the original source bytes"
+ )
+ source_size_bytes: Optional[int] = Field(
+ None, ge=0, description="Original source file size in bytes"
+ )
source_url: Optional[str] = Field(None, description="Source URL")
file_url: Optional[str] = Field(None, description="File URL")
diff --git a/packages/shared-python/shared/services/chunks/chunk_connections.py b/packages/shared-python/shared/services/chunks/chunk_connections.py
index dcfa865ec..e8bc0d3d2 100644
--- a/packages/shared-python/shared/services/chunks/chunk_connections.py
+++ b/packages/shared-python/shared/services/chunks/chunk_connections.py
@@ -3,7 +3,7 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
-from typing import Any, TypeAlias, TypedDict
+from typing import Any, Final, TypeAlias, TypedDict
from shared.utils.chunk_refs import ChunkRefSpan, extract_chunk_ref_spans
@@ -13,14 +13,27 @@ class PositionPayload(TypedDict):
end: int
+RELATION_EMBEDS: Final = "embeds"
+RELATION_EMBEDDED_BY: Final = "embedded_by"
+RELATION_RELATED: Final = "related"
+# Reserved for future producers; no code path emits these yet.
+RELATION_CROSS_DOC_ENTITY: Final = "cross_doc_entity"
+RELATION_INTRA_DOC_REF: Final = "intra_doc_ref"
+
+# Relations whose targets are expanded when hydrating or deduplicating a
+# body chunk's connected assets. Reverse and future relations are excluded so
+# they stay purely additive for existing consumers.
+FORWARD_RELATIONS: Final = frozenset({RELATION_EMBEDS, RELATION_RELATED})
+
+_ASSET_CHUNK_TYPES: Final = frozenset({"image", "table"})
+
+
class ConnectionPayload(TypedDict, total=False):
target: str
+ target_document_id: str
relation: str
ref: str
position: PositionPayload
- score: float
- keywords: list[str]
- same_as_owner: str
RelationshipRef: TypeAlias = str | ChunkRefSpan
@@ -97,7 +110,7 @@ def convert_refs_to_embed_connections(
connection: ConnectionPayload = {
"target": target_id,
- "relation": "embeds",
+ "relation": RELATION_EMBEDS,
"ref": ref_text,
}
if isinstance(start, int) and isinstance(end, int):
@@ -129,20 +142,14 @@ def normalize_connect_to_targets(
normalized_item: ConnectionPayload = {
"target": normalized_target,
- "relation": str(item.get("relation") or "related"),
+ "relation": str(item.get("relation") or RELATION_RELATED),
}
- score = item.get("score")
- if isinstance(score, (int, float)):
- normalized_item["score"] = float(score)
- keywords = item.get("keywords")
- if isinstance(keywords, list):
- normalized_item["keywords"] = [str(keyword) for keyword in keywords]
+ target_document_id = item.get("target_document_id")
+ if target_document_id:
+ normalized_item["target_document_id"] = str(target_document_id)
ref = item.get("ref")
if ref:
normalized_item["ref"] = str(ref)
- same_as_owner = item.get("same_as_owner")
- if same_as_owner:
- normalized_item["same_as_owner"] = str(same_as_owner)
position = item.get("position")
if isinstance(position, dict):
start = position.get("start")
@@ -158,9 +165,7 @@ def normalize_connect_to_targets(
normalized.append(
{
"target": normalized_target,
- "relation": "related",
- "score": 1.0,
- "keywords": [],
+ "relation": RELATION_RELATED,
}
)
return normalized
@@ -197,6 +202,37 @@ def merge_connections(
return merged
+def attach_embedded_by_connections(chunks: Sequence[Mapping[str, Any]]) -> None:
+ """Give every embedded image/table an ``embedded_by`` edge to its owner.
+
+ Mutates each asset chunk's ``metadata["connect_to"]`` in place. ``chunks``
+ must already carry their final forward ``embeds`` edges, with targets
+ normalized to chunk ids.
+ """
+ assets_by_chunk_id: dict[str, list[Mapping[str, Any]]] = {}
+ for chunk in chunks:
+ if chunk.get("type") in _ASSET_CHUNK_TYPES:
+ assets_by_chunk_id.setdefault(str(chunk["chunk_id"]), []).append(chunk)
+
+ for owner in chunks:
+ owner_id = str(owner["chunk_id"])
+ for connection in owner["metadata"].get("connect_to") or []:
+ if (
+ not isinstance(connection, dict)
+ or connection.get("relation") != RELATION_EMBEDS
+ ):
+ continue
+ target_id = str(connection.get("target") or "")
+ if target_id == owner_id:
+ continue
+ for asset in assets_by_chunk_id.get(target_id, ()):
+ asset_metadata = asset["metadata"]
+ asset_metadata["connect_to"] = merge_connections(
+ asset_metadata.get("connect_to") or [],
+ [{"target": owner_id, "relation": RELATION_EMBEDDED_BY}],
+ )
+
+
def _parse_type_relationship_refs(type_value: object) -> list[str]:
if not isinstance(type_value, str) or "\n" not in type_value:
return []
@@ -206,10 +242,10 @@ def _parse_type_relationship_refs(type_value: object) -> list[str]:
def _get_connection_key(item: ConnectionValue) -> ConnectionKey:
if not isinstance(item, dict):
- return ("", "related", "")
+ return ("", RELATION_RELATED, "")
return (
str(item.get("target") or ""),
- str(item.get("relation") or "related"),
+ str(item.get("relation") or RELATION_RELATED),
str(item.get("ref") or ""),
)
diff --git a/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py b/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
index e5ad3931f..4968b213b 100644
--- a/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
+++ b/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
@@ -12,8 +12,10 @@
from loguru import logger
from shared.services.chunks.chunk_connections import (
+ RELATION_RELATED,
ConnectionValue,
RelationshipRef,
+ attach_embedded_by_connections,
build_resource_target_map,
convert_refs_to_embed_connections,
merge_connections,
@@ -42,12 +44,12 @@ def iterrows(self) -> Iterable[tuple[object, _ParserRow]]: ...
class ChunkMetadata(TypedDict, total=False):
+ body_text_preview: str
keywords: list[str]
entities: list[dict[str, str]]
asset_title: str
summary: str
length: int
- tokens: list[str]
connect_to: list[ConnectionValue]
page_nums: list[int]
file_path: str
@@ -136,26 +138,6 @@ def _safe_parse_entities(value: object) -> list[dict[str, str]]:
return entities
-def _safe_parse_tokens(value: object) -> list[str]:
- if _is_missing(value):
- return []
- token_text = str(value).strip()
- if not token_text:
- return []
- if token_text.startswith("[") and token_text.endswith("]"):
- inner_text = token_text[1:-1].strip()
- if (inner_text.startswith("'") and inner_text.endswith("'")) or (
- inner_text.startswith('"') and inner_text.endswith('"')
- ):
- inner_text = inner_text[1:-1]
- token_text = inner_text
- if ";" in token_text:
- return [token.strip() for token in token_text.split(";") if token.strip()]
- if "->" in token_text:
- return [token.strip() for token in token_text.split("->") if token.strip()]
- return []
-
-
def _normalize_resource_ref(ref: RelationshipRef) -> str:
if isinstance(ref, dict):
ref_text = str(ref.get("ref") or "").strip()
@@ -203,9 +185,7 @@ def _parse_connect_to(value: object) -> list[ConnectionValue]:
return [
{
"target": connects_text,
- "relation": "related",
- "score": 1.0,
- "keywords": [],
+ "relation": RELATION_RELATED,
}
]
@@ -324,8 +304,6 @@ def dataframe_to_chunks(df: _ParserDataFrame | None) -> list[Dict[str, JsonValue
asset_title = str(row.get("asset_title", "") or "").strip()
if asset_title:
metadata["asset_title"] = asset_title
- if chunk_type != "page":
- metadata["tokens"] = _safe_parse_tokens(row.get("tokens"))
_merge_extra_metadata(metadata, _parse_extra_metadata(row.get("extra_metadata")))
if chunk_type == "image":
@@ -384,6 +362,7 @@ def dataframe_to_chunks(df: _ParserDataFrame | None) -> list[Dict[str, JsonValue
resource_target_map,
),
)
+ attach_embedded_by_connections(chunks)
logger.debug(f"DataFrame conversion completed: chunk count={len(chunks)}")
return cast(list[Dict[str, JsonValue]], chunks)
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/__init__.py b/packages/shared-python/shared/services/retrieval/agent_explore/__init__.py
index d313f765d..206ed702c 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/__init__.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/__init__.py
@@ -1,8 +1,9 @@
"""In-process tool-calling agentic retrieval route.
-Default agentic path when ``use_agentic`` is unset or true. Runs the
-``agent_tools`` corpus registry through an LLM tool-calling loop. Map-nav is
-archived under ``deprecated/mapnav/`` and is not a live route.
+Agentic path when ``use_agentic`` is true, or when it is unset and the query
+router selects agent. Runs the ``agent_tools`` corpus registry through an LLM
+tool-calling loop. Map-nav is archived under ``deprecated/mapnav/`` and is
+not a live route.
Which provider runs that loop (Cursor SDK or OpenAI-compatible) is the
``AGENT_EXPLORE_HARNESS`` switch resolved via ``resolve_harness()``.
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/bridge.py b/packages/shared-python/shared/services/retrieval/agent_explore/bridge.py
index 6c773c1c8..4a506c2b1 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/bridge.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/bridge.py
@@ -7,8 +7,10 @@
from __future__ import annotations
-from typing import Any
-
+from shared.services.retrieval.agent_explore.evidence_pool import (
+ Candidate,
+ pool_trace_records,
+)
from shared.services.retrieval.agent_explore.types import AgentStep
from shared.services.retrieval.trace import DecisionTraceStep
@@ -28,53 +30,49 @@ def build_decision_trace(steps: list[AgentStep]) -> list[DecisionTraceStep]:
observation: dict[str, object] = {"observation_text": observation_text}
if step.ref_status is not None:
observation["ref_status"] = step.ref_status
+ if step.candidates is not None:
+ observation["candidates"] = step.candidates
+ decision: dict[str, object] = {
+ "action": step.tool_name or "no_tool_call",
+ "args": step.tool_args,
+ "round_index": step.round_index,
+ }
+ result: dict[str, object] = {
+ "status": "error" if step.error else "ok",
+ "error": step.error,
+ }
+ if step.picked is not None:
+ result["picked"] = step.picked
+ if step.pick_rejected is not None:
+ result["pick_rejected"] = step.pick_rejected
trace_steps.append(
DecisionTraceStep(
step_index=step.step_index,
agent="agent_explore",
phase=phase,
observation=observation,
- decision={
- "action": step.tool_name or "no_tool_call",
- "args": step.tool_args,
- },
- result={
- "status": "error" if step.error else "ok",
- "error": step.error,
- },
- budget={
- "tokens_used_delta": step.tokens_used_delta,
- "tokens_used_total": step.tokens_used_total,
- },
+ decision=decision,
+ result=result,
+ budget={},
elapsed_ms=step.elapsed_ms,
)
)
return trace_steps
-def attach_ref_provenance(
+def attach_evidence_pool(
steps: list[DecisionTraceStep],
- *,
- agent_selected_refs: list[dict[str, Any]] | None,
- fallback_refs: list[dict[str, Any]],
- resolved_refs: list[dict[str, Any]],
- dropped_refs: list[dict[str, Any]],
+ pool: list[Candidate],
) -> list[DecisionTraceStep]:
- """Write ref provenance onto the existing finish TRACE step.
+ """Write the evidence pool onto the existing finish TRACE step.
- Reuses ``DecisionTraceStep.result``. If the episode never called finish,
- append one finish step so the same public TRACE shape still holds the
- four lists.
+ If the episode never called finish, append one finish step so the same
+ public TRACE shape still holds the pool.
"""
- provenance = {
- "agent_selected_refs": agent_selected_refs,
- "fallback_refs": fallback_refs,
- "resolved_refs": resolved_refs,
- "dropped_refs": dropped_refs,
- }
+ records = pool_trace_records(pool)
for step in reversed(steps):
if step.phase == "finish":
- step.result = {**step.result, **provenance}
+ step.result = {**step.result, "evidence_pool": records}
return steps
steps.append(
DecisionTraceStep(
@@ -82,8 +80,8 @@ def attach_ref_provenance(
agent="agent_explore",
phase="finish",
observation={"observation_text": "finish was not called"},
- decision={"action": "finish", "args": {"refs": None}},
- result={"status": "ok", "error": None, **provenance},
+ decision={"action": "finish", "args": {}},
+ result={"status": "ok", "error": None, "evidence_pool": records},
)
)
return steps
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/budget.py b/packages/shared-python/shared/services/retrieval/agent_explore/budget.py
index 47a0c6067..652f559d2 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/budget.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/budget.py
@@ -2,17 +2,13 @@
Standalone from ``nav/nav_token_budget.py`` on purpose — this package must
have no runtime dependency on ``nav/`` (see ``config.py``'s module
-docstring). Re-reads the same ``RETRIEVAL_NAV_TOKEN_LIMIT`` env var the plan
-calls for, so operators keep one token-limit knob across both agentic
-routes, but the counting mechanism is a fresh, request-scoped object (this
-episode runs as one async function, not a separate thread with recursive
-calls), not the nav contextvar machinery ``nav_token_budget`` needed for its
-own call shape.
+docstring). Exploration is limited by steps; the wall clock is a
+service-level guard. Tokens are not limited: ``tokens_used`` only records the
+episode total.
"""
from __future__ import annotations
-import os
import time
from dataclasses import dataclass, field
from typing import Any
@@ -22,26 +18,13 @@
AGENT_EXPLORE_WALL_CLOCK_SECONDS,
)
-_ENV_TOKEN_LIMIT = "RETRIEVAL_NAV_TOKEN_LIMIT"
-_DEFAULT_TOKEN_LIMIT = 100_000
-
-StopReason = str # one of: "token_limit" | "max_steps" | "wall_clock"
-
-
-def resolve_token_limit() -> int:
- """Always a positive limit: env override, else the shared default."""
- try:
- limit = int(os.environ.get(_ENV_TOKEN_LIMIT, "").strip())
- except ValueError:
- limit = 0
- return limit if limit > 0 else _DEFAULT_TOKEN_LIMIT
+StopReason = str # one of: "max_steps" | "wall_clock"
@dataclass
class EpisodeBudget:
- """Tracks one episode's LLM-token / step / wall-clock spend."""
+ """Tracks one episode's step / wall-clock spend and its token total."""
- token_limit: int = field(default_factory=resolve_token_limit)
max_steps: int = AGENT_EXPLORE_MAX_STEPS
wall_clock_seconds: float = AGENT_EXPLORE_WALL_CLOCK_SECONDS
tokens_used: int = 0
@@ -59,22 +42,13 @@ def record_usage(self, usage: dict[str, Any] | None) -> None:
def record_step(self) -> None:
self.steps_used += 1
+ def remaining_seconds(self) -> float:
+ return self.wall_clock_seconds - (time.monotonic() - self._started_at)
+
def exhausted(self) -> StopReason | None:
"""Return which budget dimension is exceeded, if any, else None."""
- if self.tokens_used >= self.token_limit:
- return "token_limit"
if self.steps_used >= self.max_steps:
return "max_steps"
- if time.monotonic() - self._started_at >= self.wall_clock_seconds:
+ if self.remaining_seconds() <= 0:
return "wall_clock"
return None
-
- def snapshot(self) -> dict[str, Any]:
- return {
- "token_limit": self.token_limit,
- "tokens_used": self.tokens_used,
- "max_steps": self.max_steps,
- "steps_used": self.steps_used,
- "wall_clock_seconds": self.wall_clock_seconds,
- "elapsed_seconds": round(time.monotonic() - self._started_at, 3),
- }
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/config.py b/packages/shared-python/shared/services/retrieval/agent_explore/config.py
index 258dfca1c..4cb318acb 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/config.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/config.py
@@ -8,8 +8,6 @@
from __future__ import annotations
-from shared.services.retrieval.agent_tools.registry import REF_ADDRESS_ONE_OF, REF_ADDRESS_RULE
-
AGENT_EXPLORE_MODEL = "deepseek-v4-flash"
# Model for AGENT_EXPLORE_HARNESS=cursor_sdk (harness/cursor_harness.py) —
@@ -22,11 +20,11 @@
AGENT_EXPLORE_CURSOR_MODEL = "composer-2.5"
# One LLM turn = one round-trip that may contain several parallel tool calls
-# (see episode.py).
-AGENT_EXPLORE_MAX_STEPS = 12
+# (see episode.py). Both harnesses count one explore turn as one step.
+AGENT_EXPLORE_MAX_STEPS = 15
# Wall-clock ceiling for the whole episode (LLM round-trips + tool
-# dispatch), independent of the token budget. New constant, not tuned yet.
+# dispatch). A service-level guard, never shown to the model.
AGENT_EXPLORE_WALL_CLOCK_SECONDS = 180.0
# Max tokens requested per LLM completion turn (thinking disabled — see
@@ -38,104 +36,43 @@
FINISH_TOOL_SCHEMA: dict[str, object] = {
"type": "object",
"properties": {
- "refs": {
- "type": "array",
- "description": (
- "Final cited evidence, in priority order. Each item "
- "identifies one section or chunk you have already looked at "
- "via corpus.read, corpus.assets, or corpus.query_table."
- ),
- "items": {
- "type": "object",
- "properties": {
- "document_id": {
- "type": "string",
- "description": "Document that owns the cited section or chunk.",
- },
- "section_path": {
- "type": "string",
- "description": "Cited section. Omit when chunk_id is set.",
- },
- "chunk_id": {
- "type": "string",
- "description": "Cited chunk. Omit when section_path is set.",
- },
- },
- "required": ["document_id"],
- "oneOf": REF_ADDRESS_ONE_OF,
- "additionalProperties": False,
- "description": REF_ADDRESS_RULE,
- },
- },
"notes": {
"type": "string",
"description": (
- "Optional short note on why these refs answer the query, "
- "or why none were found."
+ "Optional short note for the answer writer, or why nothing "
+ "was found."
),
},
},
- "required": ["refs"],
+ "required": [],
"additionalProperties": False,
}
FINISH_TOOL_DESCRIPTION = (
- "Call this when you have gathered enough evidence to answer the query, "
- "or when you are certain the corpus does not contain an answer. This "
- "ends the exploration — do not answer in plain text; the final answer "
- "is synthesized downstream from the refs you cite here."
+ "End the exploration now. The evidence pool becomes the final evidence."
)
-# Appended after the verbatim CORPUS_SCHEMA.md text (schema_doc.py) to form
-# this harness's system prompt. Kept out of CORPUS_SCHEMA.md itself because
-# the finish-tool loop contract is agent_explore-specific, not something the
-# MCP-facing harnesses (Cursor/Codex/Claude) need — see that file's own
-# "single source... do not duplicate" header.
-LOOP_CONTRACT_SUFFIX = f"""
+# Harness-level tool, not in agent_tools.REGISTRY, so MCP never exposes it.
+PICK_TOOL_NAME = "corpus.pick"
----
-
-## Exploration loop contract
-
-You are exploring this corpus autonomously to answer one query. Tool
-names in this prompt are the registered names (`corpus.read`). This
-function-calling loop exposes the same tools with `.` replaced by `_`
-(`corpus_read`); use that underscore form when calling. Logs keep the
-dotted name.
-
-Use the tools above to navigate; you may call several tools in one turn
-when they are independent. If a call's arguments depend on another
-call's result, do not issue them in the same turn. In particular: do
-not call `corpus.grep` in the same turn as `corpus.recall`,
-`corpus.read`, `corpus.list_documents`, `corpus.outline`,
-`corpus.node_filter`, or `corpus.assets` unless `pattern` / `patterns`
-is already known — those calls produce the term; a grep with no term is
-an empty call. Wait for the result, then grep. `corpus.list_documents`
-enumerates the entire namespace: use it only if the user explicitly
-asks to list or inventory the corpus's documents, never as the starting
-step for a content question. When you have enough evidence, call
-`{FINISH_TOOL_NAME}`
-with the `refs` you want cited as the answer — do not write the final answer
-as plain text yourself, it is synthesized downstream from your cited refs.
-If you exhaust your tool budget without a confident answer, call
-`{FINISH_TOOL_NAME}` with your best-effort `refs` (or an empty list plus a
-`notes` explanation of why nothing was found) rather than continuing to
-call other tools.
-
-`refs` is REQUIRED and must not be omitted or left empty if you called
-`corpus.read`, `corpus.assets`, or `corpus.query_table` even once during
-this exploration: copy the `document_id` and `chunk_id`/`section_path` of
-every section/chunk you looked at that supports your answer into `refs`
-before calling `{FINISH_TOOL_NAME}`. Calling `{FINISH_TOOL_NAME}` with no
-`refs` after having already read relevant content discards that evidence.
+PICK_TOOL_SCHEMA: dict[str, object] = {
+ "type": "object",
+ "properties": {
+ "pick": {
+ "type": "array",
+ "items": {"type": "string", "pattern": r"^(R\d+\.\d+|O\d+)$"},
+ "uniqueItems": True,
+ "description": (
+ "Ids from your latest corpus.read / corpus.outline results to "
+ "keep. Pass an empty list to keep none."
+ ),
+ },
+ },
+ "required": ["pick"],
+ "additionalProperties": False,
+}
-Before calling `{FINISH_TOOL_NAME}`, if you have not called `corpus.read`,
-`corpus.assets`, or `corpus.query_table` even once this exploration, you
-have not actually verified anything yet — a search tool returning
-candidates is not the same as having read them. In that case, either read
-your best candidate section first, or — only if you have positively
-confirmed there is nothing to read (e.g. a structural check came back
-with zero matching sections) — say so explicitly in `notes`. Repeatedly
-rephrasing the same search instead of reading a candidate you already
-found is not a substitute for reading it.
-"""
+PICK_TOOL_DESCRIPTION = (
+ "Pick which of your latest corpus.read / corpus.outline results join the "
+ "evidence pool. Body text you read but do not pick cannot be read again."
+)
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/dispatch.py b/packages/shared-python/shared/services/retrieval/agent_explore/dispatch.py
index f002472bd..febf2da20 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/dispatch.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/dispatch.py
@@ -20,14 +20,21 @@
from shared.services.retrieval.document_scope import DocumentScope
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
from contextlib import AbstractAsyncContextManager
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from shared.services.retrieval.corpus_revision_context import CorpusRevisionContext
-from shared.services.retrieval.agent_tools import REGISTRY, ToolBudget, ToolContext, ToolResult
+from shared.services.retrieval.agent_tools import (
+ REGISTRY,
+ Decision,
+ ReadableAddresses,
+ ToolBudget,
+ ToolContext,
+ ToolResult,
+)
DbFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
@@ -42,6 +49,8 @@ async def dispatch_tool_call(
document_scope: DocumentScope = DocumentScope(),
budget: ToolBudget | None = None,
query: str = "",
+ readable: ReadableAddresses | None = None,
+ decided: Mapping[tuple[str, str], Decision] | None = None,
) -> ToolResult:
"""Run one ``REGISTRY`` tool call against a fresh, call-scoped DB session.
@@ -52,7 +61,8 @@ async def dispatch_tool_call(
into the (call-scoped, already-closed) ``ToolContext``. ``query`` is the
end user's original query for this episode (see ``ToolContext.query``'s
docstring) — both harnesses' ``run_episode`` already receive it, so this
- is a pass-through, not a new source of truth.
+ is a pass-through, not a new source of truth. ``readable``/``decided`` are
+ the episode's ``corpus.read`` gates, passed through unchanged.
"""
try:
async with db_factory() as db:
@@ -65,6 +75,8 @@ async def dispatch_tool_call(
budget=budget or ToolBudget(),
document_scope=document_scope.narrow(list(pins)) if pins is not None else document_scope,
query=query,
+ readable=readable,
+ decided=decided,
)
return await REGISTRY.dispatch(name, tool_ctx, args)
except Exception as exc: # noqa: BLE001 - one broken tool must not kill the episode
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/evidence_pool.py b/packages/shared-python/shared/services/retrieval/agent_explore/evidence_pool.py
new file mode 100644
index 000000000..b13a53161
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/evidence_pool.py
@@ -0,0 +1,404 @@
+"""Per-round addresses, pick ids, and the episode evidence pool.
+
+Results of one round become usable only when the next round begins: their
+addresses become readable, and their pick ids open a pick phase.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import Any, Literal
+
+from shared.services.retrieval.agent_explore.budget import EpisodeBudget
+from shared.services.retrieval.agent_tools import Decision, ReadableAddresses, ToolResult
+from shared.services.retrieval.hydration.evidence_compose import group_evidence_units
+from shared.services.retrieval.hydration.row_utils import (
+ iter_connected_target_ids,
+ normalize_chunk_type,
+)
+from shared.services.retrieval.settings import ASSET_CHUNK_TYPES
+
+
+@dataclass(frozen=True)
+class Candidate:
+ handle: str
+ kind: Literal["read", "outline"]
+ document_id: str
+ source_file_name: str
+ section_path: str | None = None
+ chunk_type: str | None = None
+ chunk_ids: tuple[str, ...] = ()
+ summary: str = ""
+ extra_sections: int = 0
+ outline_lines: tuple[str, ...] = ()
+
+
+@dataclass
+class PickOutcome:
+ added: list[str] = field(default_factory=list)
+ rejected: list[dict[str, str]] = field(default_factory=list)
+
+
+class EvidencePool:
+ def __init__(self) -> None:
+ self.entries: list[Candidate] = []
+ self.round_index = 0
+ self.pick_phase = False
+ self._read_calls = 0
+ self._outline_calls = 0
+ self._chunk_keys: set[tuple[str, str]] = set()
+ self._seen_active: set[tuple[str, str]] = set()
+ self._seen_staged: set[tuple[str, str]] = set()
+ self._batch: dict[str, Candidate] = {}
+ self._batch_staged: dict[str, Candidate] = {}
+ self._batch_round = 0
+ self._decided: dict[tuple[str, str], Decision] = {}
+ self._queried_tables: dict[tuple[str, str], str] = {}
+
+ def begin_round(self) -> None:
+ self.round_index += 1
+ self._seen_active |= self._seen_staged
+ self._seen_staged = set()
+ if self._batch_staged:
+ self._batch.update(self._batch_staged)
+ self._batch_staged = {}
+ self._batch_round = self.round_index - 1
+ self.pick_phase = bool(self._batch)
+
+ def issue(self, tool_name: str, result: ToolResult) -> list[Candidate]:
+ if result.error:
+ return []
+ self._collect_seen(result)
+ self._record_queried_table(tool_name, result)
+ if tool_name == "corpus.read":
+ return self._issue_read(result)
+ if tool_name == "corpus.outline":
+ return self._issue_outline(result)
+ return []
+
+ def apply_pick(self, handles: list[str]) -> PickOutcome:
+ outcome = PickOutcome()
+ seen: set[str] = set()
+ for handle in handles:
+ if handle in seen:
+ continue
+ seen.add(handle)
+ candidate = self._batch.get(handle)
+ if candidate is None:
+ outcome.rejected.append({"handle": handle, "reason": "unknown id"})
+ continue
+ if candidate.kind == "read" and self._all_chunks_in_pool(candidate):
+ outcome.rejected.append({"handle": handle, "reason": "already in pool"})
+ continue
+ self.entries.append(candidate)
+ for chunk_id in candidate.chunk_ids:
+ self._chunk_keys.add((candidate.document_id, chunk_id))
+ outcome.added.append(handle)
+ picked = set(outcome.added)
+ for candidate in self._batch.values():
+ if candidate.kind != "read":
+ continue
+ picked_handle = candidate.handle if candidate.handle in picked else None
+ for chunk_id in candidate.chunk_ids:
+ self._decided[(candidate.document_id, chunk_id)] = Decision(
+ read_round=self._batch_round,
+ picked_handle=picked_handle,
+ handle=candidate.handle,
+ )
+ self._batch = {}
+ self.pick_phase = False
+ return outcome
+
+ def readable(self) -> ReadableAddresses:
+ return frozenset(self._seen_active)
+
+ def decided(self) -> Mapping[tuple[str, str], Decision]:
+ return MappingProxyType(dict(self._decided))
+
+ def queried_tables(self) -> Mapping[tuple[str, str], str]:
+ return MappingProxyType(self._queried_tables)
+
+ def render_candidates(self, candidates: list[Candidate]) -> str:
+ if not candidates:
+ return ""
+ if len(candidates) == 1 and candidates[0].kind == "outline":
+ return f"[pick id for this result: {candidates[0].handle} = this outline]"
+ parts: list[str] = []
+ for candidate in candidates:
+ label = candidate.section_path or candidate.handle
+ parts.append(f"{candidate.handle} = {label}")
+ return f"[pick ids for this result: {', '.join(parts)}]"
+
+ def render_pool(self) -> str:
+ if not self.entries:
+ return "evidence pool: empty"
+ lines = [f"evidence pool ({len(self.entries)}):"]
+ for candidate in self.entries:
+ if candidate.kind == "outline":
+ lines.extend(candidate.outline_lines)
+ continue
+ path = candidate.section_path or ""
+ kind = candidate.chunk_type or "text"
+ lines.append(
+ f" [{candidate.handle}] {candidate.source_file_name} / {path} [{kind}]"
+ )
+ if candidate.summary:
+ lines.append(f" summary: {candidate.summary}")
+ return "\n".join(lines)
+
+ def chunk_refs(self) -> list[dict[str, str]]:
+ return pool_chunk_refs(self.entries)
+
+ def _all_chunks_in_pool(self, candidate: Candidate) -> bool:
+ if not candidate.chunk_ids:
+ return False
+ return all(
+ (candidate.document_id, chunk_id) in self._chunk_keys
+ for chunk_id in candidate.chunk_ids
+ )
+
+ def _record_queried_table(self, tool_name: str, result: ToolResult) -> None:
+ if tool_name != "corpus.query_table":
+ return
+ document_id = str(result.payload.get("document_id") or "").strip()
+ chunk_id = str(result.payload.get("chunk_id") or "").strip()
+ table_html = result.payload.get("table_html")
+ if not document_id or not chunk_id or not isinstance(table_html, str):
+ return
+ self._queried_tables[(document_id, chunk_id)] = table_html
+
+ def _collect_seen(self, result: ToolResult) -> None:
+ for key in ("rows", "chunks"):
+ rows = result.payload.get(key)
+ if not isinstance(rows, list):
+ continue
+ for row in rows:
+ if not isinstance(row, dict):
+ continue
+ document_id = str(row.get("document_id") or "")
+ if not document_id:
+ continue
+ section_path = str(row.get("section_path") or "").strip()
+ if section_path:
+ self._seen_staged.add((document_id, section_path))
+ chunk_ids = [
+ row.get("chunk_id"),
+ *(row.get("mounted_chunk_ids") or []),
+ *iter_connected_target_ids(row),
+ ]
+ for chunk_id in chunk_ids:
+ if chunk_id:
+ self._seen_staged.add((document_id, str(chunk_id)))
+
+ def _issue_read(self, result: ToolResult) -> list[Candidate]:
+ self._read_calls += 1
+ statuses = result.payload.get("refs")
+ chunks = result.payload.get("chunks")
+ if not isinstance(statuses, list):
+ return []
+ chunk_rows = chunks if isinstance(chunks, list) else []
+ rows_by_id = {
+ str(row.get("chunk_id") or ""): row
+ for row in chunk_rows
+ if isinstance(row, dict) and row.get("chunk_id")
+ }
+ issued: list[Candidate] = []
+ for index, entry in enumerate(statuses, start=1):
+ if not isinstance(entry, dict) or entry.get("status") != "ok":
+ continue
+ chunk_ids = tuple(
+ str(item) for item in (entry.get("chunk_ids") or []) if str(item)
+ )
+ first = rows_by_id.get(chunk_ids[0]) if chunk_ids else None
+ document_id = str(entry.get("document_id") or "")
+ section_path = str(entry.get("section_path") or "").strip() or None
+ if first and not section_path:
+ section_path = str(first.get("section_path") or "").strip() or None
+ chunk_type = normalize_chunk_type((first or {}).get("chunk_type"))
+ summary = _read_summary(first, chunk_type)
+ candidate = Candidate(
+ handle=f"R{self._read_calls}.{index}",
+ kind="read",
+ document_id=document_id,
+ source_file_name=str((first or {}).get("source_file_name") or ""),
+ section_path=section_path,
+ chunk_type=chunk_type or None,
+ chunk_ids=chunk_ids,
+ summary=summary,
+ extra_sections=max(len(chunk_ids) - 1, 0),
+ )
+ issued.append(candidate)
+ self._batch_staged[candidate.handle] = candidate
+ return issued
+
+ def _issue_outline(self, result: ToolResult) -> list[Candidate]:
+ self._outline_calls += 1
+ rows = result.payload.get("rows")
+ details = result.payload.get("details")
+ if not isinstance(rows, list) or not rows:
+ return []
+ documents = details.get("documents") if isinstance(details, dict) else None
+ names: Mapping[str, Any] = documents if isinstance(documents, dict) else {}
+ handle = f"O{self._outline_calls}"
+ first = rows[0] if isinstance(rows[0], dict) else {}
+ document_id = str(first.get("document_id") or "")
+ source_file_name = str(names.get(document_id) or first.get("title") or "")
+ lines = _outline_title_tree(rows, names, handle=handle)
+ candidate = Candidate(
+ handle=handle,
+ kind="outline",
+ document_id=document_id,
+ source_file_name=source_file_name,
+ outline_lines=tuple(lines),
+ )
+ self._batch_staged[candidate.handle] = candidate
+ return [candidate]
+
+
+def render_budget(budget: EpisodeBudget) -> str:
+ return f"budget: steps {budget.steps_used}/{budget.max_steps}"
+
+
+def render_pick_outcome(outcome: PickOutcome) -> str:
+ text = f"picked: {', '.join(outcome.added) or 'none'}"
+ if outcome.rejected:
+ rejected = ", ".join(
+ f"{item['handle']} ({item['reason']})" for item in outcome.rejected
+ )
+ text += f"; rejected: {rejected}"
+ return text
+
+
+def render_trace_line(
+ index: int,
+ wire_name: str,
+ args: dict[str, Any],
+ status: str,
+) -> str:
+ encoded = json.dumps(args, ensure_ascii=False, default=str)
+ return f"{index}. {wire_name} {encoded} -> {status}"
+
+
+def pool_chunk_refs(entries: list[Candidate]) -> list[dict[str, str]]:
+ refs: list[dict[str, str]] = []
+ for candidate in entries:
+ if candidate.kind != "read":
+ continue
+ for chunk_id in candidate.chunk_ids:
+ refs.append({"document_id": candidate.document_id, "chunk_id": chunk_id})
+ return refs
+
+
+def compose_pool_evidence(
+ entries: list[Candidate],
+ assembled_rows: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+ rows_by_id = {
+ str(row.get("chunk_id") or ""): row
+ for row in assembled_rows
+ if row.get("chunk_id")
+ }
+ units: list[dict[str, Any]] = []
+ for candidate in entries:
+ if candidate.kind == "outline":
+ text = "\n".join(candidate.outline_lines)
+ if not text:
+ continue
+ units.append(
+ {
+ "document_id": candidate.document_id,
+ "source_file_name": candidate.source_file_name,
+ "section_path": "",
+ "sort_order": None,
+ "parts": [{"type": "text", "text": text}],
+ "kind": "outline",
+ }
+ )
+ continue
+ chunk_rows = sorted(
+ (rows_by_id[chunk_id] for chunk_id in candidate.chunk_ids if chunk_id in rows_by_id),
+ key=lambda row: row["sort_order"],
+ )
+ parts = [part for row in chunk_rows for part in row["composed"]]
+ if not parts:
+ continue
+ units.append(
+ {
+ "document_id": candidate.document_id,
+ "source_file_name": candidate.source_file_name,
+ "section_path": candidate.section_path or "",
+ "sort_order": chunk_rows[0]["sort_order"],
+ "parts": parts,
+ "kind": "read",
+ }
+ )
+ return group_evidence_units(units)
+
+
+def pool_trace_records(entries: list[Candidate]) -> list[dict[str, Any]]:
+ records: list[dict[str, Any]] = []
+ for candidate in entries:
+ records.append(
+ {
+ "handle": candidate.handle,
+ "kind": candidate.kind,
+ "document_id": candidate.document_id,
+ "section_path": candidate.section_path,
+ "chunk_ids": list(candidate.chunk_ids),
+ }
+ )
+ return records
+
+
+def trace_status(result: ToolResult) -> str:
+ if result.error:
+ return f"failed: {result.error}"
+ statuses = result.payload.get("refs")
+ if isinstance(statuses, list):
+ failures: list[str] = []
+ for index, entry in enumerate(statuses, start=1):
+ if isinstance(entry, dict) and entry.get("status") == "failed":
+ reason = str(entry.get("reason") or "failed")
+ failures.append(f"ref {index} failed: {reason}")
+ if failures:
+ return "partial: " + "; ".join(failures)
+ return "ok"
+
+
+def _read_summary(row: dict[str, Any] | None, chunk_type: str) -> str:
+ if not row:
+ return ""
+ if chunk_type in ASSET_CHUNK_TYPES:
+ metadata = row.get("chunk_metadata") or {}
+ if isinstance(metadata, dict):
+ return str(metadata.get("summary") or "").strip()
+ return ""
+ return str(row.get("section_summary") or "").strip()
+
+
+def _outline_title_tree(
+ rows: list[Any], names: Mapping[str, Any], *, handle: str
+) -> list[str]:
+ lines: list[str] = []
+ current_doc = ""
+ first_doc = True
+ for row in rows:
+ if not isinstance(row, dict):
+ continue
+ document_id = str(row.get("document_id") or "")
+ if document_id != current_doc:
+ current_doc = document_id
+ file_name = str(names.get(document_id) or row.get("title") or document_id)
+ if first_doc:
+ lines.append(f" [{handle}] {file_name}")
+ first_doc = False
+ else:
+ lines.append(f" {file_name}")
+ title = str(row.get("title") or "").strip()
+ if title:
+ depth = int(row.get("depth") or 0)
+ lines.append(f"{' ' * (depth + 1)}{title}")
+ return lines
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/harness/cursor_harness.py b/packages/shared-python/shared/services/retrieval/agent_explore/harness/cursor_harness.py
index ba2a3b4bd..591dba905 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/harness/cursor_harness.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/harness/cursor_harness.py
@@ -14,50 +14,35 @@
guarded import in ``_require_cursor_sdk`` raises a clear error at episode
start if ``cursor-sdk`` isn't installed.
-Architectural difference from ``openai_harness.py`` that budget enforcement
-has to work around: this harness does not control the LLM turn loop.
-``agent.send(...)`` + ``await run.wait()`` hands the *entire* multi-turn
-tool-calling loop to the Cursor SDK; the host process only sees (a)
-``execute`` callbacks for each ``corpus.*``/``finish`` tool call — run
-synchronously off the SDK's own thread and bridged back onto this event
-loop via ``asyncio.run_coroutine_threadsafe`` (mirrors the PoC's
-``_dispatch_sync``) — and (b) the terminal ``RunResult`` once ``wait()``
-returns. There is no per-turn hook to inspect budget mid-turn and force
-``tool_choice`` the way ``openai_harness.py`` does. Each budget dimension is
-therefore enforced (or explicitly not) differently here:
-
-- **``max_steps``**: a plain counter (``budget.steps_used``, incremented once
- per ``corpus.*`` dispatch — ``finish`` does not count, it ends the episode
- on its own) checked in ``_dispatch_sync`` *before* dispatching. Once the
- counter reaches ``budget.max_steps``, further ``corpus.*`` calls are not
- forwarded to ``REGISTRY`` at all — the callback returns a fixed
- "budget exhausted, call finish now" string instead. This is a real,
- synchronous cutoff (unlike the two dimensions below): no reliance on
- cancelling the SDK run from a background task.
-- **``wall_clock``**: ``asyncio.wait_for(run.wait(), timeout=budget.wall_clock_seconds)``,
- per the plan. On timeout, best-effort ``await run.cancel()`` (own short
- timeout, so a hung cancel RPC can't hang this call forever) so the
- underlying agent run actually stops server-side instead of merely being
- abandoned by this process, then the episode result is built from whatever
- ``finish``/tool-trajectory refs were captured before the timeout.
-- **``token_limit``**: **not actively enforced mid-run.** Verified by reading
- the installed ``cursor-sdk`` package's source
- (``cursor_sdk._run_base._RunBase.usage``, 2026-09) that cumulative token
- usage is incrementally accumulated from ``SDKUsageMessage`` stream events
- as ``run.wait()`` consumes them, and exposed as a live ``run.usage``
- property — so a mid-run cutoff (e.g. a polling task calling
- ``run.cancel()``) is *plausible*. It was **not** exercised against a real
- ``CURSOR_API_KEY`` run in this change (no key available in the
- implementation environment), so building an active cutoff on top of an
- unverified live-update assumption was judged higher-risk than shipping.
- ``EpisodeBudget.exhausted()``'s ``token_limit`` dimension is therefore left
- unchecked here by design; ``budget.tokens_used`` is only populated
- *post-hoc* from the terminal ``RunResult.usage`` for observability
- (``EpisodeResult.tokens_used``), after the episode has already finished.
- If ``eval-cursor-harness`` confirms ``run.usage`` does update live against
- a real key, promoting this to an active cutoff (mirroring the
- ``max_steps``/``wall_clock`` pattern above) is straightforward follow-up
- work — deliberately not done speculatively here.
+This harness does not control the LLM turn loop. ``agent.send(...)`` +
+``await run.wait()`` hands the multi-turn tool-calling loop to the Cursor
+SDK. The host process only sees ``execute`` callbacks, ``on_delta`` updates
+(driven by ``run.wait()``), and the terminal ``RunResult``.
+
+Rounds: calls the model issues in one turn share one ``model_call_id``,
+carried by the ``tool-call-started`` update. That update and the tool
+callback arrive almost together, so each callback first waits for its own
+tag (at most the remaining wall clock; without it the call fails with
+``round tag missing``, unexecuted and uncounted). A new ``model_call_id``
+begins a new pool round.
+
+Pick phase: the Cursor tool list is fixed when the agent is created, so a
+round after a successful read or outline accepts only ``corpus_pick``; every
+other call (including ``finish``) is rejected at once, unexecuted and
+uncounted. ``corpus_pick`` itself never counts as a step.
+
+Budget dimensions:
+
+- **``max_steps``**: one explore round (shared ``model_call_id``) is one
+ step, matching the OpenAI harness. Checked when a new explore round
+ starts. Once ``budget.steps_used`` reaches ``budget.max_steps``, further
+ explore rounds are not forwarded; the callback returns a fixed exhausted
+ message plus pool and budget. Same-round sibling ``corpus.*`` calls share
+ that one step and still dispatch.
+- **``wall_clock``**: ``asyncio.wait_for(run.wait(), timeout=budget.wall_clock_seconds)``.
+ On timeout, best-effort ``await run.cancel()``.
+- **tokens**: not limited; ``budget.tokens_used`` is only the episode total
+ from the terminal ``RunResult.usage``.
"""
from __future__ import annotations
@@ -78,43 +63,52 @@
FINISH_TOOL_DESCRIPTION,
FINISH_TOOL_NAME,
FINISH_TOOL_SCHEMA,
- LOOP_CONTRACT_SUFFIX,
+ PICK_TOOL_DESCRIPTION,
+ PICK_TOOL_NAME,
+ PICK_TOOL_SCHEMA,
)
from shared.services.retrieval.agent_explore.dispatch import DbFactory, dispatch_tool_call
+from shared.services.retrieval.agent_explore.evidence_pool import (
+ EvidencePool,
+ render_budget,
+ render_pick_outcome,
+)
+from shared.services.retrieval.agent_explore.prompt import AGENT_SYSTEM_PROMPT
from shared.services.retrieval.agent_explore.shared import (
- EVIDENCE_TOOL_NAMES,
- budget_status_line,
cursor_execute_content,
- finish_refs_from_args,
invalid_finish_message,
+ invalid_pick_message,
read_ref_status,
- select_episode_refs,
tool_message_content,
validate_finish_args,
+ validate_pick_args,
wire_safe_tool_name,
)
from shared.services.retrieval.agent_explore.types import AgentStep, EpisodeResult
-from shared.services.retrieval.agent_tools import (
- REGISTRY,
- ToolBudget,
- ToolResult,
- load_corpus_schema_text,
-)
+from shared.services.retrieval.agent_tools import REGISTRY, ToolBudget, ToolResult
# Grace period for the underlying agent run to actually stop, after a
# best-effort run.cancel() following a wall_clock timeout, before this
-# process gives up waiting for a terminal RunResult and falls back to
-# whatever refs the trajectory already captured. Not tuned against real
-# cancel-RPC latency yet (no CURSOR_API_KEY in the implementation
-# environment) — revisit with eval-cursor-harness data.
+# process gives up waiting for a terminal RunResult.
_CANCEL_GRACE_SECONDS = 30.0
_BUDGET_EXHAUSTED_MESSAGE = (
"error: step budget exhausted for this episode — do not call any more "
- "corpus.* tools; call finish now with whatever refs you already have "
- "(or an empty list plus a notes explanation)."
+ "corpus.* tools; call finish now."
+)
+
+_PICK_PHASE_REJECTION = (
+ "Pick phase: only corpus_pick is accepted now. Pick from R/O ids above, "
+ "or pass an empty list."
+)
+
+_PICK_PHASE_INSTRUCTION = (
+ "Pick phase: you can only call corpus_pick now. Pick the ids worth "
+ "keeping, or pass an empty list."
)
+_ROUND_TAG_MISSING = "round tag missing"
+
def _require_cursor_sdk() -> Any:
try:
@@ -129,6 +123,10 @@ def _require_cursor_sdk() -> Any:
return cursor_sdk
+def _state_tail(pool: EvidencePool, budget: EpisodeBudget) -> str:
+ return "\n\n".join([pool.render_pool(), render_budget(budget)])
+
+
class CursorHarness:
"""``Harness`` implementation over the Cursor SDK local agent."""
@@ -156,42 +154,82 @@ async def run_episode(
tool_budget = ToolBudget()
loop = asyncio.get_running_loop()
- # The Cursor SDK delivers same-turn parallel tool calls as concurrent
- # HTTP requests on separate threads (ThreadingHTTPServer in
- # cursor_sdk._tool_callback) — CORPUS_SCHEMA.md's loop contract
- # explicitly invites the model to call several independent tools in
- # one turn. Without this lock, the check-then-increment on
- # budget.steps_used below is a race: several concurrently-dispatched
- # calls can each read "not exhausted yet" before any of them
- # increments, letting more calls through than max_steps allows.
- budget_lock = threading.Lock()
+ state = threading.Condition()
+ pool = EvidencePool()
+ round_of: dict[str, str] = {}
+ current_round: dict[str, Any] = {"id": None, "pick_phase": False}
steps: list[AgentStep] = []
- trajectory_refs: list[dict[str, Any]] = []
- # None until finish specifies refs. Explicit ``refs: []`` stays [].
- # Omitted refs still fall back via select_episode_refs.
- finish_state: dict[str, Any] = {"refs": None, "notes": ""}
stop_reason = "finished"
+ def _on_delta(update: Any) -> None:
+ if getattr(update, "type", None) != "tool-call-started":
+ return
+ with state:
+ round_of[update.call_id] = update.model_call_id
+ state.notify_all()
+
+ def _enter_round(ctx: Any) -> tuple[bool | None, bool]:
+ """Wait for this call's round tag (caller holds ``state``).
+
+ Returns ``(pick_phase, is_new_round)``. ``pick_phase`` is
+ ``None`` when the tag never arrived.
+ """
+ call_id = ctx.tool_call_id
+ if not state.wait_for(
+ lambda: call_id in round_of, timeout=budget.remaining_seconds()
+ ):
+ return None, False
+ model_call_id = round_of[call_id]
+ if model_call_id != current_round["id"]:
+ pool.begin_round()
+ current_round["id"] = model_call_id
+ current_round["pick_phase"] = pool.pick_phase
+ return current_round["pick_phase"], True
+ return current_round["pick_phase"], False
+
+ def _append_step(**fields: Any) -> None:
+ steps.append(AgentStep(step_index=len(steps), **fields))
+
+ def _reject(tool_name: str, args: dict[str, Any], message: str) -> None:
+ _append_step(
+ tool_name=tool_name,
+ tool_args=args,
+ observation_text=message,
+ error=message,
+ elapsed_ms=0,
+ round_index=pool.round_index,
+ )
+
def _dispatch_sync(
- tool_name: str, args: dict[str, Any]
+ tool_name: str, args: dict[str, Any], ctx: Any
) -> str | list[dict[str, Any]]:
- with budget_lock:
- if budget.steps_used >= budget.max_steps:
- steps.append(
- AgentStep(
- step_index=len(steps),
+ with state:
+ pick_round, new_round = _enter_round(ctx)
+ round_index = pool.round_index
+ if pick_round is None:
+ _reject(tool_name, args, _ROUND_TAG_MISSING)
+ return f"error: {_ROUND_TAG_MISSING}"
+ if pick_round:
+ _reject(tool_name, args, _PICK_PHASE_REJECTION)
+ return _PICK_PHASE_REJECTION
+ if new_round:
+ if budget.steps_used >= budget.max_steps:
+ observation = (
+ _BUDGET_EXHAUSTED_MESSAGE + "\n" + _state_tail(pool, budget)
+ )
+ _append_step(
tool_name=tool_name,
tool_args=args,
- observation_text=_BUDGET_EXHAUSTED_MESSAGE,
+ observation_text=observation,
error="budget_max_steps",
elapsed_ms=0,
- tokens_used_delta=0,
- tokens_used_total=budget.tokens_used,
+ round_index=round_index,
)
- )
- return _BUDGET_EXHAUSTED_MESSAGE
- budget.record_step()
+ return observation
+ budget.record_step()
+ readable = pool.readable()
+ decided = pool.decided()
tool_started = time.perf_counter()
future = asyncio.run_coroutine_threadsafe(
dispatch_tool_call(
@@ -203,6 +241,8 @@ def _dispatch_sync(
document_scope=document_scope,
budget=tool_budget,
query=query,
+ readable=readable,
+ decided=decided,
),
loop,
)
@@ -216,29 +256,28 @@ def _dispatch_sync(
tool_name=tool_name,
max_chars=tool_budget.max_chars,
)
- # Appended per call, unlike openai_harness.py's once-per-turn
- # placement — this harness has no batched-turn concept exposed to
- # the host process (see module docstring): each corpus.* dispatch
- # is the only per-step hook available to surface budget state.
- content_with_budget = content + "\n" + budget_status_line(budget)
- observation = cursor_execute_content(
- tool_result, text=content_with_budget
- )
- steps.append(
- AgentStep(
- step_index=len(steps),
+ with state:
+ candidates = pool.issue(tool_name, tool_result)
+ tail = _state_tail(pool, budget)
+ text = content
+ if candidates:
+ text = (
+ f"{content}\n{pool.render_candidates(candidates)}\n"
+ f"{_PICK_PHASE_INSTRUCTION}"
+ )
+ text = text + "\n\n" + tail
+ observation = cursor_execute_content(tool_result, text=text)
+ with state:
+ _append_step(
tool_name=tool_name,
tool_args=args,
- observation_text=content_with_budget,
+ observation_text=text,
error=tool_result.error,
elapsed_ms=elapsed_ms,
- tokens_used_delta=0,
- tokens_used_total=budget.tokens_used,
+ round_index=round_index,
ref_status=read_ref_status(tool_name, tool_result),
+ candidates=[item.handle for item in candidates] or None,
)
- )
- if tool_name in EVIDENCE_TOOL_NAMES and not tool_result.error:
- trajectory_refs.extend(tool_result.refs)
return observation
custom_tools: dict[str, Any] = {}
@@ -247,9 +286,9 @@ def _dispatch_sync(
def _make_execute(resolved_name: str):
def execute(
- args: dict[str, Any], _ctx: Any
+ args: dict[str, Any], ctx: Any
) -> str | list[dict[str, Any]]:
- return _dispatch_sync(resolved_name, dict(args or {}))
+ return _dispatch_sync(resolved_name, dict(args or {}), ctx)
return execute
@@ -259,45 +298,65 @@ def execute(
input_schema=spec.json_schema,
)
- def finish_execute(args: dict[str, Any], _ctx: Any) -> str:
+ def pick_execute(args: dict[str, Any], ctx: Any) -> str:
args = dict(args or {})
- validation_error = validate_finish_args(args)
- if validation_error is not None:
- with budget_lock:
- steps.append(
- AgentStep(
- step_index=len(steps),
- tool_name=FINISH_TOOL_NAME,
- tool_args=args,
- observation_text=validation_error,
- error=validation_error,
- elapsed_ms=0,
- tokens_used_delta=0,
- tokens_used_total=budget.tokens_used,
- )
- )
- return json.dumps(
- {"status": "error", "error": invalid_finish_message(validation_error)}
+ with state:
+ pick_round, _new_round = _enter_round(ctx)
+ if pick_round is None:
+ _reject(PICK_TOOL_NAME, args, _ROUND_TAG_MISSING)
+ return f"error: {_ROUND_TAG_MISSING}"
+ validation_error = validate_pick_args(args)
+ if validation_error is not None:
+ message = invalid_pick_message(validation_error)
+ _reject(PICK_TOOL_NAME, args, message)
+ return f"error: {message}"
+ outcome = pool.apply_pick(list(args["pick"]))
+ observation = render_pick_outcome(outcome)
+ _append_step(
+ tool_name=PICK_TOOL_NAME,
+ tool_args=args,
+ observation_text=observation,
+ error=None,
+ elapsed_ms=0,
+ round_index=pool.round_index,
+ picked=outcome.added,
+ pick_rejected=outcome.rejected,
)
- selected = finish_refs_from_args(args)
- cited = selected if selected is not None else []
- notes = str(args.get("notes") or "")
- with budget_lock:
- finish_state["refs"] = selected
- finish_state["notes"] = notes
- steps.append(
- AgentStep(
- step_index=len(steps),
- tool_name=FINISH_TOOL_NAME,
- tool_args=args,
- observation_text=f"refs={len(cited)} notes={notes!r}",
- error=None,
- elapsed_ms=0,
- tokens_used_delta=0,
- tokens_used_total=budget.tokens_used,
+ return observation + "\n\n" + _state_tail(pool, budget)
+
+ custom_tools[wire_safe_tool_name(PICK_TOOL_NAME)] = cursor_sdk.CustomTool(
+ execute=pick_execute,
+ description=PICK_TOOL_DESCRIPTION,
+ input_schema=PICK_TOOL_SCHEMA,
+ )
+
+ def finish_execute(args: dict[str, Any], ctx: Any) -> str:
+ args = dict(args or {})
+ with state:
+ pick_round, _new_round = _enter_round(ctx)
+ if pick_round is None or pick_round:
+ message = _ROUND_TAG_MISSING if pick_round is None else _PICK_PHASE_REJECTION
+ _reject(FINISH_TOOL_NAME, args, message)
+ return json.dumps({"status": "error", "error": message})
+ validation_error = validate_finish_args(args)
+ if validation_error is not None:
+ _reject(FINISH_TOOL_NAME, args, validation_error)
+ return json.dumps(
+ {
+ "status": "error",
+ "error": invalid_finish_message(validation_error),
+ }
)
+ notes = str(args.get("notes") or "")
+ _append_step(
+ tool_name=FINISH_TOOL_NAME,
+ tool_args=args,
+ observation_text=f"pool={len(pool.entries)} notes={notes!r}",
+ error=None,
+ elapsed_ms=0,
+ round_index=pool.round_index,
)
- return json.dumps({"status": "finished", "refs": len(cited)})
+ return json.dumps({"status": "finished", "pool": len(pool.entries)})
custom_tools[FINISH_TOOL_NAME] = cursor_sdk.CustomTool(
execute=finish_execute,
@@ -305,13 +364,7 @@ def finish_execute(args: dict[str, Any], _ctx: Any) -> str:
input_schema=FINISH_TOOL_SCHEMA,
)
- system_prompt = load_corpus_schema_text() + LOOP_CONTRACT_SUFFIX
- user_prompt = (
- f"{system_prompt}\n\n---\n\nUser query:\n{query}\n\n"
- "Tool names on the wire use underscores "
- f"({', '.join(sorted(custom_tools))}). Explore with those tools, "
- "then call finish with cited refs."
- )
+ user_prompt = f"{AGENT_SYSTEM_PROMPT}\n\n---\n\nUser query:\n{query}"
result: Any = None
async with await cursor_sdk.AsyncClient.launch_bridge(
@@ -327,7 +380,9 @@ def finish_execute(args: dict[str, Any], _ctx: Any) -> str:
),
)
) as agent:
- run = await agent.send(user_prompt)
+ run = await agent.send(
+ user_prompt, cursor_sdk.SendOptions(on_delta=_on_delta)
+ )
try:
result = await asyncio.wait_for(
run.wait(), timeout=budget.wall_clock_seconds
@@ -346,18 +401,11 @@ def finish_execute(args: dict[str, Any], _ctx: Any) -> str:
result_usage_total_tokens = result.usage.total_tokens
budget.record_usage({"total_tokens": result_usage_total_tokens})
- selection = select_episode_refs(
- finish_state["refs"],
- trajectory_refs,
- str(finish_state["notes"] or ""),
- )
return EpisodeResult(
- refs=selection.refs,
- notes=selection.notes,
+ pool=list(pool.entries),
steps=steps,
stop_reason=stop_reason,
tokens_used=budget.tokens_used,
model_name=self._model,
- agent_selected_refs=selection.agent_selected_refs,
- fallback_refs=selection.fallback_refs,
+ queried_tables=dict(pool.queried_tables()),
)
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/harness/openai_harness.py b/packages/shared-python/shared/services/retrieval/agent_explore/harness/openai_harness.py
index 1d84fdabd..1db56ac54 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/harness/openai_harness.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/harness/openai_harness.py
@@ -7,45 +7,19 @@
(2026-09-08) against ``deepseek-v4-flash`` via this codebase's client:
single tool call, parallel tool calls in one turn, tool-result feedback +
final synthesis, and forced ``tool_choice`` (used for the budget-exhaustion
-cutoff below) all work.
+cutoff and the pick phase below) all work.
+
+Each turn is one round and sends only a system message plus one rebuilt
+user message (query, one-line call trace, latest-turn results with pick ids,
+evidence pool, steps). Native tool-call history is not kept.
+
+A turn after a successful read or outline is a pick phase: only
+``corpus_pick`` is offered and forced, and the turn does not count as a
+step. Tokens are only summed for the episode total.
Tool calls within one turn are dispatched sequentially through
-``dispatch.dispatch_tool_call`` (fresh DB session per call — safe for any
-harness, not just this one), not via ``asyncio.gather``: batching several
-tool calls into one LLM turn already removes the LLM round-trip per tool
-(the dominant cost); true DB-level concurrency within a turn is not
-implemented here since this provider's tool calls are handled one at a time
-by design, not because concurrent dispatch would be unsafe (it no longer is
-— see ``dispatch.py``).
-
-No import from archived map-nav modules.
-
-Two Phase 4 fixes (audited live against the eval fixture in
-``apps/worker/scripts/fixtures/changheba_archive_eval_queries.json``), both
-now shared with any other harness via ``shared.py`` except stale-message
-collapsing (fix 1), which stays here — it mutates this harness's own
-``messages: list[dict]`` history, a mechanism the Cursor SDK harness has no
-equivalent hook for:
-
-1. **Stale discovery-message collapsing** (``_TOOL_MESSAGE_FRESH_TURNS``,
- ``_collapse_stale_tool_messages``): ``messages`` only ever appended, so a
- single ``corpus.outline``/``corpus.node_filter`` call (each capped at
- ``ToolBudget.max_chars`` — currently ``EVIDENCE_TEXT_CHAR_BUDGET=12_000``,
- see ``registry.py``) was resent in full on every later turn. Verified
- live: two independent queries (q04, q06 in the eval fixture) hit
- ``RETRIEVAL_NAV_TOKEN_LIMIT`` (100k default) within 7-8 LLM turns from
- this resend alone, not from query difficulty — per-turn token cost grew
- monotonically (q04: 4.4k -> 4.8k -> 19.8k -> 21.5k -> 24.2k -> 29.6k).
- Successful evidence reads remain visible until finish. Removing them after
- two turns caused a real SpaceX query to alternate between complementary
- sections until its token budget ran out. Per-result text caps and the
- episode budget still apply.
-2. **Trajectory refs fallback** (``shared.select_episode_refs``): verified
- live that ``finish`` can be called with no ``refs`` key at all (raw
- ``function.arguments`` was literally ``'{}'``) even after the model had
- already read clearly relevant sections via ``corpus.read``. Omitted refs
- (``None``) still fall back to ``corpus.read``/``corpus.assets`` trajectory
- refs. An explicit ``refs: []`` is respected and does not fall back.
+``dispatch.dispatch_tool_call`` (fresh DB session per call). No import from
+archived map-nav modules.
"""
from __future__ import annotations
@@ -64,38 +38,38 @@
FINISH_TOOL_DESCRIPTION,
FINISH_TOOL_NAME,
FINISH_TOOL_SCHEMA,
- LOOP_CONTRACT_SUFFIX,
+ PICK_TOOL_DESCRIPTION,
+ PICK_TOOL_NAME,
+ PICK_TOOL_SCHEMA,
)
from shared.services.retrieval.agent_explore.dispatch import DbFactory, dispatch_tool_call
+from shared.services.retrieval.agent_explore.evidence_pool import (
+ EvidencePool,
+ render_budget,
+ render_pick_outcome,
+ render_trace_line,
+ trace_status,
+)
+from shared.services.retrieval.agent_explore.prompt import AGENT_SYSTEM_PROMPT
from shared.services.retrieval.agent_explore.shared import (
- EVIDENCE_TOOL_NAMES,
- budget_status_line,
build_wire_tool_name_map,
- finish_refs_from_args,
- https_image_parts,
invalid_finish_message,
- model_accepts_images,
+ invalid_pick_message,
read_ref_status,
- select_episode_refs,
tool_message_content,
validate_finish_args,
+ validate_pick_args,
wire_safe_tool_name,
)
from shared.services.retrieval.agent_explore.types import AgentStep, EpisodeResult
-from shared.services.retrieval.agent_tools import (
- REGISTRY,
- ToolBudget,
- ToolResult,
- load_corpus_schema_text,
-)
+from shared.services.retrieval.agent_tools import REGISTRY, ToolBudget, ToolResult
+
+_PICK_WIRE_NAME = wire_safe_tool_name(PICK_TOOL_NAME)
-# A discovery or failed tool message is kept in full for the turn it was produced plus this
-# many additional turns, then collapsed to a placeholder — see module
-# docstring point 1. Not tuned against a real recall-vs-token tradeoff yet;
-# 2 was chosen so a result stays fully visible for one full turn after the
-# one it was produced in (enough for the model to act on it immediately),
-# revisit with more Phase 4 data.
-_TOOL_MESSAGE_FRESH_TURNS = 2
+_PICK_PHASE_INSTRUCTION = (
+ "Pick phase: you can only call corpus_pick now. Pick the ids worth "
+ "keeping, or pass an empty list."
+)
def _resolve_client_and_model() -> tuple[Any, str]:
@@ -141,6 +115,17 @@ def _build_openai_tools() -> tuple[list[dict[str, Any]], dict[str, str]]:
return tools, name_map
+def _build_pick_tool() -> dict[str, Any]:
+ return {
+ "type": "function",
+ "function": {
+ "name": _PICK_WIRE_NAME,
+ "description": PICK_TOOL_DESCRIPTION,
+ "parameters": PICK_TOOL_SCHEMA,
+ },
+ }
+
+
def _parse_tool_arguments(raw: str | None) -> tuple[dict[str, Any], str | None]:
"""Parse one tool call's JSON arguments, or return the parse error.
@@ -160,31 +145,27 @@ def _parse_tool_arguments(raw: str | None) -> tuple[dict[str, Any], str | None]:
return parsed, None
-def _collapse_stale_tool_messages(
- messages: list[dict[str, Any]],
- tool_message_log: list[dict[str, Any]],
+def _compose_user_content(
*,
- current_turn: int,
- fresh_turns: int,
-) -> None:
- """Replace stale discovery/error messages while preserving read evidence.
-
- ``messages`` only ever grows within one episode (see module docstring
- point 1); this is what keeps that growth bounded instead of resending
- every past discovery result on every later turn. Successful evidence must
- remain visible so complementary reads can be synthesized without rereading.
- """
- for entry in tool_message_log:
- if entry["collapsed"] or entry.get("has_evidence", False):
- continue
- if current_turn - entry["turn_index"] < fresh_turns:
- continue
- messages[entry["message_index"]]["content"] = (
- f"[collapsed: {entry['tool_name']} result from turn "
- f"{entry['turn_index']} was {entry['original_chars']} chars — "
- "call the tool again if you need it back in view]"
+ query: str,
+ trace_lines: list[str],
+ latest_results: list[str],
+ latest_turn: int,
+ pool: EvidencePool,
+ budget: EpisodeBudget,
+ instruction: str,
+) -> str:
+ parts = [f"User query: {query}"]
+ if trace_lines:
+ parts.append("trace:\n" + "\n".join(f" {line}" for line in trace_lines))
+ if latest_results:
+ parts.append(
+ f"latest results (turn {latest_turn}):\n" + "\n\n".join(latest_results)
)
- entry["collapsed"] = True
+ parts.append(pool.render_pool())
+ parts.append(render_budget(budget))
+ parts.append(instruction)
+ return "\n\n".join(part for part in parts if part)
class OpenAIHarness:
@@ -203,38 +184,44 @@ async def run_episode(
tool_budget = ToolBudget()
client, model = _resolve_client_and_model()
openai_tools, tool_name_map = _build_openai_tools()
-
- system_prompt = load_corpus_schema_text() + LOOP_CONTRACT_SUFFIX
- messages: list[dict[str, Any]] = [
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": query},
- ]
-
+ pick_tool = _build_pick_tool()
+ pool = EvidencePool()
+ trace_lines: list[str] = []
+ latest_results: list[str] = []
+ latest_turn = 0
steps: list[AgentStep] = []
- finish_refs: list[dict[str, Any]] | None = None
result_notes = ""
- # Refs from every corpus.read/corpus.assets/corpus.query_table call this episode, in call
- # order — the fallback source when finish's own refs end up empty (see
- # module docstring point 2).
- trajectory_refs: list[dict[str, Any]] = []
- # One entry per appended tool-role message: {message_index, turn_index,
- # tool_name, original_chars, has_evidence, collapsed} — see _collapse_stale_tool_messages.
- tool_message_log: list[dict[str, Any]] = []
turn_index = 0
+ stop_reason = "finished"
while True:
turn_index += 1
- _collapse_stale_tool_messages(
- messages,
- tool_message_log,
- current_turn=turn_index,
- fresh_turns=_TOOL_MESSAGE_FRESH_TURNS,
- )
-
- forced_reason = budget.exhausted()
- tool_choice: Any = "auto"
- if forced_reason is not None:
- tool_choice = {"type": "function", "function": {"name": FINISH_TOOL_NAME}}
+ pool.begin_round()
+ pick_turn = pool.pick_phase
+ forced_reason = None if pick_turn else budget.exhausted()
+ if pick_turn:
+ tools = [pick_tool]
+ tool_choice: Any = {"type": "function", "function": {"name": _PICK_WIRE_NAME}}
+ else:
+ tools = openai_tools
+ tool_choice = "auto"
+ if forced_reason is not None:
+ tool_choice = {"type": "function", "function": {"name": FINISH_TOOL_NAME}}
+ messages = [
+ {"role": "system", "content": AGENT_SYSTEM_PROMPT},
+ {
+ "role": "user",
+ "content": _compose_user_content(
+ query=query,
+ trace_lines=trace_lines,
+ latest_results=latest_results,
+ latest_turn=latest_turn,
+ pool=pool,
+ budget=budget,
+ instruction=_PICK_PHASE_INSTRUCTION if pick_turn else "",
+ ),
+ },
+ ]
turn_started = time.perf_counter()
response, usage = await asyncio.to_thread(
@@ -243,13 +230,13 @@ async def run_episode(
model=model,
temperature=0.0,
max_tokens=AGENT_EXPLORE_MAX_COMPLETION_TOKENS,
- tools=openai_tools,
+ tools=tools,
tool_choice=tool_choice,
)
budget.record_usage(usage)
- budget.record_step()
+ if not pick_turn:
+ budget.record_step()
turn_elapsed_ms = int((time.perf_counter() - turn_started) * 1000)
- turn_tokens = int((usage or {}).get("total_tokens", 0) or 0)
message = response.choices[0].message
tool_calls = list(message.tool_calls or [])
@@ -265,47 +252,102 @@ async def run_episode(
observation_text=result_notes,
error=None,
elapsed_ms=turn_elapsed_ms,
- tokens_used_delta=turn_tokens,
- tokens_used_total=budget.tokens_used,
+ round_index=pool.round_index,
)
)
break
- finish_call = next(
- (tc for tc in tool_calls if tc.function.name == FINISH_TOOL_NAME), None
+ if pick_turn:
+ for call_index, tool_call in enumerate(tool_calls):
+ args, parse_error = _parse_tool_arguments(tool_call.function.arguments)
+ pick_error = parse_error or validate_pick_args(args)
+ if pick_error is not None:
+ message = invalid_pick_message(pick_error)
+ trace_lines.append(
+ render_trace_line(
+ len(trace_lines) + 1,
+ _PICK_WIRE_NAME,
+ args,
+ trace_status(ToolResult(text="", error=message)),
+ )
+ )
+ steps.append(
+ AgentStep(
+ step_index=len(steps),
+ tool_name=PICK_TOOL_NAME,
+ tool_args=args,
+ observation_text=message,
+ error=message,
+ elapsed_ms=turn_elapsed_ms if call_index == 0 else 0,
+ round_index=pool.round_index,
+ )
+ )
+ continue
+ outcome = pool.apply_pick(list(args["pick"]))
+ observation = render_pick_outcome(outcome)
+ trace_lines.append(
+ render_trace_line(
+ len(trace_lines) + 1, _PICK_WIRE_NAME, args, observation
+ )
+ )
+ steps.append(
+ AgentStep(
+ step_index=len(steps),
+ tool_name=PICK_TOOL_NAME,
+ tool_args=args,
+ observation_text=observation,
+ error=None,
+ elapsed_ms=turn_elapsed_ms if call_index == 0 else 0,
+ round_index=pool.round_index,
+ picked=outcome.added,
+ pick_rejected=outcome.rejected,
+ )
+ )
+ continue
+
+ parsed_calls: list[tuple[str, dict[str, Any], str | None]] = [
+ (
+ str(tool_call.function.name or ""),
+ *_parse_tool_arguments(tool_call.function.arguments),
+ )
+ for tool_call in tool_calls
+ ]
+
+ finish_entry = next(
+ (item for item in parsed_calls if item[0] == FINISH_TOOL_NAME),
+ None,
)
finish_error: str | None = None
- if finish_call is not None:
- args, parse_error = _parse_tool_arguments(finish_call.function.arguments)
+ finish_args: dict[str, Any] = {}
+ finish_parse_error: str | None = None
+ if finish_entry is not None:
+ _name, finish_args, finish_parse_error = finish_entry
+ parse_error = finish_parse_error
if parse_error is None:
- parse_error = validate_finish_args(args)
+ parse_error = validate_finish_args(finish_args)
if parse_error is not None and forced_reason is None:
finish_error = parse_error
- if finish_call is not None and finish_error is None:
- finish_refs = finish_refs_from_args(args) if parse_error is None else None
- cited = finish_refs if finish_refs is not None else []
+ if finish_entry is not None and finish_error is None:
result_notes = (
- parse_error if parse_error is not None else str(args.get("notes") or "")
+ finish_parse_error
+ if finish_parse_error is not None
+ else str(finish_args.get("notes") or "")
)
stop_reason = f"budget_{forced_reason}" if forced_reason else "finished"
steps.append(
AgentStep(
step_index=len(steps),
tool_name=FINISH_TOOL_NAME,
- tool_args=args,
- observation_text=f"refs={len(cited)} notes={result_notes!r}",
- error=parse_error,
+ tool_args=finish_args,
+ observation_text=f"pool={len(pool.entries)} notes={result_notes!r}",
+ error=finish_parse_error,
elapsed_ms=turn_elapsed_ms,
- tokens_used_delta=turn_tokens,
- tokens_used_total=budget.tokens_used,
+ round_index=pool.round_index,
)
)
break
if forced_reason is not None:
- # Forced tool_choice=finish but the provider returned a
- # different tool anyway (not observed in verification, but a
- # budget cutoff must never loop past). Stop here regardless.
stop_reason = f"budget_{forced_reason}"
result_notes = str(message.content or "") or (
"budget exhausted; provider did not return finish"
@@ -318,131 +360,85 @@ async def run_episode(
observation_text=result_notes,
error="forced_finish_not_honored",
elapsed_ms=turn_elapsed_ms,
- tokens_used_delta=turn_tokens,
- tokens_used_total=budget.tokens_used,
+ round_index=pool.round_index,
)
)
break
- messages.append(
- {
- "role": "assistant",
- "content": message.content or "",
- "tool_calls": [
- {
- "id": tc.id,
- "type": "function",
- "function": {
- "name": tc.function.name,
- "arguments": tc.function.arguments,
- },
- }
- for tc in tool_calls
- ],
- }
- )
- first_tool_tokens_recorded = False
- for tc in tool_calls:
+ readable = pool.readable()
+ decided = pool.decided()
+ first_step_recorded = False
+ turn_results: list[str] = []
+ for requested_name, call_args, parse_error in parsed_calls:
tool_started = time.perf_counter()
- args, parse_error = _parse_tool_arguments(tc.function.arguments)
- requested_name = str(tc.function.name or "")
canonical_name = tool_name_map.get(requested_name, requested_name)
- if tc is finish_call:
+ if requested_name == FINISH_TOOL_NAME:
tool_result = ToolResult(
text="", error=invalid_finish_message(str(finish_error))
)
+ trace_result = ToolResult(text="", error=str(finish_error))
elif parse_error is not None:
tool_result = ToolResult(text="", error=parse_error)
+ trace_result = tool_result
else:
tool_result = await dispatch_tool_call(
canonical_name,
- args,
+ call_args,
db_factory=db_factory,
user_id=user_id,
namespace=namespace,
document_scope=document_scope,
budget=tool_budget,
query=query,
+ readable=readable,
+ decided=decided,
)
+ trace_result = tool_result
tool_elapsed_ms = int((time.perf_counter() - tool_started) * 1000)
content = tool_message_content(
tool_result,
tool_name=canonical_name,
max_chars=tool_budget.max_chars,
)
- messages.append(
- {"role": "tool", "tool_call_id": tc.id, "content": content}
- )
- tool_message_index = len(messages) - 1
- if model_accepts_images(model):
- image_parts = https_image_parts(tool_result)
- if image_parts:
- messages.append(
- {"role": "user", "content": image_parts}
- )
- tool_message_log.append(
- {
- "message_index": tool_message_index,
- "turn_index": turn_index,
- "tool_name": canonical_name,
- "original_chars": len(content),
- "has_evidence": (
- canonical_name in EVIDENCE_TOOL_NAMES
- and not tool_result.error
- and bool(tool_result.refs)
- ),
- "collapsed": False,
- }
+ candidates = pool.issue(canonical_name, tool_result)
+ if candidates:
+ content = f"{content}\n{pool.render_candidates(candidates)}"
+ turn_results.append(f"{requested_name}:\n{content}")
+ trace_lines.append(
+ render_trace_line(
+ len(trace_lines) + 1,
+ requested_name,
+ call_args,
+ trace_status(trace_result),
+ )
)
- if canonical_name in EVIDENCE_TOOL_NAMES and not tool_result.error:
- trajectory_refs.extend(tool_result.refs)
- # Turn-level token usage is attributed to the first tool step in
- # this turn (the completion that decided all calls in it); the
- # rest are 0 to avoid double-counting the same LLM usage.
steps.append(
AgentStep(
step_index=len(steps),
tool_name=canonical_name,
- tool_args=args,
+ tool_args=call_args,
observation_text=content,
error=tool_result.error,
elapsed_ms=(
tool_elapsed_ms
- if first_tool_tokens_recorded
+ if first_step_recorded
else turn_elapsed_ms + tool_elapsed_ms
),
- tokens_used_delta=0 if first_tool_tokens_recorded else turn_tokens,
- tokens_used_total=budget.tokens_used,
+ round_index=pool.round_index,
ref_status=read_ref_status(canonical_name, tool_result),
+ candidates=[item.handle for item in candidates] or None,
)
)
- first_tool_tokens_recorded = True
+ first_step_recorded = True
- # Appended once per turn, to the last tool message only (not
- # every AgentStep's recorded observation_text above) — the model
- # only needs to see current remaining budget once before its next
- # completion call, not once per parallel tool call in this turn.
- last_tool = next(
- (
- message
- for message in reversed(messages)
- if message.get("role") == "tool"
- ),
- None,
- )
- if last_tool is not None:
- last_tool["content"] = (
- str(last_tool["content"]) + "\n" + budget_status_line(budget)
- )
+ latest_results = turn_results
+ latest_turn = turn_index
- selection = select_episode_refs(finish_refs, trajectory_refs, result_notes)
return EpisodeResult(
- refs=selection.refs,
- notes=selection.notes,
+ pool=list(pool.entries),
steps=steps,
stop_reason=stop_reason,
tokens_used=budget.tokens_used,
model_name=model,
- agent_selected_refs=selection.agent_selected_refs,
- fallback_refs=selection.fallback_refs,
+ queried_tables=dict(pool.queried_tables()),
)
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/prompt.py b/packages/shared-python/shared/services/retrieval/agent_explore/prompt.py
new file mode 100644
index 000000000..c84f6c17b
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/prompt.py
@@ -0,0 +1,45 @@
+"""Agent-explore system prompt.
+
+Tool-specific usage stays on each tool's own description. This module only
+holds the shared overview plus the exploration-loop contract.
+"""
+
+from __future__ import annotations
+
+from shared.services.retrieval.agent_tools import load_corpus_overview_text
+
+LOOP_RULES = """\
+## How this exploration works
+
+You collect evidence for one user question; you do not write the answer.
+Each turn, either call tools or call finish.
+
+Reading:
+- corpus.read only accepts addresses (document_id plus section_path or
+ chunk_id) that appeared in results from your earlier turns. On your first
+ turn you have no results yet, so do not call corpus.read.
+- Calls in the same turn cannot use each other's results.
+
+Evidence pool: only what you pick reaches the answer writer.
+- A corpus.read result gives each successfully read ref a pick id (R3.1,
+ R3.2, ...). A corpus.outline result gives the whole outline one pick id (O2).
+ Failed refs get no id. Other tools give no ids: to pick an image or table,
+ read it by chunk_id first.
+- After a turn where corpus.read or corpus.outline succeeds, the next turn is
+ a pick phase: you can only call corpus.pick. Pass the ids worth keeping, or
+ an empty list. Picked ids join the evidence pool.
+- Body text you read but did not pick cannot be read again.
+- Each turn you see: a trace of your past calls (tool, arguments, ok or why
+ it failed), the full results of your latest calls, the evidence pool, and
+ the steps used so far.
+
+Call finish when the pool is enough to answer, or when you are sure the
+corpus has no answer. finish ends the exploration at once.
+
+If a call needs another call's result, wait for that result instead of
+issuing both in the same turn.
+
+Tool names on the wire use "_" instead of "." (corpus_read = corpus.read).
+"""
+
+AGENT_SYSTEM_PROMPT = load_corpus_overview_text() + "\n\n" + LOOP_RULES
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/ref_resolution.py b/packages/shared-python/shared/services/retrieval/agent_explore/ref_resolution.py
deleted file mode 100644
index d244e41ea..000000000
--- a/packages/shared-python/shared/services/retrieval/agent_explore/ref_resolution.py
+++ /dev/null
@@ -1,154 +0,0 @@
-"""Resolve ``finish.refs`` into the chunk_id-bearing shape ``resolve_workflow_references`` requires.
-
-Verified live: ``resolve_workflow_references`` -> ``hydrate_referenced_chunk_rows``
-drops any ref whose lookup key has an empty ``chunk_id`` (``row_utils.
-build_reference_lookup_key`` + the ``ref_keys = [k for k in ref_keys if k[0]
-and k[1]]`` guard in ``hydration/reference.py``) — a ``{document_id,
-section_path}``-only ref silently resolves to zero ``referenced_chunks``,
-which is exactly the shape ``agent_tools.CORPUS_SCHEMA.md``/``corpus.read``
-teaches the agent to cite (``corpus.read``'s rendered ``text`` — the only
-thing the LLM ever sees — shows ``section_path``, never ``chunk_id``;
-``chunk_id`` only appears in its structured ``payload``/``refs``, which the
-LLM does not see). This module closes that gap at the harness boundary
-instead of changing what the agent is taught to cite: for any ref missing
-``chunk_id``, resolve that section's own body chunk — the same "one section,
-one body chunk" lookup ``corpus.read``'s ``section_path`` branch performs
-(``agent_tools/section_path_lookup.py``), including the same suffix fallback
-when the agent cites a path without ancestor prefixes.
-"""
-
-from __future__ import annotations
-
-from shared.services.retrieval.corpus_revision_context import CorpusRevisionContext
-
-from shared.services.retrieval.corpus_storage import CorpusStorage
-
-from shared.services.retrieval.document_scope import DocumentScope
-
-from dataclasses import dataclass, field
-from typing import Any
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from shared.services.retrieval.agent_tools.section_path_lookup import (
- resolve_section_path_anchor,
-)
-
-_BODY_CHUNK_TYPES = ("text", "page")
-
-
-@dataclass
-class FinishRefResolution:
- resolved: list[dict[str, Any]] = field(default_factory=list)
- dropped: list[dict[str, Any]] = field(default_factory=list)
-
-
-async def resolve_finish_refs(
- db: AsyncSession,
- *,
- user_id: str,
- namespace: str,
- refs: list[dict[str, Any]],
- document_scope: DocumentScope = DocumentScope(),
-) -> FinishRefResolution:
- """Return refs with ``chunk_id`` populated; dropped refs keep a reason."""
- corpusStorage: CorpusStorage = CorpusStorage.resolve_namespace(namespace)
- resolution = FinishRefResolution()
- scoped: list[dict[str, Any]] = []
- for ref in refs:
- document_id = str(ref.get("document_id") or "").strip()
- if not document_id:
- resolution.dropped.append({"ref": ref, "reason": "missing document_id"})
- continue
- if not document_scope.allows(document_id):
- resolution.dropped.append(
- {"ref": ref, "reason": f"document_id out of scope: {document_id}"}
- )
- continue
- scoped.append(ref)
-
- document_ids = {
- str(ref.get("document_id") or "").strip() for ref in scoped if ref.get("document_id")
- }
- if not document_ids:
- return resolution
-
- documents = (
- (
- await db.execute(
- select(corpusStorage.Document)
- .where(corpusStorage.Document.document_id.in_(document_ids))
- .where(corpusStorage.Document.user_id == corpusStorage.resolve_owner(user_id))
- .where(corpusStorage.Document.namespace == namespace)
- .where(corpusStorage.Document.status == "active")
- .where(document_scope.predicate(corpusStorage.Document.document_id))
- )
- )
- .scalars()
- .all()
- )
- revision_by_doc = {
- d.document_id: revision for d in documents if (revision := CorpusRevisionContext.resolve_revision(d.document_id, d.current_job_result_id))
- }
-
- for ref in scoped:
- document_id = str(ref.get("document_id") or "").strip()
- chunk_id = str(ref.get("chunk_id") or "").strip()
- if document_id and chunk_id:
- resolution.resolved.append({"document_id": document_id, "chunk_id": chunk_id})
- continue
-
- section_path = str(ref.get("section_path") or "").strip()
- job_result_id = revision_by_doc.get(document_id)
- if not job_result_id:
- resolution.dropped.append(
- {"ref": ref, "reason": f"unknown document_id: {document_id}"}
- )
- continue
- if not section_path:
- resolution.dropped.append(
- {"ref": ref, "reason": f"ref for {document_id} needs section_path or chunk_id"}
- )
- continue
-
- resolved_path, path_error = await resolve_section_path_anchor(
- db,
- document_id=document_id,
- job_result_id=job_result_id,
- section_path=section_path,
- )
- if path_error or not resolved_path:
- resolution.dropped.append(
- {
- "ref": ref,
- "reason": path_error or f"unknown section_path for {document_id}",
- }
- )
- continue
-
- row = (
- await db.execute(
- select(corpusStorage.DocumentChunk.chunk_id)
- .select_from(corpusStorage.DocumentChunk)
- .join(
- corpusStorage.DocumentSection,
- corpusStorage.DocumentSection.section_id == corpusStorage.DocumentChunk.section_id,
- )
- .where(corpusStorage.DocumentChunk.document_id == document_id)
- .where(corpusStorage.DocumentChunk.job_result_id == job_result_id)
- .where(corpusStorage.DocumentSection.section_path == resolved_path)
- .where(corpusStorage.DocumentChunk.chunk_type.in_(_BODY_CHUNK_TYPES))
- )
- ).first()
- if row is None:
- resolution.dropped.append(
- {
- "ref": ref,
- "reason": f"no body chunk for {document_id}: {resolved_path}",
- }
- )
- continue
- resolution.resolved.append({"document_id": document_id, "chunk_id": str(row[0])})
-
- return resolution
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/shared.py b/packages/shared-python/shared/services/retrieval/agent_explore/shared.py
index 47fbc60ef..0294b5f92 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/shared.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/shared.py
@@ -5,34 +5,21 @@
and, as a debug-script PoC, had started duplicating and drifting from it
instead of sharing it — see the Phase 3.5 section of
``.cursor/plans/agentic_corpus_explore_retrieval_c2c4ea21.plan.md``.
-
-Deliberately excludes anything that assumes an editable ``messages:
-list[dict]`` conversation history (that's OpenAI-harness-specific — see
-``harness/openai_harness.py``'s ``_collapse_stale_tool_messages``, which is
-NOT here because the Cursor SDK manages its own context with no equivalent
-hook exposed to the host process).
"""
from __future__ import annotations
-from dataclasses import dataclass
from typing import Any
import jsonschema
import jsonschema.validators
-from shared.services.retrieval.agent_explore.budget import EpisodeBudget
-from shared.services.retrieval.agent_explore.config import FINISH_TOOL_SCHEMA
-from shared.services.retrieval.agent_tools import ToolResult
-
-# Tools whose ToolResult.refs point at evidence the agent has actually looked
-# at (full body content), as opposed to candidate/listing refs from
-# list_documents/outline/node_filter/recall/grep — those describe *where
-# things are*, not *what was read*, and would inject unread noise into the
-# trajectory-refs fallback below if included.
-EVIDENCE_TOOL_NAMES = frozenset(
- {"corpus.read", "corpus.assets", "corpus.query_table"}
+from shared.services.retrieval.agent_explore.config import (
+ FINISH_TOOL_SCHEMA,
+ PICK_TOOL_NAME,
+ PICK_TOOL_SCHEMA,
)
+from shared.services.retrieval.agent_tools import ToolResult
# The two map-narrowing tools bound their own text at MAP_TOOL_CHAR_BUDGET by
# folding whole subtrees (agent_tools.map_render) or failing the call, so
@@ -112,6 +99,7 @@ def tool_message_content(result: ToolResult, *, tool_name: str, max_chars: int)
if tool_name in MAP_TOOL_NAMES or len(text) <= max_chars:
return text
omitted = len(text) - max_chars
+ # TODO: return a fragment for oversized corpus.read bodies, the way oversized tables do, instead of this head cut.
return (
text[:max_chars]
+ f"\n...[truncated, {omitted} more chars — narrow the scope "
@@ -119,54 +107,6 @@ def tool_message_content(result: ToolResult, *, tool_name: str, max_chars: int)
)
-@dataclass(frozen=True)
-class EpisodeRefSelection:
- """Final episode refs plus how they were chosen.
-
- ``agent_selected_refs`` is ``None`` only when finish was never called.
- An explicit empty finish stays empty and does not take the trajectory
- fallback.
- """
-
- refs: list[dict[str, Any]]
- notes: str
- agent_selected_refs: list[dict[str, Any]] | None
- fallback_refs: list[dict[str, Any]]
-
-
-def select_episode_refs(
- finish_refs: list[dict[str, Any]] | None,
- trajectory_refs: list[dict[str, Any]],
- notes: str,
-) -> EpisodeRefSelection:
- """Choose episode refs without treating ``[]`` and ``None`` as the same."""
- if finish_refs is None:
- fallback_refs = dedup_refs(trajectory_refs)
- if not fallback_refs:
- return EpisodeRefSelection(
- refs=[],
- notes=notes,
- agent_selected_refs=None,
- fallback_refs=[],
- )
- suffix = (
- "[refs auto-filled from corpus.read/corpus.assets/"
- "corpus.query_table trajectory; finish was not called]"
- )
- return EpisodeRefSelection(
- refs=fallback_refs,
- notes=(notes + " " if notes else "") + suffix,
- agent_selected_refs=None,
- fallback_refs=fallback_refs,
- )
- return EpisodeRefSelection(
- refs=list(finish_refs),
- notes=notes,
- agent_selected_refs=list(finish_refs),
- fallback_refs=[],
- )
-
-
def validate_finish_args(args: dict[str, Any]) -> str | None:
"""Validate ``finish``'s own args against ``FINISH_TOOL_SCHEMA``; ``None`` = valid.
@@ -176,10 +116,26 @@ def validate_finish_args(args: dict[str, Any]) -> str | None:
is unverified (see ``config.py``'s ``FINISH_TOOL_SCHEMA`` docstring
context), so this is the real enforcement point.
"""
+ return _schema_args_error("finish", FINISH_TOOL_SCHEMA, args)
+
+
+def validate_pick_args(args: dict[str, Any]) -> str | None:
+ """Validate ``corpus.pick`` args against ``PICK_TOOL_SCHEMA``; ``None`` = valid.
+
+ The Cursor SDK does not enforce ``input_schema`` (a live run sent
+ ``pick_ids``), so a malformed call must fail here instead of being
+ applied as an empty pick that closes the pick phase.
+ """
+ return _schema_args_error(PICK_TOOL_NAME, PICK_TOOL_SCHEMA, args)
+
+
+def _schema_args_error(
+ tool_name: str, schema: dict[str, object], args: dict[str, Any]
+) -> str | None:
if not isinstance(args, dict):
- return f"finish: arguments must be a JSON object, got {type(args).__name__}"
- validator_cls = jsonschema.validators.validator_for(FINISH_TOOL_SCHEMA)
- validator = validator_cls(FINISH_TOOL_SCHEMA)
+ return f"{tool_name}: arguments must be a JSON object, got {type(args).__name__}"
+ validator_cls = jsonschema.validators.validator_for(schema)
+ validator = validator_cls(schema)
errors = sorted(
validator.iter_errors(args), key=lambda error: [str(p) for p in error.path]
)
@@ -190,7 +146,7 @@ def validate_finish_args(args: dict[str, Any]) -> str | None:
if error.validator == "oneOf" and isinstance(error.schema, dict):
message = str(error.schema.get("description") or message)
location = "".join(f"[{p}]" if isinstance(p, int) else f".{p}" for p in error.path)
- return f"finish: invalid arguments{f' at {location}' if location else ''}: {message}"
+ return f"{tool_name}: invalid arguments{f' at {location}' if location else ''}: {message}"
def read_ref_status(tool_name: str, result: ToolResult) -> list[dict[str, Any]] | None:
@@ -205,65 +161,14 @@ def invalid_finish_message(error: str) -> str:
"""Model-facing text for a rejected ``finish`` call; the episode goes on."""
return (
f"{error}. finish was not accepted and the episode continues. Correct "
- 'form: {"refs": [{"document_id": "...", "section_path": "..."} or '
- '{"document_id": "...", "chunk_id": "..."}], "notes": "..."} — each '
- "ref names exactly one of section_path or chunk_id. Call finish again "
+ 'form: {"notes": "..."} — Call finish again '
"with fixed arguments, or keep exploring with the corpus tools."
)
-def finish_refs_from_args(args: dict[str, Any] | None) -> list[dict[str, Any]] | None:
- """Return cited refs, or ``None`` when finish omitted the ``refs`` key.
-
- ``None`` means the agent did not specify refs (never called finish, or
- called finish with ``{}``). An explicit ``refs: []`` stays an empty list
- and must not be collapsed into ``None``.
- """
- if not isinstance(args, dict) or "refs" not in args or args.get("refs") is None:
- return None
- return normalize_finish_refs(args.get("refs"))
-
-
-def normalize_finish_refs(raw: Any) -> list[dict[str, Any]]:
- """Keep only dict items with a non-empty ``document_id`` from a raw ``finish.refs``."""
- if not isinstance(raw, list):
- return []
- normalized: list[dict[str, Any]] = []
- for item in raw:
- if isinstance(item, dict) and str(item.get("document_id") or "").strip():
- normalized.append(item)
- return normalized
-
-
-def budget_status_line(budget: EpisodeBudget) -> str:
- """One-line remaining-budget summary appended to a tool observation.
-
- Neither harness previously surfaced ``EpisodeBudget``'s own counters
- (``steps_used``/``max_steps``, ``tokens_used``/``token_limit``,
- elapsed/wall_clock) to the model at all — it had no way to tell "I'm on
- step 3 of 12" from "I'm on step 11 of 12", so it could not self-regulate
- when to stop exploring and call ``finish``. This exposes the same
- ``EpisodeBudget.snapshot()`` data already used for the hard cutoff,
- reused as a soft signal the model can read every turn.
- """
- snap = budget.snapshot()
+def invalid_pick_message(error: str) -> str:
+ """Model-facing text for a rejected ``corpus.pick`` call; the pick phase goes on."""
return (
- f"[budget: steps {snap['steps_used']}/{snap['max_steps']}, "
- f"tokens {snap['tokens_used']}/{snap['token_limit']}, "
- f"elapsed {snap['elapsed_seconds']:.0f}s/{snap['wall_clock_seconds']:.0f}s]"
+ f"{error}. Nothing was picked and the pick phase continues. Correct "
+ 'form: {"pick": ["", ...]} or {"pick": []} — call corpus_pick again.'
)
-
-
-def dedup_refs(refs: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Dedup by ``(document_id, chunk_id)``, keeping first-seen order."""
- seen: set[tuple[str, str]] = set()
- deduped: list[dict[str, Any]] = []
- for ref in refs:
- document_id = str(ref.get("document_id") or "").strip()
- chunk_id = str(ref.get("chunk_id") or "").strip()
- key = (document_id, chunk_id)
- if not document_id or not chunk_id or key in seen:
- continue
- seen.add(key)
- deduped.append(ref)
- return deduped
diff --git a/packages/shared-python/shared/services/retrieval/agent_explore/types.py b/packages/shared-python/shared/services/retrieval/agent_explore/types.py
index 7d16092d7..beb71ad98 100644
--- a/packages/shared-python/shared/services/retrieval/agent_explore/types.py
+++ b/packages/shared-python/shared/services/retrieval/agent_explore/types.py
@@ -5,6 +5,8 @@
from dataclasses import dataclass, field
from typing import Any
+from shared.services.retrieval.agent_explore.evidence_pool import Candidate
+
@dataclass
class AgentStep:
@@ -16,22 +18,23 @@ class AgentStep:
observation_text: str
error: str | None
elapsed_ms: int
- tokens_used_delta: int
- tokens_used_total: int
+ round_index: int
# corpus.read only: one {ref, status, reason?} entry per requested ref,
# kept whole in the trace (observation_text is capped there).
ref_status: list[dict[str, Any]] | None = None
+ candidates: list[str] | None = None
+ # corpus.pick only.
+ picked: list[str] | None = None
+ pick_rejected: list[dict[str, str]] | None = None
@dataclass
class EpisodeResult:
"""Everything ``bridge.py`` / the route need after the episode ends."""
- refs: list[dict[str, Any]]
- notes: str
+ pool: list[Candidate]
steps: list[AgentStep] = field(default_factory=list)
stop_reason: str = "finished"
tokens_used: int = 0
model_name: str = ""
- agent_selected_refs: list[dict[str, Any]] | None = None
- fallback_refs: list[dict[str, Any]] = field(default_factory=list)
+ queried_tables: dict[tuple[str, str], str] = field(default_factory=dict)
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_OVERVIEW.md b/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_OVERVIEW.md
new file mode 100644
index 000000000..09fd4de50
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_OVERVIEW.md
@@ -0,0 +1,27 @@
+# Knowhere corpus
+
+A namespace holds documents. Each document is a tree of sections that follows
+its headings. A section is addressed by document_id + section_path;
+section_path joins the heading titles from the top level down with " / ".
+
+A section can own one body chunk:
+- text: the section's own text (Word, Markdown, Excel, ...)
+- page: PDF pages belonging to the section, as text plus a page image
+A section with child sections may have no body of its own; its content is in
+its children.
+
+Images and tables are separate chunks, addressed by document_id + chunk_id.
+Result rows show the section that contains them.
+
+Example:
+
+doc_7f3a Hypertension_Guideline.pdf
+├── 1 Overview [text]
+├── 2 Treatment (no body; see children)
+│ ├── 2 Treatment / 2.1 Lifestyle [text]
+│ └── 2 Treatment / 2.2 Drugs [text] contains table chunk_id=tbl_12
+└── 3 Follow-up [page p.14-15]
+
+Addresses from this example:
+- {document_id: "doc_7f3a", section_path: "2 Treatment / 2.2 Drugs"}
+- {document_id: "doc_7f3a", chunk_id: "tbl_12"}
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md b/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md
deleted file mode 100644
index 225848378..000000000
--- a/packages/shared-python/shared/services/retrieval/agent_tools/CORPUS_SCHEMA.md
+++ /dev/null
@@ -1,216 +0,0 @@
-# Knowhere Corpus Schema (agent-facing)
-
-Single source of truth for how an agent should understand and explore a
-Knowhere corpus through the tools in this package. This text is intended to
-be shipped verbatim as both the API `/mcp` server instructions and the
-`agent_explore` system prompt — do not duplicate it elsewhere; edit here.
-
-This describes the **published, DB-served corpus** (`documents`,
-`document_sections`, `document_chunks`, `graph_nodes`, `graph_edges`) that
-the tools below are designed to query. It is not the on-disk parse artifact schema
-(`chunks.json` / `doc_nav.json` on disk use different join keys and a
-different path separator) — that schema is for the parser pipeline, not for
-this tool set.
-
----
-
-## 1. Corpus model
-
-```text
-Namespace
- └─ Document (document_id, source_file_name, parse_track)
- └─ Section (section_id, parent_section_id, section_path, section_level, summary)
- └─ Chunk (chunk_id, chunk_type, content, chunk_metadata)
-```
-
-- **Namespace**: the retrieval scope. One namespace can hold documents parsed
- by different tracks (see §2) — do not assume a namespace is uniform.
-- **Document**: `document_id` is the stable identifier for every other tool
- call. `parse_track` is `page_memory` or `chunk` (see §2). Tool hit lines
- print `document_id` in parentheses after the filename (or alone, for
- `node_filter`); copy that value into later calls. Never pass
- `source_file_name` as `document_id`.
-- **Section**: the navigable tree. `section_path` segments are joined with
- `" / "` (space-slash-space), one segment per heading/synthetic level.
-- **Chunk**: the retrieval unit. `chunk_type` is `text`, `page`, `image`, or
- `table`. A section owns **at most one body chunk** (`text` or `page`) —
- but which sections get one is track-dependent (see §2). `image`/`table`
- chunks are **not** attached to the section they are conceptually "in" —
- see §3.
-
-Document-level relationships (`related` edges, including entity-overlap
-scoring) exist in a separate graph — see §4.
-
-## 2. Body chunks: `text` vs `page`, and the `SAME-AS` pointer
-
-A namespace can mix both tracks across documents (e.g. one PDF on the
-`page_memory` track next to one DOCX/XLSX on the `chunk` track). Treat
-`text` and `page` as the same *role* — both are a section's own body — but
-they differ in shape, and in **which sections get a body chunk at all**:
-
-- **`chunk` track** (`text` chunks, e.g. DOCX/XLSX/MD): a section owns a
- body chunk if it has any content of its own directly under its heading,
- before any child heading — this can be **any section, leaf or not**. Do
- not assume a section with children has no body text of its own; check
- whether it owns a chunk instead of assuming from tree position.
-- **`page_memory` track** (`page` chunks, PDF): only **leaf** sections own a
- body chunk. Internal/structural sections never carry a `page` chunk
- themselves — their summaries aggregate from their leaf descendants.
-
-A section owns **at most one** body chunk either way (verified against
-real published data: a section with children and a section without both
-had exactly 0 or 1, never more). So a "chunk" and "the section that owns
-it" are the same unit, not two different granularities — there is no
-separate finer-or-coarser level to choose between. The only structural
-sections with no chunk of their own are `chunk`-track sections whose
-entire content lives in their descendants.
-
-For `page` chunks specifically: one leaf section's body may span one or
-more physical pages. A page's text is stored **once**, under whichever leaf
-is first in reading order to cover that page (the "owner"). Every other
-leaf section that also covers that physical page has, in place of the
-text, a literal marker:
-
- ```text
- [SAME-AS p]
- ```
-
- This is a pointer, not a preview. If you need that page's actual text,
- resolve the marker by reading the owner section's chunk at that page
- number — do not treat the marker's absence of text as "this section has
- no content there." `page` chunks also carry `page_nums` (all physical
- pages they cover) and `page_assets` (rendered page-citation screenshots —
- these are references for citation, not separate `image` chunks).
-
- **Format trap**: `` inside the marker is written
- verbatim by the parser and stored as-is — it is the on-disk path
- (`"///..."`, plain `/`, filename
- included), **not** the DB `section_path` you get back from
- `corpus.outline` / `corpus.node_filter` / `corpus.recall` (which is
- `" / "`-joined and excludes the filename). Do not string-match the
- marker directly against a DB `section_path`. Convert it first —
- `section_path_from_chunk_path()` in `search/lexical_text.py` already
- does this conversion and is the function to reuse when implementing
- marker resolution, not a new one.
-
-## 3. Asset chunks: `image` / `table`, and `connect_to`
-
-`image` and `table` chunks are **not children of the section they visually
-belong to**. In the DB they are parked under their document's synthetic
-`Root` section. The real association to a body section is the `connect_to`
-list on the **body chunk**, not a location on the asset:
-
-- `relation: "embeds"` — the body chunk that owns/embeds this asset inline.
-- `relation: "related"` — another body chunk that shares the same source
- page as a page-track asset, without owning it (`same_as_owner` may name
- the owning section).
-
-This link is **one-directional** (body → asset). There is no stored
-asset → body back-link. `corpus.grep`, `corpus.recall`, and `corpus.assets`
-resolve the host through that body link and print the hosting
-`section_path` on the asset row. If no host is found, the row keeps
-`Root` and is marked as having no host section. Use `corpus.assets`
-`host_of` to list every host, rather than reading the asset's stored
-path.
-
-## 4. Document graph
-
-Today the graph only has **document-level** nodes (`node_kind='document'`)
-and undirected `related` edges between documents. Edge scoring prefers
-**typed-entity overlap** between the two documents' aggregated `entities`
-first; it only falls back to free-form TF-IDF keyword overlap when either
-document lacks entities. Either way the edge carries which terms matched
-(`properties.shared_entities` or `properties.shared_keywords`). There are
-no section-level or entity-level graph *nodes* — an entity is not itself a
-queryable node, and there is no entity-to-entity or entity-to-chunk edge,
-only this document-to-document rollup. Use `neighbors` for "what else is
-like this document" (and to see which shared terms justify that link), not
-for anything finer-grained than a document pair.
-
-## 5. Reserved / not yet available
-
-- **Vector**: not available yet. `recall` today is path + content BM25.
- Exact-string lookup is `grep`.
-
-## 6. Tools and how they work together
-
-Every tool exists to find the `section_path`s (or asset `chunk_id`s) that
-answer the query, then hand them to `corpus.read`. Names below are the
-registered names (`corpus.outline`). MCP clients call those names as-is.
-Exact call parameters live only in each tool's own schema/description
-(ask for that, do not memorize names here) — this section is the
-collaboration map: which tools narrow a search space, which produce hits,
-and what to do with either kind of result.
-
-```mermaid
-flowchart LR
- subgraph MapNarrowing["Narrow a map (overview / structural predicate)"]
- outline["corpus.outline"]
- node_filter["corpus.node_filter"]
- end
- subgraph LeafHits["Find hits (exact string / fuzzy / asset listing)"]
- grep["corpus.grep"]
- recall["corpus.recall"]
- assets["corpus.assets"]
- end
- outline -->|narrows scope for| LeafHits
- node_filter -->|narrows scope for| LeafHits
- MapNarrowing -->|section_path| read["corpus.read"]
- LeafHits -->|section_path or chunk_id| read
- read -->|table too large| query_table["corpus.query_table"]
- read --> finish
- query_table --> finish
- list_documents["corpus.list_documents (namespace inventory, standalone)"]
- neighbors["corpus.neighbors (related documents, standalone)"]
-```
-
-**`corpus.outline` and `corpus.node_filter`** are the two map-narrowing
-tools: both take one or more scope targets (a whole document, or a section
-and everything under it) and return the *same* row shape — every
-`section_path` in that scope, indented by level, with title/summary.
-`corpus.outline` returns that unconditionally (the outline of what is
-there); `corpus.node_filter` returns it only for the branches around a
-structural predicate match (marked as a hit), still shown as full context
-(ancestors + the entire matched subtree), not a bare list of matches. Use
-a returned `section_path` to `corpus.read` a branch, or as the scope of a
-`corpus.grep` / `corpus.recall` / `corpus.assets` call. Neither is
-callable on a single leaf section with nothing under it; `corpus.read`
-that directly instead.
-
-**`corpus.grep`, `corpus.recall`, and `corpus.assets`** are the leaf-hit
-tools: they search *within* a scope (the whole corpus, or one narrowed by
-a prior `corpus.outline`/`corpus.node_filter` call) and return rows
-pointing at specific sections/chunks, not a map. `corpus.grep` is exact
-string (several terms are any-match / OR, rows stay in document order,
-not relevance order); `corpus.recall` is fuzzy ranked search;
-`corpus.assets` lists image/table chunks by type — on its own it is an
-unfiltered listing with no way to judge relevance, so pair it with a
-prior `corpus.grep`/`corpus.recall` hit (scope to that hit's section, or
-reverse-resolve the hit's own asset references) rather than browsing
-every asset in a document. Grep does not scan table-cell HTML — a table
-only surfaces from `corpus.grep`/`corpus.recall` via its published
-summary/keywords/caption, never by a literal cell value; find the table
-via `corpus.assets` (or a hit on the surrounding body text) and use
-`corpus.read`/`corpus.query_table` to inspect its cells. Every hit row
-names a `document_id` plus either a `section_path` (body hits) or a
-`chunk_id` (image/table hits). Image/table rows show the hosting
-section, not the asset's stored `Root` path — read whichever address
-applies with `corpus.read` next, then decide pick or no-pick for that
-ref immediately from `corpus.read`'s own per-ref status.
-
-**`corpus.read`** is where body content actually gets consumed: it accepts
-many refs at once (mixing map-tool section_paths and leaf-hit chunk_ids),
-and reports each ref's outcome separately so a failed ref does not hide a
-successful one. A table `corpus.read` reports as too large points at
-**`corpus.query_table`** for cell-level `SELECT`s over that same table.
-
-**`corpus.list_documents`** and **`corpus.neighbors`** stand outside this
-flow: `corpus.list_documents` is a namespace-wide inventory (only for an
-explicit "list/inventory the documents" request, never a cold-start for a
-content question); `corpus.neighbors` returns document-level related
-edges (§4), unscoped by section.
-
-General rule: narrow with `corpus.outline`/`corpus.node_filter` before
-searching a large corpus; prefer `corpus.grep` over `corpus.recall` when
-you know the exact string you are looking for, and fall back to
-`corpus.recall` only for genuinely fuzzy questions.
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/__init__.py b/packages/shared-python/shared/services/retrieval/agent_tools/__init__.py
index 5583243f4..42b2e3573 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/__init__.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/__init__.py
@@ -2,7 +2,7 @@
See ``.cursor/plans/agentic_corpus_explore_retrieval_c2c4ea21.plan.md`` (Phase 2)
for the design. Tools in this package query the published, DB-served corpus
-described in ``CORPUS_SCHEMA.md`` (``documents`` / ``document_sections`` /
+described in ``CORPUS_OVERVIEW.md`` (``documents`` / ``document_sections`` /
``document_chunks`` / ``graph_nodes`` / ``graph_edges``) — not the on-disk
parse artifacts.
@@ -18,6 +18,8 @@
from shared.services.retrieval.agent_tools.registry import (
REGISTRY,
+ Decision,
+ ReadableAddresses,
ToolBudget,
ToolContext,
ToolRegistry,
@@ -26,7 +28,7 @@
capped_limit,
register_tool,
)
-from shared.services.retrieval.agent_tools.schema_doc import load_corpus_schema_text
+from shared.services.retrieval.agent_tools.schema_doc import load_corpus_overview_text
# Register every ``corpus.*`` tool into ``REGISTRY`` as an import side effect:
# each module under ``tools/`` calls ``@register_tool`` at import time. Kept
@@ -40,12 +42,14 @@
__all__ = [
"_tools",
"REGISTRY",
+ "Decision",
+ "ReadableAddresses",
"ToolBudget",
"ToolContext",
"ToolRegistry",
"ToolResult",
"ToolSpec",
"capped_limit",
- "load_corpus_schema_text",
+ "load_corpus_overview_text",
"register_tool",
]
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/asset_hosts.py b/packages/shared-python/shared/services/retrieval/agent_tools/asset_hosts.py
index 337793b97..55d9ee92e 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/asset_hosts.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/asset_hosts.py
@@ -2,7 +2,7 @@
Image/table chunks are stored under their document's ``Root`` section; the
real placement is ``chunk_metadata.connect_to`` on the body chunk that embeds
-them (``CORPUS_SCHEMA.md`` §3). Hosts are keyed by ``(document_id,
+them. Hosts are keyed by ``(document_id,
asset chunk_id)`` — chunk ids are content hashes and repeat across
documents — and read only from each document's current revision. When an
asset has several hosts they are kept in body order, so every tool that
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/explore_mount.py b/packages/shared-python/shared/services/retrieval/agent_tools/explore_mount.py
index c4cbaa050..b4282ac28 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/explore_mount.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/explore_mount.py
@@ -1,8 +1,9 @@
-"""Shared explore-phase mounting for ``corpus.grep`` and ``corpus.recall`` hits.
+"""Shared explore-phase mounting for ``corpus.recall`` hits.
Table/image hits are rendered with the same explore table/image functions
``corpus.read`` uses. Body hits that already list ``connect_to`` targets get
those assets rendered and labeled as mounted, not as the body match itself.
+Each returned row lists the asset ids it rendered in ``mounted_chunk_ids``.
"""
from __future__ import annotations
@@ -101,6 +102,7 @@ async def mount_explore_hits(
chunk_id = str(row.get("chunk_id") or "")
chunk_type = normalize_chunk_type(row.get("chunk_type"))
enriched_self = by_id.get(chunk_id, row)
+ mounted_ids: list[str] = []
if chunk_type == "table":
row["rendered"] = render_explore_table(
enriched_self, char_budget=char_budget
@@ -119,11 +121,14 @@ async def mount_explore_hits(
body = render_explore_table(target, char_budget=char_budget)
if body:
parts.append(f"mounted table chunk_id={target_id}:\n{body}")
+ mounted_ids.append(target_id)
elif target_type == "image":
body = _image_display_content(target)
if body:
parts.append(f"mounted image chunk_id={target_id}:\n{body}")
+ mounted_ids.append(target_id)
_append_image_media(target, media, seen_urls)
row["rendered"] = "\n\n".join(parts)
+ row["mounted_chunk_ids"] = mounted_ids
mounted.append(row)
return mounted, media
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/registry.py b/packages/shared-python/shared/services/retrieval/agent_tools/registry.py
index 282302f60..762571d61 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/registry.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/registry.py
@@ -14,7 +14,7 @@
from __future__ import annotations
import logging
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Mapping, Set
from contextlib import AbstractAsyncContextManager
from dataclasses import dataclass, field, replace
from typing import Any
@@ -73,14 +73,14 @@ class ToolBudget:
below, and notes it in ``ToolResult.text`` when the request was clamped.
Tools that promise a complete, non-truncated set by contract
(``node_filter``, ``outline``) do not apply this budget to their
- matched-set cardinality — see ``CORPUS_SCHEMA.md`` §6.
+ matched-set cardinality.
``max_chars`` caps the rendered ``ToolResult.text`` before it enters LLM
context. Applied in ``agent_explore.shared.tool_message_content`` (not
inside individual tools) so ``read`` can return full body text from the
tool while the harness still bounds what the model sees per turn. Aligned
with final evidence packing via ``EVIDENCE_TEXT_CHAR_BUDGET``
- (12_000).
+ (15_000).
"""
max_chars: int = EVIDENCE_TEXT_CHAR_BUDGET
@@ -96,6 +96,20 @@ def capped_limit(requested: int, budget: ToolBudget) -> int:
return min(requested, budget.max_items)
+# ``(document_id, section_path)`` and ``(document_id, chunk_id)`` addresses
+# an ``agent_explore`` episode received in results of earlier rounds.
+ReadableAddresses = Set[tuple[str, str]]
+
+
+@dataclass(frozen=True)
+class Decision:
+ """One body chunk an ``agent_explore`` episode read and then picked or not."""
+
+ read_round: int
+ picked_handle: str | None
+ handle: str
+
+
@dataclass
class ToolContext:
"""Per-call execution context. One instance is built per tool dispatch."""
@@ -116,6 +130,10 @@ class ToolContext:
# dropping summaries does not fit then fails that one call as "too
# large to map" instead of guessing.
query: str = ""
+ # ``corpus.read`` gates, set only by ``agent_explore``. ``None`` means no
+ # gate: an external MCP client calling one tool has no episode.
+ readable: ReadableAddresses | None = None
+ decided: Mapping[tuple[str, str], Decision] | None = None
@dataclass
@@ -208,7 +226,7 @@ def validate_tool_args(spec: ToolSpec, args: dict[str, Any]) -> str | None:
Every tool schema declares ``additionalProperties: false`` at every
object level (registered tools are expected to keep this true — see
- ``CORPUS_SCHEMA.md`` §6), so an unknown key anywhere in the argument
+ each tool schema), so an unknown key anywhere in the argument
tree fails here instead of being silently dropped or ignored downstream.
The returned message names the offending path, the tool's legal
top-level argument names, and — for an unknown-key error — the exact
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/schema_doc.py b/packages/shared-python/shared/services/retrieval/agent_tools/schema_doc.py
index 37569589f..ffd2983ab 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/schema_doc.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/schema_doc.py
@@ -1,10 +1,7 @@
-"""Single loader for ``CORPUS_SCHEMA.md`` — the agent-facing corpus schema text.
+"""Loader for ``CORPUS_OVERVIEW.md`` — the agent-facing corpus overview.
-Both harnesses (Phase 3) read through this function instead of the file
-directly, so there is exactly one place that resolves the path: the API
-``/mcp`` server's ``instructions`` and ``agent_explore``'s system prompt must
-stay byte-identical for the shared schema portion (see the module docstring
-at the top of ``CORPUS_SCHEMA.md`` — "do not duplicate it elsewhere").
+The API ``/mcp`` server instructions and ``agent_explore`` system prompt
+share this text. Tool-specific usage lives on each tool's own description.
"""
from __future__ import annotations
@@ -12,10 +9,10 @@
from functools import lru_cache
from pathlib import Path
-_SCHEMA_PATH = Path(__file__).with_name("CORPUS_SCHEMA.md")
+_OVERVIEW_PATH = Path(__file__).with_name("CORPUS_OVERVIEW.md")
@lru_cache(maxsize=1)
-def load_corpus_schema_text() -> str:
- """Return the verbatim contents of ``CORPUS_SCHEMA.md``."""
- return _SCHEMA_PATH.read_text(encoding="utf-8")
+def load_corpus_overview_text() -> str:
+ """Return the verbatim contents of ``CORPUS_OVERVIEW.md``."""
+ return _OVERVIEW_PATH.read_text(encoding="utf-8")
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/section_path_lookup.py b/packages/shared-python/shared/services/retrieval/agent_tools/section_path_lookup.py
index f15735181..54858a2b4 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/section_path_lookup.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/section_path_lookup.py
@@ -1,6 +1,6 @@
"""Shared section_path resolution for agent tools and harness bridge.
-``corpus.read`` and ``resolve_finish_refs`` both need to turn an agent-supplied
+``corpus.read`` needs to turn an agent-supplied
``section_path`` into one canonical DB path. Agents often cite a suffix (e.g.
``3 工程地质 / 3.2 覆盖层``) while the stored path includes ancestors
(``附件目录 / 3 工程地质 / 3.2 覆盖层``). Exact match alone fails silently
@@ -15,7 +15,29 @@
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
-from shared.services.retrieval.search.lexical_text import normalize_section_path
+from shared.services.retrieval.agent_tools.registry import ReadableAddresses
+from shared.services.retrieval.search.lexical_text import (
+ normalize_section_path,
+ split_section_path,
+)
+
+
+def section_path_received(
+ readable: ReadableAddresses, document_id: str, section_path: str
+) -> bool:
+ """Whether ``section_path`` runs contiguously inside a received path of the document.
+
+ Ancestor and suffix refs pass, matching how ``corpus.read`` resolves paths.
+ """
+ wanted = split_section_path(section_path)
+ size = len(wanted)
+ for seen_document_id, seen_path in readable:
+ if seen_document_id != document_id:
+ continue
+ parts = split_section_path(seen_path)
+ if any(parts[i : i + size] == wanted for i in range(len(parts) - size + 1)):
+ return True
+ return False
def paths_matching_section_ref(normalized: str, candidate_paths: list[str]) -> list[str]:
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/snippet.py b/packages/shared-python/shared/services/retrieval/agent_tools/snippet.py
index dd7619f90..f5284858b 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/snippet.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/snippet.py
@@ -122,9 +122,11 @@ def format_row(row: Mapping[str, Any]) -> str:
header = f"{indent}- [{row['kind']}]"
if row["title"]:
header += f" {row['title']}"
- header += f" | document_id={row['document_id']} section_path={row['section_path']}"
+ header += f" | document_id={row['document_id']}"
if row["hosted"] is False:
- header += " (no host section)"
+ header += " (not in any section)"
+ else:
+ header += f" section_path={row['section_path']}"
if row["chunk_id"]:
header += f" chunk_id={row['chunk_id']}"
if row["score"] is not None:
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/assets.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/assets.py
index d931b9389..06d14eb90 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/assets.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/assets.py
@@ -1,7 +1,7 @@
"""``corpus.assets`` — forward asset search and reverse asset -> hosts lookup.
Image/table chunks are parked under their document's synthetic ``Root``
-section in the DB (§3 of ``CORPUS_SCHEMA.md``); the real association to a
+section in the DB; the real association to a
body section lives in ``chunk_metadata.connect_to`` on the *body* chunk.
Both directions resolve hosts through ``agent_tools.asset_hosts`` (keyed by
document + chunk, current revision only), the same resolver ``corpus.grep``
@@ -54,16 +54,13 @@
@register_tool(
name="corpus.assets",
description=(
- "Forward search for image/table chunks by type/query, or reverse "
- "lookup: given asset chunk_ids (host_of), find which body "
- "section(s) embed or reference them via connect_to. Rows show the "
- "hosting section_path; an asset no body section embeds keeps Root "
- "and is marked '(no host section)'. A forward search alone only "
- "lists candidate assets — it does not tell you which ones matter "
- "for the current question. Pair it with a prior corpus.recall/"
- "corpus.grep hit: scope this call to that hit's section, or "
- "reverse-resolve the hit's own connect_to targets via host_of "
- "instead of browsing every asset in a document."
+ "Find image and table chunks by type or query, or, given asset "
+ "chunk_ids (host_of), list the sections that contain them. Rows "
+ "show the containing section_path; an asset that no section "
+ "contains is marked (not in any section). A plain listing does not "
+ "tell you which assets matter: scope it to a section from a prior "
+ "corpus.grep/corpus.recall hit, or use host_of on chunk_ids you "
+ "already have."
),
json_schema={
"type": "object",
@@ -156,10 +153,24 @@ async def _forward_search(
continue
metadata = chunk.chunk_metadata if isinstance(chunk.chunk_metadata, dict) else {}
summary = str(metadata.get("summary") or "").strip()
- keywords = metadata.get("keywords") or []
+ asset_title = str(metadata.get("asset_title") or "").strip()
+ entities = metadata.get("entities")
+ if isinstance(entities, list) and entities:
+ entity_terms = [
+ str(item.get("text", "")).strip()
+ for item in entities
+ if isinstance(item, dict) and str(item.get("text", "")).strip()
+ ]
+ else:
+ entity_terms = []
+ keywords = entity_terms or (metadata.get("keywords") or [])
if query:
haystack = " ".join(
- [summary.lower(), " ".join(str(k).lower() for k in keywords)]
+ [
+ summary.lower(),
+ asset_title.lower(),
+ " ".join(str(k).lower() for k in keywords),
+ ]
)
if query not in haystack:
continue
@@ -243,7 +254,7 @@ async def _reverse_lookup(
lines = [format_row(row) for row in rows]
lines.extend(
- f"- {target_id}: no host section found in this scope (asset stays under Root)"
+ f"- {target_id}: not contained in any section within this scope"
for target_id in unhosted
)
return ToolResult(
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/grep.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/grep.py
index 8789ddde9..2815cc0cc 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/grep.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/grep.py
@@ -4,12 +4,13 @@
current revision. That expression is the existing term trigram index.
The field is already published (body + filename + section path; tables
use summary/keywords/caption; images use their description).
-Reports a total match count (over the full in-scope corpus, not just the
-returned page) alongside capped snippets.
-Matching table/image chunks, and body chunks that list ``connect_to``
-targets, are mounted through the shared explore mount so the tool result
-includes rendered table/image content.
+Several terms are matched in one query (OR). Hits are then split back
+into one block per term, one row per matching section (first chunk in
+document / sort order). Terms with fewer matching sections are listed
+first so a rare term is not buried by a generic one. Visible rows stop
+at the existing ``limit``; leftover sections become one fold line per
+document with a reusable ``scope``.
Snippets are built by the shared ``agent_tools.snippet.build_snippet`` (head
+ first-match window + tail, ``...``-joined, overlap-merged). Rows render
@@ -25,13 +26,12 @@
from shared.services.retrieval.corpus_storage import CorpusStorage
import re
+from collections import defaultdict
from typing import Any
from sqlalchemy import func, or_, select
-from shared.models.database.job_result import JobResult
from shared.services.retrieval.agent_tools.asset_hosts import host_paths_for_hits
-from shared.services.retrieval.agent_tools.explore_mount import mount_explore_hits
from shared.services.retrieval.agent_tools.registry import (
ToolContext,
ToolResult,
@@ -91,23 +91,78 @@ def _term_search(terms: list[str], namespace: str = "default") -> tuple[re.Patte
)
+def _term_span(text: str, term: str) -> tuple[int, int] | None:
+ match = re.search(re.escape(term), text, flags=re.IGNORECASE)
+ return match.span() if match else None
+
+
+def _sections_for_term(
+ hits: list[dict[str, Any]], term: str, context_chars: int
+) -> list[dict[str, Any]]:
+ """Keep the first hit per ``(document_id, section_path)`` for this term.
+
+ ``hits`` must already be in document / sort order.
+ """
+ kept: dict[tuple[str, str], dict[str, Any]] = {}
+ order: list[tuple[str, str]] = []
+ for hit in hits:
+ text = str(hit["term_search_text"] or "")
+ span = _term_span(text, term)
+ if span is None:
+ continue
+ key = (str(hit["document_id"]), str(hit["section_path"] or ""))
+ if key in kept:
+ continue
+ kept[key] = {**hit, "snippet": build_snippet(text, span, hit_context=context_chars)}
+ order.append(key)
+ return [kept[key] for key in order]
+
+
+def _fold_line(source_file_name: str, document_id: str, leftover: int) -> str:
+ return (
+ f"{source_file_name} 还有 {leftover} 节命中 — "
+ f"scope=[{{document_id: {document_id}}}]"
+ )
+
+
+def _build_hit_row(
+ hit: dict[str, Any],
+ host_paths: dict[tuple[str, str], tuple[str, bool]],
+) -> dict[str, Any]:
+ chunk_type = str(hit["chunk_type"] or "").strip()
+ is_asset = chunk_type in ASSET_CHUNK_TYPES
+ key = (str(hit["document_id"]), str(hit["chunk_id"]))
+ section_path, hosted = (
+ host_paths[key] if is_asset and key in host_paths else (hit["section_path"], None)
+ )
+ return build_row(
+ kind=chunk_type or "text",
+ document_id=hit["document_id"],
+ section_path=section_path,
+ title=hit["source_file_name"],
+ chunk_id=hit["chunk_id"] if is_asset else None,
+ snippet=hit["snippet"],
+ hosted=hosted if is_asset else None,
+ )
+
+
@register_tool(
name="corpus.grep",
description=(
- "Exact string search against published term_search_text "
- "(body text, image descriptions, table summaries/keywords, plus "
- "filename and section path). Returns the total number of matching "
- "chunks plus a capped list of snippet rows (score-free — this is an "
- "exact match, not a ranked search; use corpus.recall for ranking). "
- "Table and image hits include rendered content; body hits include "
- "any connected table or image — read the hit's document_id + "
- "section_path with corpus.read next. Provide 'pattern' for one "
- "term, or 'patterns' for several candidate terms OR'd together in "
- "this single call (e.g. synonyms) — issue one call with multiple "
- "terms instead of several parallel corpus.grep calls for different "
- "terms in the same turn. Several terms are any-match (OR), and "
- "rows stay in document order — this tool does not rank by "
- "relevance. At least one of pattern/patterns is required."
+ "Exact string search over body text, image descriptions, table "
+ "summaries and keywords, file names and section paths. Table cell "
+ "values are not searched. Requires pattern or patterns — do not "
+ "call without a term. Each term is a separate block of section "
+ "rows (score-free — this is an exact match, not a ranked search; "
+ "use corpus.recall for ranking). Terms with fewer matching "
+ "sections are listed first. Rows past limit are folded per "
+ "document with a reusable scope — copy that scope onto a later "
+ "corpus.grep call to search only that document. Table and image "
+ "hits are address + snippet rows; read the hit's document_id + "
+ "section_path (or chunk_id for image/table) with corpus.read "
+ "next. Provide 'pattern' for one term, or 'patterns' for several "
+ "terms in this single call instead of several parallel "
+ "corpus.grep calls. At least one of pattern/patterns is required."
),
json_schema={
"type": "object",
@@ -120,9 +175,10 @@ def _term_search(terms: list[str], namespace: str = "default") -> tuple[re.Patte
"type": "array",
"items": {"type": "string"},
"description": (
- "Several search terms OR'd together in this one call. "
- "A chunk matches if any term hits. Results stay in "
- "document order, not relevance order."
+ "Several search terms in this one call. Each term is "
+ "listed as its own block. Terms with fewer matching "
+ "sections are shown first. This tool does not rank "
+ "by relevance."
),
},
"scope": SCOPE_SCHEMA,
@@ -139,7 +195,10 @@ def _term_search(terms: list[str], namespace: str = "default") -> tuple[re.Patte
"type": "integer",
"minimum": 1,
"default": _DEFAULT_LIMIT,
- "description": "Max rows to return. The total match count is still reported.",
+ "description": (
+ "Max section rows to show. Further matching sections "
+ "are folded per document with a reusable scope."
+ ),
},
},
"anyOf": [
@@ -179,7 +238,7 @@ async def grep(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
section_path_col=corpusStorage.DocumentSection.section_path,
)
- compiled, term_filter = _term_search(terms, ctx.namespace)
+ _compiled, term_filter = _term_search(terms, ctx.namespace)
# Match the indexed haystack first so PostgreSQL can use
# idx_document_chunks_term_trgm. Section is joined here (not later) so a
# scope's section-subtree condition can apply inside this same CTE.
@@ -191,9 +250,6 @@ async def grep(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
corpusStorage.DocumentChunk.job_result_id,
corpusStorage.DocumentChunk.chunk_type,
corpusStorage.DocumentChunk.term_search_text,
- corpusStorage.DocumentChunk.content,
- corpusStorage.DocumentChunk.file_path,
- corpusStorage.DocumentChunk.chunk_metadata,
corpusStorage.DocumentChunk.section_id,
corpusStorage.DocumentChunk.sort_order,
corpusStorage.DocumentSection.section_path,
@@ -222,113 +278,86 @@ async def grep(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
matched.c.document_id,
matched.c.chunk_type,
matched.c.term_search_text,
- matched.c.content,
- matched.c.file_path,
- matched.c.chunk_metadata,
- matched.c.job_result_id,
- JobResult.job_id,
- JobResult.document_metadata["result_raw_prefix"].as_string(),
+ matched.c.sort_order,
matched.c.section_path,
corpusStorage.Document.source_file_name,
- func.count().over().label("total_matches"),
)
.select_from(matched)
.join(corpusStorage.Document, corpusStorage.Document.document_id == matched.c.document_id)
- .outerjoin(JobResult, JobResult.id == CorpusRevisionContext.build_revision_column(corpusStorage.Document))
.where(
corpusStorage.Document.user_id == corpusStorage.resolve_owner(ctx.user_id),
corpusStorage.Document.namespace == ctx.namespace,
corpusStorage.Document.status == "active",
CorpusRevisionContext.build_revision_column(corpusStorage.Document) == matched.c.job_result_id,
)
- .order_by(matched.c.document_id, matched.c.sort_order)
- .limit(limit)
+ .order_by(matched.c.document_id, matched.c.sort_order, matched.c.chunk_id)
)
matched_rows = (await ctx.db.execute(rows_stmt)).all()
- total_matches = int(matched_rows[0][-1]) if matched_rows else 0
- results: list[dict[str, Any]] = []
+ hits: list[dict[str, Any]] = []
for (
chunk_id,
document_id,
chunk_type,
term_search_text,
- content,
- file_path,
- chunk_metadata,
- job_result_id,
- job_id,
- result_raw_prefix,
+ _sort_order,
section_path,
source_file_name,
- _total_matches,
) in matched_rows:
- text = str(term_search_text or "")
- match = compiled.search(text)
- snippet = build_snippet(
- text, match.span() if match else None, hit_context=context_chars
- )
- results.append(
+ hits.append(
{
"document_id": document_id,
"source_file_name": source_file_name,
"chunk_id": chunk_id,
"chunk_type": chunk_type,
"section_path": section_path,
- "snippet": snippet,
- "content": content,
- "file_path": file_path,
- "chunk_metadata": chunk_metadata or {},
- "job_result_id": job_result_id,
- "job_id": job_id,
- "result_raw_prefix": result_raw_prefix,
+ "term_search_text": term_search_text,
}
)
- media: list[dict[str, str]] = []
- if results:
- results, media = await mount_explore_hits(
- ctx, results, char_budget=ctx.budget.max_chars
- )
- host_paths = await host_paths_for_hits(ctx, results, scope)
+ blocks: list[tuple[int, int, str, list[dict[str, Any]]]] = []
+ for index, term in enumerate(terms):
+ sections = _sections_for_term(hits, term, context_chars)
+ if sections:
+ blocks.append((len(sections), index, term, sections))
+ blocks.sort()
- rows: list[dict[str, Any]] = []
- lines = [f"total_matches={total_matches} returned={len(results)}"]
+ shown_hits: list[dict[str, Any]] = []
+ remaining = limit
+ allocated: list[tuple[str, list[dict[str, Any]], list[dict[str, Any]]]] = []
+ for _count, _index, term, sections in blocks:
+ visible = sections[:remaining]
+ leftover = sections[remaining:]
+ remaining -= len(visible)
+ shown_hits.extend(visible)
+ allocated.append((term, visible, leftover))
+
+ host_paths = await host_paths_for_hits(ctx, shown_hits, scope)
+
+ lines: list[str] = []
if requested_limit > limit:
lines.append(f"note: capped to budget.max_items={ctx.budget.max_items}")
- for r in results:
- chunk_type = str(r["chunk_type"] or "").strip()
- is_asset = chunk_type in ASSET_CHUNK_TYPES
- key = (str(r["document_id"]), str(r["chunk_id"]))
- section_path, hosted = (
- host_paths[key] if is_asset and key in host_paths else (r["section_path"], None)
- )
- row = build_row(
- kind=chunk_type or "text",
- document_id=r["document_id"],
- section_path=section_path,
- title=r["source_file_name"],
- chunk_id=r["chunk_id"] if is_asset else None,
- snippet=r["snippet"],
- hosted=hosted if is_asset else None,
- )
- rows.append(row)
- lines.append(format_row(row))
- rendered = str(r.get("rendered") or "").strip()
- if rendered:
- lines.append(rendered)
+ rows: list[dict[str, Any]] = []
+ for term, visible, leftover in allocated:
+ lines.append(term)
+ for hit in visible:
+ row = _build_hit_row(hit, host_paths)
+ rows.append(row)
+ lines.append(format_row(row))
+ leftover_by_doc: dict[str, list[dict[str, Any]]] = defaultdict(list)
+ for hit in leftover:
+ leftover_by_doc[str(hit["document_id"])].append(hit)
+ for document_id in leftover_by_doc:
+ group = leftover_by_doc[document_id]
+ lines.append(
+ _fold_line(str(group[0]["source_file_name"] or ""), document_id, len(group))
+ )
return ToolResult(
text="\n".join(lines),
- payload={
- "rows": rows,
- "details": {
- "total_matches": total_matches,
- "capped": requested_limit > limit,
- },
- },
+ payload={"rows": rows, "details": {}},
refs=[
- {"document_id": r["document_id"], "chunk_id": r["chunk_id"]} for r in results
+ {"document_id": hit["document_id"], "chunk_id": hit["chunk_id"]}
+ for hit in shown_hits
],
- media=media,
)
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/list_documents.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/list_documents.py
index a742ee18f..a312a8ac3 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/list_documents.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/list_documents.py
@@ -1,7 +1,7 @@
"""``corpus.list_documents`` — namespace-level document overview.
-Joins ``documents`` with the document-level ``graph_nodes`` row (§4 of
-``CORPUS_SCHEMA.md``) to surface per-document keywords/summary/type-mix
+Joins ``documents`` with the document-level ``graph_nodes`` row
+to surface per-document keywords/summary/type-mix
without reading any chunk content.
"""
@@ -24,9 +24,8 @@
@register_tool(
name="corpus.list_documents",
description=(
- "List every active document in the namespace with its parse_track "
- "and, when available, document-level graph metadata (top_keywords, "
- "top_summary, chunk type mix). Use only when explicitly asked to "
+ "List every active document in the namespace with its summary, "
+ "top keywords and chunk count. Use only when explicitly asked to "
"inventory the namespace's documents, not as a first step for "
"question answering."
),
@@ -63,7 +62,6 @@ async def list_documents(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult:
"document_id": document.document_id,
"job_result_id": CorpusRevisionContext.resolve_revision(document.document_id, document.current_job_result_id),
"source_file_name": document.source_file_name,
- "parse_track": document.parse_track,
"top_keywords": props.get("top_keywords") or [],
"top_summary": props.get("top_summary") or "",
"types": props.get("types") or {},
@@ -72,7 +70,7 @@ async def list_documents(ctx: ToolContext, _args: dict[str, Any]) -> ToolResult:
documents.append(entry)
summary_line = (
f"- {entry['source_file_name']} ({entry['document_id']}, "
- f"track={entry['parse_track']}, chunks={entry['chunks_count']})"
+ f"chunks={entry['chunks_count']})"
)
if entry["top_summary"]:
summary_line += f"\n summary: {entry['top_summary']}"
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/neighbors.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/neighbors.py
index e70ce9a5e..f37f1f7fc 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/neighbors.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/neighbors.py
@@ -1,6 +1,6 @@
"""``corpus.neighbors`` — document-level ``related`` graph edges.
-Document-to-document only (§4 of ``CORPUS_SCHEMA.md``): no section- or
+Document-to-document only: no section- or
entity-level graph nodes exist yet. Edges are undirected and were written by
``DocumentGraphService.publish_document_graph`` with ``shared_entities`` (typed
entity overlap, preferred) or ``shared_keywords`` (TF-IDF fallback).
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/node_filter.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/node_filter.py
index 7df3d3459..2b84d3982 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/node_filter.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/node_filter.py
@@ -106,8 +106,8 @@
},
"chunk_types": chunk_types_schema(
"Narrow matched sections to those owning a chunk of one of "
- "these types (e.g. ['page'] to filter to page-track leaves "
- "only). Omit for no narrowing."
+ "these types (e.g. ['page'] to keep only sections whose "
+ "body is PDF pages). Omit for no narrowing."
),
},
"required": ["scope", "predicates"],
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/outline.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/outline.py
index 4c1246315..b6ac97a1a 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/outline.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/outline.py
@@ -204,7 +204,10 @@ def header(visible: int, hidden: int) -> str:
text=rendered.text,
payload={
"rows": visible_rows,
- "details": {"hidden_count": len(order) - len(visible_rows)},
+ "details": {
+ "hidden_count": len(order) - len(visible_rows),
+ "documents": source_file_name_by_doc,
+ },
},
refs=[
{"document_id": row["document_id"], "section_path": row["section_path"]}
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/query_table.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/query_table.py
index f2b1a6805..fad4f97c2 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/query_table.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/query_table.py
@@ -145,10 +145,11 @@ async def query_table(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
connection.close()
result_grid = [result_headers, *result_rows] if result_headers else []
+ table_html = grid_to_html(result_grid)
text = (
f"### {doc.source_file_name} ({document_id}) / {section_path} [table]\n"
f"columns={', '.join(columns)}\n"
- f"{grid_to_html(result_grid)}"
+ f"{table_html}"
)
return ToolResult(
text=text,
@@ -158,6 +159,7 @@ async def query_table(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
"columns": columns,
"headers": result_headers,
"rows": result_rows,
+ "table_html": table_html,
},
refs=[{"document_id": document_id, "chunk_id": chunk_id}],
)
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/read.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/read.py
index 39c36480d..d36dd841e 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/read.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/read.py
@@ -3,10 +3,10 @@
Unlike ``hydration.result_assembly.assemble_retrieval_results`` (which
down-weights ``page`` chunks to their summary — see ``page_summary``, a
deliberate trade-off for the retrieval-answer surface),
-``read`` returns the page chunk's full body content, with ``[SAME-AS
-p]`` markers resolved to the owner section's text (§2 of
-``CORPUS_SCHEMA.md``) rather than stripped or summarized. ``connect_to``
-assets are still inlined via the same placeholder mechanism as retrieval.
+``read`` returns the page chunk's full body content, with shared-page
+markers resolved to the owner section's text rather than stripped or
+summarized. Embedded images and tables are still inlined via the same
+placeholder mechanism as retrieval.
Table chunks load the stored HTML: small tables return that HTML; large
tables return row/column headers only and point at ``corpus.query_table``
(no window — GREP/recall never scan table-cell HTML, so there is no real
@@ -23,6 +23,10 @@
exact match first, then a unique segment-bound suffix match when the agent
omits ancestor segments; ambiguous suffix matches return an error listing
candidate full paths instead of picking one silently.
+
+When ``agent_explore`` sets ``ToolContext.readable``/``decided``, a ref must
+name an address from an earlier round's results, and body chunks the episode
+already picked or passed on are not read again.
"""
from __future__ import annotations
@@ -44,6 +48,7 @@
from shared.services.retrieval.agent_tools.registry import (
REF_ADDRESS_ONE_OF,
REF_ADDRESS_RULE,
+ Decision,
ToolContext,
ToolResult,
register_tool,
@@ -62,22 +67,72 @@
from shared.services.retrieval.agent_tools.section_path_lookup import (
resolve_section_path_anchor,
section_path_anchor_filter,
+ section_path_received,
section_path_subtree_filter,
)
from shared.services.retrieval.search.lexical_text import section_path_from_chunk_path
_SAME_AS_MARKER_RE = re.compile(r"\[SAME-AS (.+?) p(\d+)\]")
_BODY_CHUNK_TYPES = ("text", "page")
-# Repeated after the ref list and after the body: a long body can be cut by
-# the per-turn text cap, and the ref list alone may scroll out of view.
-_PICK_REMINDER = (
- "[pick: decide now, for each [ok] ref, whether to cite it in finish; "
- "[failed] refs cannot be cited]"
+_LOCATE_HINT = (
+ " Copy document_id and section_path or chunk_id exactly from a prior "
+ "result row, or locate the section first with corpus.outline, "
+ "corpus.grep, corpus.recall, or corpus.assets."
+)
+_NOT_RECEIVED_REASON = (
+ "not in any earlier result you have received; results from calls in "
+ "this same turn are not available yet"
)
+def _with_locate_hint(reason: str) -> str:
+ if _LOCATE_HINT.strip() in reason:
+ return reason
+ return reason + _LOCATE_HINT
+
+
+def _ref_received(ctx: ToolContext, document_id: str, ref: dict[str, Any]) -> bool:
+ if ctx.readable is None:
+ return True
+ chunk_id = str(ref.get("chunk_id") or "").strip()
+ if chunk_id:
+ return (document_id, chunk_id) in ctx.readable
+ section_path = str(ref.get("section_path") or "").strip()
+ return section_path_received(ctx.readable, document_id, section_path)
+
+
+def _decided_chunks(
+ ctx: ToolContext, document_id: str, chunk_ids: list[str]
+) -> dict[str, Decision]:
+ if ctx.decided is None:
+ return {}
+ return {
+ chunk_id: ctx.decided[(document_id, chunk_id)]
+ for chunk_id in chunk_ids
+ if (document_id, chunk_id) in ctx.decided
+ }
+
+
+def _already_read_reason(decisions: list[Decision]) -> str:
+ parts: list[str] = []
+ for decision in dict.fromkeys(decisions):
+ state = (
+ f"picked as {decision.picked_handle}"
+ if decision.picked_handle
+ else "not picked"
+ )
+ parts.append(f"round {decision.read_round} ({state})")
+ return f"already read in {', '.join(parts)}; cannot read again"
+
+
def _ref_status_line(entry: dict[str, Any]) -> str:
- tag = "[ok]" if entry["status"] == "ok" else f"[failed: {entry['reason']}]"
+ if entry["status"] == "ok":
+ omitted = entry.get("omitted_chunk_ids")
+ tag = (
+ f"[ok, {len(omitted)} already-read chunks omitted]" if omitted else "[ok]"
+ )
+ else:
+ tag = f"[failed: {entry['reason']}]"
return (
f"{tag} document_id={entry['document_id']} "
f"section_path={entry['section_path']} chunk_id={entry['chunk_id']}"
@@ -159,9 +214,9 @@ async def _resolve_same_as_markers(
for marker_text, owner_path in row_matches:
resolved = owner_content.get((document_id, owner_path), "")
if resolved:
- replacement = f"(SAME-AS {owner_path} resolved)\n{resolved}"
+ replacement = f"(page text shared with section {owner_path})\n{resolved}"
else:
- replacement = f"(SAME-AS {owner_path} — page not found)"
+ replacement = f"(page text shared with section {owner_path}: not found)"
content = content.replace(marker_text, replacement, 1)
row["content"] = content
@@ -257,15 +312,14 @@ def _add(row: dict[str, Any]) -> None:
@register_tool(
name="corpus.read",
description=(
- "Read full body content for already-located sections or chunks. "
- "Resolves page-track SAME-AS pointers to the owner section's text, "
- "inlines connect_to assets, loads table HTML (small tables in full; "
- "large tables as row/column headers plus a pointer to "
- "corpus.query_table), and converts asset/page_assets references to "
- "URLs. Use after outline/node_filter/recall/grep have located where "
- "to look. Each ref's outcome (ok, or failed with a reason) is "
- "reported separately — decide pick/no-pick for each ref right "
- "after seeing its status; a failed ref defaults to not picked."
+ "Read full content for sections (document_id + section_path) or "
+ "chunks (document_id + chunk_id) that you already located. Copy "
+ "both values exactly from a prior result row. Page text shared "
+ "with another section is filled in automatically. Images and "
+ "tables the section contains are shown inline: small tables in "
+ "full, large tables as row and column headers (then use "
+ "corpus.query_table). Each ref's outcome (ok, or failed with a "
+ "reason) is reported separately."
),
json_schema={
"type": "object",
@@ -308,12 +362,7 @@ def _add(row: dict[str, Any]) -> None:
"include_assets": {
"type": "boolean",
"default": True,
- "description": "Inline connect_to images and tables into the body.",
- },
- "resolve_same_as": {
- "type": "boolean",
- "default": True,
- "description": "Replace page-track SAME-AS markers with the owner text.",
+ "description": "Show the images and tables this section contains inline.",
},
},
"required": ["refs"],
@@ -329,7 +378,6 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
if mode not in ("self", "descendants"):
return ToolResult(text="", error=f"unsupported mode: {mode}")
include_assets = bool(args.get("include_assets", True))
- resolve_same_as_flag = bool(args.get("resolve_same_as", True))
char_budget = ctx.budget.max_chars
document_ids = {
@@ -387,10 +435,16 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
emit_items: list[tuple[str, Any]] = []
for index, ref in enumerate(refs):
document_id = str(ref.get("document_id") or "").strip()
+ if not _ref_received(ctx, document_id, ref):
+ ref_status[index]["status"] = "failed"
+ ref_status[index]["reason"] = _NOT_RECEIVED_REASON
+ continue
job_result_id = revision_by_doc.get(document_id)
if not job_result_id:
ref_status[index]["status"] = "failed"
- ref_status[index]["reason"] = f"unknown document_id: {document_id}"
+ ref_status[index]["reason"] = _with_locate_hint(
+ f"unknown document_id: {document_id}"
+ )
continue
chunk_id = str(ref.get("chunk_id") or "").strip()
section_path = str(ref.get("section_path") or "").strip()
@@ -400,7 +454,11 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
if chunk_id:
row = (
await ctx.db.execute(
- select(corpusStorage.DocumentChunk, corpusStorage.DocumentSection.section_path)
+ select(
+ corpusStorage.DocumentChunk,
+ corpusStorage.DocumentSection.section_path,
+ corpusStorage.DocumentSection.summary,
+ )
.select_from(corpusStorage.DocumentChunk)
.outerjoin(
corpusStorage.DocumentSection,
@@ -413,9 +471,16 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
).first()
if row is None:
ref_status[index]["status"] = "failed"
- ref_status[index]["reason"] = f"unknown chunk_id: {chunk_id} in {document_id}"
+ ref_status[index]["reason"] = _with_locate_hint(
+ f"unknown chunk_id: {chunk_id} in {document_id}"
+ )
+ continue
+ chunk, resolved_section_path, section_summary = row
+ decided = _decided_chunks(ctx, document_id, [chunk.chunk_id])
+ if decided:
+ ref_status[index]["status"] = "failed"
+ ref_status[index]["reason"] = _already_read_reason(list(decided.values()))
continue
- chunk, resolved_section_path = row
ref_status[index]["status"] = "ok"
ref_status[index]["chunk_ids"] = [chunk.chunk_id]
emit_items.append(
@@ -435,6 +500,7 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
"content": chunk.content,
"chunk_metadata": chunk.chunk_metadata or {},
"file_path": chunk.file_path,
+ "section_summary": str(section_summary or "").strip(),
}
],
)
@@ -443,7 +509,9 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
if not section_path:
ref_status[index]["status"] = "failed"
- ref_status[index]["reason"] = f"ref for {document_id} needs section_path or chunk_id"
+ ref_status[index]["reason"] = _with_locate_hint(
+ f"ref for {document_id} needs section_path or chunk_id"
+ )
continue
resolved_path, path_error = await resolve_section_path_anchor(
@@ -454,7 +522,9 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
)
if path_error or not resolved_path:
ref_status[index]["status"] = "failed"
- ref_status[index]["reason"] = path_error or f"unknown section_path for {document_id}"
+ ref_status[index]["reason"] = _with_locate_hint(
+ path_error or f"unknown section_path for {document_id}"
+ )
continue
emit_items.append(
(
@@ -513,12 +583,9 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
corpusStorage.DocumentChunk.section_id.in_(section_ids),
# Body chunks only (text/page). image/table chunks share a
# section_id with whichever section happens to store them
- # in the DB (always Root — CORPUS_SCHEMA.md §3), which is
- # not the same as "belonging" to that section; their real
- # association is connect_to on the body chunk, resolved
- # below via hydrate_connected_target_rows. Without this
- # filter, reading Root would return every still-unmounted
- # asset in the document as spurious top-level entries.
+ # in the DB, which is not the same as belonging to that
+ # section; their real association is on the body chunk,
+ # resolved below via hydrate_connected_target_rows.
corpusStorage.DocumentChunk.chunk_type.in_(_BODY_CHUNK_TYPES),
)
)
@@ -550,9 +617,26 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
for section in section_matches
if section.section_id in matched_ids
}
+ section_summary_by_id = {
+ section.section_id: str(section.summary or "").strip()
+ for section in section_matches
+ if section.section_id in matched_ids
+ }
+ job_chunks = [
+ chunk for chunk in batched_chunks if chunk.section_id in matched_ids
+ ]
+ decided = _decided_chunks(
+ ctx, job["document_id"], [chunk.chunk_id for chunk in job_chunks]
+ )
+ if job_chunks and all(chunk.chunk_id in decided for chunk in job_chunks):
+ ref_status[ref_index]["status"] = "failed"
+ ref_status[ref_index]["reason"] = _already_read_reason(list(decided.values()))
+ continue
+ if decided:
+ ref_status[ref_index]["omitted_chunk_ids"] = list(decided)
job_chunk_ids: list[str] = []
for chunk in batched_chunks:
- if chunk.section_id not in matched_ids:
+ if chunk.section_id not in matched_ids or chunk.chunk_id in decided:
continue
job_chunk_ids.append(chunk.chunk_id)
base_rows.append(
@@ -573,6 +657,7 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
"content": chunk.content,
"chunk_metadata": chunk.chunk_metadata or {},
"file_path": chunk.file_path,
+ "section_summary": section_summary_by_id.get(chunk.section_id, ""),
}
)
if job_chunk_ids:
@@ -580,7 +665,7 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
ref_status[ref_index]["chunk_ids"] = job_chunk_ids
else:
ref_status[ref_index]["status"] = "failed"
- ref_status[ref_index]["reason"] = (
+ ref_status[ref_index]["reason"] = _with_locate_hint(
f"no body chunk found for {job['resolved_path']} in "
f"{job['document_id']} (image/table-only or empty section)"
)
@@ -593,19 +678,15 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
error=(
"read: every ref failed, nothing was read:\n"
+ "\n".join(status_lines)
- + "\nNone of these can be picked for finish. Fix each ref "
- "(copy document_id + section_path/chunk_id exactly from an "
- "outline/node_filter/grep/recall/assets row) and retry."
),
)
- if resolve_same_as_flag:
- await _resolve_same_as_markers(
- ctx.db,
- base_rows,
- revision_by_doc=revision_by_doc,
- source_file_name_by_doc=source_file_name_by_doc,
- )
+ await _resolve_same_as_markers(
+ ctx.db,
+ base_rows,
+ revision_by_doc=revision_by_doc,
+ source_file_name_by_doc=source_file_name_by_doc,
+ )
connected_rows: list[dict[str, Any]] = []
if include_assets:
@@ -653,14 +734,13 @@ async def read(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
composed["content"] = _image_display_content(row)
assembled.append(composed)
- lines = ["refs:", *(f" {line}" for line in status_lines), _PICK_REMINDER]
+ lines = ["refs:", *(f" {line}" for line in status_lines)]
for row in assembled:
lines.append(
f"### {row.get('source_file_name')} ({row.get('document_id')}) / "
f"{row.get('section_path')} [{row.get('chunk_type')}]"
)
lines.append(str(row.get("content") or ""))
- lines.append(_PICK_REMINDER)
return ToolResult(
text="\n".join(lines),
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/recall.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/recall.py
index 6ac6bf72a..d265f163b 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/recall.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/recall.py
@@ -60,10 +60,11 @@ def _identifier_snippet(row: dict[str, Any]) -> str:
name="corpus.recall",
description=(
"Fuzzy ranked candidate search for a question when you don't know "
- "where the answer lives. Scores path and content with BM25. "
+ "where the answer lives. Ranks by how well section path and text "
+ "match the query. "
"Returns candidates with chunk_type, document_id, path and snippet. "
- "Table and image hits include rendered content; body hits include "
- "any connected table or image. Exact string / identifier lookup is "
+ "Table and image hits show their content; body hits also show the "
+ "images and tables they contain. Exact string / identifier lookup is "
"corpus.grep, not this tool. Read the candidate's document_id + "
"section_path (or chunk_id, for image/table hits) with corpus.read "
"next, not the filename."
@@ -147,7 +148,7 @@ async def recall(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
# recall regardless of what a better next step happens to be for
# this corpus/query, so it nudges the agent to change approach
# without prescribing which other tool to reach for (that's already
- # covered generically in CORPUS_SCHEMA.md §6's tool-selection table).
+ # covered generically in each other tool's own description).
lines.append(
"note: few or no candidates for this phrasing — rephrasing the "
"query and calling recall again rarely surfaces more; a "
@@ -177,6 +178,7 @@ async def recall(ctx: ToolContext, args: dict[str, Any]) -> ToolResult:
score=float(raw_score) if raw_score is not None else None,
hosted=hosted if is_asset else None,
)
+ row["mounted_chunk_ids"] = hit["mounted_chunk_ids"]
payload_rows.append(row)
lines.append(format_row(row))
rendered = str(hit.get("rendered") or "").strip()
diff --git a/packages/shared-python/shared/services/retrieval/execution/response_projection.py b/packages/shared-python/shared/services/retrieval/execution/response_projection.py
index 8878d82cf..b3019f6a3 100644
--- a/packages/shared-python/shared/services/retrieval/execution/response_projection.py
+++ b/packages/shared-python/shared/services/retrieval/execution/response_projection.py
@@ -38,6 +38,8 @@ async def project_public_retrieval_response(response: dict[str, Any]) -> dict[st
public_response['failure_reason'] = response['failure_reason']
if response.get('decision_trace') is not None:
public_response['decision_trace'] = response['decision_trace']
+ if response.get('router_decision') is not None:
+ public_response['router_decision'] = response['router_decision']
projected_rows = await enrich_rows_with_retrieval_asset_url(
response.get('results', []),
diff --git a/packages/shared-python/shared/services/retrieval/execution/routes.py b/packages/shared-python/shared/services/retrieval/execution/routes.py
index 4f206196a..e0cd1a5ae 100644
--- a/packages/shared-python/shared/services/retrieval/execution/routes.py
+++ b/packages/shared-python/shared/services/retrieval/execution/routes.py
@@ -14,12 +14,14 @@
RetrievalRouteOutcome,
)
from shared.services.retrieval.hydration.evidence_compose import (
- collect_evidence,
- flatten_parts,
+ group_evidence_units,
)
from shared.services.retrieval.hydration.result_assembly import (
assemble_retrieval_results,
)
+from shared.services.retrieval.routing import config as query_router_config
+from shared.services.retrieval.routing.classifier import classify_query_mode
+from shared.services.retrieval.routing.types import QueryMode, RouterDecision
from shared.services.retrieval.search.map_unit_discovery import map_unit_discovery
from shared.services.retrieval.search.ranking import rank_retrieval_candidates
from shared.services.retrieval.search.scoped_corpus import (
@@ -50,12 +52,24 @@ def open_agent_explore_database_context() -> AbstractAsyncContextManager[AsyncSe
def _evidence_fields(rows: list[dict]) -> dict:
- # TODO: 后面用 TypeSafe JEV 补结果重排/筛选。现在没有这一步。
- # 无论怎么做,发出去的 evidence 和 results 都是已经筛过或重排过的完整列表,不在 Knowhere 外面做。
- evidence = collect_evidence(rows)
+ # TODO: later use TypeSafe JEV for result rerank/filter. This step does not exist yet.
+ evidence = group_evidence_units(
+ [
+ {
+ "document_id": row["document_id"],
+ "source_file_name": row["source_file_name"],
+ "section_path": row["section_path"],
+ "sort_order": row["sort_order"],
+ "parts": row["composed"],
+ "kind": "read",
+ }
+ for row in rows
+ if row["composed"]
+ ]
+ )
return {
"evidence": evidence,
- "evidence_text": flatten_parts(evidence),
+ "evidence_text": "",
}
@@ -66,10 +80,39 @@ async def run_retrieval_route(
if small_corpus_outcome is not None:
return small_corpus_outcome
- # Explicit False → classic map-unit BM25 top-K. None/True → agent_explore.
- if context.use_agentic is False:
- return await _run_classic_topk_route(context)
- return await _run_agent_explore_route(context)
+ use_agentic = context.use_agentic
+ decision: RouterDecision | None = None
+ if use_agentic is None:
+ use_agentic, decision = await _resolve_auto_route(context)
+
+ if use_agentic is False:
+ outcome = await _run_classic_topk_route(context)
+ else:
+ outcome = await _run_agent_explore_route(context)
+
+ if decision is not None:
+ outcome.response["router_decision"] = decision.to_dict()
+ return outcome
+
+
+async def _resolve_auto_route(
+ context: RetrievalRouteContext,
+) -> tuple[bool, RouterDecision]:
+ if not query_router_config.QUERY_ROUTER_ENABLED:
+ decision = RouterDecision(
+ mode=QueryMode.AGENT,
+ reason="",
+ source="disabled",
+ model_name="",
+ tokens_used=0,
+ )
+ else:
+ decision = await classify_query_mode(context.query)
+ logger.info(
+ f"retrieval query router mode={decision.mode.value} "
+ f"source={decision.source} reason={decision.reason}"
+ )
+ return decision.mode == QueryMode.AGENT, decision
async def _try_run_small_corpus_route(
@@ -202,14 +245,15 @@ async def _run_agent_explore_route(
``AGENT_EXPLORE_HARNESS`` switch resolved by ``resolve_harness()``.
"""
from shared.services.retrieval.agent_explore.bridge import (
- attach_ref_provenance,
+ attach_evidence_pool,
build_decision_trace,
)
from shared.services.retrieval.agent_explore.budget import EpisodeBudget
- from shared.services.retrieval.agent_explore.harness import resolve_harness
- from shared.services.retrieval.agent_explore.ref_resolution import (
- resolve_finish_refs,
+ from shared.services.retrieval.agent_explore.evidence_pool import (
+ compose_pool_evidence,
+ pool_chunk_refs,
)
+ from shared.services.retrieval.agent_explore.harness import resolve_harness
from shared.services.retrieval.trace import TraceRecorder
# End any read transaction created by the route's pre-episode work before
@@ -229,37 +273,21 @@ async def _run_agent_explore_route(
document_scope=context.document_scope,
)
logger.info(
- "retrieval agent_explore stage=episode seconds={:.3f} refs={} "
+ "retrieval agent_explore stage=episode seconds={:.3f} pool={} "
"steps={} tokens={} stop_reason={}".format(
time.perf_counter() - episode_started,
- len(episode.refs),
+ len(episode.pool),
len(episode.steps),
episode.tokens_used,
episode.stop_reason,
)
)
- # episode.refs are document_id + section_path (what the agent actually
- # sees in tool text); resolve_workflow_references requires chunk_id —
- # see ref_resolution.py's module docstring for why this bridge exists.
decision_steps = build_decision_trace(episode.steps)
+ chunk_refs = pool_chunk_refs(episode.pool)
+ decision_steps = attach_evidence_pool(decision_steps, episode.pool)
async with open_fresh_database_context() as final_db:
- finish_resolution = await resolve_finish_refs(
- final_db,
- user_id=context.user_id,
- namespace=context.namespace,
- refs=episode.refs,
- document_scope=context.document_scope,
- )
- chunk_refs = finish_resolution.resolved
- decision_steps = attach_ref_provenance(
- decision_steps,
- agent_selected_refs=episode.agent_selected_refs,
- fallback_refs=episode.fallback_refs,
- resolved_refs=chunk_refs,
- dropped_refs=finish_resolution.dropped,
- )
resolved = await resolve_workflow_references(
db=final_db,
user_id=context.user_id,
@@ -275,7 +303,13 @@ async def _run_agent_explore_route(
exclude_sections=context.exclude_sections,
allowed_chunk_types=context.allowed_chunk_types,
revision_pins=context.revision_pins,
+ queried_tables=episode.queried_tables,
)
+ evidence = compose_pool_evidence(episode.pool, assembled_rows)
+ evidence_fields = {
+ "evidence": evidence,
+ "evidence_text": "",
+ }
selected_doc_ids = list(
{row.get("document_id", "") for row in resolved.rows if row.get("document_id")}
@@ -303,7 +337,6 @@ async def _run_agent_explore_route(
)
decision_trace = [step.to_dict() for step in decision_steps]
- evidence_fields = _evidence_fields(assembled_rows)
response = {
"namespace": context.namespace,
"query": context.query,
@@ -321,6 +354,6 @@ async def _run_agent_explore_route(
completion_label="AGENT EXPLORE RETRIEVAL",
completion_count=len(resolved.refs),
completion_detail=(
- f"chunks | evidence={len(evidence_fields['evidence_text'])} chars | router=agent_explore"
+ f"chunks | evidence={len(evidence)} parts | router=agent_explore"
),
)
diff --git a/packages/shared-python/shared/services/retrieval/graph/keywords.py b/packages/shared-python/shared/services/retrieval/graph/keywords.py
index 5486769e1..7d0b6be53 100644
--- a/packages/shared-python/shared/services/retrieval/graph/keywords.py
+++ b/packages/shared-python/shared/services/retrieval/graph/keywords.py
@@ -19,10 +19,20 @@ def normalize_keyword(keyword: str) -> str:
def extract_keywords_from_chunk_metadata(meta: dict) -> list[str]:
- """Extract keywords from chunk metadata."""
+ """Extract keywords from chunk metadata, preferring typed entities (§4.4)."""
if not isinstance(meta, dict):
return []
+ entities = meta.get('entities')
+ if isinstance(entities, list) and entities:
+ entity_texts = [
+ str(item.get('text', '')).strip()
+ for item in entities
+ if isinstance(item, dict) and str(item.get('text', '')).strip()
+ ]
+ if entity_texts:
+ return entity_texts
+
keywords = meta.get('keywords', [])
if isinstance(keywords, list) and keywords:
return [str(keyword) for keyword in keywords if keyword]
diff --git a/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py b/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py
index 7dd7b046e..b8d98ff7e 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py
@@ -11,12 +11,13 @@
from collections.abc import Mapping, Sequence
from typing import Any
-_PATH_REF_RE = re.compile(r"\[(?:images|tables)/[^\]\n]+\]")
+from shared.utils.chunk_refs import RESOURCE_PATH_REF_RE, match_resource_path_ref
+
_SAME_AS_RE = re.compile(r"\[SAME-AS [^\]]+\]")
def remove_path_placeholders(content: str) -> str:
- text = _PATH_REF_RE.sub("", content)
+ text = RESOURCE_PATH_REF_RE.sub("", content)
return _SAME_AS_RE.sub("", text)
@@ -51,12 +52,11 @@ def inline_assets_at_placeholders(
ref = str(item.get("ref") or "").strip()
placed = False
- for candidate in _ref_candidates(ref):
- if candidate and candidate in text:
- text = text.replace(candidate, f"\n{body}\n", 1)
- embedded.add(target_id)
- placed = True
- break
+ match = match_resource_path_ref(text, ref)
+ if match is not None:
+ text = f"{text[: match.start()]}\n{body}\n{text[match.end() :]}"
+ embedded.add(target_id)
+ placed = True
if not placed:
pending_append.append((target_id, body))
@@ -70,19 +70,3 @@ def inline_assets_at_placeholders(
embedded.add(target_id)
return strip_path_placeholders(text), embedded
-
-
-def _ref_candidates(ref: str) -> list[str]:
- raw = str(ref or "").strip()
- if not raw:
- return []
- out: list[str] = [raw]
- if raw.startswith("[") and raw.endswith("]"):
- inner = raw[1:-1].strip()
- if inner and inner not in out:
- out.append(inner)
- else:
- bracketed = f"[{raw}]"
- if bracketed not in out:
- out.append(bracketed)
- return out
diff --git a/packages/shared-python/shared/services/retrieval/hydration/evidence_compose.py b/packages/shared-python/shared/services/retrieval/hydration/evidence_compose.py
index 3f12020be..d0f0bd713 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/evidence_compose.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/evidence_compose.py
@@ -10,6 +10,7 @@
import base64
import tempfile
from pathlib import Path
+from collections.abc import Mapping
from typing import Any
from loguru import logger
@@ -17,6 +18,7 @@
from shared.services.retrieval.hydration.asset_inline import (
remove_path_placeholders,
)
+from shared.utils.chunk_refs import match_resource_path_ref
from shared.services.retrieval.hydration.row_utils import (
extract_page_nums,
normalize_chunk_type,
@@ -26,6 +28,7 @@
TableDownloadError,
load_table_html,
)
+from shared.services.retrieval.search.lexical_text import split_section_path
from shared.services.storage.raw_prefix_arguments import RawPrefixArguments
from shared.services.storage.result_storage import get_result_storage
@@ -41,47 +44,89 @@
def compose_evidence_parts(
row: dict[str, Any],
rows_by_chunk_id: dict[str, dict[str, Any]],
+ queried_tables: Mapping[tuple[str, str], str] | None = None,
) -> list[dict[str, Any]]:
chunk_type = normalize_chunk_type(row.get("chunk_type"))
if chunk_type == "page":
return _compose_page_parts(row)
if chunk_type == "table":
- return _compose_standalone_table_parts(row)
+ return _compose_standalone_table_parts(row, queried_tables)
if chunk_type == "image":
return _compose_standalone_image_parts(row)
- return _compose_text_parts(row, rows_by_chunk_id)
+ return _compose_text_parts(row, rows_by_chunk_id, queried_tables)
-def collect_evidence(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
- parts: list[dict[str, Any]] = []
- for row in rows:
- composed = row.get("composed")
- if isinstance(composed, list):
- revisionId: object = row.get("job_result_id")
- parts.extend(
- {**part, "job_result_id": revisionId} if revisionId else part
- for part in composed
- )
- return parts
+_GROUP_BODY_INDENT = " "
-def flatten_parts(parts: list[dict[str, Any]] | None) -> str:
- texts: list[str] = []
- for part in parts or []:
- if not isinstance(part, dict):
- continue
- if part.get("type") == "text":
- text = str(part.get("text") or "")
- if text:
- texts.append(text)
- continue
- if part.get("type") != "image":
+def group_evidence_units(units: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Group units by document and the direct parent of ``section_path``.
+
+ Each unit is ``document_id``, ``source_file_name``, ``section_path``,
+ ``sort_order``, ``parts``, and ``kind`` (``read`` or ``outline``).
+ Outline units ignore ``section_path`` and ``sort_order``.
+ """
+ documents: dict[str, dict[str, Any]] = {}
+ for index, unit in enumerate(units):
+ document = documents.setdefault(
+ unit["document_id"], {"outlines": [], "reads": {}}
+ )
+ if unit["kind"] == "outline":
+ document["outlines"].append(
+ (f"[§ {unit['source_file_name']}]", [unit["parts"]])
+ )
continue
- media_type = str(part.get("media_type") or "").strip() or "application/octet-stream"
- data = str(part.get("data") or "").strip()
- if data:
- texts.append(f"data:{media_type};base64,{data}")
- return "".join(texts)
+ parent_path = _direct_parent_path(unit["section_path"])
+ group = document["reads"].setdefault(
+ parent_path,
+ {
+ "title": f"[§ {unit['source_file_name']} / {parent_path}]",
+ "members": [],
+ },
+ )
+ group["members"].append((unit["sort_order"], index, unit["parts"]))
+
+ evidence: list[dict[str, Any]] = []
+ group_number = 0
+ for document in documents.values():
+ read_groups = sorted(
+ (
+ (min(group["members"]), group["title"], sorted(group["members"]))
+ for group in document["reads"].values()
+ ),
+ key=lambda item: item[0],
+ )
+ ordered = [*document["outlines"]] + [
+ (title, [member[2] for member in members])
+ for _first, title, members in read_groups
+ ]
+ for title, member_parts in ordered:
+ if group_number:
+ evidence.append(_text_part("\n"))
+ group_number += 1
+ evidence.append(_text_part(f"[E{group_number}] {title}"))
+ for parts in member_parts:
+ evidence.extend(_indent_member_part(part) for part in parts)
+ return evidence
+
+
+def _direct_parent_path(section_path: str) -> str:
+ parts = split_section_path(section_path)
+ if len(parts) > 1:
+ return " / ".join(parts[:-1])
+ return section_path
+
+
+def _indent_member_part(part: dict[str, Any]) -> dict[str, Any]:
+ if part["type"] != "text":
+ return part
+ return {
+ "type": "text",
+ "text": "".join(
+ (_GROUP_BODY_INDENT + line if line.strip() else line)
+ for line in part["text"].splitlines(keepends=True)
+ ),
+ }
def _compose_page_parts(row: dict[str, Any]) -> list[dict[str, Any]]:
@@ -97,8 +142,11 @@ def _compose_page_parts(row: dict[str, Any]) -> list[dict[str, Any]]:
return parts
-def _compose_standalone_table_parts(row: dict[str, Any]) -> list[dict[str, Any]]:
- html = _try_read_table_html(row)
+def _compose_standalone_table_parts(
+ row: dict[str, Any],
+ queried_tables: Mapping[tuple[str, str], str] | None = None,
+) -> list[dict[str, Any]]:
+ html = _try_read_table_html(row, queried_tables)
if html is None:
return []
return [_text_part(f"\n{html}\n")]
@@ -118,11 +166,12 @@ def _compose_standalone_image_parts(row: dict[str, Any]) -> list[dict[str, Any]]
def _compose_text_parts(
row: dict[str, Any],
rows_by_chunk_id: dict[str, dict[str, Any]],
+ queried_tables: Mapping[tuple[str, str], str] | None = None,
) -> list[dict[str, Any]]:
content = str(row.get("content") or "")
tables, images = _embed_targets(row, rows_by_chunk_id)
for _target_id, target_row, ref in tables:
- html = _try_read_table_html(target_row)
+ html = _try_read_table_html(target_row, queried_tables)
content, placed = _replace_placeholder(content, ref, "" if html is None else f"\n{html}\n")
if html is not None and not placed:
_warn_skipped(target_row, "table", "placeholder not found")
@@ -194,10 +243,10 @@ def _connections(row: dict[str, Any]) -> list[dict[str, Any]]:
def _replace_placeholder(text: str, ref: str, replacement: str) -> tuple[str, bool]:
- for candidate in _ref_candidates(ref):
- if candidate and candidate in text:
- return text.replace(candidate, replacement, 1), True
- return text, False
+ match = match_resource_path_ref(text, ref)
+ if match is None:
+ return text, False
+ return text[: match.start()] + replacement + text[match.end() :], True
def _earliest_image_placeholder(
@@ -206,42 +255,31 @@ def _earliest_image_placeholder(
) -> tuple[str, str, dict[str, Any]] | None:
best: tuple[int, int, dict[str, Any]] | None = None
for _target_id, target_row, ref in images:
- for candidate in _ref_candidates(ref):
- if not candidate:
- continue
- index = text.find(candidate)
- if index < 0:
- continue
- length = len(candidate)
- if (
- best is None
- or index < best[0]
- or (index == best[0] and length > best[1])
- ):
- best = (index, length, target_row)
+ match = match_resource_path_ref(text, ref)
+ if match is None:
+ continue
+ index = match.start()
+ length = match.end() - match.start()
+ if (
+ best is None
+ or index < best[0]
+ or (index == best[0] and length > best[1])
+ ):
+ best = (index, length, target_row)
if best is None:
return None
index, length, target_row = best
return text[:index], text[index + length :], target_row
-def _ref_candidates(ref: str) -> list[str]:
- raw = str(ref or "").strip()
- if not raw:
- return []
- out = [raw]
- if raw.startswith("[") and raw.endswith("]"):
- inner = raw[1:-1].strip()
- if inner and inner not in out:
- out.append(inner)
- else:
- bracketed = f"[{raw}]"
- if bracketed not in out:
- out.append(bracketed)
- return out
-
-
-def _try_read_table_html(row: dict[str, Any]) -> str | None:
+def _try_read_table_html(
+ row: dict[str, Any],
+ queried_tables: Mapping[tuple[str, str], str] | None = None,
+) -> str | None:
+ if queried_tables is not None:
+ key = (str(row.get("document_id") or ""), str(row.get("chunk_id") or ""))
+ if key in queried_tables:
+ return queried_tables[key].strip()
try:
html = load_table_html(row).strip()
except TableDownloadError as exc:
diff --git a/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py b/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py
index 1cc3219fa..f62f6745c 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py
@@ -27,6 +27,7 @@ async def assemble_retrieval_results(
document_scope: DocumentScope = DocumentScope(),
allowed_chunk_types: set[str] | None = None,
revision_pins: Mapping[str, str] | None = None,
+ queried_tables: Mapping[tuple[str, str], str] | None = None,
) -> list[dict[str, Any]]:
scoped_rows = filter_excluded_rows(
rows,
@@ -78,6 +79,7 @@ async def assemble_retrieval_results(
assembled_row['composed'] = compose_evidence_parts(
assembled_row,
rows_by_chunk_id,
+ queried_tables=queried_tables,
)
assembled.append(assembled_row)
return assembled
diff --git a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
index eaff7cb70..b8f6b8d4c 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
@@ -1,5 +1,9 @@
from __future__ import annotations
+from shared.services.chunks.chunk_connections import (
+ FORWARD_RELATIONS,
+ RELATION_RELATED,
+)
from shared.services.retrieval.document_scope import DocumentScope
from typing import Any
@@ -14,6 +18,7 @@
'content',
'content_source',
'score',
+ 'importance',
'asset_url',
'source_chunk_path',
'file_path',
@@ -95,6 +100,8 @@ def iter_connected_target_ids(row: dict[str, Any]) -> list[str]:
for item in metadata.get('connect_to') or []:
if not isinstance(item, dict):
continue
+ if str(item.get('relation') or RELATION_RELATED) not in FORWARD_RELATIONS:
+ continue
target_id = str(item.get('target') or '').strip()
if target_id:
target_ids.append(target_id)
diff --git a/packages/shared-python/shared/services/retrieval/hydration/table_grid.py b/packages/shared-python/shared/services/retrieval/hydration/table_grid.py
index 668d041ea..482bb8642 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/table_grid.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/table_grid.py
@@ -196,6 +196,8 @@ def normalize_select_sql(sql: str) -> str | None:
return text
+# TODO: later adapt nested / multi-level row and column headers.
+# Current mapping is first row as columns and the first cell of each data row as row_header.
def sql_columns(grid: list[list[str]]) -> list[str]:
if not grid:
return ["row_header"]
diff --git a/packages/shared-python/shared/services/retrieval/routing/__init__.py b/packages/shared-python/shared/services/retrieval/routing/__init__.py
new file mode 100644
index 000000000..9049724e6
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/routing/__init__.py
@@ -0,0 +1 @@
+"""Query-mode router: classify an unset ``use_agentic`` query as agent or classic."""
diff --git a/packages/shared-python/shared/services/retrieval/routing/classifier.py b/packages/shared-python/shared/services/retrieval/routing/classifier.py
new file mode 100644
index 000000000..4519f47dd
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/routing/classifier.py
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any
+
+from loguru import logger
+
+from shared.services.retrieval.routing import config as routing_config
+from shared.services.retrieval.routing.prompt import QUERY_ROUTER_SYSTEM_PROMPT
+from shared.services.retrieval.routing.types import QueryMode, RouterDecision
+
+
+def _resolve_client_and_model() -> tuple[Any, str]:
+ from shared.services.ai.llm_overrides import resolve_text
+ from shared.services.ai.openai_compatible_client_sync import get_openai_client
+
+ requested = routing_config.QUERY_ROUTER_MODEL
+ effective_model, api_key, api_url = resolve_text(requested)
+ model = effective_model or requested
+ client = get_openai_client(model=model, api_key=api_key, api_url=api_url)
+ return client, model
+
+
+def _fallback_decision(*, model_name: str, tokens_used: int = 0) -> RouterDecision:
+ return RouterDecision(
+ mode=QueryMode.AGENT,
+ reason="",
+ source="fallback",
+ model_name=model_name,
+ tokens_used=tokens_used,
+ )
+
+
+def _parse_decision(
+ content: str, *, model_name: str, tokens_used: int
+) -> RouterDecision:
+ try:
+ parsed = json.loads(content)
+ except ValueError:
+ return _fallback_decision(model_name=model_name, tokens_used=tokens_used)
+ if not isinstance(parsed, dict):
+ return _fallback_decision(model_name=model_name, tokens_used=tokens_used)
+ mode_raw = str(parsed.get("mode") or "").strip().lower()
+ if mode_raw not in {QueryMode.AGENT.value, QueryMode.CLASSIC.value}:
+ return _fallback_decision(model_name=model_name, tokens_used=tokens_used)
+ return RouterDecision(
+ mode=QueryMode(mode_raw),
+ reason=str(parsed.get("reason") or ""),
+ source="llm",
+ model_name=model_name,
+ tokens_used=tokens_used,
+ )
+
+
+async def classify_query_mode(query: str) -> RouterDecision:
+ model_name = ""
+ try:
+ client, model_name = _resolve_client_and_model()
+ content, usage = await asyncio.to_thread(
+ client.chat_completion_with_usage,
+ [
+ {"role": "system", "content": QUERY_ROUTER_SYSTEM_PROMPT},
+ {"role": "user", "content": query},
+ ],
+ model=model_name,
+ temperature=0.0,
+ max_tokens=routing_config.QUERY_ROUTER_MAX_TOKENS,
+ timeout=routing_config.QUERY_ROUTER_TIMEOUT_SECONDS,
+ response_format={"type": "json_object"},
+ )
+ tokens_used = int((usage or {}).get("total_tokens", 0) or 0)
+ return _parse_decision(
+ str(content or ""),
+ model_name=model_name,
+ tokens_used=tokens_used,
+ )
+ except Exception as exc:
+ logger.warning("retrieval query router fallback to agent: {}", exc)
+ return _fallback_decision(model_name=model_name)
diff --git a/packages/shared-python/shared/services/retrieval/routing/config.py b/packages/shared-python/shared/services/retrieval/routing/config.py
new file mode 100644
index 000000000..203dae277
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/routing/config.py
@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import os
+
+from shared.services.retrieval.agent_explore.config import AGENT_EXPLORE_MODEL
+
+QUERY_ROUTER_ENABLED_ENV = "RETRIEVAL_QUERY_ROUTER_ENABLED"
+QUERY_ROUTER_MODEL_ENV = "RETRIEVAL_QUERY_ROUTER_MODEL"
+
+QUERY_ROUTER_TIMEOUT_SECONDS = 10
+QUERY_ROUTER_MAX_TOKENS = 256
+
+
+def _env_flag(name: str, default: bool) -> bool:
+ raw = os.environ.get(name)
+ if raw is None or not str(raw).strip():
+ return default
+ return str(raw).strip().lower() in {"1", "true", "yes", "on"}
+
+
+QUERY_ROUTER_ENABLED = _env_flag(QUERY_ROUTER_ENABLED_ENV, True)
+QUERY_ROUTER_MODEL = (
+ os.environ.get(QUERY_ROUTER_MODEL_ENV, "").strip() or AGENT_EXPLORE_MODEL
+)
diff --git a/packages/shared-python/shared/services/retrieval/routing/prompt.py b/packages/shared-python/shared/services/retrieval/routing/prompt.py
new file mode 100644
index 000000000..e536ccc1f
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/routing/prompt.py
@@ -0,0 +1,10 @@
+QUERY_ROUTER_SYSTEM_PROMPT = """\
+You classify one retrieval query into a search mode. Reply with json only.
+
+Modes:
+- agent: the query needs multi-hop reasoning, cross-document synthesis, comparison, aggregation, causal or how/why analysis, or an open-ended investigation.
+- classic: the query is looking up a single fact, a specific value, a named detail, or a keyword-style locate request such as what X is or where to find Y.
+
+Return a json object:
+{"mode":"agent"|"classic","reason":"at most 30 characters"}
+"""
diff --git a/packages/shared-python/shared/services/retrieval/routing/types.py b/packages/shared-python/shared/services/retrieval/routing/types.py
new file mode 100644
index 000000000..c0b17e9d9
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/routing/types.py
@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+
+class QueryMode(str, Enum):
+ AGENT = "agent"
+ CLASSIC = "classic"
+
+
+@dataclass(frozen=True)
+class RouterDecision:
+ mode: QueryMode
+ reason: str
+ source: str
+ model_name: str
+ tokens_used: int
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "mode": self.mode.value,
+ "reason": self.reason,
+ "source": self.source,
+ "model_name": self.model_name,
+ "tokens_used": self.tokens_used,
+ }
diff --git a/packages/shared-python/shared/services/retrieval/search/lexical_text.py b/packages/shared-python/shared/services/retrieval/search/lexical_text.py
index 176dbf746..fb1e7fcde 100644
--- a/packages/shared-python/shared/services/retrieval/search/lexical_text.py
+++ b/packages/shared-python/shared/services/retrieval/search/lexical_text.py
@@ -63,12 +63,14 @@ def build_content_lexical_text(chunk: dict[str, Any]) -> Optional[str]:
return None
metadata = chunk.get("metadata") or {}
tokens = metadata.get("tokens") if isinstance(metadata, dict) else None
- if isinstance(tokens, list):
+ if isinstance(tokens, list) and tokens:
token_text = " ".join(
str(token).strip() for token in tokens if str(token).strip()
)
else:
- token_text = ""
+ token_text = " ".join(
+ PublicationPreparationCache.tokenize(content, dedupe=True)
+ )
lexical_parts = [part for part in [content, token_text] if part]
return "\n".join(lexical_parts) if lexical_parts else content
@@ -207,18 +209,23 @@ def _table_search_source_text(chunk: dict[str, Any]) -> str:
if not isinstance(metadata, dict):
metadata = {}
- keyword_value = metadata.get("keywords") or chunk.get("keywords") or []
- if isinstance(keyword_value, list):
- keyword_text = " ".join(
- str(keyword).strip()
- for keyword in keyword_value
- if str(keyword).strip()
- )
+ entity_text = _chunk_entity_text(chunk)
+ if entity_text:
+ keyword_text = entity_text
else:
- keyword_text = str(keyword_value or "").replace(";", " ").strip()
+ keyword_value = metadata.get("keywords") or chunk.get("keywords") or []
+ if isinstance(keyword_value, list):
+ keyword_text = " ".join(
+ str(keyword).strip()
+ for keyword in keyword_value
+ if str(keyword).strip()
+ )
+ else:
+ keyword_text = str(keyword_value or "").replace(";", " ").strip()
parts = [
str(metadata.get("summary") or chunk.get("summary") or "").strip(),
+ str(metadata.get("asset_title") or chunk.get("asset_title") or "").strip(),
keyword_text,
str(metadata.get("caption") or chunk.get("caption") or "").strip(),
]
diff --git a/packages/shared-python/shared/services/retrieval/search/ranking.py b/packages/shared-python/shared/services/retrieval/search/ranking.py
index eb8d25105..7d26fa8c8 100644
--- a/packages/shared-python/shared/services/retrieval/search/ranking.py
+++ b/packages/shared-python/shared/services/retrieval/search/ranking.py
@@ -164,6 +164,7 @@ def rank_candidates_by_path(
)
or 0.0
)
+ row["importance"] = row["importance_raw_score"]
apply_importance_multiplier(list(merged.values()))
has_agent_results = len(routed_rows) > 0
diff --git a/packages/shared-python/shared/services/retrieval/settings.py b/packages/shared-python/shared/services/retrieval/settings.py
index 4768177a1..85ae9dc34 100644
--- a/packages/shared-python/shared/services/retrieval/settings.py
+++ b/packages/shared-python/shared/services/retrieval/settings.py
@@ -3,9 +3,9 @@
INTERNAL_RECALL_K_MULTIPLIER = 2
DEFAULT_TOP_K = 10
-# Final evidence / tool-observation text budget (characters). Used by
+# Agent-visible tool-observation text budget (characters). Used by
# agent tool-loop harness caps (``ToolBudget.max_chars``).
-EVIDENCE_TEXT_CHAR_BUDGET = 12_000
+EVIDENCE_TEXT_CHAR_BUDGET = 15_000
# Explore-phase table policy (user-specified). A table is large when either
# axis meets this size; the same number caps query_table SELECT rows.
diff --git a/packages/shared-python/shared/services/retrieval/trace/recorder.py b/packages/shared-python/shared/services/retrieval/trace/recorder.py
index 8b5156a3b..93821265a 100644
--- a/packages/shared-python/shared/services/retrieval/trace/recorder.py
+++ b/packages/shared-python/shared/services/retrieval/trace/recorder.py
@@ -107,8 +107,6 @@ async def create_run(self) -> None:
def record_decision_trace_step(self, step: DecisionTraceStep) -> None:
"""Buffer a DB trace row derived from the public decision trace step."""
result_status = str(step.result.get("status") or "unknown")
- budget = step.budget or {}
- tokens = int(budget.get("tokens_used_delta") or 0)
model = (
step.result.get("model")
or (step.decision or {}).get("model")
@@ -145,7 +143,6 @@ def record_decision_trace_step(self, step: DecisionTraceStep) -> None:
"observation_payload_keys": list(step.observation.keys()),
"latency_ms": int(step.elapsed_ms or 0),
"error": step.result.get("error"),
- "tokens_used": tokens,
"selected_paths": selected_paths,
"selected_doc_ids": selected_docs,
"model_name": str(model).strip() if model else None,
@@ -164,7 +161,6 @@ def record_budget_stop(self, reason: str) -> None:
"observation_payload_keys": [],
"latency_ms": 0,
"error": None,
- "tokens_used": 0,
"selected_paths": None,
"selected_doc_ids": None,
"model_name": None,
@@ -188,10 +184,7 @@ async def complete(
return
total_latency = int((time.monotonic() - self._start_time) * 1000)
- summed_tokens = sum(int(step.get("tokens_used", 0) or 0) for step in self._steps)
- run_token_count = (
- int(token_count) if token_count is not None else summed_tokens
- )
+ run_token_count = int(token_count) if token_count is not None else None
try:
from shared.models.database.document import RetrievalStep
@@ -206,12 +199,11 @@ async def complete(
observation={
"status": step_data["observation_status"],
"payload_keys": step_data["observation_payload_keys"],
- "tokens_used": step_data.get("tokens_used", 0),
},
selected_paths=step_data.get("selected_paths"),
selected_doc_ids=step_data.get("selected_doc_ids"),
latency_ms=step_data["latency_ms"],
- token_count=step_data.get("tokens_used", 0),
+ token_count=None,
model_name=step_data.get("model_name") or model_name,
error=step_data.get("error"),
created_at=step_data["created_at"],
diff --git a/packages/shared-python/shared/services/storage/zip_chunk_schema.py b/packages/shared-python/shared/services/storage/zip_chunk_schema.py
index 7dc0be09e..3478cb358 100644
--- a/packages/shared-python/shared/services/storage/zip_chunk_schema.py
+++ b/packages/shared-python/shared/services/storage/zip_chunk_schema.py
@@ -79,8 +79,6 @@ def format_chunks(
if chunk_type == "text":
metadata.update(
{
- "tokens": existing_metadata.get("tokens")
- or chunk.get("tokens", 0),
"keywords": existing_metadata.get("keywords")
or chunk.get("keywords", []),
"connect_to": _build_embed_connect_to(
@@ -95,10 +93,16 @@ def format_chunks(
elif chunk_type == "image":
if image_info:
metadata["file_path"] = image_info["file_path"]
+ if original_name := existing_metadata.get("original_name"):
+ metadata["original_name"] = original_name
metadata["keywords"] = existing_metadata.get("keywords") or chunk.get(
"keywords", []
)
- metadata["tokens"] = []
+ metadata["connect_to"] = existing_metadata.get("connect_to") or []
+ if asset_region := _normalize_asset_region(
+ existing_metadata.get("asset_region")
+ ):
+ metadata["asset_region"] = asset_region
elif chunk_type == "table":
metadata["file_path"] = _resolve_table_file_path(
chunk=chunk,
@@ -110,7 +114,6 @@ def format_chunks(
metadata["keywords"] = existing_metadata.get("keywords") or chunk.get(
"keywords", []
)
- metadata["tokens"] = []
metadata["connect_to"] = _build_embed_connect_to(
chunk=chunk,
chunk_type_str=chunk_type_str,
@@ -124,6 +127,8 @@ def format_chunks(
page_assets = _normalize_page_assets(existing_metadata.get("page_assets"))
if page_assets:
metadata["page_assets"] = page_assets
+ if isinstance(existing_metadata.get("body_text_preview"), str):
+ metadata["body_text_preview"] = existing_metadata["body_text_preview"]
formatted_chunk = {
"chunk_id": chunk_id,
@@ -153,6 +158,10 @@ def _base_chunk_metadata(
"summary": existing_metadata.get("summary") or chunk.get("summary", ""),
"page_nums": existing_metadata.get("page_nums", []),
}
+ if entities := existing_metadata.get("entities"):
+ metadata["entities"] = entities
+ if asset_title := existing_metadata.get("asset_title"):
+ metadata["asset_title"] = asset_title
return metadata
@@ -230,9 +239,55 @@ def _normalize_page_assets(value: Any) -> list[dict[str, Any]]:
return assets
+def _normalize_asset_region(value: Any) -> dict[str, Any] | None:
+ if not isinstance(value, dict):
+ return None
+ page_num = _positive_int_or_none(value.get("page_num"))
+ bbox = _bbox_or_none(value.get("bbox"))
+ coord_origin = str(value.get("coord_origin") or "").strip()
+ coord_system = str(value.get("coord_system") or "").strip()
+ page_width = _positive_int_or_none(value.get("page_width"))
+ page_height = _positive_int_or_none(value.get("page_height"))
+ if (
+ page_num is None
+ or bbox is None
+ or not coord_origin
+ or not coord_system
+ or page_width is None
+ or page_height is None
+ ):
+ return None
+ region: dict[str, Any] = {
+ "page_num": page_num,
+ "bbox": bbox,
+ "coord_origin": coord_origin,
+ "coord_system": coord_system,
+ "page_width": page_width,
+ "page_height": page_height,
+ }
+ # Only the page track crops with a margin; the text track does not crop.
+ if (crop_margin_px := _non_negative_int_or_none(value.get("crop_margin_px"))) is not None:
+ region["crop_margin_px"] = crop_margin_px
+ return region
+
+
+def _bbox_or_none(value: Any) -> list[int] | None:
+ if not isinstance(value, list) or len(value) != 4:
+ return None
+ try:
+ return [int(item) for item in value]
+ except (TypeError, ValueError):
+ return None
+
+
def _positive_int_or_none(value: Any) -> int | None:
+ number = _non_negative_int_or_none(value)
+ return number if number is not None and number > 0 else None
+
+
+def _non_negative_int_or_none(value: Any) -> int | None:
try:
number = int(value)
except (TypeError, ValueError):
return None
- return number if number > 0 else None
+ return number if number >= 0 else None
diff --git a/packages/shared-python/shared/tests/test_agent_explore_evidence_pool.py b/packages/shared-python/shared/tests/test_agent_explore_evidence_pool.py
new file mode 100644
index 000000000..d4783182d
--- /dev/null
+++ b/packages/shared-python/shared/tests/test_agent_explore_evidence_pool.py
@@ -0,0 +1,572 @@
+"""Evidence pool rounds / issue / pick / render / compose."""
+
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test")
+os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test")
+os.environ.setdefault("S3_BUCKET_NAME", "test-uploads")
+os.environ.setdefault("S3_ACCESS_KEY_ID", "test")
+os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test")
+os.environ.setdefault("S3_TEMP_PATH", "/tmp")
+
+from shared.services.retrieval.agent_explore.budget import EpisodeBudget
+from shared.services.retrieval.agent_explore.evidence_pool import (
+ Candidate,
+ EvidencePool,
+ PickOutcome,
+ compose_pool_evidence,
+ render_budget,
+ render_pick_outcome,
+ render_trace_line,
+ trace_status,
+)
+from shared.services.retrieval.agent_tools import Decision, ToolResult
+from shared.services.retrieval.hydration.evidence_compose import compose_evidence_parts
+from shared.services.retrieval.agent_tools.section_path_lookup import (
+ section_path_received,
+)
+
+
+def _ok_read(*, refs: list[dict], chunks: list[dict]) -> ToolResult:
+ return ToolResult(text="read", payload={"refs": refs, "chunks": chunks})
+
+
+def _read_one(section_path: str, chunk_ids: list[str]) -> ToolResult:
+ return _ok_read(
+ refs=[
+ {
+ "status": "ok",
+ "document_id": "doc_a",
+ "section_path": section_path,
+ "chunk_ids": chunk_ids,
+ }
+ ],
+ chunks=[
+ {
+ "chunk_id": chunk_id,
+ "document_id": "doc_a",
+ "source_file_name": "guide.pdf",
+ "section_path": section_path,
+ "chunk_type": "text",
+ "section_summary": section_path,
+ }
+ for chunk_id in chunk_ids
+ ],
+ )
+
+
+def _outline() -> ToolResult:
+ return ToolResult(
+ text="outline",
+ payload={
+ "rows": [
+ {
+ "document_id": "doc_a",
+ "section_path": "guide.pdf / 1 Overview",
+ "title": "1 Overview",
+ "depth": 0,
+ }
+ ],
+ "details": {"documents": {"doc_a": "guide.pdf"}},
+ },
+ )
+
+
+def test_issue_read_only_numbers_successful_refs_at_original_index() -> None:
+ pool = EvidencePool()
+ result = _ok_read(
+ refs=[
+ {
+ "status": "failed",
+ "document_id": "doc_a",
+ "section_path": "missing",
+ "chunk_ids": [],
+ },
+ {
+ "status": "ok",
+ "document_id": "doc_a",
+ "section_path": "2 Treatment",
+ "chunk_ids": ["c2"],
+ },
+ ],
+ chunks=[
+ {
+ "chunk_id": "c2",
+ "document_id": "doc_a",
+ "source_file_name": "guide.pdf",
+ "section_path": "2 Treatment",
+ "chunk_type": "text",
+ "section_summary": "First-line drugs",
+ }
+ ],
+ )
+ issued = pool.issue("corpus.read", result)
+ assert [item.handle for item in issued] == ["R1.2"]
+ assert issued[0].summary == "First-line drugs"
+ assert issued[0].chunk_ids == ("c2",)
+
+
+def test_ids_issued_this_round_take_effect_next_round() -> None:
+ pool = EvidencePool()
+ pool.begin_round()
+ pool.issue("corpus.read", _read_one("A", ["c1"]))
+ assert pool.pick_phase is False
+ same_round = pool.apply_pick(["R1.1"])
+ assert same_round.added == []
+ assert same_round.rejected == [{"handle": "R1.1", "reason": "unknown id"}]
+
+ pool.begin_round()
+ assert pool.round_index == 2
+ assert pool.pick_phase is True
+ next_round = pool.apply_pick(["R1.1"])
+ assert next_round.added == ["R1.1"]
+
+
+def test_pick_phase_lasts_until_pick_is_called() -> None:
+ pool = EvidencePool()
+ pool.begin_round()
+ pool.issue("corpus.outline", _outline())
+ pool.begin_round()
+ assert pool.pick_phase is True
+ pool.begin_round()
+ assert pool.pick_phase is True
+ pool.apply_pick([])
+ assert pool.pick_phase is False
+ pool.begin_round()
+ assert pool.pick_phase is False
+
+
+def test_apply_pick_rejects_unknown_duplicate_and_already_in_pool() -> None:
+ pool = EvidencePool()
+ pool.begin_round()
+ pool.issue(
+ "corpus.read",
+ _ok_read(
+ refs=[
+ {
+ "status": "ok",
+ "document_id": "doc_a",
+ "section_path": "A",
+ "chunk_ids": ["c1"],
+ },
+ {
+ "status": "ok",
+ "document_id": "doc_a",
+ "section_path": "A",
+ "chunk_ids": ["c1"],
+ },
+ ],
+ chunks=[
+ {
+ "chunk_id": "c1",
+ "document_id": "doc_a",
+ "source_file_name": "guide.pdf",
+ "section_path": "A",
+ "chunk_type": "text",
+ "section_summary": "A",
+ }
+ ],
+ ),
+ )
+ pool.begin_round()
+ outcome = pool.apply_pick(["R1.1", "R1.1", "R9.9", "R1.2"])
+ assert outcome.added == ["R1.1"]
+ assert outcome.rejected == [
+ {"handle": "R9.9", "reason": "unknown id"},
+ {"handle": "R1.2", "reason": "already in pool"},
+ ]
+
+
+def test_unpicked_read_chunks_are_decided_and_unpicked_outline_is_dropped() -> None:
+ pool = EvidencePool()
+ pool.begin_round()
+ pool.issue("corpus.read", _read_one("A", ["c1", "c2"]))
+ pool.issue("corpus.read", _read_one("B", ["c3"]))
+ pool.issue("corpus.outline", _outline())
+ pool.begin_round()
+ outcome = pool.apply_pick(["R2.1"])
+ assert outcome.added == ["R2.1"]
+ assert [item.handle for item in pool.entries] == ["R2.1"]
+ assert dict(pool.decided()) == {
+ ("doc_a", "c1"): Decision(read_round=1, picked_handle=None, handle="R1.1"),
+ ("doc_a", "c2"): Decision(read_round=1, picked_handle=None, handle="R1.1"),
+ ("doc_a", "c3"): Decision(read_round=1, picked_handle="R2.1", handle="R2.1"),
+ }
+ pool.begin_round()
+ assert pool.pick_phase is False
+ assert pool.apply_pick(["O1"]).rejected == [{"handle": "O1", "reason": "unknown id"}]
+
+
+def test_seen_addresses_become_readable_when_next_round_begins() -> None:
+ pool = EvidencePool()
+ pool.begin_round()
+ pool.issue(
+ "corpus.grep",
+ ToolResult(
+ text="hits",
+ payload={
+ "rows": [
+ {
+ "document_id": "doc_a",
+ "section_path": "guide.pdf / 2 Treatment",
+ "chunk_id": None,
+ "mounted_chunk_ids": ["tbl_1"],
+ },
+ {
+ "document_id": "doc_a",
+ "section_path": "guide.pdf / Root",
+ "chunk_id": "img_1",
+ "mounted_chunk_ids": [],
+ },
+ ]
+ },
+ ),
+ )
+ pool.issue(
+ "corpus.read",
+ _ok_read(
+ refs=[
+ {
+ "status": "ok",
+ "document_id": "doc_b",
+ "section_path": "notes.pdf / Intro",
+ "chunk_ids": ["c9"],
+ }
+ ],
+ chunks=[
+ {
+ "chunk_id": "c9",
+ "document_id": "doc_b",
+ "section_path": "notes.pdf / Intro",
+ "chunk_type": "text",
+ "chunk_metadata": {"connect_to": [{"target": "img_9"}]},
+ }
+ ],
+ ),
+ )
+ assert pool.readable() == frozenset()
+ pool.begin_round()
+ assert pool.readable() == {
+ ("doc_a", "guide.pdf / 2 Treatment"),
+ ("doc_a", "tbl_1"),
+ ("doc_a", "guide.pdf / Root"),
+ ("doc_a", "img_1"),
+ ("doc_b", "notes.pdf / Intro"),
+ ("doc_b", "c9"),
+ ("doc_b", "img_9"),
+ }
+
+
+def test_section_path_received_accepts_ancestor_and_suffix_paths() -> None:
+ readable = {("doc_a", "guide.pdf / 1 Overview / 1.1 Findings")}
+ assert section_path_received(readable, "doc_a", "guide.pdf / 1 Overview")
+ assert section_path_received(readable, "doc_a", "1 Overview / 1.1 Findings")
+ assert section_path_received(readable, "doc_a", "1.1 Findings")
+ assert not section_path_received(readable, "doc_a", "guide.pdf / 1.1 Findings")
+ assert not section_path_received(readable, "doc_a", "2 Treatment")
+ assert not section_path_received(readable, "doc_b", "1 Overview")
+
+
+def test_outline_tree_is_titles_and_indent_only() -> None:
+ pool = EvidencePool()
+ issued = pool.issue(
+ "corpus.outline",
+ ToolResult(
+ text="outline",
+ payload={
+ "rows": [
+ {
+ "document_id": "doc_a",
+ "title": "1 Overview",
+ "depth": 0,
+ },
+ {
+ "document_id": "doc_a",
+ "title": "2 Treatment",
+ "depth": 0,
+ },
+ {
+ "document_id": "doc_a",
+ "title": "2.1 Lifestyle",
+ "depth": 1,
+ },
+ ],
+ "details": {"documents": {"doc_a": "Hypertension_Guideline.pdf"}},
+ },
+ ),
+ )
+ assert issued[0].handle == "O1"
+ assert issued[0].outline_lines == (
+ " [O1] Hypertension_Guideline.pdf",
+ " 1 Overview",
+ " 2 Treatment",
+ " 2.1 Lifestyle",
+ )
+ rendered = pool.render_candidates(issued)
+ assert rendered == "[pick id for this result: O1 = this outline]"
+
+
+def test_descendants_keeps_anchor_summary_and_extra_sections() -> None:
+ pool = EvidencePool()
+ issued = pool.issue(
+ "corpus.read",
+ _ok_read(
+ refs=[
+ {
+ "status": "ok",
+ "document_id": "doc_a",
+ "section_path": "2 Treatment",
+ "chunk_ids": ["c_anchor", "c_child"],
+ }
+ ],
+ chunks=[
+ {
+ "chunk_id": "c_anchor",
+ "document_id": "doc_a",
+ "source_file_name": "guide.pdf",
+ "section_path": "2 Treatment",
+ "chunk_type": "text",
+ "section_summary": "anchor summary",
+ },
+ {
+ "chunk_id": "c_child",
+ "document_id": "doc_a",
+ "source_file_name": "guide.pdf",
+ "section_path": "2 Treatment / 2.1 Lifestyle",
+ "chunk_type": "text",
+ "section_summary": "child summary",
+ },
+ ],
+ ),
+ )
+ assert issued[0].summary == "anchor summary"
+ assert issued[0].extra_sections == 1
+ assert issued[0].chunk_ids == ("c_anchor", "c_child")
+
+
+def test_render_pool_and_empty() -> None:
+ pool = EvidencePool()
+ assert pool.render_pool() == "evidence pool: empty"
+ outline = pool.issue(
+ "corpus.outline",
+ ToolResult(
+ text="outline",
+ payload={
+ "rows": [{"document_id": "doc_a", "title": "1 Overview", "depth": 0}],
+ "details": {"documents": {"doc_a": "Hypertension_Guideline.pdf"}},
+ },
+ ),
+ )
+ pool.begin_round()
+ pool.apply_pick(["O1"])
+ text = pool.render_pool()
+ assert text.startswith("evidence pool (1):")
+ assert " [O1] Hypertension_Guideline.pdf" in text
+ assert " 1 Overview" in text
+ assert pool.render_candidates(outline) == "[pick id for this result: O1 = this outline]"
+
+
+def test_render_budget_and_trace_line_ok_partial_failed() -> None:
+ budget = EpisodeBudget(max_steps=12, wall_clock_seconds=180)
+ budget.record_step()
+ budget.record_usage({"total_tokens": 12034})
+ assert render_budget(budget) == "budget: steps 1/12"
+ assert budget.tokens_used == 12034
+
+ ok = render_trace_line(
+ 1,
+ "corpus_outline",
+ {"scope": [{"document_id": "doc_a"}]},
+ trace_status(ToolResult(text="ok")),
+ )
+ assert ok == '1. corpus_outline {"scope": [{"document_id": "doc_a"}]} -> ok'
+
+ failed = render_trace_line(
+ 3,
+ "corpus_grep",
+ {"patterns": ["ACEI"]},
+ trace_status(
+ ToolResult(text="", error="corpus.grep: unknown argument(s) ['max_results']")
+ ),
+ )
+ assert failed.endswith(" -> failed: corpus.grep: unknown argument(s) ['max_results']")
+
+ partial = render_trace_line(
+ 2,
+ "corpus_read",
+ {"refs": [{"document_id": "doc_a", "section_path": "A"}]},
+ trace_status(
+ ToolResult(
+ text="partial",
+ payload={
+ "refs": [
+ {"status": "ok"},
+ {"status": "failed", "reason": "unknown section_path for doc_a"},
+ ]
+ },
+ )
+ ),
+ )
+ assert " -> partial: ref 2 failed: unknown section_path for doc_a" in partial
+
+
+def test_render_pick_outcome() -> None:
+ assert render_pick_outcome(PickOutcome()) == "picked: none"
+ assert (
+ render_pick_outcome(
+ PickOutcome(
+ added=["R1.1", "O2"],
+ rejected=[{"handle": "R9.9", "reason": "unknown id"}],
+ )
+ )
+ == "picked: R1.1, O2; rejected: R9.9 (unknown id)"
+ )
+
+
+def test_compose_pool_evidence_keeps_pick_order_and_outline_fragment() -> None:
+ entries = [
+ Candidate(
+ handle="O1",
+ kind="outline",
+ document_id="doc_a",
+ source_file_name="guide.pdf",
+ outline_lines=(" [O1] guide.pdf", " 1 Overview"),
+ ),
+ Candidate(
+ handle="R2.1",
+ kind="read",
+ document_id="doc_a",
+ source_file_name="guide.pdf",
+ section_path="2 Treatment",
+ chunk_ids=("c1", "missing"),
+ ),
+ ]
+ assembled = [
+ {
+ "chunk_id": "c1",
+ "sort_order": 4,
+ "composed": [{"type": "text", "text": "treatment body"}],
+ }
+ ]
+ evidence = compose_pool_evidence(entries, assembled)
+ assert evidence == [
+ {"type": "text", "text": "[E1] [§ guide.pdf]"},
+ {"type": "text", "text": " [O1] guide.pdf\n 1 Overview"},
+ {"type": "text", "text": "\n"},
+ {"type": "text", "text": "[E2] [§ guide.pdf / 2 Treatment]"},
+ {"type": "text", "text": " treatment body"},
+ ]
+
+
+def test_issue_ignores_other_tools_and_errors() -> None:
+ pool = EvidencePool()
+ assert pool.issue("corpus.grep", ToolResult(text="hits")) == []
+ assert pool.issue("corpus.read", ToolResult(text="", error="bad args")) == []
+ assert pool.issue("corpus.assets", ToolResult(text="assets")) == []
+
+
+def test_query_table_records_last_successful_html_and_issues_nothing() -> None:
+ pool = EvidencePool()
+ first = ToolResult(
+ text="first",
+ payload={
+ "document_id": "doc_a",
+ "chunk_id": "t1",
+ "table_html": "
"
+ },
+ )
+ composed_text = "".join(part["text"] for part in parts if part["type"] == "text")
+ assert "tablet" in composed_text
+ assert "FULL TABLE SHOULD NOT APPEAR" not in composed_text
+ assert "[tables/" not in composed_text
+
+
def test_standalone_image_is_bytes(monkeypatch) -> None:
monkeypatch.setattr(
"shared.services.retrieval.hydration.evidence_compose._try_read_image_artifact",
@@ -176,6 +218,43 @@ async def test_text_result_keeps_placeholders_and_composes_assets(monkeypatch) -
assert "[images/" not in composed_text
+@pytest.mark.asyncio
+async def test_assemble_uses_queried_subtable_and_keeps_full_without_map() -> None:
+ full_html = "