From 9c1beddc7b55d06578d4de8342bd36a0063d69f5 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Thu, 29 Oct 2026 11:41:44 +0100 Subject: [PATCH 01/50] Added biblioreads support --- README.md | 24 + biblioreads.json | 765 ++++++++++++++++++ cli_input.go | 17 + cli_input_test.go | 67 ++ internal/datasource/biblioreads/cache.go | 65 ++ internal/datasource/biblioreads/client.go | 297 +++++++ .../datasource/biblioreads/client_test.go | 509 ++++++++++++ .../biblioreads/fixtures/book-scraper.json | 60 ++ .../biblioreads/fixtures/reviews-scraper.json | 672 +++++++++++++++ .../biblioreads/fixtures/search-books.json | 19 + internal/datasource/biblioreads/model.go | 108 +++ internal/datasource/httpjson/httpjson.go | 72 ++ internal/datasource/httpjson/httpjson_test.go | 77 ++ internal/datasource/wikidata/httpclient.go | 55 +- .../datasource/wikidata/httpclient_test.go | 22 - internal/index/bleve.go | 39 +- internal/index/bleve_document_read.go | 25 +- internal/index/bleve_document_write.go | 4 + internal/index/bleve_goodreads_write.go | 323 ++++++++ internal/index/bleve_goodreads_write_test.go | 513 ++++++++++++ internal/index/bleve_textrank_write.go | 26 +- internal/index/document.go | 11 + internal/index/goodreads_fields_test.go | 86 ++ internal/webserver/controller.go | 4 +- .../controller/document/controller.go | 7 +- .../webserver/controller/document/detail.go | 1 + .../controller/document/goodreads.go | 69 ++ internal/webserver/document_detail_test.go | 2 +- internal/webserver/document_goodreads_test.go | 246 ++++++ .../embedded/js/reader-reading-mode.js | 33 + .../webserver/embedded/translations/de.yml | 2 + .../webserver/embedded/translations/es.yml | 2 + .../webserver/embedded/translations/fr.yml | 2 + .../webserver/embedded/translations/ru.yml | 2 + .../embedded/views/document/detail.html | 2 +- .../views/partials/document-goodreads.html | 1 + .../views/partials/document-metadata.html | 3 + .../js-tests/reader-reading-mode.test.mjs | 208 +++++ internal/webserver/routes.go | 1 + internal/webserver/upload_test.go | 2 +- internal/webserver/webserver.go | 1 + internal/webserver/webserver_test.go | 2 +- main.go | 26 +- 43 files changed, 4373 insertions(+), 99 deletions(-) create mode 100644 biblioreads.json create mode 100644 cli_input_test.go create mode 100644 internal/datasource/biblioreads/cache.go create mode 100644 internal/datasource/biblioreads/client.go create mode 100644 internal/datasource/biblioreads/client_test.go create mode 100644 internal/datasource/biblioreads/fixtures/book-scraper.json create mode 100644 internal/datasource/biblioreads/fixtures/reviews-scraper.json create mode 100644 internal/datasource/biblioreads/fixtures/search-books.json create mode 100644 internal/datasource/biblioreads/model.go create mode 100644 internal/datasource/httpjson/httpjson.go create mode 100644 internal/datasource/httpjson/httpjson_test.go create mode 100644 internal/index/bleve_goodreads_write.go create mode 100644 internal/index/bleve_goodreads_write_test.go create mode 100644 internal/index/goodreads_fields_test.go create mode 100644 internal/webserver/controller/document/goodreads.go create mode 100644 internal/webserver/document_goodreads_test.go create mode 100644 internal/webserver/embedded/js/reader-reading-mode.js create mode 100644 internal/webserver/embedded/views/partials/document-goodreads.html create mode 100644 internal/webserver/js-tests/reader-reading-mode.test.mjs diff --git a/README.md b/README.md index a68a9569..e9de8da3 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ A personal documents server, Coreander indexes the documents (EPUBs and PDFs wit * Upload documents through the web interface. * Download as kepub (epub for Kobo devices) converted on the fly thanks to [Kepubify](https://github.com/pgaskin/kepubify). * Gather information about authors from [Wikidata](https://wikidata.org). +* Show Goodreads ratings and book and author links through [BiblioReads](#biblioreads-integration). ## Installation @@ -85,6 +86,26 @@ Refer to [Caddy documentation](https://caddyserver.com/docs) for more informatio When using a reverse proxy, include its client-facing port in `FQDN`, for example `FQDN=coreander.example.com:443` for the HTTPS setup above (or `:80` for HTTP), so generated links do not use Coreander's internal listening port. +### BiblioReads integration + +Coreander uses [BiblioReads](https://github.com/nesaku/biblioreads), an alternative Goodreads front-end, to find Goodreads book and author links and display book ratings on document detail pages. The integration is enabled by default and does not require an API key or a Goodreads account. + +Coreander searches using the title and first author, then retries with the title alone if no result qualifies. A result must match both the title and one of the document's authors, ignoring capitalization, accents, punctuation, and whitespace differences (including spacing between author initials). Trailing series annotations such as `(Series name, #1)` and omnibus ranges such as `(#1-#3)` are ignored when comparing titles, but arbitrary subtitles and standalone volume numbers such as `(#1)` are not removed. Exact normalized titles are preferred over titles with an added series annotation. Similar-looking titles alone are not enough: if no result passes these checks, no Goodreads link or rating is shown. Matches may still refer to a different edition of the same book. + +Coreander looks up documents in the background after indexing and can also look up a document when its detail page is opened. Ratings load asynchronously, without blocking the rest of the page. Previously saved lookup outcomes, including books with no match, are reused; to re-evaluate them with the current matching rules, restart Coreander with `--force-indexing` (or `FORCE_INDEXING=true`). This reindexes the whole library and repeats Goodreads lookups. + +By default, requests are spread across built-in public BiblioReads instances, with automatic failover when an instance is unavailable. To use your own instance or a different set of public instances, pass a comma-separated list: + +```sh +coreander --biblioreads-instances=https://br.example.org,https://br2.example.org /absolute/path/to/library +``` + +Alternatively, set `BIBLIOREADS_INSTANCES` to the same comma-separated list. The flag may also be repeated, overrides the environment variable, and replaces the built-in instance list. Instance URLs must use HTTP or HTTPS. + +Successful responses are cached in memory for 24 hours by default. Change this with `--biblioreads-cache-ttl` or `BIBLIOREADS_CACHE_TTL`, in seconds; set it to `0` to disable response caching. Discovered book and author links are stored in the document index. + +Lookups send document titles and author names to the configured BiblioReads instances. To disable the integration, use `--disable-biblioreads` or set `DISABLE_BIBLIOREADS=true`. If no instance can serve a request, Goodreads information may be unavailable, but the rest of Coreander remains usable. + ### Email Some features rely on having an SMTP email service set up, and won't be available otherwise: @@ -175,6 +196,9 @@ In case both a flag and its equivalent environment variable are passed, flag tak |`--client-dynamic-image-cache-ttl` |`CLIENT_DYNAMIC_IMAGE_CACHE_TTL` | Client-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 86400 (24 hours). |`--server-static-cache-ttl` |`SERVER_STATIC_CACHE_TTL` | Server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 31536000 (1 year). |`--server-dynamic-image-cache-ttl` |`SERVER_DYNAMIC_IMAGE_CACHE_TTL` | Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 86400 (24 hours). +|`--disable-biblioreads` |`DISABLE_BIBLIOREADS` | Disable fetching Goodreads data (book and author links) from BiblioReads. Defaults to false. +|`--biblioreads-instances` |`BIBLIOREADS_INSTANCES` | Comma-separated HTTP or HTTPS BiblioReads instance URLs, replacing the built-in public instances. The flag may be repeated. For example: `--biblioreads-instances=https://br.example.org,https://br2.example.org`. Uses the built-in instances when omitted. +|`--biblioreads-cache-ttl` |`BIBLIOREADS_CACHE_TTL` | Duration in seconds for which BiblioReads responses are cached in memory. Set to 0 to disable caching. Defaults to 86400 (24 hours). |`-f` or `--force-indexing` |`FORCE_INDEXING` | Whether to force indexing already indexed documents or not. Defaults to false. |`-s` or `--jwt-secret` |`JWT_SECRET` | String to use to sign JWTs. |`-a` or `--require-auth` |`REQUIRE_AUTH` | Require authentication to access the application if true. Defaults to false. diff --git a/biblioreads.json b/biblioreads.json new file mode 100644 index 00000000..9b9694b2 --- /dev/null +++ b/biblioreads.json @@ -0,0 +1,765 @@ +//POST https://biblioreads.eu.org/api/search/books +//PAYLOAD +//{queryURL: "https://www.goodreads.com/search?q=La%20Reina%20Drag%C3%B3n%20Alice%20Borchardt"} +//RESPONSE +{ + "status": "Received", + "source": "https://github.com/nesaku/biblioreads", + "apiURL": "https://www.goodreads.com/book/auto_complete?format=json&q=La%20Reina%20Drag%C3%B3n%20Alice%20Borchardt", + "searchType": "books", + "numberOfResults": "1 results", + "result": [ + { + "id": 1, + "cover": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/books/1332599049i/12207444._SX50_.jpg", + "title": "La Reina Dragon (La leyenda de Guinevere, #1)", + "bookURL": "/book/show/12207444-la-reina-dragon", + "author": "Alice Borchardt", + "authorURL": "/author/show/151155.Alice_Borchardt", + "rating": "3.82" + } + ], + "lastScraped": "2026-10-28T17:45:31.333Z" +} + +//POST https://biblioreads.eu.org/api/book-scraper +// PAYLOAD +// {queryURL: "https://www.goodreads.com/book/show/12207444-la-reina-dragon"} +// RESPONSE +{ + "status": "Received", + "statusCode": 200, + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + "legacyBookID": 12207444, + "bookID": "kca://book/amzn1.gr.book.v1.4FDu7eHal9NDfA3jGNhOXw", + "resourceID": "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", + "cover": "https://m.media-amazon.com/images/S/compressed.photo.goodreads.com/books/1332599049i/12207444.jpg", + "series": "Tales of Guinevere", + "seriesURL": "/series/50072-tales-of-guinevere", + "seriesPosition": "1", + "workURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + "title": "La Reina Dragon", + "author": [ + { + "id": 1, + "name": "Alice Borchardt", + "role": "Author", + "url": "/author/show/151155.Alice_Borchardt" + } + ], + "rating": "3.82", + "ratingCount": "2,285", + "reviewsCount": "115", + "desc": "Arturo se dio la vuelta y vino resueltamente hacia nosotros. Estaba espléndido, y nunca olvidaré que en ese momento comencé a amarlo. Y ya nunca dejé de hacerlo. Nunca nadie me ha provocado tanta pena, dolor y alegría como él. Ni siquiera por mis hijos he sentido el amor que le profeso todavía ahora. Y creo que aunque hubiera sabido lo que nos deparaba el futuro (preocupaciones, tormentos, batallas y penas), aun así levaría entregado mi corazón tal como hice entonces.\r\n\r\nAlice Brondchardt entra en el reino de la leyenda artúrica y lo hace suyo. La reina Dragón es el primer volumen de la trilogía que recrea el mundo de Camelot con la reina Guinevere convertida en una guerrea sagaz y obstinada.\r\n\r\nNacida en un entorno en el que los enfrentamientos son constantes y las armas están siempre presentes en la vida de hombres y mujeres, Guinevere, hija de una poderosa reina pagana, es una amenaza para su gente y una preciada presa para el temible mago Merlín.\r\n\r\nEnviada a u lugar secreto, crece bajo la protección de un hombre-lobo y de un druida de agrio carácter. Pero ni siquiera en las remotas costas de Escocia, donde los dragones velan por ella, está a salvo del todopoderoso mago. Merlín conoce el destino de la bella joven y no se detendrá ante nada para evitar el cumplimiento de la profecía, puesto que si Guinevere y Arturo llegan a reinar, llevarán un paz al país que convertirá a Merlín, hambriento de poder, en un personaje secundario.", + "genres": [ + "Fantasy", + "Arthurian", + "Fiction", + "Historical Fiction", + "Dragons", + "Historical", + "Mythology", + "Magic", + "Science Fiction Fantasy", + "Retellings" + ], + "bookEdition": "452 pages, Hardcover", + "publishDate": "Wed Jan 01 2003", + "publisher": "Timun Mas", + "language": "Spanish; Castilian", + "asin": "8448033183", + "isbn": "8448033183", + "isbn13": "9788448033187", + "related": [], + "reviews": [], + "reviewBreakdown": { + "rating1": "72", + "rating2": "170", + "rating3": "586", + "rating4": "731", + "rating5": "726" + }, + "quotes": "4", + "quotesURL": "/work/quotes/2440721", + "questions": "1", + "questionsURL": "/book/258666/questions", + "lastScraped": "2026-10-28T18:34:27.525Z" +} + +// POST https://biblioreads.eu.org/api/reviews-scraper +// PAYLOAD +// {resourceID: "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", legacyBookID: 12207444,…} +// RESPONSE +{ + "status": "Received", + "statusCode": 200, + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/book/show/12207444/reviews", + "totalCount": 115, + "nextPageToken": "NTgsMTcwNTAxNjc4NjQzNA", + "prevPageToken": "MTgwMSwxMjE0MTUxODU5MDAw", + "reviews": [ + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.gH7Sf4_G4iz72wvI_yVVyg", + "reviewer": { + "id": 1259454, + "name": "Shawna Astley", + "url": "/user/show/1259454-shawna-astley", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1214150161i/1259454._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 42, + "followerCount": 0 + }, + "rating": 5, + "text": "Women fighting men with swords - whats not to like? ", + "createdAt": "Sun Jun 22 2008", + "updatedAt": "Tue Jan 07 2025", + "spoiler": false, + "likeCount": 18, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/25118938" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.be94OXB9ezWIlFZvdGMKvA", + "reviewer": { + "id": 2478293, + "name": "Zeek", + "url": "/user/show/2478293-zeek", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1339154824i/2478293.jpg", + "isAuthor": false, + "reviewCount": 939, + "followerCount": 149 + }, + "rating": 3, + "text": "Although I liked this tale overall, there were pieces of it that bogged the story down for me.

The Dragon Queen is yet another entry into the tales of Arthur and The Knights of the Round Table. For those who love their Arthurian legends with Merlin as the big bad- then this book's for you. For those who don't-then Run away!

TDQ centers around Guinevere, the warrior queen of Arthur and in Borchardt's version she is indeed a warrior. Guinevere, has the beauty of previous Arthurian tales, but in TDQ she's not the sweet little misguided star-crossed lover of Lancelot that we usually see her portrayed to be. Courageous and bold she has power in her right hand where ferocious fire emanates (grand super power that!), and healing in her left. Moreover, arrows flyout her bow as if an extension of her arm, she communicates with dragons- as well with the dead- displays prophetic abilities, and audaciously defeats asundry of evil beasts that Merlin flings at her.

The story begins with Maeniel, Guinevere's soon to be foster father. Maeniel, otherwise knows as the Gray Watcher, is a shapeshifting man/wolf who has personally felt the destructive powers of Merlin. Deciding that being a wolf was much preferred to living with the humans, he escapes to the wilds and begins a family. It is during his time with the wolves that Guinevere falls into his care, and the Gray Watcher, along with the sorcerer Dugald- who Merlin chased into hiding long ago- take up the charge of raising her. The Gray Watcher's wolf mate nurses
Guinevere from the time they find her left on a cliff and later a woman joins them who will teach her the ways of her people- a people she never knew.

As a child, Guinevere helps her village to defeat a marauding group of seafarers- and it is here where we see her potential for greatness begin to come forth. Guinevere grows in stature, wisdom and power and, though still only a young teen through much of the story, she quickly finds herself thrust into peril. Merlin wants her dead, and though the author details many things in TDQ, I never quite got the gist of his motive for seeking to destroy her so passionately. The only thing I can figure is that she must be the one who either precipitates or actually carries out Merlin's eventual destruction.

Many other side characters show up to aid or engage her in battle. There are all kinds of fantasy creatures- including mythological gods- that she meets up with and, eventually, she meets her future in the boy of Arthur.

Arthur is just as strong as she- and closer to her age then what I've seen portrayed before. Set long before Camalot, he's young and only discovering who and what he's eventually going to be. Much of the middle of the book is from his POV, where he meets the Lady of the Lake. I was disappointed however, that at the end of TDQ, he and Guinevere do not meet up again. (I suppose that will come in later installments.)

The jumping around of POV's was rather confusing for me and the author lost me a bit until it began to pick up again toward the end. The book is full of detailed descriptions of the world Borchardt has created and some of it became overwhelming. To me her descriptions dragged the story down and lost me for most of the middle of the book. But, as I said before, by the end I was back with her.

This is definitely sequel bait- the story is far from over. I'm interested to see where the author- who was Anne Rice's sister, btw- will take it.", + "createdAt": "Thu May 20 2010", + "updatedAt": "Sun May 24 2026", + "spoiler": false, + "likeCount": 10, + "commentCount": 4, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/103546143" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.sC0IQDlAXY4LbkJ3LTJHvA", + "reviewer": { + "id": 80386185, + "name": "Rial", + "url": "/user/show/80386185-rial", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1545932910i/80386185._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 30, + "followerCount": 0 + }, + "rating": 1, + "text": "I did not like this book. However, it is my first 1 Star, so it's got that going for it.

Why did I read this? It was selected for my wife and roommate's book club, and I parasitically piggyback a lot of their books (gives me something to talk about...). It's Arthurian! It's from Guinivere's perspective! Magic! Celtic! Post 'Fall of the Western Empire'! These things and more!

No, it's terrible. Why did I keep reading it? It did take me on a journey: I experienced all the 'Stages of Grief' (some of them more than once.

Denial: Ha, it's so weird, but it'll get better! The chapters are long, but that's okay! Weird, the perspective shifted...?
Anger: I liked the other character; this one is dumb. She's a Mary Sue! WTF? That just worked out for her with zero effort! Okay, she needs to show some growth...
Bargaining: ...Okay, the parts with Arthur aren't bad, maybe?
Anger: Why are they having revenge sex in the forest?
Denial: Arthur is cool, yes, he's very cool... :|
Depression: I really shouldn't be reading this anymore; I think it's actively hurting me.
Acceptance: There is only book. Oh yeah, I guess your shoes can do that. Yep, you're definitely super fucking hot, and everyone wants to be or be with you. Sure why not? I don't have enough brain power to think it should be any different. After all, you're the best, Gwen!

But the real joy of this book is in not starting it. If you made that mistake and you're too masochistic to give up, then finishing it is an exultant experience

The contents are one thing, but they aren't even well served by the writing. Transitions are abrupt and jarring. I back-read multiple times to try and figure out where the hell I was, only to realize there wasn't any explanation; it just got there and it was fine (it wasn't fine).

This thing had such potential: Arthurian set in the 'proper' time frame, the Christianization of Britannia and the clashes with pagan beliefs that ensue, Celtic and Brythonic myth woven throughout. But it all reads so poorly in the end. The author is far too verbose and floral with her language. First person POV ruins any sense of actual accomplishment because we only see the character succeeding; no real growth or change is experienced. I find Divine Right to be problematic, and this book is very much into the romanticism of a destined ruler (both for Arthur and for Gwen).

Oh yeah, Merlin is evil and helps Arthur's mom torment Arthur, I guess? Like, if you hate the kid so much just kill him? Oh wait, 'destiny'...

Tedious tripe, glorified garbage, wretched rubbish. What a waste of time.", + "createdAt": "Mon Nov 05 2018", + "updatedAt": "Fri Jun 10 2022", + "spoiler": false, + "likeCount": 6, + "commentCount": 2, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2578561727" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.rwxYpuQEynxP2f5dSLFmPQ", + "reviewer": { + "id": 90733221, + "name": "Monique Mihalitsianos", + "url": "/user/show/90733221-monique-mihalitsianos", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1773156601i/90733221._UX200_CR0,29,200,200_.jpg", + "isAuthor": true, + "reviewCount": 88, + "followerCount": 9 + }, + "rating": 3, + "text": "This book was splendid in the story-telling sense. The story was epic and magical and engaging. Also, this is, by far, the best Guinevere I have encountered in modern fiction so far. She was strong but not brass, brave but not stupid, powerful yet humble, compassionate but not weak, feminine yet full of agency. Perhaps what I liked the most about her was that she was in communion with animals and gods alike and knew her place in the wheel of existence. She was wise like that, but never overbearingly so. I loved her. I wanted to *be* her.

Arthur I also liked very much, and I liked that this was a tortured Arthur who had suffered child abuse at the hands of her horrible queen mother Igrane (*the* worst evil queen ever) but was still good and noble and could not lie and never feared danger. I liked his relationship with his father, Uther, the High King, which was refreshingly healthy and loving. Merlin I hated. This is a book with a bad Merlin in it, which I don't like as a trope. I much prefer my Merlins to be good and Gandalf-like, but in this case it worked to have him be so evil and act as the villain. The black magic he wrought was one of the sickest and most horrifying ones I've ever read in a story.

Alice Borchardt seems to know her history well, so much so in fact that it's almost like if she lived in this period of time (Roman - Saxon) in a past life. I, for one, appreciated all of the extraneous detail relating to the customs and mores of the time because it made the world-building that much better.

The magic system in this book was refreshingly deep and believable. So many magic systems in recent fantasy books are light and superficial, as if they had been thought up in brainstorming sessions instead of being fully inspired. Not so with Borchardt's story; the magic here seemed inspired by the author's own psyche and thus, was multilayered and poliphacetic, changeable and malleable but also essentially true to its own core.

I also liked that an existencial and historical struggle was hinted at: the convergence between paganism and Christianity, if there can be a convergence at all. It was only hinted at here but I wouldn't mind if this theme in particular were elaborated more in the second book (The Raven Warrior) because it is of particular interest to me. On some level I think it was also of particular interest to Borchardt, as well. I am a huge fan of Anne Rice's (Alice Borchardt's sister) Vampire Chronicles, and Rice definitely waxes philosophical about her own struggles with different faith systems and beliefs in her own fiction, much more so than Borchardt, who chose to focus more on the story. But I wouldn't have minded if she had expanded on this in her fiction, just like her sister did. But then again, I am of a philosophical bent and like these kinds of tangents.

Why the three-star rating, then, if I liked the book so much? Well, the story was golden, but the writing was confusing at times, especially at the beginning and end. Sometimes it was almost stream-of-consciousness like, and I got lost and confused. At other times, when a character referred to somebody else as 'she', for example, I didn't know who they were talking about, because there were several 'she's' they could have been referring to. So some more specificity in this and other similar instances would have been appreciated. Sometimes, also, the writing got too dream-like, too airy fairy and dreamy...and perhaps more concrete writing would have served the book better overall. I also suspect this book could have had tighter editing. I know for a fact Anne Rice is not a fan of people editing her writing, and I don't know if Alice Borchardt felt the same as her sister does in this matter, but I for one think good editing is absolutely crucial in any book.

Putting this aside, though, I can say that I thoroughly enjoyed this book and am looking forward to reading the second and final installment. This was meant to be a trilogy, but unfortunately, the author died from illness before she could complete the third book, so two books is all we have. I will read her other books as well, though. She's a gem of a story-teller, at least in my estimation.", + "createdAt": "Wed Mar 18 2020", + "updatedAt": "Thu Nov 30 2023", + "spoiler": false, + "likeCount": 3, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3208179930" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.XvL2nHk7Bt4tULNXUE2yaA", + "reviewer": { + "id": 2207539, + "name": "Steen", + "url": "/user/show/2207539-steen", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1408254648i/2207539._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 473, + "followerCount": 4 + }, + "rating": 1, + "text": "This was a did not finish book for me.

So after reading about 185 pages of The Dragon Queen by Alice Borchardt I have decided to thrown it down and stop reading. I thought I would like it being about Quinevere and all but I hate it. It's not like the Arthur legends I've ever known. But that's not really even it though. I could handle a different take on the stories ( I still have yet to read any books about Arthur that are out there that are more well known) it's just the way she has written it. She jumps from one place to another and it's like when did that happen? Also sometimes when there are a few people talking 'she says...' your like who is talking? Which one is saying what. UGH. I hate not finishing a book but I really can't continue to read it anymore.

Having said that I really have no urge to read Bequiled which is also by her. I read the first in that series and while it was better written then The Dragon Queen I really don't love the characters in the first book to continue to read the second one anymore. Not after trying to get through The Dragon Queen and hating it. It's sad but yeah. I love Anne Rice's books and hoped that I would equally like her sister's books. But I guess they are just not for me.

I've read reviews on these two books and it sounds like her werewolf books are pretty good but I think I may wait a while to even attempt to try those ones.

So now its off to the library today to pick up some more to read.", + "createdAt": "Sat Apr 11 2009", + "updatedAt": "Wed Jan 23 2019", + "spoiler": false, + "likeCount": 3, + "commentCount": 3, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/52264098" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.rZ5icbmFfLHPKpEQOm2kNw", + "reviewer": { + "id": 41814516, + "name": "Alberto García", + "url": "/user/show/41814516-alberto-garc-a", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1508706260i/41814516._UX200_CR0,33,200,200_.jpg", + "isAuthor": true, + "reviewCount": 45, + "followerCount": 40 + }, + "rating": 4, + "text": "Voto de confianza... esta m¡¡bien escrito y la revisión de la leyenda artúrica es muy interesante. Al final, en algunos capítulos, se le va un poco la pinza con salidas algo paranoicas que me han dejado bastante descolocado... lo que es seguro es que no me lo esperaba... pero tampoco comprendo demasiado la razón para ello. Veremos cómo acaba y si se queda en una anécdota.", + "createdAt": "Fri Jun 21 2019", + "updatedAt": "Mon Sep 14 2026", + "spoiler": false, + "likeCount": 3, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2796875885" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.G3-a2b8-oopoAbpH6RmbsA", + "reviewer": { + "id": 73527602, + "name": "Rachel", + "url": "/user/show/73527602-rachel", + "avatar": "", + "isAuthor": false, + "reviewCount": 13, + "followerCount": 0 + }, + "rating": 3, + "text": "This was like reading someone's dream. The ideas were all loosely connected but the transitions were weird and random or completely non-existent. However, the characters and concepts were fun and interesting and I did like this book.", + "createdAt": "Fri Jan 04 2019", + "updatedAt": "Mon Nov 17 2025", + "spoiler": false, + "likeCount": 4, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2582029737" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.juoLRbi8R_urzbj6jgRAww", + "reviewer": { + "id": 101256289, + "name": "Malischa Nijenhuis", + "url": "/user/show/101256289-malischa-nijenhuis", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1773261867i/101256289._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 1222, + "followerCount": 32 + }, + "rating": 5, + "text": "Eigenlijks lees ik nu een boek van mijn man maar wauw wat is dit een gave uitgave en boek. Nu op naar boek 2", + "createdAt": "Sat Mar 21 2020", + "updatedAt": "Mon Mar 23 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3238259324" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.GLjLEKHIsU-RrmRdU73hGQ", + "reviewer": { + "id": 3504881, + "name": "Suzanne", + "url": "/user/show/3504881-suzanne", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1294105241i/3504881.jpg", + "isAuthor": false, + "reviewCount": 546, + "followerCount": 12 + }, + "rating": 2, + "text": "I'm very sorry to say, I could not finish this book. I REALLY wanted to be in love with this story. I even purchased the 2nd book from Thrift Books soon after I started this one. I figured they would become part of my Arthurian obsession, and I would need to own them. It started out with a great concept, and I liked the differences in the legend that I hadn't read before. I wanted to know more about the different characters, but it was so choppy, from one point of view to another back and forth, I couldn't keep track of which character I was reading about.

It just dragged on and on and on and on....it was trial after trial after trial. The characters would accomplish one win and suddenly face another trial. I felt like I was reading a bad rendition of The Odyssey. I REALLY wanted to love it. But reading became a chore, and I fell into a reading slump. The book sat in one spot for a couple weeks, and I felt it hanging over my head. I finally decided enough was enough. I wanted to be eager to read again. So I took out the bookmark, and this is going on my \"couldn't finish\" list.

I will donate the set to a Free Little Library somewhere, and I hope someone else will get some pleasure out of them.", + "createdAt": "Sun Oct 14 2018", + "updatedAt": "Fri Oct 30 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 2, + "recommendFor": null, + "shelf": "did-not-finish", + "reviewURL": "/review/show/2522667985" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.73ge1ENF_VjVNC1CSJb8iw", + "reviewer": { + "id": 67537582, + "name": "Kaylee", + "url": "/user/show/67537582-kaylee", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1627453497i/67537582._UX200_CR0,1,200,200_.jpg", + "isAuthor": false, + "reviewCount": 793, + "followerCount": 33 + }, + "rating": 1, + "text": "I do have to start by saying that this is the first Arthurian legend I can recall reading. I don't think that really affected my POV but...

Extremely hard to get through from the beginning. Kept hoping it would get better but nope. Not only was the switching between third and first POV annoying but it was made worse by the main character. Did not want her voice at all. It was also hard to tell what was going on/who was speaking at times (during the third POV of course). In the beginning I went back to the and re-read things but quickly learned there was no point. Sometimes things arrived randomly and others I realized I had just spaced out because of all the needless words.  You could take out half the words in this book and be all the better off for it...

Also if I had to read \"As I said...\" one more time...!

I can't remember the last time I didn't finish a book and if I hadn't had to read this because of my book club I don't think I would have finished. Says everything in a nutshell.

So much more but I'll just leave it at that. Extremely glad to be done with this book!", + "createdAt": "Fri Nov 09 2018", + "updatedAt": "Fri Oct 30 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 2, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2577596943" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.6kd8QVi2sBrNhUslyz1aUQ", + "reviewer": { + "id": 54738977, + "name": "TigerLily ", + "url": "/user/show/54738977-tigerlily", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1460164836i/54738977.jpg", + "isAuthor": false, + "reviewCount": 133, + "followerCount": 3 + }, + "rating": 5, + "text": "Really enjoyable, a page Turner that I didn't want to be finished. I am looking forward to reading the 2nd book.", + "createdAt": "Sun Jan 10 2021", + "updatedAt": "Sun Mar 21 2021", + "spoiler": false, + "likeCount": 2, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3732026445" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.njxZJZUHk2BVK8mcTJp2aQ", + "reviewer": { + "id": 40251914, + "name": "Elizabeth Sanders", + "url": "/user/show/40251914-elizabeth-sanders", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1530132382i/40251914.jpg", + "isAuthor": false, + "reviewCount": 38, + "followerCount": 0 + }, + "rating": 4, + "text": "It's an interesting retelling of Guinevere and Arthur. I loved every bit of it and can't wait to read the sequel.", + "createdAt": "Thu Jul 04 2019", + "updatedAt": "Thu Jul 11 2019", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2871843590" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.HegnVxYZnLTsNGxJeVV2cA", + "reviewer": { + "id": 11384547, + "name": "Lyndsay MacDonald", + "url": "/user/show/11384547-lyndsay-macdonald", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1371265927i/11384547.jpg", + "isAuthor": false, + "reviewCount": 38, + "followerCount": 0 + }, + "rating": 2, + "text": "This book was weird. I think it might have been tolerable if the author didn't try to weave it into the Arthurian legend (ie. if she made up original characters), but as it is I just couldn't get into it. There were some parts/scenes that I did enjoy, and others that I didn't... but overall I found it very confusing, and often found myself asking \"What is happening right now?\" and \"Why?\"

The story line flip-flopped around a lot... the characters were not wholly believable (Guinevere did NOT seem like a 13 year old girl, by any stretch of the imagination).. and I found parts of the plot silly and unrelated.

I won't be reading any of the other books in this sequence.", + "createdAt": "Wed Jan 22 2014", + "updatedAt": "Fri Aug 12 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/831334767" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.oazwOZoGnu8yIQJ_48FVMw", + "reviewer": { + "id": 27373770, + "name": "Andrea Krivak", + "url": "/user/show/27373770-andrea-krivak", + "avatar": "", + "isAuthor": false, + "reviewCount": 26, + "followerCount": 1 + }, + "rating": 2, + "text": "I think you have to really like arthurian legends to read this book. I liked it in the beginning, it started telling the story of Geunivere as a young girl and the \"what if\" this was her upbringing. There were wizards, men who turned into wolves and of course, merlin and morgana. About halfway through she meets Arthur, and then the book divides into the telling of each of their lives and they each stray into their own mystic journeys. To me, it spent too much time on these mystic journeys and by the end, i was ready to be done with this whole story. Not interesting enough ending for me to read into the sequel.", + "createdAt": "Mon Mar 24 2014", + "updatedAt": "Fri Aug 12 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/890822661" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.IxTTRac3W8gbXoHFdfTu5g", + "reviewer": { + "id": 1009204, + "name": "Cindy", + "url": "/user/show/1009204-cindy", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1228465282i/1009204._UX200_CR0,28,200,200_.jpg", + "isAuthor": false, + "reviewCount": 856, + "followerCount": 105 + }, + "rating": 1, + "text": "Maybe some day I'll come back to this book, but I just don't think I can read this book. I find myself wondering if I'm not following something of if this book is just beyond my reading. The writing is in a way that makes it seems like, wow we want the read to think this author is just so descriptive and great. Only it becomes confusing and bored. If you aren't familiar with the story of Guinevere then there is no point in reading it because nothing is explained and you are just sitting there wondering what the heck is going on. ", + "createdAt": "Wed Aug 27 2008", + "updatedAt": "Thu Jan 25 2018", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/31374042" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1._BHlRCbTwy-DJRq5mCTQnw", + "reviewer": { + "id": 4804714, + "name": "Angela", + "url": "/user/show/4804714-angela", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1294946776i/4804714.jpg", + "isAuthor": false, + "reviewCount": 129, + "followerCount": 6 + }, + "rating": 3, + "text": "I enjoyed this book up until the last third and then it got way too much involved with Arthur. It didn't quite wrap up the way I had expected and I would suggest not reading it unless you want to continue on with the series. I may read the sequel at a later time but I need a break from the acid trip like sections with Arthur.", + "createdAt": "Thu Mar 28 2013", + "updatedAt": "Tue Aug 09 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/572964102" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.P4s-Ysv7_6bKQAupxDvCwA", + "reviewer": { + "id": 4018173, + "name": "Misty", + "url": "/user/show/4018173-misty", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1286250306i/4018173._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 307, + "followerCount": 0 + }, + "rating": 2, + "text": "I really wanted to like this book.

About halfway through, I got bored and tried to push through it, but ultimately gave up. There was so much description and not enough happening.

Also, considering this was titled the Dragon Queen, I expected more dragons. Their lack of presence was disappointing.", + "createdAt": "Tue Nov 26 2013", + "updatedAt": "Thu Jun 23 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "could-not-finish", + "reviewURL": "/review/show/774467120" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.2E0pSS7MyztIPHdoIbH4Nw", + "reviewer": { + "id": 120860, + "name": "Lassarina Aoibhell", + "url": "/user/show/120860-lassarina-aoibhell", + "avatar": "", + "isAuthor": false, + "reviewCount": 487, + "followerCount": 5 + }, + "rating": 1, + "text": "I am DNFing this book on page 84 and we're going to go on a wild ride about why.

NB: If you are a scholar of Arthuriana, history, religion, or....well, almost anything, you might want to put down your drink. I'm not responsible for the state of your monitor.

We begin with the cover copy: Daughter of a mighty pagan queen, Guinevere has become a threat to her people and a prize to the dreaded sorcerer Merlin. Sent into hiding, she grows up under the protection of a shapeshifting man-wolf and an ornery Druid. But even on the remote coast of Scotland, where dragons feed and watch over her, she is not safe. Merlin knows the young beauty's destiny--and he will stop at nothing to prevent what has been foretold...

I also need to strongly highlight one of the blurbs. Like, really strongly. \"In addition to being exceptionally well written, [The Dragon Queen] has enormous integrity: All of the stories--of Arthur's parentage, of Merlin, of the characters' daily struggles--are well grounded in the history, politics, and religions of the time.....This is a trilogy to watch.\" --San Jose Mercury News

I want you to look at that quote carefully, and then come on a ride with me.

I will acknowledge up front my own reading comprehension fail, because I had forgotten that the shapeshifter was actually in the blurb and was yelling about \"what the fuck.\". I'm still saying \"what the fuck,\" just about plenty of other things.

Let's start with: It takes until page 48 for Guinevere to show up in any real sense. 500 page book; the main character doesn't actually appear until 10% in. Don't like that. Instead we are following around Dugald, the druid, and Maeniel, the shapeshifter. He's allegedly from Gaul. We begin at Tintagel in the year 470 CE, where Merlin and some of his cronies murder Vortigen. Some kind of magical snake is involved. (Aside: absolutely no problem with Merlin as the villain, particularly in anything allegedly centering the women of this tale.) Maeniel the shifter teleports??? out of the carnage, finds himself a she-wolf who has recently given birth, and shacks up with her. They encounter Dugald and the toddler Guinevere, who is currently disguised as a boy. For some reason, although everyone complains about disliking the Romans, we re-enact the founding of Rome, with Maeniel's mate suckling little Guinevere. Why does the she-wolf agree to this? Who can say. Dugald, meanwhile, talks about the influence of St. Patrick. Yes, that one. \"We don't call ourselves [druids] any longer. Not since Patrick came to Ireland bearing Christ's message.\"

....ten pages ago we called Merlin a chief druid. Also. It's the year 470. We don't have exact dates of the saint's life, but we know it was roughly in that century. More importantly, it was the Romans who banned druidic practices, fucking 400 years earlier (Augustus had no chill.) So there is no way that Patrick's actions in Ireland are affecting the order in England. Remember that blurb I highlighted? HMMMM. ALSO WHY ARE YOU, A GUY IN CORNWALL, TALKING ABOUT PATRICK. why. what is even your deal.

Also also, in this variation of the story (?), apparently, Dugald had a philosophical disagreement with Merlin and \"lost,\" so he was expected by the rules of his order to commit suicide because he lost.

Me: I do not remember reading this in any of the books I consumed during my obsessive period about Celtic histories and cultures and archaeology. (Wiki and checking with knowledgeable friends suggests I am not alone in not remembering this.)

If nothing else, that seems like a giant fucking waste of a 20 year apprenticeship, man.

(At this point I also stopped to verify my memory that Dugald and Maeniel do not appear in other Arthurian sources. I seem to be correct. And like, yeah, we add characters to stories all the time, I'm just. why are you making me deal with men in a book about Guinevere!)

There's also a point where young Guinevere, Dugald, and Maeniel are involved in helping to fight off a ship of slavers, which conveniently results in there being a woman to join the household and teach Guinevere How To Be A Girl. I should specify that the captain of said slaver's ship was apparently named Cymry. Why is he literally named \"Welsh\"? fuck if I know.

There's also a particular kind of ambient misogyny that I find tiresome, and like, I know we're in the fifth century and the Romans were assholes, and also I am aware that Celtic cultures were not paragons of equality and virtue, but the available evidence both of Romans being bitchy about how ass-backward Celtic societies were (omg they let their women do things!!!!) and the remnants of brehon law allocating specific property to women after divorce and categorizing a type of marriage as \"she married down and gets to keep all her stuff\" [wildly paraphrased] I'm just like

what. what are you doing, ma'am

and once again I'm going to refer back to that blurb. About being well grounded in the history, politics, and religions of the time.

I do not recommend actually reading this unless you want to play a drinking game, in which case, may I suggest that the drink you use should perhaps be water?", + "createdAt": "Thu Apr 09 2026", + "updatedAt": "Thu Apr 09 2026", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "did-not-finish", + "reviewURL": "/review/show/1761732484" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.9X0QSJx7BxHSTHSctKMkwg", + "reviewer": { + "id": 11348002, + "name": "Laural", + "url": "/user/show/11348002-laural", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1349329030i/11348002._UX200_CR0,5,200,200_.jpg", + "isAuthor": false, + "reviewCount": 36, + "followerCount": 5 + }, + "rating": 2, + "text": "I got about halfway through this book. Someday I'll give it another chance, or maybe try to figure out what exactly the author did wrong. A sword weilding Gweneviere? It could have been great. I thought it would be after reading the description on the back of the gorgeous cover. ", + "createdAt": "Thu Oct 04 2012", + "updatedAt": "Thu Jun 23 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/427412066" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.p0mSKsHft_BTv2vScLHOsQ", + "reviewer": { + "id": 1250008, + "name": "Giuila", + "url": "/user/show/1250008-giuila", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1244141968i/1250008._UY200_CR2,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 61, + "followerCount": 0 + }, + "rating": 0, + "text": "... t read", + "createdAt": "Thu Jul 17 2008", + "updatedAt": "Fri Mar 24 2017", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "to-read", + "reviewURL": "/review/show/27520982" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.QMO0-GkHiX0UEYcsOaO7lw", + "reviewer": { + "id": 1431136, + "name": "Matt", + "url": "/user/show/1431136-matt", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1218856549i/1431136._UY200_CR25,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 6, + "followerCount": 0 + }, + "rating": 5, + "text": "an overall awesome book", + "createdAt": "Sat Aug 16 2008", + "updatedAt": "Fri Mar 24 2017", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/30279062" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.qOdgrbVKU-XMQR5DSbHfDQ", + "reviewer": { + "id": 16979552, + "name": "Sarah Garner", + "url": "/user/show/16979552-sarah-garner", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1408284741i/16979552._UX200_CR0,53,200,200_.jpg", + "isAuthor": false, + "reviewCount": 84, + "followerCount": 18 + }, + "rating": 2, + "text": "I struggled with this book and I couldn't finish it, although it was a great idea, I didn't love this book.", + "createdAt": "Sat Sep 20 2014", + "updatedAt": "Thu Aug 20 2020", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/1059961768" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.JjlmKD1eCF28HlpY4KmCGw", + "reviewer": { + "id": 34744007, + "name": "Michael D.", + "url": "/user/show/34744007-michael-d", + "avatar": "", + "isAuthor": false, + "reviewCount": 81, + "followerCount": 1 + }, + "rating": 2, + "text": "I had this book about 20 years ago, tried to read it, but I just couldn't get into it. I got another copy used and gave it another try. While not as bad as my first experience in the first 20 pages, I found in the ensuing pages that the narrative is quite disjointed, and timelines are fluid and unclear to me. Perhaps that was Borchardt's intent, but I was confused by it. Additionally, it seems that characters are magically transported to strange places without any transition that I could discern. This narrative was hard to follow.

The book begins in a scene in 470 in Tintigal (not Tintagil, the famous castle in Cornwall where Arthur was conceived and born in the traditional romance). Ostensibly, the main character is a wolfman (a wolf who changes into a man when he chooses to) named Maeniel, and he is a friend to the British king Vortigen, who is not the villain he is normally portrayed as in history and the romances. Merlin shows up with a contingent of Saxons, and a battle ensues in which Vortigen is slain and Maeniel is knocked unconscious by an unknown cudgel. In the next scene, he wakes up on a hillside far away, presumably the far north, near Hadrian's Wall, and he becomes the mate of a she-wolf, who bears him a son, Black Leg (who will become Lancelot in the second novel The Raven Warrior). Sometime later, we find that 70 years have passed since Vortigen's death. Oddly, there is a baby girl in the nearby village, Guenevere (Borchardt spells her name differently depending on who is addressing her), who is one of the major characters of this series, the eponymous destined Dragon Queen. When Guenevere is 15, we are then told that 100 years have passed since Vortigen died. Confusion abounds.

This story had occasional high points, with modern fantastical treatments of the transformation duels, but I really had a hard time following the flow of Borchardt's narrative. I am a pretty astute reader, so I find it difficult to believe this was my fault. Borchardt seemed to be playing coy with the unfolding of events, and that was annoying. She was the sister of the better-known Anne Rice, author of the Vampire Chronicles, so I tried hard to give this a chance.

Borchardt intended to make this a trilogy, but the passed away in 2007 before the third book in the trilogy could be published. I don't know if anyone picked up her notes with the intent to complete her work, and I can't imagine that there was a demand for it. I do intend to slog through The Raven Warrior, as it is my rule to read books through and evaluate them honestly. If they're bad, I point out why. The lack of comprehensible transitions in time and space are my primary complaint with this novel. I hope that The Raven Warrior is more cognizant of walking me through how characters get from Point A to Point B than this one was.
", + "createdAt": "Thu Nov 09 2023", + "updatedAt": "Thu Nov 09 2023", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5944805120" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v2.2025072400.a7efbaf3-dd61-4567-b4e1-d3f344fabb0c", + "reviewer": { + "id": 42506425, + "name": "Michelle M", + "url": "/user/show/42506425-michelle-m", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1715615833i/42506425._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 380, + "followerCount": 3 + }, + "rating": 5, + "text": "Honestly? 5/6 or 4.75/6
I really loved this book! It's hard to describe what exactly I loved about it though lol. It had a very ethereal quality to it... The magic in this world is just a fact of life, some peple have it, while others don't, and yet, it didn't feel fantastical. Everyone knows about magic and most peoples still believe in the gods and things even though \"The Church\" (christianity) is present on the British Isles.

Borchardt grounds the story in the time right after Rome leaves the Isles, though they are still very much an influence on the peoples. It was a very mystical vibe to the whole story, kind of like there is a shroud of fog that is the magic of the world. IDK, like I said, hard to describe lol.

I loved Guinevere, and I liked how different characters referred to her by various spellings/pronouciations thoughout the story, as if they were speaking different languages. It is a very different take on the Arthur/Guinevere story, and very much an action oriented one, with Guinevere being the focus of the action and not just a passive bystander.

I though the portrayal of the other major characters interesting, and they were given roles that they are not usual given in the story. Arthur is \"good\", Merlin and Ingrane are the \"evil\" ones, Guinevere is strong willed and knows what she wants and sets out to get it. She doesn't seem like the type of person who will betray Arthur with Lancelot, but we'll just have to find out! (I hope she doesn't). The portrayal of these characters (and other named characters) made me do some research to try to get a better picture of if they had ever been portayed like this before. My search was unsucessful lol, as they were very diffefrent from \"traditional\" stories, but I haven't really read many retellings, so I cant be sure if they are exclusive to this story.

My only compaint with this book is that I found out it was supposed to be a trilogy!!!! And YET there are only two books! T.T *cries* I know the author can't help that she passed away, but book 2 came out in 2003 and she passed in 2007! That seems like a decent amount of time to finish the third book (I'm being very ungrateful, I know I am, I just lament what never was). Also apparently Anne Rice of vampire fame is Borchardt's sister?!?!?! She doesn't have like an early draft of her sister's just laying around??? T.T

I will def be reading the next one! (this one took me so long because I paused to participate in a read a thon and then got distracted by other books lol and then I got busy with work so I didn't have as much time to read)", + "createdAt": "Thu Jul 24 2025", + "updatedAt": "Thu Jul 24 2025", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/7714101862" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.HXGbQg0020aQJjD9G6pW2A", + "reviewer": { + "id": 163241100, + "name": "Alex", + "url": "/user/show/163241100-alex", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1763011908i/163241100._UY200_CR11,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 98, + "followerCount": 0 + }, + "rating": 2, + "text": "I enjoyed Guinevere’s character early on in the book. It was interesting watching her being raised alongside her foster brother, Black Leg, by a wolf shifter (Maniel) and a she-wolf. I do appreciate her relationship with the She Wolf, who Guinevere calls Mother. The thing that really kept me going through this book was Guinevere’s interactions with the dragons of the land and her connection to the land. I was also intrigued by some of the same conversations about religion in this book since Guinevere is raised in nature and has regular interactions with a Druid and some interactions with Christiany.
All this being said I struggled with the writing. There are huge chunks of it that are confusing since it’s not always clear who is being talked about when he, she or they are mentioned considering there might be multiple people mentioned previously in the scene. Transitions between moments are also rough. I was questioning when things changed and what was being talked about. The long descriptions of things really didn’t help either and me zoning out as I read.
It was also a bit rough to read this because of how things were worded/certain sciences being really uncomfortable for seemingly no reason out of nowhere. The dynamics between Igraine and Merlin’s relationship being one them. Scenes with them were sexual with zero warning and it wasn’t just limited to them. It happened a lot with various characters. Things were also made pretty uncomfortable with minors even if it was a scene that wasn’t intended to uncomfortably sexual, it was made that way by directly saying things like they “were far too young for even a dream of passion.” With how confusing the writing was I couldn’t even tell if there was a reason for why these sort of scenes were written in detail outside of making the audience feel uncomfortable and actively dislike the characters who are at fault for these behaviors.", + "createdAt": "Sun May 24 2026", + "updatedAt": "Sun May 24 2026", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5900548383" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.VoNZbtbhrTyL1Fhg43jpGw", + "reviewer": { + "id": 42064438, + "name": "Briana", + "url": "/user/show/42064438-briana", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1715717411i/42064438._UY200_CR5,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 161, + "followerCount": 1 + }, + "rating": 2, + "text": "This book was not for me. I had high hopes, the summary sounded interesting. It took me much longer than usual to finish this book as I struggled to enjoy it and I will not be picking up the second installment. I almost DNF'd this book several times but decided to stick with it.

The first half of the book felt like it was the author's freewriting development of the book, which she never bothered to go back and actually edit into a cohesive and understandable story. It was like being is some kind of dream where everything was hazy and only loosely connected. Sometimes you ended up in an entirely different location or time without even realizing it. It was also almost impossible to understand who was being discussed as the author tends to avoid using names and simply use pronouns leaving the reader to try and figure out (usually without success) who the \"she\" is that is being discussed. It was also very interesting that the author (or editor) chose to spell Guinevere's name differently throughout the book.

The second half was a little better and is the only reason I didn't give this book a one star. I don't mind Merlin as the antagonist as some other reviewers do and I found his character interesting. However, I did not really like the relationship between Arthur and Guinevere. I'm not sure how I am supposed to be convinced that this was some fated love story when they met for all of maybe an hour total. It seemed more insta-lust than anything else and there was no relationship development at all after that.

Overall I spent my time either being very confused and a little bored or kind of confused and mildly interested.", + "createdAt": "Thu Aug 15 2024", + "updatedAt": "Thu Aug 15 2024", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/6728900786" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.IoZW54HX8Qad9w5IgbQB9A", + "reviewer": { + "id": 6424389, + "name": "Joy Pouros", + "url": "/user/show/6424389-joy-pouros", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1567889312i/6424389._UY200_CR47,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 125, + "followerCount": 2 + }, + "rating": 3, + "text": "2.5 stars.

I REALLY like the character of Guinevere, but there were other parts of the story I didn't enjoy.

The writing style can be very winding and even abstract in some ways. There are whole parts you can skim without missing much. There aren't a ton of people, but enough to get them confused, especially a couple with similar names.

I'm always very frustrated with stories that try to have strong female characters but still have an overly sexually charged world. There are so many wonderfully creative parts of this world and then we default back to several tired tropes. There's just no need.

Some chapters are 30 pages, some are 5. It doesn't appear based on POV or time, so why is this so inconsistent?

And finally... I understand this is the first book of the trilogy, but this book just felt like a long prologue. Other series the first book is still a worthwhile read and entertaining story. This is more of an origin story before you fully appreciate the main story. So there are very mixed feelings about continuing the series because while I love Guinevere, there's an awful lot to overlook. ", + "createdAt": "Tue Aug 30 2022", + "updatedAt": "Tue Aug 30 2022", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/4955253716" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.ahVbGwMWksFfbjRS9c6Vbg", + "reviewer": { + "id": 29405146, + "name": "Anita", + "url": "/user/show/29405146-anita", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1407269712i/29405146._CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 614, + "followerCount": 3 + }, + "rating": 1, + "text": "First of all, I love Arthurian legends. Second, I always finish books, constantly hopeful that the story will improve, but I must confess that I skimmed this book to the end. I just cannot read any more!
This story has no cohesion, no main narrator and skips from one location to another mid-chapter. Plus there are men suddenly shapeshifting into wolves, and back again; no reasons given; no context for the intrigue and killings taking place; and a young girl child, supposedly Guinevere of Arthurian legend, being suckled by a she-wolf after being kidnapped.
Imaginative description is great in a fantasy story; confusion is not. This book is one long puzzle - a maze with apparently no direction, interspersed with some flowery descriptions of sunsets and misty mornings; visions of ravens and dragons (fishing); and a cast of hundreds of unrelated and irrelevant characters. Reading this book feels akin to experiencing one of those dreams in which you are lost and desperately trying to find your way back home.
Need I add that I did not enjoy this book, and will not be reading the sequel?", + "createdAt": "Fri Oct 30 2020", + "updatedAt": "Fri Oct 30 2020", + "spoiler": true, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3618732305" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.zo7-0bYwOUS0LlPXyqeuXQ", + "reviewer": { + "id": 3198878, + "name": "Stevie", + "url": "/user/show/3198878-stevie", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1560478233i/3198878._UX200_CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 71, + "followerCount": 0 + }, + "rating": 2, + "text": "Rating this book was a challenge. It was an impulse grab. I suspected it would be wonderfully terrible or terribly wonderful and there was only one way to find out.

What I liked: Arthurian legend from Guin's point of view. Cool. That awkward time period in history when they were both Christian but still observed many pagan rites. It had all the names I knew, and some of the tropes I knew: Vortigen, the witch in the well, so on and so forth from various movies I've seen.

What I disliked: the writing. Some of the dialogue wasn't needed. The descriptions could be too much in places and entirely lacking in others. Who is speaking? What do they look like? They were a human in a tower and now they are a fish? When? How? Transitions don't exist. This makes it hard to follow.

It was a wild ride, and I hung on until the end because I was curious as to how it would turn out. Turns out there's a sequel...I don't know that I have that in me...", + "createdAt": "Sat Mar 21 2020", + "updatedAt": "Sat Mar 21 2020", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3240571912" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.E7XotAu3DvsavlUw1u62eA", + "reviewer": { + "id": 107938758, + "name": "Danai", + "url": "/user/show/107938758-danai", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1705239736i/107938758._UX200_CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 133, + "followerCount": 0 + }, + "rating": 3, + "text": "I didn't love it, nor did I hate it.

Everything was magical, mystical, and mysterious; Borchardt painted an imaginative world with interesting and diverse characters. She evidently has a deep knowledge of the British Isles, its history, and its lore.

But the writing was, at times, completely chaotic and confusing. As if someone was just spilling their string of thoughts without sufficient context or details - or too many of the wrong details. More than once, I found myself confused by abrupt changes in the situation, only to learn - through narration - that a memory was being shared.

This book didn't capture me nor engross me by any means, but it did beckon me to come back and finish it. If I come across the next book in the series, I will read it simply out of curiosity - what happens next for our heroes and villains? But I won't go hunting for a copy of it anytime soon...", + "createdAt": "Thu Jan 11 2024", + "updatedAt": "Thu Jan 11 2024", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5638508977" + } + ], + "lastScraped": "2026-10-28T18:38:19.808Z" +} \ No newline at end of file diff --git a/cli_input.go b/cli_input.go index f707b116..1f0ed76f 100644 --- a/cli_input.go +++ b/cli_input.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "net/url" "github.com/alecthomas/kong" ) @@ -82,6 +83,11 @@ type CLIInput struct { ServerStaticCacheTTL int `env:"SERVER_STATIC_CACHE_TTL" default:"31536000" name:"server-static-cache-ttl" help:"Server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 1 year (31536000 seconds)."` // ServerDynamicImageCacheTTL defines the server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours. ServerDynamicImageCacheTTL int `env:"SERVER_DYNAMIC_IMAGE_CACHE_TTL" default:"86400" name:"server-dynamic-image-cache-ttl" help:"Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours (86400 seconds)."` + // DisableBiblioreads turns off fetching Goodreads data from BiblioReads. + DisableBiblioreads bool `env:"DISABLE_BIBLIOREADS" default:"false" name:"disable-biblioreads" help:"Disable fetching Goodreads data (book and author links) from BiblioReads."` + BiblioreadsInstances []string `env:"BIBLIOREADS_INSTANCES" name:"biblioreads-instances" sep:"," help:"Comma-separated BiblioReads instance URLs. Overrides the built-in public instances. May be repeated."` + // BiblioreadsCacheTTL defines how long responses from BiblioReads are cached in memory, in seconds. Defaults to 24 hours. + BiblioreadsCacheTTL int `env:"BIBLIOREADS_CACHE_TTL" default:"86400" name:"biblioreads-cache-ttl" help:"Duration in seconds for which BiblioReads responses are cached in memory. Set to 0 to disable caching. Defaults to 24 hours (86400 seconds)."` // ShareCommentMaxSize defines the maximum length for share comments in characters. Defaults to 280. ShareCommentMaxSize int `env:"SHARE_COMMENT_MAX_SIZE" short:"m" default:"280" name:"share-comment-max-size" help:"Maximum length for share comments in characters. Defaults to 280."` // ShareMaxRecipients defines the maximum number of recipients allowed when sharing a document. Defaults to 10. @@ -120,6 +126,17 @@ func (c CLIInput) Validate() error { if c.ShareCommentMaxSize < 0 { return fmt.Errorf("share-comment-max-size must not be negative, got %v", c.ShareCommentMaxSize) } + // BiblioreadsCacheTTL must not be negative: 0 is the documented way to + // disable caching, so a negative value is most likely a typo. + if c.BiblioreadsCacheTTL < 0 { + return fmt.Errorf("biblioreads-cache-ttl must not be negative, got %v", c.BiblioreadsCacheTTL) + } + for _, instance := range c.BiblioreadsInstances { + u, err := url.Parse(instance) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" { + return fmt.Errorf("biblioreads-instances must contain absolute HTTP or HTTPS URLs, got %q", instance) + } + } // MinPasswordLength must be positive: 0 or negative disables the minimum // length check entirely, silently allowing empty passwords. if c.MinPasswordLength <= 0 { diff --git a/cli_input_test.go b/cli_input_test.go new file mode 100644 index 00000000..0073ec69 --- /dev/null +++ b/cli_input_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "reflect" + "strings" + "testing" + + "github.com/alecthomas/kong" +) + +func TestBiblioreadsInstances(t *testing.T) { + tests := []struct { + name string + args []string + env string + want []string + }{ + {name: "defaults"}, + {name: "single", args: []string{"--biblioreads-instances=https://br.example.org"}, want: []string{"https://br.example.org"}}, + {name: "comma separated", args: []string{"--biblioreads-instances=https://br.example.org,http://localhost:8080/"}, want: []string{"https://br.example.org", "http://localhost:8080/"}}, + {name: "repeated", args: []string{"--biblioreads-instances=https://br.example.org", "--biblioreads-instances=https://br2.example.org"}, want: []string{"https://br.example.org", "https://br2.example.org"}}, + {name: "environment", env: "https://br.example.org,https://br2.example.org", want: []string{"https://br.example.org", "https://br2.example.org"}}, + {name: "flag overrides environment", env: "https://env.example.org", args: []string{"--biblioreads-instances=https://cli.example.org"}, want: []string{"https://cli.example.org"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("BIBLIOREADS_INSTANCES", tt.env) + var input CLIInput + parser := newCLIParser(t, &input) + if _, err := parser.Parse(append(tt.args, t.TempDir())); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(input.BiblioreadsInstances, tt.want) { + t.Errorf("instances = %v, want %v", input.BiblioreadsInstances, tt.want) + } + }) + } +} + +func TestBiblioreadsInstancesRejectInvalidURLs(t *testing.T) { + for _, instance := range []string{"br.example.org", "/relative", "ftp://br.example.org", "https://", "https://bad host", "https://br.example.org,invalid"} { + t.Run(instance, func(t *testing.T) { + var input CLIInput + parser := newCLIParser(t, &input) + _, err := parser.Parse([]string{"--biblioreads-instances=" + instance, t.TempDir()}) + if err == nil || !strings.Contains(err.Error(), "biblioreads-instances") { + t.Fatalf("expected biblioreads-instances validation error, got %v", err) + } + }) + } +} + +func newCLIParser(t *testing.T, input *CLIInput) *kong.Kong { + t.Helper() + parser, err := kong.New(input, kong.Vars{ + "defaultMaxSimilarityCandidates": "0", + "defaultMinSimilarityScoreRatio": "0", + "defaultMaxSimilarityPhrases": "0", + "defaultCommonTextRankEntryRatio": "0", + "defaultMinCommonTextRankAbsoluteCount": "0", + "defaultPruneChangeTriggerRatio": "0", + }) + if err != nil { + t.Fatal(err) + } + return parser +} diff --git a/internal/datasource/biblioreads/cache.go b/internal/datasource/biblioreads/cache.go new file mode 100644 index 00000000..d4427df0 --- /dev/null +++ b/internal/datasource/biblioreads/cache.go @@ -0,0 +1,65 @@ +package biblioreads + +import ( + "sync" + "time" +) + +// DefaultCacheTTL is how long successful responses are cached by default. +const DefaultCacheTTL = 24 * time.Hour + +// cacheSweepInterval sets how often expired entries are purged, so entries that are never +// requested again don't stay in memory forever. +const cacheSweepInterval = 10 * time.Minute + +type cacheEntry struct { + body []byte + expiresAt time.Time +} + +// responseCache is an in-memory cache of raw response bodies with a fixed TTL. +type responseCache struct { + ttl time.Duration + mu sync.Mutex + entries map[string]cacheEntry + lastSweep time.Time +} + +func newResponseCache(ttl time.Duration) *responseCache { + return &responseCache{ + ttl: ttl, + entries: make(map[string]cacheEntry), + lastSweep: time.Now(), + } +} + +func (c *responseCache) get(key string) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + entry, ok := c.entries[key] + if !ok { + return nil, false + } + if !time.Now().Before(entry.expiresAt) { + delete(c.entries, key) + return nil, false + } + return entry.body, true +} + +func (c *responseCache) set(key string, body []byte) { + c.mu.Lock() + defer c.mu.Unlock() + + now := time.Now() + if now.Sub(c.lastSweep) >= cacheSweepInterval { + for k, entry := range c.entries { + if !now.Before(entry.expiresAt) { + delete(c.entries, k) + } + } + c.lastSweep = now + } + c.entries[key] = cacheEntry{body: body, expiresAt: now.Add(c.ttl)} +} diff --git a/internal/datasource/biblioreads/client.go b/internal/datasource/biblioreads/client.go new file mode 100644 index 00000000..efc39c54 --- /dev/null +++ b/internal/datasource/biblioreads/client.go @@ -0,0 +1,297 @@ +// Package biblioreads implements a client for the BiblioReads API +// (https://github.com/nesaku/biblioreads), an alternative Goodreads front-end. +package biblioreads + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math/rand/v2" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/svera/coreander/v5/internal/datasource/httpjson" +) + +const ( + goodreadsBaseURL = "https://www.goodreads.com" + + searchBooksPath = "/api/search/books" + bookScraperPath = "/api/book-scraper" + reviewsScraperPath = "/api/reviews-scraper" +) + +// DefaultInstances lists public BiblioReads instances. +var DefaultInstances = []string{ + "https://biblioreads.eu.org", + "https://biblioreads.mooo.com", + "https://biblioreads.lunar.icu", + "https://biblioreads.ducks.party", + "https://biblioreads.privacyredirect.com", + "https://br.bloat.cat", + "https://biblioreads.utilibre.org", +} + +// ErrUnavailable is returned when no instance could serve a request (network errors, rate limits, +// non-JSON responses), as opposed to an instance answering with an error of its own. +var ErrUnavailable = errors.New("biblioreads: all instances failed") + +type Client struct { + instances []string + httpClient *http.Client + cache *responseCache + + mu sync.Mutex + cooldownUntil map[string]time.Time +} + +type options struct { + instances []string + httpClient *http.Client + cacheTTL time.Duration +} + +type Option func(*options) + +// WithInstances sets the BiblioReads instances to use. An empty list keeps DefaultInstances. +func WithInstances(instances []string) Option { + return func(o *options) { + if len(instances) > 0 { + o.instances = instances + } + } +} + +// WithHTTPClient sets the HTTP client used for requests. A nil client keeps the default one, +// which has a 15 seconds timeout. +func WithHTTPClient(httpClient *http.Client) Option { + return func(o *options) { + if httpClient != nil { + o.httpClient = httpClient + } + } +} + +// WithCacheTTL sets how long successful responses are cached. A zero or negative ttl disables caching. +func WithCacheTTL(ttl time.Duration) Option { + return func(o *options) { + o.cacheTTL = ttl + } +} + +// NewClient returns a client that spreads requests across instances. Each request starts at a +// random instance and fails over to the next ones if it is unavailable (network error, 429 or a +// non-JSON response). Unavailable instances are skipped until their Retry-After time, or +// httpjson.DefaultRetryAfter, has passed. Successful responses are cached in memory for +// DefaultCacheTTL unless changed with WithCacheTTL. +func NewClient(opts ...Option) *Client { + o := options{ + instances: DefaultInstances, + httpClient: &http.Client{Timeout: 15 * time.Second}, + cacheTTL: DefaultCacheTTL, + } + for _, opt := range opts { + opt(&o) + } + + normalized := make([]string, len(o.instances)) + for i, instance := range o.instances { + normalized[i] = strings.TrimRight(instance, "/") + } + + c := &Client{ + instances: normalized, + httpClient: o.httpClient, + cooldownUntil: make(map[string]time.Time), + } + if o.cacheTTL > 0 { + c.cache = newResponseCache(o.cacheTTL) + } + return c +} + +type queryURLPayload struct { + QueryURL string `json:"queryURL"` +} + +type reviewsPayload struct { + ResourceID string `json:"resourceID"` + LegacyBookID int `json:"legacyBookID"` + NextPageToken string `json:"nextPageToken,omitempty"` +} + +// SearchBooks searches Goodreads books matching query. +func (c *Client) SearchBooks(ctx context.Context, query string) (SearchBooksResponse, error) { + var res SearchBooksResponse + payload := queryURLPayload{ + QueryURL: goodreadsBaseURL + "/search?q=" + escapeQuery(query), + } + err := c.post(ctx, searchBooksPath, payload, &res) + return res, err +} + +// Book returns the details of the Goodreads book identified by id, which can be +// either a numeric ID ("12207444") or an ID with slug ("12207444-la-reina-dragon"). +func (c *Client) Book(ctx context.Context, id string) (BookResponse, error) { + var res BookResponse + payload := queryURLPayload{ + QueryURL: goodreadsBaseURL + "/book/show/" + url.PathEscape(id), + } + err := c.post(ctx, bookScraperPath, payload, &res) + return res, err +} + +// Reviews returns a page of reviews for the work identified by resourceID and legacyBookID +// (both available in BookResponse). Pass an empty nextPageToken to get the first page. +func (c *Client) Reviews(ctx context.Context, resourceID string, legacyBookID int, nextPageToken string) (ReviewsResponse, error) { + var res ReviewsResponse + payload := reviewsPayload{ + ResourceID: resourceID, + LegacyBookID: legacyBookID, + NextPageToken: nextPageToken, + } + err := c.post(ctx, reviewsScraperPath, payload, &res) + return res, err +} + +// unavailableError means the instance can't serve requests right now and another one should be tried. +type unavailableError struct { + endpoint string + retryAfter time.Duration + reason error +} + +func (e *unavailableError) Error() string { + return fmt.Sprintf("%s: %v", e.endpoint, e.reason) +} + +func (e *unavailableError) Unwrap() error { + return e.reason +} + +func (c *Client) post(ctx context.Context, path string, payload, dest any) error { + body, err := json.Marshal(payload) + if err != nil { + return err + } + + cacheKey := path + "\x00" + string(body) + if c.cache != nil { + if cached, ok := c.cache.get(cacheKey); ok { + return json.Unmarshal(cached, dest) + } + } + + var errs []error + for _, instance := range c.candidates() { + endpoint := instance + path + respBody, err := c.postToInstance(ctx, endpoint, body) + if err == nil { + c.clearCooldown(instance) + if err := json.Unmarshal(respBody, dest); err != nil { + return fmt.Errorf("biblioreads: error decoding response from %s: %w", endpoint, err) + } + if c.cache != nil { + c.cache.set(cacheKey, respBody) + } + return nil + } + var unavailable *unavailableError + if !errors.As(err, &unavailable) { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + c.setCooldown(instance, unavailable.retryAfter) + errs = append(errs, err) + } + return fmt.Errorf("%w: %w", ErrUnavailable, errors.Join(errs...)) +} + +// candidates returns the instances that are not cooling down, starting at a random one. +// If all of them are cooling down, the one whose cooldown ends first is returned, so +// requests don't fail without trying when every instance had a transient error. +func (c *Client) candidates() []string { + c.mu.Lock() + defer c.mu.Unlock() + + now := time.Now() + start := rand.IntN(len(c.instances)) + candidates := make([]string, 0, len(c.instances)) + soonest := "" + for i := range c.instances { + instance := c.instances[(start+i)%len(c.instances)] + until, coolingDown := c.cooldownUntil[instance] + if !coolingDown || !now.Before(until) { + candidates = append(candidates, instance) + continue + } + if soonest == "" || until.Before(c.cooldownUntil[soonest]) { + soonest = instance + } + } + if len(candidates) == 0 { + return []string{soonest} + } + return candidates +} + +func (c *Client) setCooldown(instance string, d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.cooldownUntil[instance] = time.Now().Add(d) +} + +func (c *Client) clearCooldown(instance string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.cooldownUntil, instance) +} + +// postToInstance returns the body of a successful response. +func (c *Client) postToInstance(ctx context.Context, endpoint string, body []byte) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := httpjson.Do(c.httpClient, req) + if err != nil { + return nil, &unavailableError{endpoint: endpoint, retryAfter: httpjson.DefaultRetryAfter, reason: err} + } + + if resp.StatusCode == http.StatusTooManyRequests { + return nil, &unavailableError{ + endpoint: endpoint, + retryAfter: httpjson.ParseRetryAfter(resp.Header.Get("Retry-After")), + reason: fmt.Errorf("status code %d", resp.StatusCode), + } + } + + // BiblioReads always answers with JSON, even on errors (e.g. a 504 for a book that + // doesn't exist), so anything else comes from a broken instance or a proxy in front of it. + if !json.Valid(resp.Body) { + return nil, &unavailableError{ + endpoint: endpoint, + retryAfter: httpjson.DefaultRetryAfter, + reason: fmt.Errorf("status code %d with non-JSON response", resp.StatusCode), + } + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("biblioreads: request to %s failed with status code %d: %s", endpoint, resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + +// escapeQuery escapes a search string using %20 for spaces, as Goodreads does. +func escapeQuery(query string) string { + return strings.ReplaceAll(url.QueryEscape(query), "+", "%20") +} diff --git a/internal/datasource/biblioreads/client_test.go b/internal/datasource/biblioreads/client_test.go new file mode 100644 index 00000000..bb53abaf --- /dev/null +++ b/internal/datasource/biblioreads/client_test.go @@ -0,0 +1,509 @@ +package biblioreads_test + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +type capturedRequest struct { + method string + path string + contentType string + payload map[string]any +} + +func newMockServer(t *testing.T, status int, fixture string, captured *capturedRequest) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + captured.method = r.Method + captured.path = r.URL.Path + captured.contentType = r.Header.Get("Content-Type") + if err := json.NewDecoder(r.Body).Decode(&captured.payload); err != nil { + t.Errorf("couldn't decode request payload: %v", err) + } + w.WriteHeader(status) + if fixture == "" { + _, _ = w.Write([]byte(`{"status":"Error - Invalid Query"}`)) + return + } + contents, err := os.ReadFile("fixtures/" + fixture) + if err != nil { + t.Fatalf("couldn't read fixture %s: %v", fixture, err) + } + _, _ = w.Write(contents) + })) + t.Cleanup(server.Close) + return server +} + +func assertRequest(t *testing.T, captured capturedRequest, path string, payload map[string]any) { + t.Helper() + if captured.method != http.MethodPost { + t.Errorf("expected POST, got %s", captured.method) + } + if captured.path != path { + t.Errorf("expected path %s, got %s", path, captured.path) + } + if captured.contentType != "application/json" { + t.Errorf("expected application/json content type, got %s", captured.contentType) + } + if len(captured.payload) != len(payload) { + t.Errorf("expected payload %v, got %v", payload, captured.payload) + } + for key, value := range payload { + if captured.payload[key] != value { + t.Errorf("expected payload[%s] = %v, got %v", key, value, captured.payload[key]) + } + } +} + +func TestSearchBooks(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusOK, "search-books.json", &captured) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + res, err := client.SearchBooks(context.Background(), "La Reina Dragón Alice Borchardt") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertRequest(t, captured, "/api/search/books", map[string]any{ + "queryURL": "https://www.goodreads.com/search?q=La%20Reina%20Drag%C3%B3n%20Alice%20Borchardt", + }) + + if len(res.Result) != 1 { + t.Fatalf("expected 1 result, got %d", len(res.Result)) + } + got := res.Result[0] + if got.Title != "La Reina Dragon (La leyenda de Guinevere, #1)" || + got.BookURL != "/book/show/12207444-la-reina-dragon" || + got.Author != "Alice Borchardt" || + got.Rating != "3.82" || + got.ID != 1 { + t.Errorf("unexpected result: %+v", got) + } + if res.NumberOfResults != "1 results" || res.SearchType != "books" { + t.Errorf("unexpected response metadata: %+v", res) + } +} + +func TestBook(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusOK, "book-scraper.json", &captured) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + res, err := client.Book(context.Background(), "12207444-la-reina-dragon") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertRequest(t, captured, "/api/book-scraper", map[string]any{ + "queryURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + }) + + if res.LegacyBookID != 12207444 { + t.Errorf("expected legacy book ID 12207444, got %d", res.LegacyBookID) + } + if res.ResourceID != "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g" { + t.Errorf("unexpected resource ID %s", res.ResourceID) + } + if res.Title != "La Reina Dragon" || res.Series != "Tales of Guinevere" || res.SeriesPosition != "1" { + t.Errorf("unexpected book data: %+v", res) + } + if len(res.Author) != 1 || res.Author[0].Name != "Alice Borchardt" || res.Author[0].Role != "Author" { + t.Errorf("unexpected authors: %+v", res.Author) + } + if len(res.Genres) != 10 || res.Genres[0] != "Fantasy" { + t.Errorf("unexpected genres: %v", res.Genres) + } + if res.ISBN13 != "9788448033187" || res.Language != "Spanish; Castilian" { + t.Errorf("unexpected book data: %+v", res) + } + if res.ReviewBreakdown.Rating5 != "726" { + t.Errorf("unexpected review breakdown: %+v", res.ReviewBreakdown) + } +} + +func TestReviews(t *testing.T) { + t.Run("first page", func(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusOK, "reviews-scraper.json", &captured) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + res, err := client.Reviews(context.Background(), "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", 12207444, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertRequest(t, captured, "/api/reviews-scraper", map[string]any{ + "resourceID": "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", + "legacyBookID": float64(12207444), + }) + + if res.TotalCount != 115 || res.NextPageToken != "NTgsMTcwNTAxNjc4NjQzNA" { + t.Errorf("unexpected response metadata: %+v", res) + } + if len(res.Reviews) == 0 { + t.Fatal("expected reviews, got none") + } + got := res.Reviews[0] + if got.Rating != 5 || got.Shelf != "read" || got.LikeCount != 18 || got.RecommendFor != nil { + t.Errorf("unexpected review: %+v", got) + } + if got.Reviewer.ID != 1259454 || got.Reviewer.Name != "Shawna Astley" || got.Reviewer.ReviewCount != 42 { + t.Errorf("unexpected reviewer: %+v", got.Reviewer) + } + }) + + t.Run("next page", func(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusOK, "reviews-scraper.json", &captured) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "token"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertRequest(t, captured, "/api/reviews-scraper", map[string]any{ + "resourceID": "kca://work/abc", + "legacyBookID": float64(1), + "nextPageToken": "token", + }) + }) +} + +func TestErrorStatus(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusNotFound, "", &captured) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + if _, err := client.Book(context.Background(), "0"); err == nil { + t.Error("expected error on non-200 response, got nil") + } +} + +func newCountingServer(t *testing.T, status int, body string, hits *atomic.Int32) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + +func TestFailover(t *testing.T) { + t.Run("unavailable instances are skipped", func(t *testing.T) { + var unavailableHits, rateLimitedHits, htmlHits, workingHits atomic.Int32 + unavailable := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &unavailableHits) + rateLimited := newCountingServer(t, http.StatusTooManyRequests, `{"status":"Too many requests"}`, &rateLimitedHits) + html := newCountingServer(t, http.StatusOK, "Parked domain", &htmlHits) + working := newCountingServer(t, http.StatusOK, `{"status":"Received","title":"La Reina Dragon"}`, &workingHits) + down := httptest.NewServer(http.NotFoundHandler()) + down.Close() + + instances := []string{down.URL, unavailable.URL, rateLimited.URL, html.URL, working.URL} + for range 20 { + client := biblioreads.NewClient(biblioreads.WithInstances(instances), biblioreads.WithCacheTTL(0)) + res, err := client.Book(context.Background(), "12207444") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.Title != "La Reina Dragon" { + t.Fatalf("unexpected title %q", res.Title) + } + } + if workingHits.Load() != 20 { + t.Errorf("expected working instance to be hit 20 times, got %d", workingHits.Load()) + } + }) + + t.Run("requests start at different instances", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + first := newCountingServer(t, http.StatusOK, `{"status":"Received"}`, &firstHits) + second := newCountingServer(t, http.StatusOK, `{"status":"Received"}`, &secondHits) + + client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + for range 100 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected load to be spread, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) + + t.Run("JSON error responses are not retried", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) + second := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) + + client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + _, err := client.Book(context.Background(), "0") + if err == nil { + t.Fatal("expected error, got nil") + } + if errors.Is(err, biblioreads.ErrUnavailable) { + t.Errorf("expected a non-ErrUnavailable error, got %v", err) + } + if hits.Load() != 1 { + t.Errorf("expected a single request, got %d", hits.Load()) + } + }) + + t.Run("error when all instances fail", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) + second := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) + + client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected ErrUnavailable, got %v", err) + } + if hits.Load() != 2 { + t.Errorf("expected both instances to be tried, got %d requests", hits.Load()) + } + }) + + t.Run("cancelled context stops failover", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusOK, `{}`, &hits) + second := newCountingServer(t, http.StatusOK, `{}`, &hits) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(ctx, "1"); err == nil { + t.Fatal("expected error, got nil") + } + if hits.Load() != 0 { + t.Errorf("expected no requests, got %d", hits.Load()) + } + }) +} + +func TestCooldown(t *testing.T) { + t.Run("failing instance is skipped on later requests", func(t *testing.T) { + var badHits, goodHits atomic.Int32 + bad := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &badHits) + good := newCountingServer(t, http.StatusOK, `{"status":"Received"}`, &goodHits) + + client := biblioreads.NewClient(biblioreads.WithInstances([]string{bad.URL, good.URL}), biblioreads.WithCacheTTL(0)) + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if badHits.Load() > 1 { + t.Errorf("expected failing instance to be hit at most once, got %d", badHits.Load()) + } + if goodHits.Load() != 20 { + t.Errorf("expected working instance to be hit 20 times, got %d", goodHits.Load()) + } + }) + + t.Run("instance is used again after Retry-After", func(t *testing.T) { + var limitedHits, otherHits atomic.Int32 + limited := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if limitedHits.Add(1) == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = w.Write([]byte(`{"status":"Received"}`)) + })) + t.Cleanup(limited.Close) + other := newCountingServer(t, http.StatusOK, `{"status":"Received"}`, &otherHits) + + client := biblioreads.NewClient(biblioreads.WithInstances([]string{limited.URL, other.URL}), biblioreads.WithCacheTTL(0)) + for limitedHits.Load() == 0 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if limitedHits.Load() != 1 { + t.Fatalf("expected rate limited instance to be skipped during cooldown, got %d hits", limitedHits.Load()) + } + + time.Sleep(1100 * time.Millisecond) + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if limitedHits.Load() == 1 { + t.Error("expected rate limited instance to be used again after its cooldown") + } + }) + + t.Run("instance is probed when all are cooling down", func(t *testing.T) { + var hits atomic.Int32 + flaky := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte(`{"status":"Received"}`)) + })) + t.Cleanup(flaky.Close) + + client := biblioreads.NewClient(biblioreads.WithInstances([]string{flaky.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(context.Background(), "1"); err == nil { + t.Fatal("expected error on first request, got nil") + } + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("expected cooling down instance to be probed, got %v", err) + } + }) + + t.Run("JSON error responses don't trigger cooldown", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + notFound := `{"status":"Error - Book not found"}` + first := newCountingServer(t, http.StatusGatewayTimeout, notFound, &firstHits) + second := newCountingServer(t, http.StatusGatewayTimeout, notFound, &secondHits) + + client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + for range 20 { + if _, err := client.Book(context.Background(), "0"); err == nil { + t.Fatal("expected error, got nil") + } + } + if firstHits.Load()+secondHits.Load() != 20 { + t.Errorf("expected one request per call, got %d", firstHits.Load()+secondHits.Load()) + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected no instance to be benched, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) + + t.Run("cancelled requests don't trigger cooldown", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + first := newCountingServer(t, http.StatusOK, `{}`, &firstHits) + second := newCountingServer(t, http.StatusOK, `{}`, &secondHits) + + client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for range 5 { + _, _ = client.Book(ctx, "1") + } + for range 40 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected no instance to be benched, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) +} + +func TestCache(t *testing.T) { + t.Run("successful responses are cached per request", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","title":"La Reina Dragon","genres":["Fantasy"]}`, &hits) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + + first, err := client.Book(context.Background(), "1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + first.Genres[0] = "modified" + + second, err := client.Book(context.Background(), "1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hits.Load() != 1 { + t.Errorf("expected cached response to be used, got %d requests", hits.Load()) + } + if second.Title != "La Reina Dragon" || second.Genres[0] != "Fantasy" { + t.Errorf("expected an unmodified copy of the cached response, got %+v", second) + } + + if _, err := client.Book(context.Background(), "2"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.SearchBooks(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, ""); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "token"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hits.Load() != 5 { + t.Errorf("expected different requests not to share cache entries, got %d requests", hits.Load()) + } + }) + + t.Run("entries expire after the TTL", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received"}`, &hits) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(100*time.Millisecond)) + + for range 3 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if hits.Load() != 1 { + t.Fatalf("expected 1 request before expiry, got %d", hits.Load()) + } + + time.Sleep(150 * time.Millisecond) + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hits.Load() != 2 { + t.Errorf("expected expired entry to be refreshed, got %d requests", hits.Load()) + } + }) + + t.Run("errors are not cached", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusGatewayTimeout, `{"status":"Error - Book not found"}`, &hits) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + + for range 3 { + if _, err := client.Book(context.Background(), "0"); err == nil { + t.Fatal("expected error, got nil") + } + } + if hits.Load() != 3 { + t.Errorf("expected every failed request to reach the server, got %d requests", hits.Load()) + } + }) + + t.Run("caching can be disabled", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received"}`, &hits) + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(0)) + + for range 3 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if hits.Load() != 3 { + t.Errorf("expected 3 requests with caching disabled, got %d", hits.Load()) + } + }) +} diff --git a/internal/datasource/biblioreads/fixtures/book-scraper.json b/internal/datasource/biblioreads/fixtures/book-scraper.json new file mode 100644 index 00000000..5b8abc6a --- /dev/null +++ b/internal/datasource/biblioreads/fixtures/book-scraper.json @@ -0,0 +1,60 @@ +{ + "status": "Received", + "statusCode": 200, + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + "legacyBookID": 12207444, + "bookID": "kca://book/amzn1.gr.book.v1.4FDu7eHal9NDfA3jGNhOXw", + "resourceID": "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", + "cover": "https://m.media-amazon.com/images/S/compressed.photo.goodreads.com/books/1332599049i/12207444.jpg", + "series": "Tales of Guinevere", + "seriesURL": "/series/50072-tales-of-guinevere", + "seriesPosition": "1", + "workURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + "title": "La Reina Dragon", + "author": [ + { + "id": 1, + "name": "Alice Borchardt", + "role": "Author", + "url": "/author/show/151155.Alice_Borchardt" + } + ], + "rating": "3.82", + "ratingCount": "2,285", + "reviewsCount": "115", + "desc": "Arturo se dio la vuelta y vino resueltamente hacia nosotros. Estaba espléndido, y nunca olvidaré que en ese momento comencé a amarlo. Y ya nunca dejé de hacerlo. Nunca nadie me ha provocado tanta pena, dolor y alegría como él. Ni siquiera por mis hijos he sentido el amor que le profeso todavía ahora. Y creo que aunque hubiera sabido lo que nos deparaba el futuro (preocupaciones, tormentos, batallas y penas), aun así levaría entregado mi corazón tal como hice entonces.\r\n\r\nAlice Brondchardt entra en el reino de la leyenda artúrica y lo hace suyo. La reina Dragón es el primer volumen de la trilogía que recrea el mundo de Camelot con la reina Guinevere convertida en una guerrea sagaz y obstinada.\r\n\r\nNacida en un entorno en el que los enfrentamientos son constantes y las armas están siempre presentes en la vida de hombres y mujeres, Guinevere, hija de una poderosa reina pagana, es una amenaza para su gente y una preciada presa para el temible mago Merlín.\r\n\r\nEnviada a u lugar secreto, crece bajo la protección de un hombre-lobo y de un druida de agrio carácter. Pero ni siquiera en las remotas costas de Escocia, donde los dragones velan por ella, está a salvo del todopoderoso mago. Merlín conoce el destino de la bella joven y no se detendrá ante nada para evitar el cumplimiento de la profecía, puesto que si Guinevere y Arturo llegan a reinar, llevarán un paz al país que convertirá a Merlín, hambriento de poder, en un personaje secundario.", + "genres": [ + "Fantasy", + "Arthurian", + "Fiction", + "Historical Fiction", + "Dragons", + "Historical", + "Mythology", + "Magic", + "Science Fiction Fantasy", + "Retellings" + ], + "bookEdition": "452 pages, Hardcover", + "publishDate": "Wed Jan 01 2003", + "publisher": "Timun Mas", + "language": "Spanish; Castilian", + "asin": "8448033183", + "isbn": "8448033183", + "isbn13": "9788448033187", + "related": [], + "reviews": [], + "reviewBreakdown": { + "rating1": "72", + "rating2": "170", + "rating3": "586", + "rating4": "731", + "rating5": "726" + }, + "quotes": "4", + "quotesURL": "/work/quotes/2440721", + "questions": "1", + "questionsURL": "/book/258666/questions", + "lastScraped": "2026-10-28T18:34:27.525Z" +} diff --git a/internal/datasource/biblioreads/fixtures/reviews-scraper.json b/internal/datasource/biblioreads/fixtures/reviews-scraper.json new file mode 100644 index 00000000..1d7908c5 --- /dev/null +++ b/internal/datasource/biblioreads/fixtures/reviews-scraper.json @@ -0,0 +1,672 @@ +{ + "status": "Received", + "statusCode": 200, + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/book/show/12207444/reviews", + "totalCount": 115, + "nextPageToken": "NTgsMTcwNTAxNjc4NjQzNA", + "prevPageToken": "MTgwMSwxMjE0MTUxODU5MDAw", + "reviews": [ + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.gH7Sf4_G4iz72wvI_yVVyg", + "reviewer": { + "id": 1259454, + "name": "Shawna Astley", + "url": "/user/show/1259454-shawna-astley", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1214150161i/1259454._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 42, + "followerCount": 0 + }, + "rating": 5, + "text": "Women fighting men with swords - whats not to like? ", + "createdAt": "Sun Jun 22 2008", + "updatedAt": "Tue Jan 07 2025", + "spoiler": false, + "likeCount": 18, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/25118938" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.be94OXB9ezWIlFZvdGMKvA", + "reviewer": { + "id": 2478293, + "name": "Zeek", + "url": "/user/show/2478293-zeek", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1339154824i/2478293.jpg", + "isAuthor": false, + "reviewCount": 939, + "followerCount": 149 + }, + "rating": 3, + "text": "Although I liked this tale overall, there were pieces of it that bogged the story down for me.

The Dragon Queen is yet another entry into the tales of Arthur and The Knights of the Round Table. For those who love their Arthurian legends with Merlin as the big bad- then this book's for you. For those who don't-then Run away!

TDQ centers around Guinevere, the warrior queen of Arthur and in Borchardt's version she is indeed a warrior. Guinevere, has the beauty of previous Arthurian tales, but in TDQ she's not the sweet little misguided star-crossed lover of Lancelot that we usually see her portrayed to be. Courageous and bold she has power in her right hand where ferocious fire emanates (grand super power that!), and healing in her left. Moreover, arrows flyout her bow as if an extension of her arm, she communicates with dragons- as well with the dead- displays prophetic abilities, and audaciously defeats asundry of evil beasts that Merlin flings at her.

The story begins with Maeniel, Guinevere's soon to be foster father. Maeniel, otherwise knows as the Gray Watcher, is a shapeshifting man/wolf who has personally felt the destructive powers of Merlin. Deciding that being a wolf was much preferred to living with the humans, he escapes to the wilds and begins a family. It is during his time with the wolves that Guinevere falls into his care, and the Gray Watcher, along with the sorcerer Dugald- who Merlin chased into hiding long ago- take up the charge of raising her. The Gray Watcher's wolf mate nurses
Guinevere from the time they find her left on a cliff and later a woman joins them who will teach her the ways of her people- a people she never knew.

As a child, Guinevere helps her village to defeat a marauding group of seafarers- and it is here where we see her potential for greatness begin to come forth. Guinevere grows in stature, wisdom and power and, though still only a young teen through much of the story, she quickly finds herself thrust into peril. Merlin wants her dead, and though the author details many things in TDQ, I never quite got the gist of his motive for seeking to destroy her so passionately. The only thing I can figure is that she must be the one who either precipitates or actually carries out Merlin's eventual destruction.

Many other side characters show up to aid or engage her in battle. There are all kinds of fantasy creatures- including mythological gods- that she meets up with and, eventually, she meets her future in the boy of Arthur.

Arthur is just as strong as she- and closer to her age then what I've seen portrayed before. Set long before Camalot, he's young and only discovering who and what he's eventually going to be. Much of the middle of the book is from his POV, where he meets the Lady of the Lake. I was disappointed however, that at the end of TDQ, he and Guinevere do not meet up again. (I suppose that will come in later installments.)

The jumping around of POV's was rather confusing for me and the author lost me a bit until it began to pick up again toward the end. The book is full of detailed descriptions of the world Borchardt has created and some of it became overwhelming. To me her descriptions dragged the story down and lost me for most of the middle of the book. But, as I said before, by the end I was back with her.

This is definitely sequel bait- the story is far from over. I'm interested to see where the author- who was Anne Rice's sister, btw- will take it.", + "createdAt": "Thu May 20 2010", + "updatedAt": "Sun May 24 2026", + "spoiler": false, + "likeCount": 10, + "commentCount": 4, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/103546143" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.sC0IQDlAXY4LbkJ3LTJHvA", + "reviewer": { + "id": 80386185, + "name": "Rial", + "url": "/user/show/80386185-rial", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1545932910i/80386185._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 30, + "followerCount": 0 + }, + "rating": 1, + "text": "I did not like this book. However, it is my first 1 Star, so it's got that going for it.

Why did I read this? It was selected for my wife and roommate's book club, and I parasitically piggyback a lot of their books (gives me something to talk about...). It's Arthurian! It's from Guinivere's perspective! Magic! Celtic! Post 'Fall of the Western Empire'! These things and more!

No, it's terrible. Why did I keep reading it? It did take me on a journey: I experienced all the 'Stages of Grief' (some of them more than once.

Denial: Ha, it's so weird, but it'll get better! The chapters are long, but that's okay! Weird, the perspective shifted...?
Anger: I liked the other character; this one is dumb. She's a Mary Sue! WTF? That just worked out for her with zero effort! Okay, she needs to show some growth...
Bargaining: ...Okay, the parts with Arthur aren't bad, maybe?
Anger: Why are they having revenge sex in the forest?
Denial: Arthur is cool, yes, he's very cool... :|
Depression: I really shouldn't be reading this anymore; I think it's actively hurting me.
Acceptance: There is only book. Oh yeah, I guess your shoes can do that. Yep, you're definitely super fucking hot, and everyone wants to be or be with you. Sure why not? I don't have enough brain power to think it should be any different. After all, you're the best, Gwen!

But the real joy of this book is in not starting it. If you made that mistake and you're too masochistic to give up, then finishing it is an exultant experience

The contents are one thing, but they aren't even well served by the writing. Transitions are abrupt and jarring. I back-read multiple times to try and figure out where the hell I was, only to realize there wasn't any explanation; it just got there and it was fine (it wasn't fine).

This thing had such potential: Arthurian set in the 'proper' time frame, the Christianization of Britannia and the clashes with pagan beliefs that ensue, Celtic and Brythonic myth woven throughout. But it all reads so poorly in the end. The author is far too verbose and floral with her language. First person POV ruins any sense of actual accomplishment because we only see the character succeeding; no real growth or change is experienced. I find Divine Right to be problematic, and this book is very much into the romanticism of a destined ruler (both for Arthur and for Gwen).

Oh yeah, Merlin is evil and helps Arthur's mom torment Arthur, I guess? Like, if you hate the kid so much just kill him? Oh wait, 'destiny'...

Tedious tripe, glorified garbage, wretched rubbish. What a waste of time.", + "createdAt": "Mon Nov 05 2018", + "updatedAt": "Fri Jun 10 2022", + "spoiler": false, + "likeCount": 6, + "commentCount": 2, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2578561727" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.rwxYpuQEynxP2f5dSLFmPQ", + "reviewer": { + "id": 90733221, + "name": "Monique Mihalitsianos", + "url": "/user/show/90733221-monique-mihalitsianos", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1773156601i/90733221._UX200_CR0,29,200,200_.jpg", + "isAuthor": true, + "reviewCount": 88, + "followerCount": 9 + }, + "rating": 3, + "text": "This book was splendid in the story-telling sense. The story was epic and magical and engaging. Also, this is, by far, the best Guinevere I have encountered in modern fiction so far. She was strong but not brass, brave but not stupid, powerful yet humble, compassionate but not weak, feminine yet full of agency. Perhaps what I liked the most about her was that she was in communion with animals and gods alike and knew her place in the wheel of existence. She was wise like that, but never overbearingly so. I loved her. I wanted to *be* her.

Arthur I also liked very much, and I liked that this was a tortured Arthur who had suffered child abuse at the hands of her horrible queen mother Igrane (*the* worst evil queen ever) but was still good and noble and could not lie and never feared danger. I liked his relationship with his father, Uther, the High King, which was refreshingly healthy and loving. Merlin I hated. This is a book with a bad Merlin in it, which I don't like as a trope. I much prefer my Merlins to be good and Gandalf-like, but in this case it worked to have him be so evil and act as the villain. The black magic he wrought was one of the sickest and most horrifying ones I've ever read in a story.

Alice Borchardt seems to know her history well, so much so in fact that it's almost like if she lived in this period of time (Roman - Saxon) in a past life. I, for one, appreciated all of the extraneous detail relating to the customs and mores of the time because it made the world-building that much better.

The magic system in this book was refreshingly deep and believable. So many magic systems in recent fantasy books are light and superficial, as if they had been thought up in brainstorming sessions instead of being fully inspired. Not so with Borchardt's story; the magic here seemed inspired by the author's own psyche and thus, was multilayered and poliphacetic, changeable and malleable but also essentially true to its own core.

I also liked that an existencial and historical struggle was hinted at: the convergence between paganism and Christianity, if there can be a convergence at all. It was only hinted at here but I wouldn't mind if this theme in particular were elaborated more in the second book (The Raven Warrior) because it is of particular interest to me. On some level I think it was also of particular interest to Borchardt, as well. I am a huge fan of Anne Rice's (Alice Borchardt's sister) Vampire Chronicles, and Rice definitely waxes philosophical about her own struggles with different faith systems and beliefs in her own fiction, much more so than Borchardt, who chose to focus more on the story. But I wouldn't have minded if she had expanded on this in her fiction, just like her sister did. But then again, I am of a philosophical bent and like these kinds of tangents.

Why the three-star rating, then, if I liked the book so much? Well, the story was golden, but the writing was confusing at times, especially at the beginning and end. Sometimes it was almost stream-of-consciousness like, and I got lost and confused. At other times, when a character referred to somebody else as 'she', for example, I didn't know who they were talking about, because there were several 'she's' they could have been referring to. So some more specificity in this and other similar instances would have been appreciated. Sometimes, also, the writing got too dream-like, too airy fairy and dreamy...and perhaps more concrete writing would have served the book better overall. I also suspect this book could have had tighter editing. I know for a fact Anne Rice is not a fan of people editing her writing, and I don't know if Alice Borchardt felt the same as her sister does in this matter, but I for one think good editing is absolutely crucial in any book.

Putting this aside, though, I can say that I thoroughly enjoyed this book and am looking forward to reading the second and final installment. This was meant to be a trilogy, but unfortunately, the author died from illness before she could complete the third book, so two books is all we have. I will read her other books as well, though. She's a gem of a story-teller, at least in my estimation.", + "createdAt": "Wed Mar 18 2020", + "updatedAt": "Thu Nov 30 2023", + "spoiler": false, + "likeCount": 3, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3208179930" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.XvL2nHk7Bt4tULNXUE2yaA", + "reviewer": { + "id": 2207539, + "name": "Steen", + "url": "/user/show/2207539-steen", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1408254648i/2207539._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 473, + "followerCount": 4 + }, + "rating": 1, + "text": "This was a did not finish book for me.

So after reading about 185 pages of The Dragon Queen by Alice Borchardt I have decided to thrown it down and stop reading. I thought I would like it being about Quinevere and all but I hate it. It's not like the Arthur legends I've ever known. But that's not really even it though. I could handle a different take on the stories ( I still have yet to read any books about Arthur that are out there that are more well known) it's just the way she has written it. She jumps from one place to another and it's like when did that happen? Also sometimes when there are a few people talking 'she says...' your like who is talking? Which one is saying what. UGH. I hate not finishing a book but I really can't continue to read it anymore.

Having said that I really have no urge to read Bequiled which is also by her. I read the first in that series and while it was better written then The Dragon Queen I really don't love the characters in the first book to continue to read the second one anymore. Not after trying to get through The Dragon Queen and hating it. It's sad but yeah. I love Anne Rice's books and hoped that I would equally like her sister's books. But I guess they are just not for me.

I've read reviews on these two books and it sounds like her werewolf books are pretty good but I think I may wait a while to even attempt to try those ones.

So now its off to the library today to pick up some more to read.", + "createdAt": "Sat Apr 11 2009", + "updatedAt": "Wed Jan 23 2019", + "spoiler": false, + "likeCount": 3, + "commentCount": 3, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/52264098" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.rZ5icbmFfLHPKpEQOm2kNw", + "reviewer": { + "id": 41814516, + "name": "Alberto García", + "url": "/user/show/41814516-alberto-garc-a", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1508706260i/41814516._UX200_CR0,33,200,200_.jpg", + "isAuthor": true, + "reviewCount": 45, + "followerCount": 40 + }, + "rating": 4, + "text": "Voto de confianza... esta m¡¡bien escrito y la revisión de la leyenda artúrica es muy interesante. Al final, en algunos capítulos, se le va un poco la pinza con salidas algo paranoicas que me han dejado bastante descolocado... lo que es seguro es que no me lo esperaba... pero tampoco comprendo demasiado la razón para ello. Veremos cómo acaba y si se queda en una anécdota.", + "createdAt": "Fri Jun 21 2019", + "updatedAt": "Mon Sep 14 2026", + "spoiler": false, + "likeCount": 3, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2796875885" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.G3-a2b8-oopoAbpH6RmbsA", + "reviewer": { + "id": 73527602, + "name": "Rachel", + "url": "/user/show/73527602-rachel", + "avatar": "", + "isAuthor": false, + "reviewCount": 13, + "followerCount": 0 + }, + "rating": 3, + "text": "This was like reading someone's dream. The ideas were all loosely connected but the transitions were weird and random or completely non-existent. However, the characters and concepts were fun and interesting and I did like this book.", + "createdAt": "Fri Jan 04 2019", + "updatedAt": "Mon Nov 17 2025", + "spoiler": false, + "likeCount": 4, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2582029737" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.juoLRbi8R_urzbj6jgRAww", + "reviewer": { + "id": 101256289, + "name": "Malischa Nijenhuis", + "url": "/user/show/101256289-malischa-nijenhuis", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1773261867i/101256289._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 1222, + "followerCount": 32 + }, + "rating": 5, + "text": "Eigenlijks lees ik nu een boek van mijn man maar wauw wat is dit een gave uitgave en boek. Nu op naar boek 2", + "createdAt": "Sat Mar 21 2020", + "updatedAt": "Mon Mar 23 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3238259324" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.GLjLEKHIsU-RrmRdU73hGQ", + "reviewer": { + "id": 3504881, + "name": "Suzanne", + "url": "/user/show/3504881-suzanne", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1294105241i/3504881.jpg", + "isAuthor": false, + "reviewCount": 546, + "followerCount": 12 + }, + "rating": 2, + "text": "I'm very sorry to say, I could not finish this book. I REALLY wanted to be in love with this story. I even purchased the 2nd book from Thrift Books soon after I started this one. I figured they would become part of my Arthurian obsession, and I would need to own them. It started out with a great concept, and I liked the differences in the legend that I hadn't read before. I wanted to know more about the different characters, but it was so choppy, from one point of view to another back and forth, I couldn't keep track of which character I was reading about.

It just dragged on and on and on and on....it was trial after trial after trial. The characters would accomplish one win and suddenly face another trial. I felt like I was reading a bad rendition of The Odyssey. I REALLY wanted to love it. But reading became a chore, and I fell into a reading slump. The book sat in one spot for a couple weeks, and I felt it hanging over my head. I finally decided enough was enough. I wanted to be eager to read again. So I took out the bookmark, and this is going on my \"couldn't finish\" list.

I will donate the set to a Free Little Library somewhere, and I hope someone else will get some pleasure out of them.", + "createdAt": "Sun Oct 14 2018", + "updatedAt": "Fri Oct 30 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 2, + "recommendFor": null, + "shelf": "did-not-finish", + "reviewURL": "/review/show/2522667985" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.73ge1ENF_VjVNC1CSJb8iw", + "reviewer": { + "id": 67537582, + "name": "Kaylee", + "url": "/user/show/67537582-kaylee", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1627453497i/67537582._UX200_CR0,1,200,200_.jpg", + "isAuthor": false, + "reviewCount": 793, + "followerCount": 33 + }, + "rating": 1, + "text": "I do have to start by saying that this is the first Arthurian legend I can recall reading. I don't think that really affected my POV but...

Extremely hard to get through from the beginning. Kept hoping it would get better but nope. Not only was the switching between third and first POV annoying but it was made worse by the main character. Did not want her voice at all. It was also hard to tell what was going on/who was speaking at times (during the third POV of course). In the beginning I went back to the and re-read things but quickly learned there was no point. Sometimes things arrived randomly and others I realized I had just spaced out because of all the needless words.  You could take out half the words in this book and be all the better off for it...

Also if I had to read \"As I said...\" one more time...!

I can't remember the last time I didn't finish a book and if I hadn't had to read this because of my book club I don't think I would have finished. Says everything in a nutshell.

So much more but I'll just leave it at that. Extremely glad to be done with this book!", + "createdAt": "Fri Nov 09 2018", + "updatedAt": "Fri Oct 30 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 2, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2577596943" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.6kd8QVi2sBrNhUslyz1aUQ", + "reviewer": { + "id": 54738977, + "name": "TigerLily ", + "url": "/user/show/54738977-tigerlily", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1460164836i/54738977.jpg", + "isAuthor": false, + "reviewCount": 133, + "followerCount": 3 + }, + "rating": 5, + "text": "Really enjoyable, a page Turner that I didn't want to be finished. I am looking forward to reading the 2nd book.", + "createdAt": "Sun Jan 10 2021", + "updatedAt": "Sun Mar 21 2021", + "spoiler": false, + "likeCount": 2, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3732026445" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.njxZJZUHk2BVK8mcTJp2aQ", + "reviewer": { + "id": 40251914, + "name": "Elizabeth Sanders", + "url": "/user/show/40251914-elizabeth-sanders", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1530132382i/40251914.jpg", + "isAuthor": false, + "reviewCount": 38, + "followerCount": 0 + }, + "rating": 4, + "text": "It's an interesting retelling of Guinevere and Arthur. I loved every bit of it and can't wait to read the sequel.", + "createdAt": "Thu Jul 04 2019", + "updatedAt": "Thu Jul 11 2019", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2871843590" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.HegnVxYZnLTsNGxJeVV2cA", + "reviewer": { + "id": 11384547, + "name": "Lyndsay MacDonald", + "url": "/user/show/11384547-lyndsay-macdonald", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1371265927i/11384547.jpg", + "isAuthor": false, + "reviewCount": 38, + "followerCount": 0 + }, + "rating": 2, + "text": "This book was weird. I think it might have been tolerable if the author didn't try to weave it into the Arthurian legend (ie. if she made up original characters), but as it is I just couldn't get into it. There were some parts/scenes that I did enjoy, and others that I didn't... but overall I found it very confusing, and often found myself asking \"What is happening right now?\" and \"Why?\"

The story line flip-flopped around a lot... the characters were not wholly believable (Guinevere did NOT seem like a 13 year old girl, by any stretch of the imagination).. and I found parts of the plot silly and unrelated.

I won't be reading any of the other books in this sequence.", + "createdAt": "Wed Jan 22 2014", + "updatedAt": "Fri Aug 12 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/831334767" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.oazwOZoGnu8yIQJ_48FVMw", + "reviewer": { + "id": 27373770, + "name": "Andrea Krivak", + "url": "/user/show/27373770-andrea-krivak", + "avatar": "", + "isAuthor": false, + "reviewCount": 26, + "followerCount": 1 + }, + "rating": 2, + "text": "I think you have to really like arthurian legends to read this book. I liked it in the beginning, it started telling the story of Geunivere as a young girl and the \"what if\" this was her upbringing. There were wizards, men who turned into wolves and of course, merlin and morgana. About halfway through she meets Arthur, and then the book divides into the telling of each of their lives and they each stray into their own mystic journeys. To me, it spent too much time on these mystic journeys and by the end, i was ready to be done with this whole story. Not interesting enough ending for me to read into the sequel.", + "createdAt": "Mon Mar 24 2014", + "updatedAt": "Fri Aug 12 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/890822661" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.IxTTRac3W8gbXoHFdfTu5g", + "reviewer": { + "id": 1009204, + "name": "Cindy", + "url": "/user/show/1009204-cindy", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1228465282i/1009204._UX200_CR0,28,200,200_.jpg", + "isAuthor": false, + "reviewCount": 856, + "followerCount": 105 + }, + "rating": 1, + "text": "Maybe some day I'll come back to this book, but I just don't think I can read this book. I find myself wondering if I'm not following something of if this book is just beyond my reading. The writing is in a way that makes it seems like, wow we want the read to think this author is just so descriptive and great. Only it becomes confusing and bored. If you aren't familiar with the story of Guinevere then there is no point in reading it because nothing is explained and you are just sitting there wondering what the heck is going on. ", + "createdAt": "Wed Aug 27 2008", + "updatedAt": "Thu Jan 25 2018", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/31374042" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1._BHlRCbTwy-DJRq5mCTQnw", + "reviewer": { + "id": 4804714, + "name": "Angela", + "url": "/user/show/4804714-angela", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1294946776i/4804714.jpg", + "isAuthor": false, + "reviewCount": 129, + "followerCount": 6 + }, + "rating": 3, + "text": "I enjoyed this book up until the last third and then it got way too much involved with Arthur. It didn't quite wrap up the way I had expected and I would suggest not reading it unless you want to continue on with the series. I may read the sequel at a later time but I need a break from the acid trip like sections with Arthur.", + "createdAt": "Thu Mar 28 2013", + "updatedAt": "Tue Aug 09 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/572964102" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.P4s-Ysv7_6bKQAupxDvCwA", + "reviewer": { + "id": 4018173, + "name": "Misty", + "url": "/user/show/4018173-misty", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1286250306i/4018173._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 307, + "followerCount": 0 + }, + "rating": 2, + "text": "I really wanted to like this book.

About halfway through, I got bored and tried to push through it, but ultimately gave up. There was so much description and not enough happening.

Also, considering this was titled the Dragon Queen, I expected more dragons. Their lack of presence was disappointing.", + "createdAt": "Tue Nov 26 2013", + "updatedAt": "Thu Jun 23 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "could-not-finish", + "reviewURL": "/review/show/774467120" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.2E0pSS7MyztIPHdoIbH4Nw", + "reviewer": { + "id": 120860, + "name": "Lassarina Aoibhell", + "url": "/user/show/120860-lassarina-aoibhell", + "avatar": "", + "isAuthor": false, + "reviewCount": 487, + "followerCount": 5 + }, + "rating": 1, + "text": "I am DNFing this book on page 84 and we're going to go on a wild ride about why.

NB: If you are a scholar of Arthuriana, history, religion, or....well, almost anything, you might want to put down your drink. I'm not responsible for the state of your monitor.

We begin with the cover copy: Daughter of a mighty pagan queen, Guinevere has become a threat to her people and a prize to the dreaded sorcerer Merlin. Sent into hiding, she grows up under the protection of a shapeshifting man-wolf and an ornery Druid. But even on the remote coast of Scotland, where dragons feed and watch over her, she is not safe. Merlin knows the young beauty's destiny--and he will stop at nothing to prevent what has been foretold...

I also need to strongly highlight one of the blurbs. Like, really strongly. \"In addition to being exceptionally well written, [The Dragon Queen] has enormous integrity: All of the stories--of Arthur's parentage, of Merlin, of the characters' daily struggles--are well grounded in the history, politics, and religions of the time.....This is a trilogy to watch.\" --San Jose Mercury News

I want you to look at that quote carefully, and then come on a ride with me.

I will acknowledge up front my own reading comprehension fail, because I had forgotten that the shapeshifter was actually in the blurb and was yelling about \"what the fuck.\". I'm still saying \"what the fuck,\" just about plenty of other things.

Let's start with: It takes until page 48 for Guinevere to show up in any real sense. 500 page book; the main character doesn't actually appear until 10% in. Don't like that. Instead we are following around Dugald, the druid, and Maeniel, the shapeshifter. He's allegedly from Gaul. We begin at Tintagel in the year 470 CE, where Merlin and some of his cronies murder Vortigen. Some kind of magical snake is involved. (Aside: absolutely no problem with Merlin as the villain, particularly in anything allegedly centering the women of this tale.) Maeniel the shifter teleports??? out of the carnage, finds himself a she-wolf who has recently given birth, and shacks up with her. They encounter Dugald and the toddler Guinevere, who is currently disguised as a boy. For some reason, although everyone complains about disliking the Romans, we re-enact the founding of Rome, with Maeniel's mate suckling little Guinevere. Why does the she-wolf agree to this? Who can say. Dugald, meanwhile, talks about the influence of St. Patrick. Yes, that one. \"We don't call ourselves [druids] any longer. Not since Patrick came to Ireland bearing Christ's message.\"

....ten pages ago we called Merlin a chief druid. Also. It's the year 470. We don't have exact dates of the saint's life, but we know it was roughly in that century. More importantly, it was the Romans who banned druidic practices, fucking 400 years earlier (Augustus had no chill.) So there is no way that Patrick's actions in Ireland are affecting the order in England. Remember that blurb I highlighted? HMMMM. ALSO WHY ARE YOU, A GUY IN CORNWALL, TALKING ABOUT PATRICK. why. what is even your deal.

Also also, in this variation of the story (?), apparently, Dugald had a philosophical disagreement with Merlin and \"lost,\" so he was expected by the rules of his order to commit suicide because he lost.

Me: I do not remember reading this in any of the books I consumed during my obsessive period about Celtic histories and cultures and archaeology. (Wiki and checking with knowledgeable friends suggests I am not alone in not remembering this.)

If nothing else, that seems like a giant fucking waste of a 20 year apprenticeship, man.

(At this point I also stopped to verify my memory that Dugald and Maeniel do not appear in other Arthurian sources. I seem to be correct. And like, yeah, we add characters to stories all the time, I'm just. why are you making me deal with men in a book about Guinevere!)

There's also a point where young Guinevere, Dugald, and Maeniel are involved in helping to fight off a ship of slavers, which conveniently results in there being a woman to join the household and teach Guinevere How To Be A Girl. I should specify that the captain of said slaver's ship was apparently named Cymry. Why is he literally named \"Welsh\"? fuck if I know.

There's also a particular kind of ambient misogyny that I find tiresome, and like, I know we're in the fifth century and the Romans were assholes, and also I am aware that Celtic cultures were not paragons of equality and virtue, but the available evidence both of Romans being bitchy about how ass-backward Celtic societies were (omg they let their women do things!!!!) and the remnants of brehon law allocating specific property to women after divorce and categorizing a type of marriage as \"she married down and gets to keep all her stuff\" [wildly paraphrased] I'm just like

what. what are you doing, ma'am

and once again I'm going to refer back to that blurb. About being well grounded in the history, politics, and religions of the time.

I do not recommend actually reading this unless you want to play a drinking game, in which case, may I suggest that the drink you use should perhaps be water?", + "createdAt": "Thu Apr 09 2026", + "updatedAt": "Thu Apr 09 2026", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "did-not-finish", + "reviewURL": "/review/show/1761732484" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.9X0QSJx7BxHSTHSctKMkwg", + "reviewer": { + "id": 11348002, + "name": "Laural", + "url": "/user/show/11348002-laural", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1349329030i/11348002._UX200_CR0,5,200,200_.jpg", + "isAuthor": false, + "reviewCount": 36, + "followerCount": 5 + }, + "rating": 2, + "text": "I got about halfway through this book. Someday I'll give it another chance, or maybe try to figure out what exactly the author did wrong. A sword weilding Gweneviere? It could have been great. I thought it would be after reading the description on the back of the gorgeous cover. ", + "createdAt": "Thu Oct 04 2012", + "updatedAt": "Thu Jun 23 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/427412066" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.p0mSKsHft_BTv2vScLHOsQ", + "reviewer": { + "id": 1250008, + "name": "Giuila", + "url": "/user/show/1250008-giuila", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1244141968i/1250008._UY200_CR2,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 61, + "followerCount": 0 + }, + "rating": 0, + "text": "... t read", + "createdAt": "Thu Jul 17 2008", + "updatedAt": "Fri Mar 24 2017", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "to-read", + "reviewURL": "/review/show/27520982" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.QMO0-GkHiX0UEYcsOaO7lw", + "reviewer": { + "id": 1431136, + "name": "Matt", + "url": "/user/show/1431136-matt", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1218856549i/1431136._UY200_CR25,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 6, + "followerCount": 0 + }, + "rating": 5, + "text": "an overall awesome book", + "createdAt": "Sat Aug 16 2008", + "updatedAt": "Fri Mar 24 2017", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/30279062" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.qOdgrbVKU-XMQR5DSbHfDQ", + "reviewer": { + "id": 16979552, + "name": "Sarah Garner", + "url": "/user/show/16979552-sarah-garner", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1408284741i/16979552._UX200_CR0,53,200,200_.jpg", + "isAuthor": false, + "reviewCount": 84, + "followerCount": 18 + }, + "rating": 2, + "text": "I struggled with this book and I couldn't finish it, although it was a great idea, I didn't love this book.", + "createdAt": "Sat Sep 20 2014", + "updatedAt": "Thu Aug 20 2020", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/1059961768" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.JjlmKD1eCF28HlpY4KmCGw", + "reviewer": { + "id": 34744007, + "name": "Michael D.", + "url": "/user/show/34744007-michael-d", + "avatar": "", + "isAuthor": false, + "reviewCount": 81, + "followerCount": 1 + }, + "rating": 2, + "text": "I had this book about 20 years ago, tried to read it, but I just couldn't get into it. I got another copy used and gave it another try. While not as bad as my first experience in the first 20 pages, I found in the ensuing pages that the narrative is quite disjointed, and timelines are fluid and unclear to me. Perhaps that was Borchardt's intent, but I was confused by it. Additionally, it seems that characters are magically transported to strange places without any transition that I could discern. This narrative was hard to follow.

The book begins in a scene in 470 in Tintigal (not Tintagil, the famous castle in Cornwall where Arthur was conceived and born in the traditional romance). Ostensibly, the main character is a wolfman (a wolf who changes into a man when he chooses to) named Maeniel, and he is a friend to the British king Vortigen, who is not the villain he is normally portrayed as in history and the romances. Merlin shows up with a contingent of Saxons, and a battle ensues in which Vortigen is slain and Maeniel is knocked unconscious by an unknown cudgel. In the next scene, he wakes up on a hillside far away, presumably the far north, near Hadrian's Wall, and he becomes the mate of a she-wolf, who bears him a son, Black Leg (who will become Lancelot in the second novel The Raven Warrior). Sometime later, we find that 70 years have passed since Vortigen's death. Oddly, there is a baby girl in the nearby village, Guenevere (Borchardt spells her name differently depending on who is addressing her), who is one of the major characters of this series, the eponymous destined Dragon Queen. When Guenevere is 15, we are then told that 100 years have passed since Vortigen died. Confusion abounds.

This story had occasional high points, with modern fantastical treatments of the transformation duels, but I really had a hard time following the flow of Borchardt's narrative. I am a pretty astute reader, so I find it difficult to believe this was my fault. Borchardt seemed to be playing coy with the unfolding of events, and that was annoying. She was the sister of the better-known Anne Rice, author of the Vampire Chronicles, so I tried hard to give this a chance.

Borchardt intended to make this a trilogy, but the passed away in 2007 before the third book in the trilogy could be published. I don't know if anyone picked up her notes with the intent to complete her work, and I can't imagine that there was a demand for it. I do intend to slog through The Raven Warrior, as it is my rule to read books through and evaluate them honestly. If they're bad, I point out why. The lack of comprehensible transitions in time and space are my primary complaint with this novel. I hope that The Raven Warrior is more cognizant of walking me through how characters get from Point A to Point B than this one was.
", + "createdAt": "Thu Nov 09 2023", + "updatedAt": "Thu Nov 09 2023", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5944805120" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v2.2025072400.a7efbaf3-dd61-4567-b4e1-d3f344fabb0c", + "reviewer": { + "id": 42506425, + "name": "Michelle M", + "url": "/user/show/42506425-michelle-m", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1715615833i/42506425._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 380, + "followerCount": 3 + }, + "rating": 5, + "text": "Honestly? 5/6 or 4.75/6
I really loved this book! It's hard to describe what exactly I loved about it though lol. It had a very ethereal quality to it... The magic in this world is just a fact of life, some peple have it, while others don't, and yet, it didn't feel fantastical. Everyone knows about magic and most peoples still believe in the gods and things even though \"The Church\" (christianity) is present on the British Isles.

Borchardt grounds the story in the time right after Rome leaves the Isles, though they are still very much an influence on the peoples. It was a very mystical vibe to the whole story, kind of like there is a shroud of fog that is the magic of the world. IDK, like I said, hard to describe lol.

I loved Guinevere, and I liked how different characters referred to her by various spellings/pronouciations thoughout the story, as if they were speaking different languages. It is a very different take on the Arthur/Guinevere story, and very much an action oriented one, with Guinevere being the focus of the action and not just a passive bystander.

I though the portrayal of the other major characters interesting, and they were given roles that they are not usual given in the story. Arthur is \"good\", Merlin and Ingrane are the \"evil\" ones, Guinevere is strong willed and knows what she wants and sets out to get it. She doesn't seem like the type of person who will betray Arthur with Lancelot, but we'll just have to find out! (I hope she doesn't). The portrayal of these characters (and other named characters) made me do some research to try to get a better picture of if they had ever been portayed like this before. My search was unsucessful lol, as they were very diffefrent from \"traditional\" stories, but I haven't really read many retellings, so I cant be sure if they are exclusive to this story.

My only compaint with this book is that I found out it was supposed to be a trilogy!!!! And YET there are only two books! T.T *cries* I know the author can't help that she passed away, but book 2 came out in 2003 and she passed in 2007! That seems like a decent amount of time to finish the third book (I'm being very ungrateful, I know I am, I just lament what never was). Also apparently Anne Rice of vampire fame is Borchardt's sister?!?!?! She doesn't have like an early draft of her sister's just laying around??? T.T

I will def be reading the next one! (this one took me so long because I paused to participate in a read a thon and then got distracted by other books lol and then I got busy with work so I didn't have as much time to read)", + "createdAt": "Thu Jul 24 2025", + "updatedAt": "Thu Jul 24 2025", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/7714101862" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.HXGbQg0020aQJjD9G6pW2A", + "reviewer": { + "id": 163241100, + "name": "Alex", + "url": "/user/show/163241100-alex", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1763011908i/163241100._UY200_CR11,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 98, + "followerCount": 0 + }, + "rating": 2, + "text": "I enjoyed Guinevere’s character early on in the book. It was interesting watching her being raised alongside her foster brother, Black Leg, by a wolf shifter (Maniel) and a she-wolf. I do appreciate her relationship with the She Wolf, who Guinevere calls Mother. The thing that really kept me going through this book was Guinevere’s interactions with the dragons of the land and her connection to the land. I was also intrigued by some of the same conversations about religion in this book since Guinevere is raised in nature and has regular interactions with a Druid and some interactions with Christiany.
All this being said I struggled with the writing. There are huge chunks of it that are confusing since it’s not always clear who is being talked about when he, she or they are mentioned considering there might be multiple people mentioned previously in the scene. Transitions between moments are also rough. I was questioning when things changed and what was being talked about. The long descriptions of things really didn’t help either and me zoning out as I read.
It was also a bit rough to read this because of how things were worded/certain sciences being really uncomfortable for seemingly no reason out of nowhere. The dynamics between Igraine and Merlin’s relationship being one them. Scenes with them were sexual with zero warning and it wasn’t just limited to them. It happened a lot with various characters. Things were also made pretty uncomfortable with minors even if it was a scene that wasn’t intended to uncomfortably sexual, it was made that way by directly saying things like they “were far too young for even a dream of passion.” With how confusing the writing was I couldn’t even tell if there was a reason for why these sort of scenes were written in detail outside of making the audience feel uncomfortable and actively dislike the characters who are at fault for these behaviors.", + "createdAt": "Sun May 24 2026", + "updatedAt": "Sun May 24 2026", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5900548383" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.VoNZbtbhrTyL1Fhg43jpGw", + "reviewer": { + "id": 42064438, + "name": "Briana", + "url": "/user/show/42064438-briana", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1715717411i/42064438._UY200_CR5,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 161, + "followerCount": 1 + }, + "rating": 2, + "text": "This book was not for me. I had high hopes, the summary sounded interesting. It took me much longer than usual to finish this book as I struggled to enjoy it and I will not be picking up the second installment. I almost DNF'd this book several times but decided to stick with it.

The first half of the book felt like it was the author's freewriting development of the book, which she never bothered to go back and actually edit into a cohesive and understandable story. It was like being is some kind of dream where everything was hazy and only loosely connected. Sometimes you ended up in an entirely different location or time without even realizing it. It was also almost impossible to understand who was being discussed as the author tends to avoid using names and simply use pronouns leaving the reader to try and figure out (usually without success) who the \"she\" is that is being discussed. It was also very interesting that the author (or editor) chose to spell Guinevere's name differently throughout the book.

The second half was a little better and is the only reason I didn't give this book a one star. I don't mind Merlin as the antagonist as some other reviewers do and I found his character interesting. However, I did not really like the relationship between Arthur and Guinevere. I'm not sure how I am supposed to be convinced that this was some fated love story when they met for all of maybe an hour total. It seemed more insta-lust than anything else and there was no relationship development at all after that.

Overall I spent my time either being very confused and a little bored or kind of confused and mildly interested.", + "createdAt": "Thu Aug 15 2024", + "updatedAt": "Thu Aug 15 2024", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/6728900786" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.IoZW54HX8Qad9w5IgbQB9A", + "reviewer": { + "id": 6424389, + "name": "Joy Pouros", + "url": "/user/show/6424389-joy-pouros", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1567889312i/6424389._UY200_CR47,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 125, + "followerCount": 2 + }, + "rating": 3, + "text": "2.5 stars.

I REALLY like the character of Guinevere, but there were other parts of the story I didn't enjoy.

The writing style can be very winding and even abstract in some ways. There are whole parts you can skim without missing much. There aren't a ton of people, but enough to get them confused, especially a couple with similar names.

I'm always very frustrated with stories that try to have strong female characters but still have an overly sexually charged world. There are so many wonderfully creative parts of this world and then we default back to several tired tropes. There's just no need.

Some chapters are 30 pages, some are 5. It doesn't appear based on POV or time, so why is this so inconsistent?

And finally... I understand this is the first book of the trilogy, but this book just felt like a long prologue. Other series the first book is still a worthwhile read and entertaining story. This is more of an origin story before you fully appreciate the main story. So there are very mixed feelings about continuing the series because while I love Guinevere, there's an awful lot to overlook. ", + "createdAt": "Tue Aug 30 2022", + "updatedAt": "Tue Aug 30 2022", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/4955253716" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.ahVbGwMWksFfbjRS9c6Vbg", + "reviewer": { + "id": 29405146, + "name": "Anita", + "url": "/user/show/29405146-anita", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1407269712i/29405146._CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 614, + "followerCount": 3 + }, + "rating": 1, + "text": "First of all, I love Arthurian legends. Second, I always finish books, constantly hopeful that the story will improve, but I must confess that I skimmed this book to the end. I just cannot read any more!
This story has no cohesion, no main narrator and skips from one location to another mid-chapter. Plus there are men suddenly shapeshifting into wolves, and back again; no reasons given; no context for the intrigue and killings taking place; and a young girl child, supposedly Guinevere of Arthurian legend, being suckled by a she-wolf after being kidnapped.
Imaginative description is great in a fantasy story; confusion is not. This book is one long puzzle - a maze with apparently no direction, interspersed with some flowery descriptions of sunsets and misty mornings; visions of ravens and dragons (fishing); and a cast of hundreds of unrelated and irrelevant characters. Reading this book feels akin to experiencing one of those dreams in which you are lost and desperately trying to find your way back home.
Need I add that I did not enjoy this book, and will not be reading the sequel?", + "createdAt": "Fri Oct 30 2020", + "updatedAt": "Fri Oct 30 2020", + "spoiler": true, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3618732305" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.zo7-0bYwOUS0LlPXyqeuXQ", + "reviewer": { + "id": 3198878, + "name": "Stevie", + "url": "/user/show/3198878-stevie", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1560478233i/3198878._UX200_CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 71, + "followerCount": 0 + }, + "rating": 2, + "text": "Rating this book was a challenge. It was an impulse grab. I suspected it would be wonderfully terrible or terribly wonderful and there was only one way to find out.

What I liked: Arthurian legend from Guin's point of view. Cool. That awkward time period in history when they were both Christian but still observed many pagan rites. It had all the names I knew, and some of the tropes I knew: Vortigen, the witch in the well, so on and so forth from various movies I've seen.

What I disliked: the writing. Some of the dialogue wasn't needed. The descriptions could be too much in places and entirely lacking in others. Who is speaking? What do they look like? They were a human in a tower and now they are a fish? When? How? Transitions don't exist. This makes it hard to follow.

It was a wild ride, and I hung on until the end because I was curious as to how it would turn out. Turns out there's a sequel...I don't know that I have that in me...", + "createdAt": "Sat Mar 21 2020", + "updatedAt": "Sat Mar 21 2020", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3240571912" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.E7XotAu3DvsavlUw1u62eA", + "reviewer": { + "id": 107938758, + "name": "Danai", + "url": "/user/show/107938758-danai", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1705239736i/107938758._UX200_CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 133, + "followerCount": 0 + }, + "rating": 3, + "text": "I didn't love it, nor did I hate it.

Everything was magical, mystical, and mysterious; Borchardt painted an imaginative world with interesting and diverse characters. She evidently has a deep knowledge of the British Isles, its history, and its lore.

But the writing was, at times, completely chaotic and confusing. As if someone was just spilling their string of thoughts without sufficient context or details - or too many of the wrong details. More than once, I found myself confused by abrupt changes in the situation, only to learn - through narration - that a memory was being shared.

This book didn't capture me nor engross me by any means, but it did beckon me to come back and finish it. If I come across the next book in the series, I will read it simply out of curiosity - what happens next for our heroes and villains? But I won't go hunting for a copy of it anytime soon...", + "createdAt": "Thu Jan 11 2024", + "updatedAt": "Thu Jan 11 2024", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5638508977" + } + ], + "lastScraped": "2026-10-28T18:38:19.808Z" +} \ No newline at end of file diff --git a/internal/datasource/biblioreads/fixtures/search-books.json b/internal/datasource/biblioreads/fixtures/search-books.json new file mode 100644 index 00000000..3c3de16c --- /dev/null +++ b/internal/datasource/biblioreads/fixtures/search-books.json @@ -0,0 +1,19 @@ +{ + "status": "Received", + "source": "https://github.com/nesaku/biblioreads", + "apiURL": "https://www.goodreads.com/book/auto_complete?format=json&q=La%20Reina%20Drag%C3%B3n%20Alice%20Borchardt", + "searchType": "books", + "numberOfResults": "1 results", + "result": [ + { + "id": 1, + "cover": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/books/1332599049i/12207444._SX50_.jpg", + "title": "La Reina Dragon (La leyenda de Guinevere, #1)", + "bookURL": "/book/show/12207444-la-reina-dragon", + "author": "Alice Borchardt", + "authorURL": "/author/show/151155.Alice_Borchardt", + "rating": "3.82" + } + ], + "lastScraped": "2026-10-28T17:45:31.333Z" +} diff --git a/internal/datasource/biblioreads/model.go b/internal/datasource/biblioreads/model.go new file mode 100644 index 00000000..55de7137 --- /dev/null +++ b/internal/datasource/biblioreads/model.go @@ -0,0 +1,108 @@ +package biblioreads + +type SearchBooksResponse struct { + Status string `json:"status"` + Source string `json:"source"` + APIURL string `json:"apiURL"` + SearchType string `json:"searchType"` + NumberOfResults string `json:"numberOfResults"` + Result []SearchResult `json:"result"` + LastScraped string `json:"lastScraped"` +} + +type SearchResult struct { + ID int `json:"id"` + Cover string `json:"cover"` + Title string `json:"title"` + BookURL string `json:"bookURL"` + Author string `json:"author"` + AuthorURL string `json:"authorURL"` + Rating string `json:"rating"` +} + +type BookResponse struct { + Status string `json:"status"` + StatusCode int `json:"statusCode"` + Source string `json:"source"` + ScrapeURL string `json:"scrapeURL"` + LegacyBookID int `json:"legacyBookID"` + BookID string `json:"bookID"` + ResourceID string `json:"resourceID"` + Cover string `json:"cover"` + Series string `json:"series"` + SeriesURL string `json:"seriesURL"` + SeriesPosition string `json:"seriesPosition"` + WorkURL string `json:"workURL"` + Title string `json:"title"` + Author []BookAuthor `json:"author"` + Rating string `json:"rating"` + RatingCount string `json:"ratingCount"` + ReviewsCount string `json:"reviewsCount"` + Desc string `json:"desc"` + Genres []string `json:"genres"` + BookEdition string `json:"bookEdition"` + PublishDate string `json:"publishDate"` + Publisher string `json:"publisher"` + Language string `json:"language"` + ASIN string `json:"asin"` + ISBN string `json:"isbn"` + ISBN13 string `json:"isbn13"` + ReviewBreakdown ReviewBreakdown `json:"reviewBreakdown"` + Quotes string `json:"quotes"` + QuotesURL string `json:"quotesURL"` + Questions string `json:"questions"` + QuestionsURL string `json:"questionsURL"` + LastScraped string `json:"lastScraped"` +} + +type BookAuthor struct { + ID int `json:"id"` + Name string `json:"name"` + Role string `json:"role"` + URL string `json:"url"` +} + +type ReviewBreakdown struct { + Rating1 string `json:"rating1"` + Rating2 string `json:"rating2"` + Rating3 string `json:"rating3"` + Rating4 string `json:"rating4"` + Rating5 string `json:"rating5"` +} + +type ReviewsResponse struct { + Status string `json:"status"` + StatusCode int `json:"statusCode"` + Source string `json:"source"` + ScrapeURL string `json:"scrapeURL"` + TotalCount int `json:"totalCount"` + NextPageToken string `json:"nextPageToken"` + PrevPageToken string `json:"prevPageToken"` + Reviews []Review `json:"reviews"` + LastScraped string `json:"lastScraped"` +} + +type Review struct { + ID string `json:"id"` + Reviewer Reviewer `json:"reviewer"` + Rating int `json:"rating"` + Text string `json:"text"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Spoiler bool `json:"spoiler"` + LikeCount int `json:"likeCount"` + CommentCount int `json:"commentCount"` + RecommendFor *string `json:"recommendFor"` + Shelf string `json:"shelf"` + ReviewURL string `json:"reviewURL"` +} + +type Reviewer struct { + ID int `json:"id"` + Name string `json:"name"` + URL string `json:"url"` + Avatar string `json:"avatar"` + IsAuthor bool `json:"isAuthor"` + ReviewCount int `json:"reviewCount"` + FollowerCount int `json:"followerCount"` +} diff --git a/internal/datasource/httpjson/httpjson.go b/internal/datasource/httpjson/httpjson.go new file mode 100644 index 00000000..100c931d --- /dev/null +++ b/internal/datasource/httpjson/httpjson.go @@ -0,0 +1,72 @@ +// Package httpjson contains helpers shared by the datasource clients that consume JSON HTTP APIs. +package httpjson + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" +) + +const ( + UserAgent = "Mozilla/5.0 (compatible; coreander/1.0; +https://github.com/svera/coreander)" + + // DefaultRetryAfter is used when a Retry-After header is missing or invalid. + DefaultRetryAfter = 60 * time.Second +) + +type Response struct { + StatusCode int + Header http.Header + Body []byte +} + +// Do sends req with the default headers and reads the whole response body. +func Do(client *http.Client, req *http.Request) (Response, error) { + req.Header.Set("User-Agent", UserAgent) + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return Response{}, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return Response{}, err + } + return Response{StatusCode: resp.StatusCode, Header: resp.Header, Body: body}, nil +} + +// Decode unmarshals the response body into dest, returning an error if the status code is not 200. +func (r Response) Decode(dest any) error { + if r.StatusCode != http.StatusOK { + return fmt.Errorf("request failed with status code %d: %s", r.StatusCode, r.Body) + } + return json.Unmarshal(r.Body, dest) +} + +// ParseRetryAfter returns the wait time specified by a Retry-After header value, which can be +// either a number of seconds or an HTTP date. DefaultRetryAfter is returned if it can't be used. +func ParseRetryAfter(header string) time.Duration { + if header == "" { + return DefaultRetryAfter + } + if seconds, err := strconv.Atoi(header); err == nil { + if seconds <= 0 { + return DefaultRetryAfter + } + return time.Duration(seconds) * time.Second + } + if retryTime, err := http.ParseTime(header); err == nil { + wait := time.Until(retryTime) + if wait <= 0 { + return DefaultRetryAfter + } + return wait + } + return DefaultRetryAfter +} diff --git a/internal/datasource/httpjson/httpjson_test.go b/internal/datasource/httpjson/httpjson_test.go new file mode 100644 index 00000000..d44c33ed --- /dev/null +++ b/internal/datasource/httpjson/httpjson_test.go @@ -0,0 +1,77 @@ +package httpjson_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/httpjson" +) + +func TestParseRetryAfter(t *testing.T) { + t.Run("seconds", func(t *testing.T) { + if got := httpjson.ParseRetryAfter("30"); got != 30*time.Second { + t.Fatalf("expected 30s, got %s", got) + } + }) + + t.Run("http date", func(t *testing.T) { + retryTime := time.Now().Add(45 * time.Second).UTC() + got := httpjson.ParseRetryAfter(retryTime.Format(http.TimeFormat)) + if got < 44*time.Second || got > 46*time.Second { + t.Fatalf("expected about 45s, got %s", got) + } + }) + + t.Run("missing header uses default", func(t *testing.T) { + if got := httpjson.ParseRetryAfter(""); got != httpjson.DefaultRetryAfter { + t.Fatalf("expected default %s, got %s", httpjson.DefaultRetryAfter, got) + } + }) + + t.Run("invalid header uses default", func(t *testing.T) { + for _, header := range []string{"-5", "0", "soon"} { + if got := httpjson.ParseRetryAfter(header); got != httpjson.DefaultRetryAfter { + t.Fatalf("expected default %s for %q, got %s", httpjson.DefaultRetryAfter, header, got) + } + } + }) +} + +func TestDo(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("User-Agent") != httpjson.UserAgent { + t.Errorf("unexpected User-Agent %q", r.Header.Get("User-Agent")) + } + if r.Header.Get("Accept") != "application/json" { + t.Errorf("unexpected Accept %q", r.Header.Get("Accept")) + } + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTeapot) + _, _ = w.Write([]byte(`{"a":1}`)) + })) + defer server.Close() + + req, _ := http.NewRequest(http.MethodGet, server.URL, nil) + resp, err := httpjson.Do(server.Client(), req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusTeapot || resp.Header.Get("Retry-After") != "5" || string(resp.Body) != `{"a":1}` { + t.Errorf("unexpected response %+v", resp) + } +} + +func TestDecode(t *testing.T) { + var dest struct{ A int } + if err := (httpjson.Response{StatusCode: http.StatusOK, Body: []byte(`{"a":1}`)}).Decode(&dest); err != nil || dest.A != 1 { + t.Errorf("expected A=1 without error, got %d, %v", dest.A, err) + } + if err := (httpjson.Response{StatusCode: http.StatusNotFound, Body: []byte(`{}`)}).Decode(&dest); err == nil { + t.Error("expected error on non-200 status") + } + if err := (httpjson.Response{StatusCode: http.StatusOK, Body: []byte(``)}).Decode(&dest); err == nil { + t.Error("expected error on invalid JSON") + } +} diff --git a/internal/datasource/wikidata/httpclient.go b/internal/datasource/wikidata/httpclient.go index 18d90a0f..f2e0b5f6 100644 --- a/internal/datasource/wikidata/httpclient.go +++ b/internal/datasource/wikidata/httpclient.go @@ -1,17 +1,13 @@ package wikidata import ( - "encoding/json" - "fmt" - "io" "log" "net/http" - "strconv" "sync" "time" -) -const defaultRetryAfter = 60 * time.Second + "github.com/svera/coreander/v5/internal/datasource/httpjson" +) var apiHTTPClient = newRateLimitedHTTPClient() @@ -35,33 +31,18 @@ func (c *rateLimitedHTTPClient) getJSON(url string, dest any) error { if err != nil { return err } - for key, value := range wikidataAPIHeaders() { - req.Header.Set(key, value) - } - resp, err := c.client.Do(req) + resp, err := httpjson.Do(c.client, req) if err != nil { return err } if resp.StatusCode == http.StatusTooManyRequests { - _ = resp.Body.Close() c.waitUntil(resp.Header.Get("Retry-After")) continue } - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - return fmt.Errorf("request failed with status code %d: %s", resp.StatusCode, body) - } - - body, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if err != nil { - return err - } - return json.Unmarshal(body, dest) + return resp.Decode(dest) } } @@ -71,7 +52,7 @@ func (c *rateLimitedHTTPClient) waitUntil(retryAfterHeader string) { c.mu.Lock() var rateLimitWait time.Duration if retryAfterHeader != "" { - rateLimitWait = parseRetryAfter(retryAfterHeader) + rateLimitWait = httpjson.ParseRetryAfter(retryAfterHeader) resumeAt := time.Now().Add(rateLimitWait) if resumeAt.After(c.resumeAt) { c.resumeAt = resumeAt @@ -86,29 +67,3 @@ func (c *rateLimitedHTTPClient) waitUntil(retryAfterHeader string) { time.Sleep(wait) } } - -func parseRetryAfter(header string) time.Duration { - if header == "" { - return defaultRetryAfter - } - if seconds, err := strconv.Atoi(header); err == nil { - if seconds <= 0 { - return defaultRetryAfter - } - return time.Duration(seconds) * time.Second - } - if retryTime, err := http.ParseTime(header); err == nil { - wait := time.Until(retryTime) - if wait <= 0 { - return defaultRetryAfter - } - return wait - } - return defaultRetryAfter -} - -func wikidataAPIHeaders() map[string]string { - return map[string]string{ - "User-Agent": "Mozilla/5.0 (compatible; coreander/1.0; +https://github.com/svera/coreander)", - } -} diff --git a/internal/datasource/wikidata/httpclient_test.go b/internal/datasource/wikidata/httpclient_test.go index 61fcfa30..116e07fb 100644 --- a/internal/datasource/wikidata/httpclient_test.go +++ b/internal/datasource/wikidata/httpclient_test.go @@ -8,28 +8,6 @@ import ( "time" ) -func TestParseRetryAfter(t *testing.T) { - t.Run("seconds", func(t *testing.T) { - if got := parseRetryAfter("30"); got != 30*time.Second { - t.Fatalf("expected 30s, got %s", got) - } - }) - - t.Run("http date", func(t *testing.T) { - retryTime := time.Now().Add(45 * time.Second).UTC() - got := parseRetryAfter(retryTime.Format(http.TimeFormat)) - if got < 44*time.Second || got > 46*time.Second { - t.Fatalf("expected about 45s, got %s", got) - } - }) - - t.Run("missing header uses default", func(t *testing.T) { - if got := parseRetryAfter(""); got != defaultRetryAfter { - t.Fatalf("expected default %s, got %s", defaultRetryAfter, got) - } - }) -} - func TestRateLimitedHTTPClientWaitsForRetryAfter(t *testing.T) { var requests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/index/bleve.go b/internal/index/bleve.go index da4cc666..2777e690 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -30,7 +30,7 @@ import ( // DocumentVersion identifies the mapping used for indexing documents. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. -const DocumentVersion = "v13" +const DocumentVersion = "v14" // AuthorVersion identifies the mapping used for indexing authors. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. @@ -182,6 +182,12 @@ type Config struct { // to already be resolved (e.g. via ResolveMetadataWorkers); <= 0 falls // back to 1. TextRankEnrichWorkers int + // GoodReadsSource, if set, is used to fill each document's GoodReads URLs + // in the background (see EnrichGoodReads). Nil disables it. + GoodReadsSource GoodReadsSource + // GoodReadsEnrichInterval is the minimum time between two GoodReads + // lookups; <= 0 falls back to DefaultGoodReadsEnrichInterval. + GoodReadsEnrichInterval time.Duration } type BleveIndexer struct { @@ -215,6 +221,13 @@ type BleveIndexer struct { // textRankEnrichJobs feeds documents queued by indexFile to a fixed-size // pool of long-lived worker goroutines; see runTextRankEnrichWorker. textRankEnrichJobs chan Document + // goodReadsEnrichJobs feeds IDs of documents queued by indexFile to + // runGoodReadsEnrichWorker; only used when goodReadsSource is set. + goodReadsEnrichJobs chan string + goodReadsSource GoodReadsSource + goodReadsInterval time.Duration + goodReadsThrottleMu sync.Mutex + goodReadsLastRequest time.Time // fileLocks holds a *sync.Mutex per document ID, serializing concurrent // indexFile calls for the same file (upload vs. file watcher reacting to // the same write). @@ -319,12 +332,22 @@ func NewBleve(documentsIndex bleve.Index, authorsIndex bleve.Index, fs afero.Fs, minCommonTextRankAbsoluteCount: minCommonTextRankAbsoluteCount, pruneChangeTriggerRatio: pruneChangeTriggerRatio, textRankEnrichJobs: make(chan Document, textRankEnrichWorkers), + goodReadsSource: cfg.GoodReadsSource, + goodReadsInterval: cfg.GoodReadsEnrichInterval, } for range textRankEnrichWorkers { go b.runTextRankEnrichWorker() } + if b.goodReadsInterval <= 0 { + b.goodReadsInterval = DefaultGoodReadsEnrichInterval + } + if b.goodReadsSource != nil { + b.goodReadsEnrichJobs = make(chan string, 1) + go b.runGoodReadsEnrichWorker() + } + return b } @@ -376,6 +399,14 @@ func CreateDocumentsMapping() mapping.IndexMapping { keywordFieldMapping := bleve.NewKeywordFieldMapping() + // Stored only, so the field can be read back but never matches a search. + // IncludeInAll=false is redundant with Index=false today, but keeps the + // field out of the composite _all field explicitly. + storedOnlyFieldMapping := bleve.NewKeywordFieldMapping() + storedOnlyFieldMapping.Index = false + storedOnlyFieldMapping.IncludeInAll = false + storedOnlyFieldMapping.DocValues = false + simpleTextFieldMapping := bleve.NewTextFieldMapping() simpleTextFieldMapping.Analyzer = defaultAnalyzer @@ -434,6 +465,9 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.TypeMapping[lang].AddFieldMappingsAt("Illustrations", numericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsAuthorURL", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) } indexMapping.DefaultMapping.DefaultAnalyzer = defaultAnalyzer @@ -458,6 +492,9 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("Illustrations", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsAuthorURL", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) return indexMapping } diff --git a/internal/index/bleve_document_read.go b/internal/index/bleve_document_read.go index 48ee2b13..9994ab51 100644 --- a/internal/index/bleve_document_read.go +++ b/internal/index/bleve_document_read.go @@ -1023,6 +1023,10 @@ func hydrateDocument(match *search.DocumentMatch) Document { textRankEnriched = match.Fields["TextRankEnriched"].(bool) } + goodReadsBookURL, _ := match.Fields["GoodReadsBookURL"].(string) + goodReadsAuthorURL, _ := match.Fields["GoodReadsAuthorURL"].(string) + goodReadsEnriched, _ := match.Fields["GoodReadsEnriched"].(bool) + doc := Document{ ID: match.ID, Metadata: metadata.Metadata{ @@ -1040,15 +1044,18 @@ func hydrateDocument(match *search.DocumentMatch) Document { Illustrations: illustrations, Format: match.Fields["Format"].(string), }, - Slug: match.Fields["Slug"].(string), - AuthorsSlugs: slicer(match.Fields["AuthorsSlugs"]), - IllustratorsSlugs: illustratorsSlugs, - SeriesSlug: match.Fields["SeriesSlug"].(string), - SubjectsSlugs: slicer(match.Fields["SubjectsSlugs"]), - AddedOn: addedOn, - TextRankPhrases: textRankPhrases, - TextRankWords: textRankWords, - TextRankEnriched: textRankEnriched, + Slug: match.Fields["Slug"].(string), + AuthorsSlugs: slicer(match.Fields["AuthorsSlugs"]), + IllustratorsSlugs: illustratorsSlugs, + SeriesSlug: match.Fields["SeriesSlug"].(string), + SubjectsSlugs: slicer(match.Fields["SubjectsSlugs"]), + AddedOn: addedOn, + TextRankPhrases: textRankPhrases, + TextRankWords: textRankWords, + TextRankEnriched: textRankEnriched, + GoodReadsBookURL: goodReadsBookURL, + GoodReadsAuthorURL: goodReadsAuthorURL, + GoodReadsEnriched: goodReadsEnriched, } return doc diff --git a/internal/index/bleve_document_write.go b/internal/index/bleve_document_write.go index 110a892a..fa594df4 100644 --- a/internal/index/bleve_document_write.go +++ b/internal/index/bleve_document_write.go @@ -145,7 +145,10 @@ func (b *BleveIndexer) indexFileLocked(file string) (string, error) { } document.Metadata = job.meta document.TextRankEnriched = !b.supportsTextRank(file) + documents := []Document{document} b.documentsMu.Lock() + b.preserveGoodReadsLocked(documents) + document = documents[0] err = b.documentsIdx.Index(id, document) b.documentsMu.Unlock() if err != nil { @@ -181,6 +184,7 @@ func (b *BleveIndexer) indexFileLocked(file string) (string, error) { if !document.TextRankEnriched { b.scheduleTextRankEnrichment(document) } + b.scheduleGoodReadsEnrichment(document.ID) b.maybePruneForLibraryChange() diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go new file mode 100644 index 00000000..1cf55be5 --- /dev/null +++ b/internal/index/bleve_goodreads_write.go @@ -0,0 +1,323 @@ +package index + +import ( + "context" + "errors" + "fmt" + "log" + "regexp" + "strings" + "time" + "unicode" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "golang.org/x/text/unicode/norm" +) + +const ( + // DefaultGoodReadsEnrichInterval is the minimum time between two GoodReads lookups. + DefaultGoodReadsEnrichInterval = time.Second + goodReadsRequestTimeout = 2 * time.Minute +) + +// GoodReadsSource searches books on Goodreads; implemented by *biblioreads.Client. +type GoodReadsSource interface { + SearchBooks(ctx context.Context, query string) (biblioreads.SearchBooksResponse, error) +} + +var goodReadsFields = []string{"GoodReadsBookURL", "GoodReadsAuthorURL", "GoodReadsEnriched"} + +// Strip named series annotations and omnibus ranges, not standalone volume numbers. +var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) + +func normalizeGoodReadsName(value string) string { + var result strings.Builder + for _, r := range norm.NFD.String(strings.ToLower(value)) { + switch { + case unicode.Is(unicode.Mn, r): + continue + case unicode.IsLetter(r), unicode.IsNumber(r): + result.WriteRune(r) + default: + result.WriteByte(' ') + } + } + return strings.Join(strings.Fields(result.String()), " ") +} + +func goodReadsTitle(value string) string { + return normalizeGoodReadsName(goodReadsSeriesSuffix.ReplaceAllString(value, "")) +} + +func matchingGoodReadsBook(document Document, results []biblioreads.SearchResult) (biblioreads.SearchResult, bool) { + title := goodReadsTitle(document.Title) + if title == "" { + return biblioreads.SearchResult{}, false + } + var best biblioreads.SearchResult + bestScore := 0 + for _, result := range results { + if result.BookURL == "" || goodReadsTitle(result.Title) != title { + continue + } + author := strings.ReplaceAll(normalizeGoodReadsName(result.Author), " ", "") + if author == "" { + continue + } + for _, name := range document.Authors { + if strings.ReplaceAll(normalizeGoodReadsName(name), " ", "") != author { + continue + } + score := 1 + if normalizeGoodReadsName(result.Title) == normalizeGoodReadsName(document.Title) { + score = 2 + } + if score > bestScore { + best, bestScore = result, score + } + break + } + } + return best, bestScore > 0 +} + +// goodReadsQuery returns the Goodreads search string for document: its title +// followed by its first author, if any. +func goodReadsQuery(document Document) string { + q := strings.TrimSpace(document.Title) + if len(document.Authors) > 0 { + q = strings.TrimSpace(q + " " + document.Authors[0]) + } + return q +} + +func documentsNeedingGoodReadsQuery() query.Query { + notEnriched := bleve.NewBoolFieldQuery(false) + notEnriched.SetField("GoodReadsEnriched") + return notEnriched +} + +func (b *BleveIndexer) documentsNeedingGoodReads(size int) ([]Document, uint64, error) { + searchReq := bleve.NewSearchRequestOptions(documentsNeedingGoodReadsQuery(), size, 0, false) + searchReq.Fields = []string{"*"} + b.documentsMu.RLock() + searchResult, err := b.documentsIdx.Search(searchReq) + b.documentsMu.RUnlock() + if err != nil { + return nil, 0, err + } + documents := make([]Document, 0, len(searchResult.Hits)) + for _, hit := range searchResult.Hits { + documents = append(documents, hydrateDocument(hit)) + } + return documents, searchResult.Total, nil +} + +// EnrichGoodReads looks up on Goodreads every document not processed yet (see +// Document.GoodReadsEnriched), storing the book and author URLs of a matching +// search result. Lookups are throttled (see Config.GoodReadsEnrichInterval). +// It stops at the first error that means Goodreads can't be reached right +// now, leaving the remaining documents for the next run. +func (b *BleveIndexer) EnrichGoodReads(batchSize int) error { + if b.goodReadsSource == nil { + return nil + } + + _, total, err := b.documentsNeedingGoodReads(0) + if err != nil || total == 0 { + return err + } + log.Printf("Enriching %d documents with GoodReads data", total) + + // Bounds the loop if a document's write keeps failing, which would leave + // it in documentsNeedingGoodReads' results forever. + maxIterations := int(total) + 10 + for iterations := 0; ; iterations++ { + if iterations >= maxIterations { + return fmt.Errorf("GoodReads enrichment did not finish after %d iterations, some documents may be stuck unenriched", iterations) + } + // Enriched documents drop out of the query, so it always starts from 0. + chunk, _, err := b.documentsNeedingGoodReads(batchSize) + if err != nil { + return err + } + if len(chunk) == 0 { + break + } + for _, document := range chunk { + if _, err := b.enrichGoodReads(context.Background(), document, true); err != nil { + return err + } + } + } + + log.Printf("GoodReads enrichment finished") + return nil +} + +// GoodReadsBookURL returns the Goodreads book path of document (see +// Document.GoodReadsBookURL). If the document hasn't been looked up yet, it +// searches it right away (without waiting for the background enrichment's +// throttle) and persists the result, so the background enrichment skips it. +// Returns an empty string if there's no match or GoodReads is disabled. +func (b *BleveIndexer) GoodReadsBookURL(ctx context.Context, document Document) (string, error) { + if document.GoodReadsBookURL != "" || document.GoodReadsEnriched || b.goodReadsSource == nil { + return document.GoodReadsBookURL, nil + } + return b.enrichGoodReads(ctx, document, false) +} + +// enrichGoodReads searches document on Goodreads, unless it already has a +// book URL, and persists the first result's URLs, returning the book one. +// Only errors meaning Goodreads is unreachable are returned; in that case +// the document is left unenriched to be retried later. +func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document, throttle bool) (string, error) { + q := goodReadsQuery(document) + bookURL, authorURL := document.GoodReadsBookURL, document.GoodReadsAuthorURL + if bookURL == "" && document.Title != "" { + ctx, cancel := context.WithTimeout(ctx, goodReadsRequestTimeout) + defer cancel() + queries := []string{q} + if title := strings.TrimSpace(document.Title); title != q { + queries = append(queries, title) + } + for _, query := range queries { + if throttle { + b.waitGoodReadsTurn() + } + res, err := b.goodReadsSource.SearchBooks(ctx, query) + switch { + case errors.Is(err, biblioreads.ErrUnavailable), errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): + return "", fmt.Errorf("error searching %q on GoodReads: %w", query, err) + case err != nil: + log.Printf("Error searching %q on GoodReads: %s\n", query, err) + default: + if match, ok := matchingGoodReadsBook(document, res.Result); ok { + bookURL, authorURL = match.BookURL, match.AuthorURL + } + } + if bookURL != "" { + break + } + } + } + + // Re-read the document under the write lock, so concurrent writes of other + // fields (e.g. TextRank enrichment) aren't overwritten with stale values. + b.documentsMu.Lock() + defer b.documentsMu.Unlock() + current, found, err := b.documentByIndexIDLocked(document.ID) + if err != nil { + log.Printf("Error reading document %s for GoodReads enrichment: %s\n", document.ID, err) + return bookURL, nil + } + // Removed, or reindexed with a different title/author in the meantime + // (which schedules its own lookup). + if !found || goodReadsQuery(current) != q { + return bookURL, nil + } + // Looked up concurrently (e.g. by the background enrichment and a detail + // page request at the same time): keep the stored result. + if current.GoodReadsEnriched { + return current.GoodReadsBookURL, nil + } + if current.GoodReadsBookURL != "" { + bookURL, authorURL = current.GoodReadsBookURL, current.GoodReadsAuthorURL + } + current.GoodReadsBookURL = bookURL + current.GoodReadsAuthorURL = authorURL + current.GoodReadsEnriched = true + if err := b.documentsIdx.Index(current.ID, current); err != nil { + log.Printf("Error indexing GoodReads-enriched document %s: %s\n", current.ID, err) + } + return bookURL, nil +} + +// waitGoodReadsTurn blocks until at least goodReadsInterval has passed since +// the previous GoodReads lookup, shared by EnrichGoodReads and the per-file +// worker. +func (b *BleveIndexer) waitGoodReadsTurn() { + b.goodReadsThrottleMu.Lock() + defer b.goodReadsThrottleMu.Unlock() + if wait := time.Until(b.goodReadsLastRequest.Add(b.goodReadsInterval)); wait > 0 { + time.Sleep(wait) + } + b.goodReadsLastRequest = time.Now() +} + +// runGoodReadsEnrichWorker looks up documents queued by indexFile for the +// indexer's lifetime. +func (b *BleveIndexer) runGoodReadsEnrichWorker() { + for id := range b.goodReadsEnrichJobs { + document, err := b.documentByIndexID(id) + if err != nil { + log.Printf("Error reading document %s for GoodReads enrichment: %s\n", id, err) + continue + } + if document.ID == "" || document.GoodReadsEnriched { + continue + } + if _, err := b.enrichGoodReads(context.Background(), document, true); err != nil { + log.Println(err) + } + } +} + +// scheduleGoodReadsEnrichment queues the document with the given ID for +// lookup from a throwaway goroutine, so indexFile never blocks on it. +func (b *BleveIndexer) scheduleGoodReadsEnrichment(id string) { + if b.goodReadsSource == nil { + return + } + go func() { + b.goodReadsEnrichJobs <- id + }() +} + +// documentByIndexIDLocked is documentByIndexID for callers already holding documentsMu. +func (b *BleveIndexer) documentByIndexIDLocked(id string) (Document, bool, error) { + searchReq := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) + searchReq.Fields = []string{"*"} + searchResult, err := b.documentsIdx.Search(searchReq) + if err != nil || len(searchResult.Hits) == 0 { + return Document{}, false, err + } + return hydrateDocument(searchResult.Hits[0]), true, nil +} + +// preserveGoodReadsLocked copies the currently stored GoodReads fields into +// documents, so writers that rebuilt them from an earlier read don't discard +// a lookup persisted in the meantime. Must be called with documentsMu held +// for writing, right before writing documents back. +func (b *BleveIndexer) preserveGoodReadsLocked(documents []Document) { + if len(documents) == 0 { + return + } + ids := make([]string, len(documents)) + for i, document := range documents { + ids[i] = document.ID + } + searchReq := bleve.NewSearchRequestOptions(bleve.NewDocIDQuery(ids), len(ids), 0, false) + searchReq.Fields = goodReadsFields + searchResult, err := b.documentsIdx.Search(searchReq) + if err != nil { + log.Printf("Error reading stored GoodReads fields: %s\n", err) + return + } + stored := make(map[string]Document, len(searchResult.Hits)) + for _, hit := range searchResult.Hits { + bookURL, _ := hit.Fields["GoodReadsBookURL"].(string) + authorURL, _ := hit.Fields["GoodReadsAuthorURL"].(string) + enriched, _ := hit.Fields["GoodReadsEnriched"].(bool) + stored[hit.ID] = Document{GoodReadsBookURL: bookURL, GoodReadsAuthorURL: authorURL, GoodReadsEnriched: enriched} + } + for i := range documents { + if s, ok := stored[documents[i].ID]; ok { + documents[i].GoodReadsBookURL = s.GoodReadsBookURL + documents[i].GoodReadsAuthorURL = s.GoodReadsAuthorURL + documents[i].GoodReadsEnriched = s.GoodReadsEnriched + } + } +} diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go new file mode 100644 index 00000000..99c9fc74 --- /dev/null +++ b/internal/index/bleve_goodreads_write_test.go @@ -0,0 +1,513 @@ +package index + +import ( + "context" + "errors" + "image" + "slices" + "sync" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/metadata" +) + +type fakeGoodReadsSource struct { + mu sync.Mutex + queries []string + results []biblioreads.SearchResult + byQuery map[string][]biblioreads.SearchResult + queryErrors map[string]error + err error +} + +func (f *fakeGoodReadsSource) SearchBooks(_ context.Context, query string) (biblioreads.SearchBooksResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.queries = append(f.queries, query) + if f.byQuery != nil { + return biblioreads.SearchBooksResponse{Result: f.byQuery[query]}, f.queryErrors[query] + } + return biblioreads.SearchBooksResponse{Result: f.results}, f.err +} + +func (f *fakeGoodReadsSource) Queries() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.queries) +} + +type goodReadsTestReader struct{} + +func (goodReadsTestReader) Metadata(string) (metadata.Metadata, error) { + return metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt", "Second Author"}, Format: "PDF"}, nil +} + +func (goodReadsTestReader) Cover(string, int) (image.Image, error) { + return nil, nil +} + +var goodReadsTestResults = []biblioreads.SearchResult{ + {Title: "La Reina Dragon", Author: "Alice Borchardt", BookURL: "/book/show/12207444-la-reina-dragon", AuthorURL: "/author/show/151155.Alice_Borchardt"}, + {BookURL: "/book/show/2-other", AuthorURL: "/author/show/2.Other"}, +} + +const ( + wantGoodReadsBookURL = "/book/show/12207444-la-reina-dragon" + wantGoodReadsAuthorURL = "/author/show/151155.Alice_Borchardt" + goodReadsTestID = "book.pdf" +) + +func newGoodReadsTestIndexer(t *testing.T, source GoodReadsSource) *BleveIndexer { + t.Helper() + idx, _ := newTestTextRankIndexer(t, Config{GoodReadsSource: source, GoodReadsEnrichInterval: time.Millisecond}) + appFS := afero.NewMemMapFs() + if err := afero.WriteFile(appFS, "lib/book.pdf", []byte(""), 0644); err != nil { + t.Fatal(err) + } + idx.fs = appFS + idx.libraryPath = "lib" + idx.reader = map[string]metadata.Reader{".pdf": goodReadsTestReader{}} + return idx +} + +func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { + source := &fakeGoodReadsSource{results: goodReadsTestResults} + idx := newGoodReadsTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + + doc, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + if doc.GoodReadsBookURL != wantGoodReadsBookURL || doc.GoodReadsAuthorURL != wantGoodReadsAuthorURL { + t.Errorf("unexpected URLs %q and %q", doc.GoodReadsBookURL, doc.GoodReadsAuthorURL) + } + if !doc.GoodReadsEnriched { + t.Error("expected document to be marked as GoodReads-enriched") + } + if want := []string{"La Reina Dragon Alice Borchardt"}; !slices.Equal(source.Queries(), want) { + t.Errorf("expected queries %q, got %q", want, source.Queries()) + } + + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + if len(source.Queries()) != 1 { + t.Errorf("expected enriched documents not to be searched again, got queries %q", source.Queries()) + } +} + +func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { + for name, source := range map[string]*fakeGoodReadsSource{ + "no results": {}, + "non-transient err": {err: errors.New("biblioreads: request failed with status code 504")}, + "wrong title": {results: []biblioreads.SearchResult{{Title: "Another book", Author: "Alice Borchardt", BookURL: "/book/show/1"}}}, + "wrong author": {results: []biblioreads.SearchResult{{Title: "La Reina Dragon", Author: "Another writer", BookURL: "/book/show/1"}}}, + } { + t.Run(name, func(t *testing.T) { + idx := newGoodReadsTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + doc, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != "" || doc.GoodReadsAuthorURL != "" { + t.Errorf("expected document enriched with no URLs, got %+v", doc) + } + if want := []string{"La Reina Dragon Alice Borchardt", "La Reina Dragon"}; !slices.Equal(source.Queries(), want) { + t.Errorf("expected both searches before giving up, got %q", source.Queries()) + } + }) + } +} + +func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { + source := &fakeGoodReadsSource{err: biblioreads.ErrUnavailable} + idx := newGoodReadsTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + + if err := idx.EnrichGoodReads(10); !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected ErrUnavailable, got %v", err) + } + doc, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + if doc.GoodReadsEnriched { + t.Error("expected document not to be marked as GoodReads-enriched") + } + + source.mu.Lock() + source.err = nil + source.results = goodReadsTestResults + source.mu.Unlock() + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + if doc, _ = idx.documentByIndexID(goodReadsTestID); doc.GoodReadsBookURL != wantGoodReadsBookURL { + t.Errorf("expected document to be enriched on retry, got %q", doc.GoodReadsBookURL) + } +} + +func TestNewFileSchedulesGoodReadsEnrichment(t *testing.T) { + source := &fakeGoodReadsSource{results: goodReadsTestResults} + idx := newGoodReadsTestIndexer(t, source) + + if _, err := idx.NewFile("new.pdf", []byte("")); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(2 * time.Second) + for { + doc, err := idx.documentByIndexID("new.pdf") + if err != nil { + t.Fatal(err) + } + if doc.GoodReadsEnriched { + if doc.GoodReadsBookURL != wantGoodReadsBookURL { + t.Errorf("unexpected book URL %q", doc.GoodReadsBookURL) + } + return + } + if time.Now().After(deadline) { + t.Fatal("timed out waiting for background GoodReads enrichment") + } + time.Sleep(10 * time.Millisecond) + } +} + +// TestTextRankWritesPreserveGoodReadsFields guards against TextRank +// enrichment, which writes back documents read before ranking them, +// discarding GoodReads URLs persisted in the meantime, and vice versa. +func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { + source := &fakeGoodReadsSource{results: goodReadsTestResults} + idx := newGoodReadsTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + stale, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + stale.TextRankEnriched = false + stale.TextRankWords = []string{"dragon"} + idx.enrichTextRankAndReindex(stale) + + doc, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != wantGoodReadsBookURL { + t.Errorf("expected GoodReads fields to survive a TextRank write, got %+v", doc) + } + if !doc.TextRankEnriched { + t.Error("expected TextRank write to be persisted") + } + + // A GoodReads write must keep TextRank fields written after the document was read. + stale, _ = idx.documentByIndexID(goodReadsTestID) + stale.GoodReadsEnriched = false + doc.TextRankWords = []string{"guinevere"} + idx.documentsMu.Lock() + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + idx.documentsMu.Unlock() + if _, err := idx.enrichGoodReads(context.Background(), stale, true); err != nil { + t.Fatal(err) + } + if doc, _ = idx.documentByIndexID(goodReadsTestID); !slices.Equal(doc.TextRankWords, []string{"guinevere"}) { + t.Errorf("expected TextRank words to survive a GoodReads write, got %q", doc.TextRankWords) + } +} + +func TestEnrichGoodReadsWithoutSourceIsNoop(t *testing.T) { + idx := newGoodReadsTestIndexer(t, nil) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + if _, err := idx.NewFile("new.pdf", []byte("")); err != nil { + t.Fatal(err) + } + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + doc, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + if doc.GoodReadsEnriched { + t.Error("expected document not to be GoodReads-enriched when the source is disabled") + } +} + +func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { + source := &fakeGoodReadsSource{results: goodReadsTestResults} + idx := newGoodReadsTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + doc, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + doc.GoodReadsBookURL = "/book/show/1-existing" + doc.GoodReadsAuthorURL = "/author/show/1.Existing" + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + + if len(source.Queries()) != 0 { + t.Errorf("expected no search for a document with a link, got %q", source.Queries()) + } + doc, err = idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != "/book/show/1-existing" || doc.GoodReadsAuthorURL != "/author/show/1.Existing" { + t.Errorf("expected existing link kept and document marked enriched, got %+v", doc) + } +} + +func TestGoodReadsBookURL(t *testing.T) { + source := &fakeGoodReadsSource{results: goodReadsTestResults} + idx := newGoodReadsTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + doc, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + + got, err := idx.GoodReadsBookURL(context.Background(), doc) + if err != nil { + t.Fatal(err) + } + if got != wantGoodReadsBookURL { + t.Errorf("expected %q, got %q", wantGoodReadsBookURL, got) + } + stored, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + if !stored.GoodReadsEnriched || stored.GoodReadsBookURL != wantGoodReadsBookURL { + t.Errorf("expected the link to be persisted, got %+v", stored) + } + + if got, _ := idx.GoodReadsBookURL(context.Background(), stored); got != wantGoodReadsBookURL { + t.Errorf("expected stored link %q, got %q", wantGoodReadsBookURL, got) + } + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + if len(source.Queries()) != 1 { + t.Errorf("expected a single search, got %q", source.Queries()) + } +} + +func TestGoodReadsBookURLReturnsErrorWhenUnavailable(t *testing.T) { + source := &fakeGoodReadsSource{err: biblioreads.ErrUnavailable} + idx := newGoodReadsTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + doc, _ := idx.documentByIndexID(goodReadsTestID) + if _, err := idx.GoodReadsBookURL(context.Background(), doc); !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected ErrUnavailable, got %v", err) + } + if doc, _ = idx.documentByIndexID(goodReadsTestID); doc.GoodReadsEnriched { + t.Error("expected document to stay unenriched for a later retry") + } +} + +func TestMatchingGoodReadsBook(t *testing.T) { + correct := biblioreads.SearchResult{ + Title: "El señor de los anillos", Author: "J.R.R. Tolkien", + BookURL: "/book/show/37910810", AuthorURL: "/author/show/656983.J_R_R_Tolkien", + } + parody := biblioreads.SearchResult{ + Title: "El sopor de los anillos", Author: "The Harvard Lampoon", + BookURL: "/book/show/7297403-el-sopor-de-los-anillos", + } + tests := []struct { + name string + title string + authors []string + results []biblioreads.SearchResult + want string + }{ + {name: "correct result after parody", results: []biblioreads.SearchResult{parody, correct}, want: correct.BookURL}, + {name: "parody rejected", results: []biblioreads.SearchResult{parody}}, + {name: "similar title with correct author rejected", results: []biblioreads.SearchResult{{Title: parody.Title, Author: correct.Author, BookURL: parody.BookURL}}}, + {name: "wrong author rejected", results: []biblioreads.SearchResult{{Title: correct.Title, Author: parody.Author, BookURL: parody.BookURL}}}, + {name: "normalization", title: " EL SEN\u0303OR DE LOS ANILLOS! ", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, + {name: "series suffix", results: []biblioreads.SearchResult{{Title: correct.Title + " (El señor de los anillos, #1)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "document series suffix", title: correct.Title + " (Middle Earth, #1)", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, + {name: "omnibus range", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1-#3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "omnibus range with dash", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1–3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "document omnibus range", title: correct.Title + " (#1-#3)", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, + {name: "standalone volume retained", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1)", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "exact preferred", results: []biblioreads.SearchResult{{Title: correct.Title + " (Middle Earth, #1)", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: correct.BookURL}, + {name: "subtitle retained", results: []biblioreads.SearchResult{{Title: correct.Title + ": A parody", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "volume retained", results: []biblioreads.SearchResult{{Title: correct.Title + " 2", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "arbitrary parenthetical retained", results: []biblioreads.SearchResult{{Title: correct.Title + " (A parody)", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "missing result author", results: []biblioreads.SearchResult{{Title: correct.Title, BookURL: correct.BookURL}}}, + {name: "missing result title", results: []biblioreads.SearchResult{{Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "missing URL", results: []biblioreads.SearchResult{{Title: correct.Title, Author: correct.Author}}}, + {name: "any document author", authors: []string{"Another Writer", "J. R. R. Tolkien"}, results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, + {name: "no author", authors: []string{}, results: []biblioreads.SearchResult{correct}}, + {name: "no results"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + title, authors := tt.title, tt.authors + if title == "" { + title = correct.Title + } + if authors == nil { + authors = []string{"J. R. R. Tolkien"} + } + match, ok := matchingGoodReadsBook(Document{Metadata: metadata.Metadata{Title: title, Authors: authors}}, tt.results) + if match.BookURL != tt.want || ok != (tt.want != "") { + t.Fatalf("got %q, %v; want %q", match.BookURL, ok, tt.want) + } + }) + } +} + +func TestGoodReadsTitleFallback(t *testing.T) { + const title = "El señor de los anillos" + correct := biblioreads.SearchResult{Title: title, Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810"} + parody := biblioreads.SearchResult{Title: "El sopor de los anillos", Author: "The Harvard Lampoon", BookURL: "/book/show/7297403"} + for _, fallbackUnavailable := range []bool{false, true} { + name := "matched" + if fallbackUnavailable { + name = "unavailable" + } + t.Run(name, func(t *testing.T) { + source := &fakeGoodReadsSource{byQuery: map[string][]biblioreads.SearchResult{ + title + " J. R. R. Tolkien": {parody}, + title: {parody, correct}, + }} + idx := newGoodReadsTestIndexer(t, source) + doc := Document{ID: goodReadsTestID, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + if fallbackUnavailable { + source.queryErrors = map[string]error{title: biblioreads.ErrUnavailable} + } + got, err := idx.GoodReadsBookURL(context.Background(), doc) + if fallbackUnavailable { + if !errors.Is(err, biblioreads.ErrUnavailable) || got != "" { + t.Fatalf("expected unavailable error, got %q, %v", got, err) + } + stored, readErr := idx.documentByIndexID(doc.ID) + if readErr != nil || stored.GoodReadsEnriched { + t.Fatalf("expected document left for retry, got %+v, %v", stored, readErr) + } + source.queryErrors = nil + got, err = idx.GoodReadsBookURL(context.Background(), stored) + } + if err != nil || got != correct.BookURL { + t.Fatalf("got %q, %v; want %q", got, err, correct.BookURL) + } + want := []string{title + " J. R. R. Tolkien", title} + if fallbackUnavailable { + want = append(want, want...) + } + if !slices.Equal(source.Queries(), want) { + t.Fatalf("queries = %q, want %q", source.Queries(), want) + } + }) + } +} + +func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { + const title = "El señor de los anillos" + const wantURL = "/book/show/37910810-el-se-or-de-los-anillos" + // Titles and ordering from BiblioReads' actual title-only search response. + results := []biblioreads.SearchResult{ + {Title: "La comunidad del anillo (El señor de los anillos, #1)", Author: "J.R.R. Tolkien", BookURL: "/book/show/222947.La_comunidad_del_anillo"}, + {Title: "Las dos torres (El señor de los anillos, #2)", Author: "J.R.R. Tolkien", BookURL: "/book/show/15280.Las_dos_torres"}, + {Title: "El retorno del rey (El señor de los anillos, #3)", Author: "J.R.R. Tolkien", BookURL: "/book/show/63358.El_retorno_del_rey"}, + {Title: "El señor de los anillos (#1-#3)", Author: "J.R.R. Tolkien", BookURL: wantURL, AuthorURL: "/author/show/656983.J_R_R_Tolkien"}, + {Title: "Apéndices (El Señor de los Anillos, #4)", Author: "J.R.R. Tolkien", BookURL: "/book/show/54694.Ap_ndices"}, + } + source := &fakeGoodReadsSource{byQuery: map[string][]biblioreads.SearchResult{title: results}} + idx := newGoodReadsTestIndexer(t, source) + doc := Document{ID: goodReadsTestID, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + got, err := idx.GoodReadsBookURL(context.Background(), doc) + if err != nil || got != wantURL { + t.Fatalf("got %q, %v; want %q", got, err, wantURL) + } + stored, err := idx.documentByIndexID(doc.ID) + if err != nil { + t.Fatal(err) + } + if !stored.GoodReadsEnriched || stored.GoodReadsBookURL != wantURL || stored.GoodReadsAuthorURL != results[3].AuthorURL { + t.Fatalf("expected persisted omnibus match, got %+v", stored) + } + if want := []string{title + " J. R. R. Tolkien", title}; !slices.Equal(source.Queries(), want) { + t.Fatalf("queries = %q, want %q", source.Queries(), want) + } +} + +func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { + source := &fakeGoodReadsSource{results: goodReadsTestResults} + idx := newGoodReadsTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + doc, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + doc.GoodReadsBookURL = "/book/show/1-wrong" + doc.GoodReadsAuthorURL = "/author/show/1.Wrong" + doc.GoodReadsEnriched = true + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + doc, err = idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + if doc.GoodReadsBookURL != wantGoodReadsBookURL || doc.GoodReadsAuthorURL != wantGoodReadsAuthorURL { + t.Fatalf("expected refreshed links, got %q and %q", doc.GoodReadsBookURL, doc.GoodReadsAuthorURL) + } +} diff --git a/internal/index/bleve_textrank_write.go b/internal/index/bleve_textrank_write.go index ad9335b1..f0f38104 100644 --- a/internal/index/bleve_textrank_write.go +++ b/internal/index/bleve_textrank_write.go @@ -85,8 +85,10 @@ func (b *BleveIndexer) scheduleTextRankEnrichment(document Document) { // waiting on TextRank. func (b *BleveIndexer) enrichTextRankAndReindex(document Document) { enriched := b.rankDocument(document) + documents := []Document{enriched} b.documentsMu.Lock() - err := b.documentsIdx.Index(enriched.ID, enriched) + b.preserveGoodReadsLocked(documents) + err := b.documentsIdx.Index(enriched.ID, documents[0]) b.documentsMu.Unlock() if err != nil { log.Printf("Error indexing TextRank-enriched document %s: %s\n", enriched.ID, err) @@ -263,13 +265,14 @@ func (b *BleveIndexer) EnrichTextRankKeywords(batchSize, workers int) error { enriched := b.rankDocuments(chunk, workers) + b.documentsMu.Lock() + b.preserveGoodReadsLocked(enriched) batch := b.documentsIdx.NewBatch() for _, document := range enriched { if err := batch.Index(document.ID, document); err != nil { log.Printf("Error indexing enriched document %s: %s\n", document.ID, err) } } - b.documentsMu.Lock() err = b.documentsIdx.Batch(batch) b.documentsMu.Unlock() if err != nil { @@ -529,7 +532,7 @@ func (b *BleveIndexer) rewriteCommonTextRankEntries(docCount uint64, batchSize i return pruned, err } - batch := b.documentsIdx.NewBatch() + var documents []Document for _, hit := range docSearchResult.Hits { document := hydrateDocument(hit) p, ok := toPrune[document.ID] @@ -538,20 +541,25 @@ func (b *BleveIndexer) rewriteCommonTextRankEntries(docCount uint64, batchSize i } document.TextRankPhrases = p.phrases document.TextRankWords = p.words + documents = append(documents, document) + } + + b.documentsMu.Lock() + b.preserveGoodReadsLocked(documents) + batch := b.documentsIdx.NewBatch() + for _, document := range documents { if err := batch.Index(document.ID, document); err != nil { log.Printf("Error indexing document %s while pruning common TextRank entries: %s\n", document.ID, err) continue } pruned++ } - if batch.Size() > 0 { - b.documentsMu.Lock() err = b.documentsIdx.Batch(batch) - b.documentsMu.Unlock() - if err != nil { - return pruned, err - } + } + b.documentsMu.Unlock() + if err != nil { + return pruned, err } b.recordPruningProgress(len(result.Hits)) } diff --git a/internal/index/document.go b/internal/index/document.go index 07dfe299..5ef44084 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -76,6 +76,17 @@ type Document struct { // in the background. See documentsNeedingTextRank, which drives that pass // off this field so it survives (and resumes correctly across) restarts. TextRankEnriched bool + // GoodReadsBookURL and GoodReadsAuthorURL link the document to its + // Goodreads book and author pages. They are stored but not indexed (see + // CreateDocumentsMapping), so they never affect search results. They are + // filled in the background by the GoodReads enrichment (see + // EnrichGoodReads), with paths relative to https://www.goodreads.com + // (e.g. "/book/show/12207444-la-reina-dragon"), as returned by BiblioReads. + GoodReadsBookURL string + GoodReadsAuthorURL string + // GoodReadsEnriched is false until the GoodReads enrichment has looked + // this document up, whether or not a match was found. + GoodReadsEnriched bool } func (d Document) SupportsAnnotations() bool { diff --git a/internal/index/goodreads_fields_test.go b/internal/index/goodreads_fields_test.go new file mode 100644 index 00000000..ad0dc68b --- /dev/null +++ b/internal/index/goodreads_fields_test.go @@ -0,0 +1,86 @@ +package index_test + +import ( + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" +) + +func TestGoodReadsURLsAreStoredButNotSearchable(t *testing.T) { + documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, index.Config{}) + + const ( + bookURL = "/book/show/12207444-zyxwvbook" + authorURL = "/author/show/151155.Zyxwv_Author" + ) + for _, lang := range []string{"es", ""} { + doc := index.Document{ + ID: "doc-" + lang, + Slug: "doc-" + lang, + Metadata: metadata.Metadata{ + Title: "La Reina Dragon", + Authors: []string{"Alice Borchardt"}, + Language: lang, + Format: "EPUB", + }, + AuthorsSlugs: []string{"alice-borchardt"}, + GoodReadsBookURL: bookURL, + GoodReadsAuthorURL: authorURL, + } + if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + + got, err := idx.Document(doc.Slug) + if err != nil { + t.Fatal(err) + } + if got.GoodReadsBookURL != bookURL || got.GoodReadsAuthorURL != authorURL { + t.Errorf("language %q: expected stored URLs to be returned, got %q and %q", lang, got.GoodReadsBookURL, got.GoodReadsAuthorURL) + } + } + + for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155"} { + res, err := idx.Search(index.SearchFields{Keywords: keywords}, 1, 10) + if err != nil { + t.Fatal(err) + } + if res.TotalHits() != 0 { + t.Errorf("expected no results searching keywords %q, got %d", keywords, res.TotalHits()) + } + } + + queries := map[string]bleve.SearchRequest{ + "book URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsBookURL", bookURL)), + "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsAuthorURL", authorURL)), + "_all match": *bleve.NewSearchRequest(bleve.NewMatchQuery("zyxwvbook")), + "_all term": *bleve.NewSearchRequest(bleve.NewTermQuery(bookURL)), + "query string field": *bleve.NewSearchRequest(bleve.NewQueryStringQuery(`GoodReadsBookURL:"` + bookURL + `"`)), + } + for name, req := range queries { + res, err := documentsIndexMem.Search(&req) + if err != nil { + t.Fatal(err) + } + if res.Total != 0 { + t.Errorf("%s: expected no hits, got %d", name, res.Total) + } + } +} + +func fieldTermQuery(field, term string) *query.TermQuery { + q := bleve.NewTermQuery(term) + q.SetField(field) + return q +} diff --git a/internal/webserver/controller.go b/internal/webserver/controller.go index 0eeb4802..bd2362e3 100644 --- a/internal/webserver/controller.go +++ b/internal/webserver/controller.go @@ -30,7 +30,7 @@ type Controllers struct { Series *series.Controller } -func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender Sender, appFs afero.Fs, dataSource author.DataSource) Controllers { +func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender Sender, appFs afero.Fs, dataSource author.DataSource, bookSource document.BookSource) Controllers { usersRepository := &model.UserRepository{DB: db} invitationsRepository := &model.InvitationRepository{DB: db} highlightsRepository := &model.HighlightRepository{DB: db, Idx: idx, IllustratedMinAmount: cfg.IllustratedMinAmount} @@ -111,7 +111,7 @@ func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender S Users: user.NewController(usersRepository, invitationsRepository, usersCfg, sender, translator), Completed: completed.NewController(readingRepository, idx), Highlights: highlight.NewController(highlightsRepository, readingRepository, usersRepository, sender, cfg.WordsPerMinute, idx), - Documents: document.NewController(highlightsRepository, usersRepository, readingRepository, annotationsRepository, sender, idx, appFs, documentsCfg, translator), + Documents: document.NewController(highlightsRepository, usersRepository, readingRepository, annotationsRepository, sender, idx, appFs, documentsCfg, translator, bookSource), Home: home.NewController(highlightsRepository, readingRepository, sender, idx, homeCfg), Authors: authorsController, Search: search.NewController(highlightsRepository, readingRepository, sender, idx, searchCfg), diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index d3a77edb..df217bf9 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -1,6 +1,7 @@ package document import ( + "context" "image" "time" @@ -35,6 +36,7 @@ type IdxReaderWriter interface { Languages() ([]string, error) Subjects() (map[string][]string, error) Author(slug, lang string) (index.Author, error) + GoodReadsBookURL(ctx context.Context, document index.Document) (string, error) } type highlightsRepository interface { @@ -88,9 +90,11 @@ type Controller struct { config Config appFs afero.Fs translator i18n.Translator + bookSource BookSource } -func NewController(hlRepository highlightsRepository, usersRepository usersRepository, readingRepository readingRepository, annotationsRepository annotationsRepository, sender Sender, idx IdxReaderWriter, appFs afero.Fs, cfg Config, translator i18n.Translator) *Controller { +// NewController creates a document controller. bookSource may be nil, which disables Goodreads data. +func NewController(hlRepository highlightsRepository, usersRepository usersRepository, readingRepository readingRepository, annotationsRepository annotationsRepository, sender Sender, idx IdxReaderWriter, appFs afero.Fs, cfg Config, translator i18n.Translator, bookSource BookSource) *Controller { return &Controller{ hlRepository: hlRepository, usersRepository: usersRepository, @@ -101,5 +105,6 @@ func NewController(hlRepository highlightsRepository, usersRepository usersRepos config: cfg, appFs: appFs, translator: translator, + bookSource: bookSource, } } diff --git a/internal/webserver/controller/document/detail.go b/internal/webserver/controller/document/detail.go index 29a49a6e..393f2ba7 100644 --- a/internal/webserver/controller/document/detail.go +++ b/internal/webserver/controller/document/detail.go @@ -71,6 +71,7 @@ func (d *Controller) Detail(c fiber.Ctx) error { "AuthorSummaries": authorSummaries, "IllustratorSummaries": illustratorSummaries, "Debug": c.RequestCtx().QueryArgs().Has("debug"), + "GoodReadsEnabled": d.bookSource != nil, }, "layout") } diff --git a/internal/webserver/controller/document/goodreads.go b/internal/webserver/controller/document/goodreads.go new file mode 100644 index 00000000..0c528c8f --- /dev/null +++ b/internal/webserver/controller/document/goodreads.go @@ -0,0 +1,69 @@ +package document + +import ( + "context" + "log" + "strings" + "time" + + "github.com/gofiber/fiber/v3" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +const ( + goodReadsBaseURL = "https://www.goodreads.com" + goodReadsBookPrefix = "/book/show/" + goodReadsBookTimeout = 30 * time.Second +) + +// BookSource retrieves book details from Goodreads; implemented by *biblioreads.Client. +type BookSource interface { + Book(ctx context.Context, id string) (biblioreads.BookResponse, error) +} + +// GoodReads renders the Goodreads rating shown in a document's detail page +// metadata. It's loaded asynchronously via htmx, so slow BiblioReads +// responses don't block the page. Renders nothing if the rating can't be +// retrieved. +func (d *Controller) GoodReads(c fiber.Ctx) error { + if d.bookSource == nil { + return c.SendString("") + } + + document, err := d.idx.Document(c.Params("slug")) + if err != nil { + log.Println(err) + return c.SendString("") + } + if document.Slug == "" { + return c.SendString("") + } + + ctx, cancel := context.WithTimeout(c.Context(), goodReadsBookTimeout) + defer cancel() + // Searches the document first if it hasn't been linked to Goodreads yet. + bookURL, err := d.idx.GoodReadsBookURL(ctx, document) + if err != nil { + log.Println(err) + return c.SendString("") + } + id, ok := strings.CutPrefix(bookURL, goodReadsBookPrefix) + if !ok || id == "" { + return c.SendString("") + } + + book, err := d.bookSource.Book(ctx, id) + if err != nil { + log.Printf("Error retrieving GoodReads book %s: %s\n", id, err) + return c.SendString("") + } + if book.Rating == "" { + return c.SendString("") + } + + return c.Render("partials/document-goodreads", fiber.Map{ + "Rating": book.Rating, + "RatingCount": book.RatingCount, + "URL": goodReadsBaseURL + bookURL, + }) +} diff --git a/internal/webserver/document_detail_test.go b/internal/webserver/document_detail_test.go index d79a8947..623ad0f0 100644 --- a/internal/webserver/document_detail_test.go +++ b/internal/webserver/document_detail_test.go @@ -186,7 +186,7 @@ func TestRelatedDocumentHeadersShowFullTotals(t *testing.T) { cfg := defaultTestConfig() sender := &infrastructure.NoEmail{} - controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, nil) + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, nil, nil) app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) t.Cleanup(func() { _ = app.Shutdown() }) diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go new file mode 100644 index 00000000..e9748b97 --- /dev/null +++ b/internal/webserver/document_goodreads_test.go @@ -0,0 +1,246 @@ +package webserver_test + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" + + "github.com/PuerkitoBio/goquery" + "github.com/blevesearch/bleve/v2" + "github.com/gofiber/fiber/v3" + "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" + "github.com/svera/coreander/v5/internal/webserver" + "github.com/svera/coreander/v5/internal/webserver/controller/document" + "github.com/svera/coreander/v5/internal/webserver/infrastructure" + "github.com/svera/coreander/v5/internal/webserver/model" +) + +type fakeBookSource struct { + mu sync.Mutex + ids []string + queries []string + book biblioreads.BookResponse + results []biblioreads.SearchResult + err error +} + +func (f *fakeBookSource) SearchBooks(_ context.Context, query string) (biblioreads.SearchBooksResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.queries = append(f.queries, query) + return biblioreads.SearchBooksResponse{Result: f.results}, nil +} + +func (f *fakeBookSource) Book(_ context.Context, id string) (biblioreads.BookResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.ids = append(f.ids, id) + return f.book, f.err +} + +func newGoodReadsTestApp(t *testing.T, source *fakeBookSource) (*fiber.App, *index.BleveIndexer) { + t.Helper() + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + + documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + appFS := afero.NewMemMapFs() + var bookSource document.BookSource + idxCfg := index.Config{} + if source != nil { + bookSource = source + idxCfg.GoodReadsSource = source + } + idx := index.NewBleve(documents, authors, appFS, testLibraryDir, nil, idxCfg) + t.Cleanup(func() { _ = idx.Close() }) + + for _, doc := range []index.Document{ + { + ID: "linked.epub", Slug: "linked", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, + GoodReadsBookURL: "/book/show/12207444-la-reina-dragon", + GoodReadsEnriched: true, + }, + { + ID: "unlinked.epub", Slug: "unlinked", + Metadata: metadata.Metadata{Title: "Unlinked", Authors: []string{"Jane Doe", "John Doe"}, Format: "EPUB"}, + AuthorsSlugs: []string{"jane-doe", "john-doe"}, + }, + { + ID: "unmatched.epub", Slug: "unmatched", + Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, + GoodReadsEnriched: true, + }, + } { + if err := documents.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + } + + cfg := defaultTestConfig() + sender := &infrastructure.NoEmail{} + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, nil, bookSource) + app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) + t.Cleanup(func() { _ = app.Shutdown() }) + return app, idx +} + +func getBody(t *testing.T, app *fiber.App, path string) string { + t.Helper() + response, err := app.Test(mustGetRequest(t, path)) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("%s: status = %d, want %d", path, response.StatusCode, http.StatusOK) + } + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func goodReadsPlaceholders(t *testing.T, html string) *goquery.Selection { + t.Helper() + page, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + if err != nil { + t.Fatal(err) + } + return page.Find(`li[hx-get$="/goodreads"]`) +} + +func TestDocumentDetailGoodReadsRating(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{Rating: "3.82", RatingCount: "2,285"}} + app, _ := newGoodReadsTestApp(t, source) + + placeholder := goodReadsPlaceholders(t, getBody(t, app, "/documents/linked")) + if placeholder.Length() != 1 { + t.Fatalf("expected one GoodReads placeholder, got %d", placeholder.Length()) + } + if url, _ := placeholder.Attr("hx-get"); url != "/documents/linked/goodreads" { + t.Errorf("unexpected placeholder URL %q", url) + } + if placeholder.ParentsFiltered(`[id="document-metadata-linked"]`).Length() != 1 { + t.Error("expected the placeholder inside the document metadata list") + } + + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + if err != nil { + t.Fatal(err) + } + link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if strings.TrimSpace(link.Text()) != "3.82" { + t.Errorf("expected rating link with 3.82, got %q", link.Text()) + } + if !strings.Contains(page.Find("li").Text(), "2,285") { + t.Errorf("expected rating count in %q", page.Find("li").Text()) + } + if len(source.ids) != 1 || source.ids[0] != "12207444-la-reina-dragon" { + t.Errorf("expected Book to be called with the stored id, got %q", source.ids) + } + if len(source.queries) != 0 { + t.Errorf("expected no search for a linked document, got %q", source.queries) + } +} + +func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { + source := &fakeBookSource{ + book: biblioreads.BookResponse{Rating: "4.10"}, + results: []biblioreads.SearchResult{{Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", AuthorURL: "/author/show/1.Jane_Doe"}, {BookURL: "/book/show/2-other"}}, + } + app, idx := newGoodReadsTestApp(t, source) + + if got := goodReadsPlaceholders(t, getBody(t, app, "/documents/unlinked")).Length(); got != 1 { + t.Fatalf("expected a placeholder for a document not looked up yet, got %d", got) + } + + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unlinked/goodreads"))) + if err != nil { + t.Fatal(err) + } + if got := strings.TrimSpace(page.Find(`a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.10" { + t.Errorf("expected rating link with 4.10, got %q", got) + } + if len(source.queries) != 1 || source.queries[0] != "Unlinked Jane Doe" { + t.Errorf("expected a single search with title and first author, got %q", source.queries) + } + if len(source.ids) != 1 || source.ids[0] != "1-unlinked" { + t.Errorf("expected Book to be called with the found id, got %q", source.ids) + } + + stored, err := idx.Document("unlinked") + if err != nil { + t.Fatal(err) + } + if stored.GoodReadsBookURL != "/book/show/1-unlinked" || stored.GoodReadsAuthorURL != "/author/show/1.Jane_Doe" || !stored.GoodReadsEnriched { + t.Errorf("expected the found link to be stored, got %+v", stored) + } + + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + getBody(t, app, "/documents/unlinked/goodreads") + if len(source.queries) != 1 { + t.Errorf("expected the stored link to be reused without searching again, got %q", source.queries) + } +} + +func TestDocumentDetailGoodReadsSkipsUnmatchedDocuments(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{Rating: "3.82"}} + app, _ := newGoodReadsTestApp(t, source) + if got := goodReadsPlaceholders(t, getBody(t, app, "/documents/unmatched")).Length(); got != 0 { + t.Errorf("expected no placeholder for a document already looked up without match, got %d", got) + } + if body := strings.TrimSpace(getBody(t, app, "/documents/unmatched/goodreads")); body != "" { + t.Errorf("expected empty response, got %q", body) + } + if len(source.queries) != 0 || len(source.ids) != 0 { + t.Errorf("expected no calls, got queries %q and ids %q", source.queries, source.ids) + } +} + +func TestDocumentDetailGoodReadsRatingRendersNothingWhenUnavailable(t *testing.T) { + for name, source := range map[string]*fakeBookSource{ + "error": {err: errors.New("unavailable")}, + "no rating": {}, + } { + t.Run(name, func(t *testing.T) { + app, _ := newGoodReadsTestApp(t, source) + if body := strings.TrimSpace(getBody(t, app, "/documents/linked/goodreads")); body != "" { + t.Errorf("expected empty response, got %q", body) + } + }) + } +} + +func TestDocumentDetailGoodReadsDisabled(t *testing.T) { + app, _ := newGoodReadsTestApp(t, nil) + for _, slug := range []string{"linked", "unlinked"} { + if got := goodReadsPlaceholders(t, getBody(t, app, "/documents/"+slug)).Length(); got != 0 { + t.Errorf("%s: expected no placeholder when GoodReads is disabled, got %d", slug, got) + } + if body := strings.TrimSpace(getBody(t, app, "/documents/"+slug+"/goodreads")); body != "" { + t.Errorf("%s: expected empty response, got %q", slug, body) + } + } +} diff --git a/internal/webserver/embedded/js/reader-reading-mode.js b/internal/webserver/embedded/js/reader-reading-mode.js new file mode 100644 index 00000000..376b5f98 --- /dev/null +++ b/internal/webserver/embedded/js/reader-reading-mode.js @@ -0,0 +1,33 @@ +export function bindReaderReadingMode(view) { + const document = view.ownerDocument + const content = document.createElement('main') + content.id = 'reader-reading-content' + content.hidden = true + document.body.append(content) + let previousDocuments = [] + + view.addEventListener('relocate', () => { + const documents = view.renderer.getContents().map(({ doc }) => doc).filter(Boolean) + if (documents.length === previousDocuments.length && + documents.every((doc, index) => doc === previousDocuments[index])) return + + const paragraphs = [] + for (const doc of documents) { + // Use rendered text, not EPUB markup, scripts, hidden notes, or reader overlays. + const text = doc.body?.innerText ?? + [...doc.querySelectorAll('text')].map(node => node.textContent).join('\n') + for (const line of text.split(/\n+/)) { + if (!line.trim()) continue + const paragraph = document.createElement('p') + paragraph.textContent = line.trim() + paragraph.lang = doc.documentElement.lang + paragraph.dir = doc.documentElement.dir + paragraphs.push(paragraph) + } + } + content.replaceChildren(...paragraphs) + content.hidden = paragraphs.length === 0 + previousDocuments = documents + }) + return content +} diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index de06e22e..6d01b874 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -404,3 +404,5 @@ "A newer version of this document is available.": "Eine neuere Version dieses Dokuments ist verfügbar." "This document is no longer available. Its offline copy has been removed.": "Dieses Dokument ist nicht mehr verfügbar. Seine Offline-Kopie wurde entfernt." "Reload document": "Dokument neu laden" +"Goodreads rating": "Goodreads-Bewertung" +"%s ratings": "%s Bewertungen" diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index a14d6f25..04a2e6c6 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -400,3 +400,5 @@ "A newer version of this document is available.": "Hay una versión más reciente de este documento disponible." "This document is no longer available. Its offline copy has been removed.": "Este documento ya no está disponible. Se ha eliminado su copia sin conexión." "Reload document": "Recargar documento" +"Goodreads rating": "Puntuación en Goodreads" +"%s ratings": "%s valoraciones" diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index 71859694..55442a89 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -402,3 +402,5 @@ "A newer version of this document is available.": "Une version plus récente de ce document est disponible." "This document is no longer available. Its offline copy has been removed.": "Ce document n'est plus disponible. Sa copie hors connexion a été supprimée." "Reload document": "Recharger le document" +"Goodreads rating": "Note Goodreads" +"%s ratings": "%s notes" diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index ae8fbbd6..506656f7 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -396,3 +396,5 @@ "A newer version of this document is available.": "Доступна новая версия этого документа." "This document is no longer available. Its offline copy has been removed.": "Этот документ больше недоступен. Его копия для чтения без подключения к сети удалена." "Reload document": "Перезагрузить документ" +"Goodreads rating": "Рейтинг Goodreads" +"%s ratings": "Оценок: %s" diff --git a/internal/webserver/embedded/views/document/detail.html b/internal/webserver/embedded/views/document/detail.html index a309e652..7bba26ee 100644 --- a/internal/webserver/embedded/views/document/detail.html +++ b/internal/webserver/embedded/views/document/detail.html @@ -32,7 +32,7 @@

{{template "partials/actions" dict "Lang" .Lang "Document" .Document "Session" .Session "FQDN" .fqdn "EmailSendingConfigured" .EmailSendingConfigured "DefaultAction" .DefaultAction "CanShare" .CanShare "PreferredEpub" .PreferredEpub "EmailFrom" .EmailFrom "ShareMaxRecipients" .ShareMaxRecipients "ShareCommentMaxSize" .ShareCommentMaxSize "ButtonSize" "btn-lg" "ButtonStyle" "btn-primary"}}
- {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount}} + {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount "GoodReadsEnabled" .GoodReadsEnabled}}
diff --git a/internal/webserver/embedded/views/partials/document-goodreads.html b/internal/webserver/embedded/views/partials/document-goodreads.html new file mode 100644 index 00000000..d049d28b --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-goodreads.html @@ -0,0 +1 @@ +
  • {{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}
    • diff --git a/internal/webserver/embedded/views/partials/document-metadata.html b/internal/webserver/embedded/views/partials/document-metadata.html index 8d32a311..4477c833 100644 --- a/internal/webserver/embedded/views/partials/document-metadata.html +++ b/internal/webserver/embedded/views/partials/document-metadata.html @@ -12,6 +12,9 @@ {{ if .Document.Pages }}
  • {{.Document.Pages}}
    • {{ end }} + {{ if and .GoodReadsEnabled (or .Document.GoodReadsBookURL (not .Document.GoodReadsEnriched)) }} +
    • + {{ end }} {{ if and .IllustratedMinAmount (ge .Document.Illustrations .IllustratedMinAmount) }}
  • {{t .Lang "Illustrated"}}
    • {{ end }} diff --git a/internal/webserver/js-tests/reader-reading-mode.test.mjs b/internal/webserver/js-tests/reader-reading-mode.test.mjs new file mode 100644 index 00000000..308c7816 --- /dev/null +++ b/internal/webserver/js-tests/reader-reading-mode.test.mjs @@ -0,0 +1,208 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' + +const source = await readFile(new URL('../embedded/js/reader.js', import.meta.url), 'utf8') +const template = await readFile(new URL('../embedded/views/document/reader.html', import.meta.url), 'utf8') +const css = await readFile(new URL('../embedded/css/reader.css', import.meta.url), 'utf8') +const mirrorSource = await readFile(new URL('../embedded/js/reader-reading-mode.js', import.meta.url), 'utf8') +const { bindReaderReadingMode } = await import( + `data:text/javascript;base64,${Buffer.from(mirrorSource).toString('base64')}`) + +// Exercise the reader's actual load handler without bootstrapping its toolbar or network sync. +const onLoad = source.match(/ #onLoad\(\{ detail: \{ doc \} \}\) \{([\s\S]*?)\n \}/)[1] +const openStart = source.match(/ async open\(file\) \{([\s\S]*?) bindReaderWheel/)[1] +const makeReader = new Function('document', 'window', 'bindReaderReadingMode', ` + return new class { + #handleKeydown(event) { this.keys.push(event.key) } + #onLoad({ detail: { doc } }) { ${onLoad} } + keys = [] + async open(file) { ${openStart} } + }() +`) + +function chapter(text = 'Chapter text.\n\nAnother paragraph.') { + const doc = new EventTarget() + doc.body = { innerText: text } + doc.documentElement = { lang: 'en', dir: 'ltr' } + return { doc } +} + +function setup() { + const view = new EventTarget() + const attributes = new Map() + const chapters = [chapter(), chapter('Next chapter.')] + let active = [chapters[0]] + const relocate = () => view.dispatchEvent(new Event('relocate')) + const load = index => { + active = [chapters[index]] + view.dispatchEvent(Object.assign(new Event('load'), { + detail: { doc: chapters[index].doc, index }, + })) + relocate() + } + view.renderer = { getContents: () => active } + view.setAttribute = (name, value) => attributes.set(name, value) + view.open = async () => load(0) + const appended = [] + const document = { + createElement: tag => { + if (tag === 'foliate-view') return view + return { + tagName: tag.toUpperCase(), + children: [], + replaceChildren(...children) { this.children = children }, + } + }, + body: { append: element => appended.push(element) }, + getElementById: () => ({ value: 'book' }), + } + view.ownerDocument = document + return { + reader: makeReader(document, { localStorage: {} }, bindReaderReadingMode), + view, attributes, chapters, appended, load, relocate, + setContents: contents => { active = contents }, + } +} + +test('initial chapter text is exposed in the outer main document before initialization', async () => { + const { reader, view, attributes, appended } = setup() + await reader.open({}) + assert.equal(appended[0], view) + const main = appended[1] + assert.equal(main.tagName, 'MAIN') + assert.equal(main.hidden, false) + assert.deepEqual(main.children.map(p => p.textContent), ['Chapter text.', 'Another paragraph.']) + assert.equal(main.children[0].lang, 'en') + assert.equal(attributes.get('tabindex'), '0') +}) + +test('navigation replaces the chapter copy without accumulating old text or changing keyboard behavior', async () => { + const { reader, chapters, load, appended, relocate } = setup() + await reader.open({}) + const originalParagraphs = appended[1].children + relocate() + assert.equal(appended[1].children, originalParagraphs) + load(1) + assert.deepEqual(appended[1].children.map(p => p.textContent), ['Next chapter.']) + for (const { doc } of chapters) { + doc.dispatchEvent(Object.assign(new Event('keydown'), { key: 'ArrowRight' })) + } + assert.deepEqual(reader.keys, ['ArrowRight', 'ArrowRight']) +}) + +test('SVG fixed-layout documents expose only text, and empty pages clear the copy', async () => { + const { reader, chapters, appended, setContents, relocate } = setup() + const { doc } = chapters[0] + doc.body = null + doc.querySelectorAll = selector => { + assert.equal(selector, 'text') + return [{ textContent: 'SVG page text.' }] + } + await reader.open({}) + assert.deepEqual(appended[1].children.map(p => p.textContent), ['SVG page text.']) + setContents([chapter(' \n ')]) + relocate() + assert.equal(appended[1].hidden, true) + assert.deepEqual(appended[1].children, []) +}) + +test('spreads copy both documents in order and treat book text as text rather than markup', async () => { + const { reader, appended, setContents, relocate } = setup() + await reader.open({}) + setContents([chapter(''), chapter('Second page.')]) + relocate() + assert.deepEqual(appended[1].children.map(p => p.textContent), + ['', 'Second page.']) +}) + +test('chapter mirror has on-screen layout instead of clipped or one-pixel text bounds', () => { + const styles = css.match(/#reader-reading-content \{([^}]+)\}/)[1] + assert.match(styles, /inset: 48px 0;/) + assert.match(styles, /z-index: -1;/) + assert.doesNotMatch(styles, /clip-path|width: 1px|height: 1px/) + assert.match(css, /foliate-view \{\s*background: Canvas;/) +}) + +test('error icon and return link stay hidden until a document fails to load', () => { + assert.match(template, / diff --git a/internal/webserver/embedded/views/partials/author-summary.html b/internal/webserver/embedded/views/partials/author-summary.html index 820b1bf2..a039fff3 100644 --- a/internal/webserver/embedded/views/partials/author-summary.html +++ b/internal/webserver/embedded/views/partials/author-summary.html @@ -86,12 +86,7 @@

    {{.Author.Name}}

    {{t .Lang "Go to Wikipedia entry"}} {{end}} - {{if .ServiceLink.URL}} -
  • - {{if .ServiceLink.Icon}}{{end}} - {{t .Lang "Go to %s author page" .ServiceLink.Source}} -
    • - {{end}} + {{template "partials/author-service-link" dict "Lang" .Lang "ServiceLink" .ServiceLink}} {{end}} From ac2e4652c2278df5c63ec18a6b83b07ad2a70050 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Sat, 31 Oct 2026 10:34:34 +0100 Subject: [PATCH 10/50] Improve biblioreads request performance, review language dropdown --- internal/datasource/biblioreads/client.go | 123 +++--- .../datasource/biblioreads/client_test.go | 128 ++++--- internal/datasource/biblioreads/model.go | 2 +- .../datasource/biblioreads/throttle_test.go | 80 ++++ internal/index/bleve.go | 28 +- internal/index/bleve_document_read.go | 18 + internal/index/bleve_goodreads_write.go | 174 +++++++-- internal/index/bleve_goodreads_write_test.go | 89 ++++- internal/index/document.go | 26 +- internal/index/goodreads_concurrency_test.go | 65 ++++ internal/index/goodreads_fields_test.go | 16 +- internal/reviews/biblioreads.go | 19 + internal/reviews/model.go | 23 +- .../webserver/controller/document/detail.go | 10 + .../webserver/controller/document/reviews.go | 37 +- .../webserver/controller/document/score.go | 17 +- internal/webserver/controller/search/parse.go | 30 +- internal/webserver/document_goodreads_test.go | 361 +++++++++++++++++- internal/webserver/embedded/css/display.css | 25 ++ .../webserver/embedded/translations/de.yml | 3 + .../webserver/embedded/translations/es.yml | 3 + .../webserver/embedded/translations/fr.yml | 3 + .../webserver/embedded/translations/ru.yml | 3 + .../embedded/views/document/detail.html | 6 +- .../views/partials/docs-list-item.html | 2 +- .../partials/document-metadata-filters.html | 26 ++ .../views/partials/document-metadata.html | 3 + .../partials/document-reviews-header.html | 14 + .../views/partials/document-reviews.html | 30 +- .../views/partials/document-score.html | 10 +- .../webserver/model/highlight_repository.go | 10 + .../webserver/model/highlight_search_test.go | 32 ++ internal/webserver/search_test.go | 34 ++ internal/webserver/service_links_test.go | 4 +- 34 files changed, 1232 insertions(+), 222 deletions(-) create mode 100644 internal/datasource/biblioreads/throttle_test.go create mode 100644 internal/index/goodreads_concurrency_test.go create mode 100644 internal/webserver/embedded/views/partials/document-reviews-header.html create mode 100644 internal/webserver/model/highlight_search_test.go diff --git a/internal/datasource/biblioreads/client.go b/internal/datasource/biblioreads/client.go index 3b4c3e8a..03443653 100644 --- a/internal/datasource/biblioreads/client.go +++ b/internal/datasource/biblioreads/client.go @@ -11,6 +11,7 @@ import ( "math/rand/v2" "net/http" "net/url" + "sort" "strings" "sync" "time" @@ -48,18 +49,29 @@ type Client struct { httpClient *http.Client cache *responseCache - mu sync.Mutex - cooldownUntil map[string]time.Time + mu sync.Mutex + cooldownUntil map[string]time.Time + nextRequest map[string]time.Time + requestInterval time.Duration } type options struct { - instances []string - httpClient *http.Client - cacheTTL time.Duration + instances []string + httpClient *http.Client + cacheTTL time.Duration + requestInterval time.Duration } type Option func(*options) +// WithRequestInterval sets the minimum spacing between request starts to each instance. +// Zero disables spacing; the default is one second. +func WithRequestInterval(interval time.Duration) Option { + return func(o *options) { + o.requestInterval = max(interval, 0) + } +} + // WithInstances sets the BiblioReads instances to use. An empty list keeps DefaultInstances. func WithInstances(instances []string) Option { return func(o *options) { @@ -86,16 +98,18 @@ func WithCacheTTL(ttl time.Duration) Option { } } -// NewClient returns a client that spreads requests across instances. Each request starts at a -// random instance and fails over to the next ones if it is unavailable (network error, 429 or a -// non-JSON response). Unavailable instances are skipped until their Retry-After time, or +// NewClient returns a client that spreads requests across instances, preferring +// instances ready to accept a request and randomizing ties. Requests to each +// instance are spaced at least one second apart by default, including failover +// attempts. Unavailable instances are skipped until their Retry-After time, or // httpjson.DefaultRetryAfter, has passed. Successful responses are cached in memory for // DefaultCacheTTL unless changed with WithCacheTTL. func NewClient(opts ...Option) *Client { o := options{ - instances: DefaultInstances, - httpClient: &http.Client{Timeout: 15 * time.Second}, - cacheTTL: DefaultCacheTTL, + instances: DefaultInstances, + httpClient: &http.Client{Timeout: 15 * time.Second}, + cacheTTL: DefaultCacheTTL, + requestInterval: time.Second, } for _, opt := range opts { opt(&o) @@ -107,9 +121,11 @@ func NewClient(opts ...Option) *Client { } c := &Client{ - instances: normalized, - httpClient: o.httpClient, - cooldownUntil: make(map[string]time.Time), + instances: normalized, + httpClient: o.httpClient, + cooldownUntil: make(map[string]time.Time), + nextRequest: make(map[string]time.Time), + requestInterval: o.requestInterval, } if o.cacheTTL > 0 { c.cache = newResponseCache(o.cacheTTL) @@ -205,7 +221,17 @@ func (c *Client) post(ctx context.Context, path string, payload, dest any) error var errs []error var incompleteResponse []byte - for _, instance := range c.candidates() { + tried := make(map[string]bool) + for len(tried) < len(c.instances) { + instance, err := c.acquireInstance(ctx, tried) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + errs = append(errs, err) + break + } + tried[instance] = true endpoint := instance + path respBody, err := c.postToInstance(ctx, endpoint, body) if err == nil { @@ -226,7 +252,6 @@ func (c *Client) post(ctx context.Context, path string, payload, dest any) error } } if err == nil { - c.clearCooldown(instance) reviews, isReviews := dest.(*ReviewsResponse) if c.cache != nil && (!isReviews || len(reviews.Reviews) > 0) { c.cache.set(cacheKey, respBody) @@ -293,32 +318,46 @@ func supportsReviewLanguage(version string) bool { return semver.IsValid(version) && semver.Compare(version, "v4.4.0") >= 0 } -// candidates returns the instances that are not cooling down, starting at a random one. -// If all of them are cooling down, the one whose cooldown ends first is returned, so -// requests don't fail without trying when every instance had a transient error. -func (c *Client) candidates() []string { - c.mu.Lock() - defer c.mu.Unlock() - - now := time.Now() - start := rand.IntN(len(c.instances)) - candidates := make([]string, 0, len(c.instances)) - soonest := "" - for i := range c.instances { - instance := c.instances[(start+i)%len(c.instances)] - until, coolingDown := c.cooldownUntil[instance] - if !coolingDown || !now.Before(until) { - candidates = append(candidates, instance) - continue +// acquireInstance rechecks availability after waiting, so cancellation does not +// reserve a future slot and concurrent requests cannot take the same slot. +func (c *Client) acquireInstance(ctx context.Context, tried map[string]bool) (string, error) { + for { + if err := ctx.Err(); err != nil { + return "", err } - if soonest == "" || until.Before(c.cooldownUntil[soonest]) { - soonest = instance + c.mu.Lock() + start := rand.IntN(len(c.instances)) + candidates := make([]string, 0, len(c.instances)) + now := time.Now() + for i := range c.instances { + instance := c.instances[(start+i)%len(c.instances)] + if !tried[instance] && !now.Before(c.cooldownUntil[instance]) { + candidates = append(candidates, instance) + } + } + sort.SliceStable(candidates, func(i, j int) bool { + return c.nextRequest[candidates[i]].Before(c.nextRequest[candidates[j]]) + }) + if len(candidates) == 0 { + c.mu.Unlock() + return "", fmt.Errorf("%w: no untried instance is outside its cooldown", ErrUnavailable) + } + instance := candidates[0] + wait := time.Until(c.nextRequest[instance]) + if wait <= 0 { + c.nextRequest[instance] = time.Now().Add(c.requestInterval) + c.mu.Unlock() + return instance, nil + } + c.mu.Unlock() + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return "", ctx.Err() + case <-timer.C: } } - if len(candidates) == 0 { - return []string{soonest} - } - return candidates } func (c *Client) setCooldown(instance string, d time.Duration) { @@ -327,12 +366,6 @@ func (c *Client) setCooldown(instance string, d time.Duration) { c.cooldownUntil[instance] = time.Now().Add(d) } -func (c *Client) clearCooldown(instance string) { - c.mu.Lock() - defer c.mu.Unlock() - delete(c.cooldownUntil, instance) -} - // postToInstance returns the body of a successful response. func (c *Client) postToInstance(ctx context.Context, endpoint string, body []byte) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) diff --git a/internal/datasource/biblioreads/client_test.go b/internal/datasource/biblioreads/client_test.go index 45d474e6..d46e97ad 100644 --- a/internal/datasource/biblioreads/client_test.go +++ b/internal/datasource/biblioreads/client_test.go @@ -25,6 +25,10 @@ type capturedRequest struct { const validBookResponse = `{"status":"Received","statusCode":200,"rating":"3.92","resourceID":"work-id","legacyBookID":5898652}` +func newTestClient(opts ...biblioreads.Option) *biblioreads.Client { + return biblioreads.NewClient(append([]biblioreads.Option{biblioreads.WithRequestInterval(0)}, opts...)...) +} + func newMockServer(t *testing.T, status int, fixture string, captured *capturedRequest) *httptest.Server { t.Helper() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -73,7 +77,7 @@ func assertRequest(t *testing.T, captured capturedRequest, path string, payload func TestSearchBooks(t *testing.T) { var captured capturedRequest server := newMockServer(t, http.StatusOK, "search-books.json", &captured) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) res, err := client.SearchBooks(context.Background(), "La Reina Dragón Alice Borchardt") if err != nil { @@ -126,7 +130,7 @@ func TestBookResponseRecovery(t *testing.T) { defer first.Close() second := httptest.NewServer(handler) defer second.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL})) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) for range 2 { book, err := client.Book(context.Background(), "5898652-la-cinta-roja") if err != nil || book.Rating != "3.92" || book.ResourceID != "work-id" || book.LegacyBookID != 5898652 { @@ -146,7 +150,7 @@ func TestBookResponseRecovery(t *testing.T) { func TestIncompleteBookResponsesAreNotCached(t *testing.T) { var hits atomic.Int32 server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92"}`, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) for range 2 { book, err := client.Book(context.Background(), "5898652") if err != nil || book.Rating != "3.92" { @@ -175,7 +179,7 @@ func TestIncompleteBookFallbackPrefersScore(t *testing.T) { defer first.Close() second := httptest.NewServer(handler) defer second.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL})) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) for range 2 { book, err := client.Book(context.Background(), "5898652") if err != nil || book.Rating != "3.92" || book.Title != "Scored result" { @@ -195,7 +199,7 @@ func TestIncompleteBookFallbackPrefersScore(t *testing.T) { func TestBook(t *testing.T) { var captured capturedRequest server := newMockServer(t, http.StatusOK, "book-scraper.json", &captured) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) res, err := client.Book(context.Background(), "12207444-la-reina-dragon") if err != nil { @@ -242,7 +246,7 @@ func TestAuthor(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var captured capturedRequest server := newMockServer(t, http.StatusOK, "author-info.json", &captured) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) res, err := client.Author(context.Background(), tt.id) if err != nil { @@ -286,18 +290,24 @@ func TestAuthorErrors(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var hits atomic.Int32 server := newCountingServer(t, tt.status, tt.body, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) for range 2 { - _, err := client.Author(context.Background(), "151155") + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + _, err := client.Author(ctx, "151155") + cancel() if err == nil { t.Fatal("expected error, got nil") } - if errors.Is(err, biblioreads.ErrUnavailable) != tt.unavailable { + if errors.Is(err, biblioreads.ErrUnavailable) != tt.unavailable && !errors.Is(err, context.DeadlineExceeded) { t.Errorf("expected unavailable=%v, got %v", tt.unavailable, err) } } - if hits.Load() != 2 { + wantHits := int32(2) + if tt.unavailable { + wantHits = 1 + } + if hits.Load() != wantHits { t.Errorf("expected errors not to be cached, got %d requests", hits.Load()) } }) @@ -315,14 +325,17 @@ func TestReviewsInvalidResponses(t *testing.T) { t.Run(body, func(t *testing.T) { var hits atomic.Int32 server := newCountingServer(t, http.StatusOK, body, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) for range 2 { - if _, err := client.Reviews(context.Background(), "work", 1, "", ""); !errors.Is(err, biblioreads.ErrUnavailable) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + _, err := client.Reviews(ctx, "work", 1, "", "") + cancel() + if !errors.Is(err, biblioreads.ErrUnavailable) && !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("expected unavailable error, got %v", err) } } - if hits.Load() != 2 { - t.Fatal("invalid responses must not be cached") + if hits.Load() != 1 { + t.Fatal("invalid responses must trigger cooldown") } }) } @@ -345,7 +358,7 @@ func TestReviewsFailover(t *testing.T) { defer first.Close() second := httptest.NewServer(handler) defer second.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL})) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) response, err := client.Reviews(context.Background(), "work", 1, "", "") if err != nil || len(response.Reviews) == 0 || hits.Load() != 2 { t.Fatalf("expected failover to working instance, got %d requests, %+v, %v", hits.Load(), response, err) @@ -369,7 +382,7 @@ func TestReviewsEmptyResponseRecovery(t *testing.T) { defer first.Close() second := httptest.NewServer(handler) defer second.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL})) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) for range 2 { reviews, err := client.Reviews(context.Background(), "work", 1, "", "") if err != nil || len(reviews.Reviews) == 0 { @@ -384,7 +397,7 @@ func TestReviewsEmptyResponseRecovery(t *testing.T) { func TestReviewsEmptyResponsesAreNotCached(t *testing.T) { var hits atomic.Int32 server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"totalCount":0,"reviews":[]}`, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) for range 2 { response, err := client.Reviews(context.Background(), "work", 1, "", "") if err != nil || len(response.Reviews) != 0 { @@ -409,7 +422,7 @@ func TestReviewsEmptyFallbackSurvivesInvalidResponse(t *testing.T) { defer first.Close() second := httptest.NewServer(handler) defer second.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL})) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) response, err := client.Reviews(context.Background(), "work", 1, "", "es") if err != nil || len(response.Reviews) != 0 || response.NextPageToken != "empty-page" { t.Fatalf("expected original empty fallback without fields from the invalid response, got %+v, %v", response, err) @@ -443,7 +456,7 @@ func TestReviewsLanguageVersionCompatibility(t *testing.T) { } })) defer server.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) for range 2 { _, err := client.Reviews(context.Background(), "work", 1, "", "es") if tt.supported && err != nil { @@ -453,10 +466,7 @@ func TestReviewsLanguageVersionCompatibility(t *testing.T) { t.Fatalf("expected unsupported instance to fail, got %v", err) } } - wantHits := int32(2) - if tt.supported { - wantHits = 1 - } + wantHits := int32(1) if hits.Load() != wantHits { t.Fatalf("expected %d requests, got %d", wantHits, hits.Load()) } @@ -483,7 +493,7 @@ func TestReviewsLanguageFailsOverFromLegacyInstance(t *testing.T) { defer first.Close() second := httptest.NewServer(handler) defer second.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL})) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) for range 2 { response, err := client.Reviews(context.Background(), "work", 1, "", "es") if err != nil { @@ -521,7 +531,7 @@ func TestReviewsLanguageCache(t *testing.T) { }) })) defer server.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) for _, language := range []string{"es", "en", "es", "en"} { response, err := client.Reviews(context.Background(), "work", 1, "", language) if err != nil { @@ -537,10 +547,29 @@ func TestReviewsLanguageCache(t *testing.T) { } func TestReviews(t *testing.T) { + t.Run("fractional ratings", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"totalCount":4,"reviews":[{"rating":0.25},{"rating":3.5},{"rating":4.75},{"rating":5}]}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + response, err := client.Reviews(context.Background(), "work", 1, "", "") + if err != nil { + t.Fatal(err) + } + want := []float64{0.25, 3.5, 4.75, 5} + if len(response.Reviews) != len(want) { + t.Fatalf("expected %d reviews, got %d", len(want), len(response.Reviews)) + } + for i, rating := range want { + if response.Reviews[i].Rating != rating { + t.Errorf("review %d: expected rating %g, got %g", i, rating, response.Reviews[i].Rating) + } + } + }) + t.Run("first page", func(t *testing.T) { var captured capturedRequest server := newMockServer(t, http.StatusOK, "reviews-scraper.json", &captured) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) res, err := client.Reviews(context.Background(), "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", 12207444, "", "") if err != nil { @@ -581,7 +610,7 @@ func TestReviews(t *testing.T) { _, _ = w.Write([]byte(`{"version":"BiblioReads v4.4.0","status":"Received","statusCode":200,"totalCount":1,"reviews":[{"text":"Una reseña"}]}`)) })) defer server.Close() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "token", "es"); err != nil { t.Fatalf("unexpected error: %v", err) @@ -599,7 +628,7 @@ func TestReviews(t *testing.T) { func TestErrorStatus(t *testing.T) { var captured capturedRequest server := newMockServer(t, http.StatusNotFound, "", &captured) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) if _, err := client.Book(context.Background(), "0"); err == nil { t.Error("expected error on non-200 response, got nil") @@ -629,7 +658,7 @@ func TestFailover(t *testing.T) { instances := []string{down.URL, unavailable.URL, rateLimited.URL, html.URL, working.URL} for range 20 { - client := biblioreads.NewClient(biblioreads.WithInstances(instances), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances(instances), biblioreads.WithCacheTTL(0)) res, err := client.Book(context.Background(), "12207444") if err != nil { t.Fatalf("unexpected error: %v", err) @@ -648,7 +677,7 @@ func TestFailover(t *testing.T) { first := newCountingServer(t, http.StatusOK, validBookResponse, &firstHits) second := newCountingServer(t, http.StatusOK, validBookResponse, &secondHits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) for range 100 { if _, err := client.Book(context.Background(), "1"); err != nil { t.Fatalf("unexpected error: %v", err) @@ -664,7 +693,7 @@ func TestFailover(t *testing.T) { first := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) second := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) _, err := client.Book(context.Background(), "0") if err == nil { t.Fatal("expected error, got nil") @@ -682,7 +711,7 @@ func TestFailover(t *testing.T) { first := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) second := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } @@ -698,7 +727,7 @@ func TestFailover(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) if _, err := client.Book(ctx, "1"); err == nil { t.Fatal("expected error, got nil") } @@ -714,7 +743,7 @@ func TestCooldown(t *testing.T) { bad := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &badHits) good := newCountingServer(t, http.StatusOK, validBookResponse, &goodHits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{bad.URL, good.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{bad.URL, good.URL}), biblioreads.WithCacheTTL(0)) for range 20 { if _, err := client.Book(context.Background(), "1"); err != nil { t.Fatalf("unexpected error: %v", err) @@ -741,7 +770,7 @@ func TestCooldown(t *testing.T) { t.Cleanup(limited.Close) other := newCountingServer(t, http.StatusOK, validBookResponse, &otherHits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{limited.URL, other.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{limited.URL, other.URL}), biblioreads.WithCacheTTL(0)) for limitedHits.Load() == 0 { if _, err := client.Book(context.Background(), "1"); err != nil { t.Fatalf("unexpected error: %v", err) @@ -767,21 +796,26 @@ func TestCooldown(t *testing.T) { } }) - t.Run("instance is probed when all are cooling down", func(t *testing.T) { + t.Run("respects cooldown when all instances are cooling down", func(t *testing.T) { var hits atomic.Int32 flaky := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if hits.Add(1) == 1 { - w.WriteHeader(http.StatusServiceUnavailable) + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) return } _, _ = w.Write([]byte(validBookResponse)) })) t.Cleanup(flaky.Close) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{flaky.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{flaky.URL}), biblioreads.WithCacheTTL(0)) if _, err := client.Book(context.Background(), "1"); err == nil { t.Fatal("expected error on first request, got nil") } + if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected unavailable during cooldown, got %v", err) + } + time.Sleep(1100 * time.Millisecond) if _, err := client.Book(context.Background(), "1"); err != nil { t.Fatalf("expected cooling down instance to be probed, got %v", err) } @@ -793,7 +827,7 @@ func TestCooldown(t *testing.T) { first := newCountingServer(t, http.StatusNotFound, notFound, &firstHits) second := newCountingServer(t, http.StatusNotFound, notFound, &secondHits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) for range 20 { if _, err := client.Book(context.Background(), "0"); err == nil { t.Fatal("expected error, got nil") @@ -812,7 +846,7 @@ func TestCooldown(t *testing.T) { first := newCountingServer(t, http.StatusOK, validBookResponse, &firstHits) second := newCountingServer(t, http.StatusOK, validBookResponse, &secondHits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) ctx, cancel := context.WithCancel(context.Background()) cancel() for range 5 { @@ -833,7 +867,7 @@ func TestCache(t *testing.T) { t.Run("successful responses are cached per request", func(t *testing.T) { var hits atomic.Int32 server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92","resourceID":"work-id","legacyBookID":5898652,"title":"La Reina Dragon","genres":["Fantasy"],"totalCount":1,"reviews":[{"id":"review-1"}]}`, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) first, err := client.Book(context.Background(), "1") if err != nil { @@ -877,7 +911,7 @@ func TestCache(t *testing.T) { t.Run("entries expire after the TTL", func(t *testing.T) { var hits atomic.Int32 server := newCountingServer(t, http.StatusOK, validBookResponse, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(100*time.Millisecond)) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(100*time.Millisecond)) for range 3 { if _, err := client.Book(context.Background(), "1"); err != nil { @@ -897,25 +931,25 @@ func TestCache(t *testing.T) { } }) - t.Run("errors are not cached", func(t *testing.T) { + t.Run("errors put the instance into cooldown", func(t *testing.T) { var hits atomic.Int32 server := newCountingServer(t, http.StatusGatewayTimeout, `{"status":"Error - Book not found"}`, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL})) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) for range 3 { if _, err := client.Book(context.Background(), "0"); err == nil { t.Fatal("expected error, got nil") } } - if hits.Load() != 3 { - t.Errorf("expected every failed request to reach the server, got %d requests", hits.Load()) + if hits.Load() != 1 { + t.Errorf("expected subsequent requests to respect cooldown, got %d requests", hits.Load()) } }) t.Run("caching can be disabled", func(t *testing.T) { var hits atomic.Int32 server := newCountingServer(t, http.StatusOK, validBookResponse, &hits) - client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(0)) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(0)) for range 3 { if _, err := client.Book(context.Background(), "1"); err != nil { diff --git a/internal/datasource/biblioreads/model.go b/internal/datasource/biblioreads/model.go index 2b08cc17..bc8ad4bd 100644 --- a/internal/datasource/biblioreads/model.go +++ b/internal/datasource/biblioreads/model.go @@ -122,7 +122,7 @@ type ReviewsResponse struct { type Review struct { ID string `json:"id"` Reviewer Reviewer `json:"reviewer"` - Rating int `json:"rating"` + Rating float64 `json:"rating"` Text string `json:"text"` CreatedAt string `json:"createdAt"` UpdatedAt string `json:"updatedAt"` diff --git a/internal/datasource/biblioreads/throttle_test.go b/internal/datasource/biblioreads/throttle_test.go new file mode 100644 index 00000000..d54bbb7b --- /dev/null +++ b/internal/datasource/biblioreads/throttle_test.go @@ -0,0 +1,80 @@ +package biblioreads_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sort" + "sync" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +func TestRequestsAreThrottledPerInstance(t *testing.T) { + const interval = 200 * time.Millisecond + var mu sync.Mutex + arrivals := make([][]time.Time, 2) + var instances []string + for i := range arrivals { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + arrivals[i] = append(arrivals[i], time.Now()) + mu.Unlock() + _, _ = w.Write([]byte(validBookResponse)) + })) + t.Cleanup(server.Close) + instances = append(instances, server.URL) + } + client := biblioreads.NewClient(biblioreads.WithInstances(instances), + biblioreads.WithCacheTTL(0), biblioreads.WithRequestInterval(interval)) + var wg sync.WaitGroup + for range 6 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Errorf("Book: %v", err) + } + }() + } + wg.Wait() + for i, times := range arrivals { + if len(times) != 3 { + t.Fatalf("instance %d: expected three requests, got %d", i, len(times)) + } + sort.Slice(times, func(i, j int) bool { return times[i].Before(times[j]) }) + for j := 1; j < len(times); j++ { + if gap := times[j].Sub(times[j-1]); gap < interval-20*time.Millisecond { + t.Errorf("instance %d: request gap %s is below %s", i, gap, interval) + } + } + } + gap := arrivals[0][0].Sub(arrivals[1][0]) + if gap.Abs() >= interval/2 { + t.Errorf("independent instances should start concurrently, gap %s", gap) + } +} + +func TestThrottleWaitHonorsCancellation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(validBookResponse)) + })) + defer server.Close() + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), + biblioreads.WithCacheTTL(0), biblioreads.WithRequestInterval(time.Second)) + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := client.Book(ctx, "2"); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected deadline exceeded during throttle wait, got %v", err) + } + if time.Since(start) >= 500*time.Millisecond { + t.Fatal("cancellation did not interrupt the throttle wait") + } +} diff --git a/internal/index/bleve.go b/internal/index/bleve.go index 47f40e87..fd3b1be0 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -30,7 +30,7 @@ import ( // DocumentVersion identifies the mapping used for indexing documents. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. -const DocumentVersion = "v15" +const DocumentVersion = "v14" // AuthorVersion identifies the mapping used for indexing authors. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. @@ -185,9 +185,6 @@ type Config struct { // GoodReadsSource, if set, is used to fill each document's GoodReads URLs // in the background (see EnrichGoodReads). Nil disables it. GoodReadsSource GoodReadsSource - // GoodReadsEnrichInterval is the minimum time between two GoodReads - // lookups; <= 0 falls back to DefaultGoodReadsEnrichInterval. - GoodReadsEnrichInterval time.Duration } type BleveIndexer struct { @@ -223,11 +220,9 @@ type BleveIndexer struct { textRankEnrichJobs chan Document // goodReadsEnrichJobs feeds IDs of documents queued by indexFile to // runGoodReadsEnrichWorker; only used when goodReadsSource is set. - goodReadsEnrichJobs chan string - goodReadsSource GoodReadsSource - goodReadsInterval time.Duration - goodReadsThrottleMu sync.Mutex - goodReadsLastRequest time.Time + goodReadsEnrichJobs chan string + goodReadsSource GoodReadsSource + goodReadsSlots chan struct{} // fileLocks holds a *sync.Mutex per document ID, serializing concurrent // indexFile calls for the same file (upload vs. file watcher reacting to // the same write). @@ -333,19 +328,18 @@ func NewBleve(documentsIndex bleve.Index, authorsIndex bleve.Index, fs afero.Fs, pruneChangeTriggerRatio: pruneChangeTriggerRatio, textRankEnrichJobs: make(chan Document, textRankEnrichWorkers), goodReadsSource: cfg.GoodReadsSource, - goodReadsInterval: cfg.GoodReadsEnrichInterval, + goodReadsSlots: make(chan struct{}, goodReadsEnrichWorkers), } for range textRankEnrichWorkers { go b.runTextRankEnrichWorker() } - if b.goodReadsInterval <= 0 { - b.goodReadsInterval = DefaultGoodReadsEnrichInterval - } if b.goodReadsSource != nil { - b.goodReadsEnrichJobs = make(chan string, 1) - go b.runGoodReadsEnrichWorker() + b.goodReadsEnrichJobs = make(chan string, goodReadsEnrichWorkers) + for range goodReadsEnrichWorkers { + go b.runGoodReadsEnrichWorker() + } } return b @@ -471,6 +465,8 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.TypeMapping[lang].AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsRating", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsRatingValue", numericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) @@ -499,6 +495,8 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsRating", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsRatingValue", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) diff --git a/internal/index/bleve_document_read.go b/internal/index/bleve_document_read.go index 1576ea47..afebb639 100644 --- a/internal/index/bleve_document_read.go +++ b/internal/index/bleve_document_read.go @@ -226,6 +226,20 @@ func (b *BleveIndexer) addFilters(searchFields SearchFields, filtersQuery *query bq.SetBoost(0) filtersQuery.AddQuery(bq) } + if searchFields.GoodReadsRatingFrom > 0 || searchFields.GoodReadsRatingTo > 0 { + min := math.SmallestNonzeroFloat64 + var max *float64 + if searchFields.GoodReadsRatingFrom > 0 { + min = searchFields.GoodReadsRatingFrom + } + if searchFields.GoodReadsRatingTo > 0 { + max = &searchFields.GoodReadsRatingTo + } + ratingQuery := newInclusiveNumericRangeQuery(&min, max) + ratingQuery.SetField("GoodReadsRatingValue") + ratingQuery.SetBoost(0) + filtersQuery.AddQuery(ratingQuery) + } if searchFields.IllustratedOnly && b.illustratedMinAmount > 0 { minIllustrations := float64(b.illustratedMinAmount) q := bleve.NewNumericRangeQuery(&minIllustrations, nil) @@ -1024,6 +1038,8 @@ func hydrateDocument(match *search.DocumentMatch) Document { } goodReadsBookURL, _ := match.Fields["GoodReadsBookURL"].(string) + goodReadsRating, _ := match.Fields["GoodReadsRating"].(string) + goodReadsRatingValue, _ := match.Fields["GoodReadsRatingValue"].(float64) goodReadsResourceID, _ := match.Fields["GoodReadsResourceID"].(string) goodReadsLegacyBookID, _ := match.Fields["GoodReadsLegacyBookID"].(float64) goodReadsEnriched, _ := match.Fields["GoodReadsEnriched"].(bool) @@ -1055,6 +1071,8 @@ func hydrateDocument(match *search.DocumentMatch) Document { TextRankWords: textRankWords, TextRankEnriched: textRankEnriched, GoodReadsBookURL: goodReadsBookURL, + GoodReadsRating: goodReadsRating, + GoodReadsRatingValue: goodReadsRatingValue, GoodReadsResourceID: goodReadsResourceID, GoodReadsLegacyBookID: int(goodReadsLegacyBookID), GoodReadsEnriched: goodReadsEnriched, diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go index e5a919ac..5d611179 100644 --- a/internal/index/bleve_goodreads_write.go +++ b/internal/index/bleve_goodreads_write.go @@ -5,8 +5,11 @@ import ( "errors" "fmt" "log" + "math" "regexp" + "strconv" "strings" + "sync" "time" "unicode" @@ -17,9 +20,8 @@ import ( ) const ( - // DefaultGoodReadsEnrichInterval is the minimum time between two GoodReads lookups. - DefaultGoodReadsEnrichInterval = time.Second - goodReadsRequestTimeout = 2 * time.Minute + goodReadsEnrichWorkers = 4 + goodReadsRequestTimeout = 2 * time.Minute ) // GoodReadsSource searches books on Goodreads; implemented by *biblioreads.Client. @@ -27,13 +29,15 @@ type GoodReadsSource interface { SearchBooks(ctx context.Context, query string) (biblioreads.SearchBooksResponse, error) } -var goodReadsFields = []string{"GoodReadsBookURL", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} +var goodReadsFields = []string{"GoodReadsBookURL", "GoodReadsRating", "GoodReadsRatingValue", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} // Strip named series annotations and omnibus ranges, not standalone volume numbers. var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) var goodReadsSubtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+`) +var goodReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) + func normalizeGoodReadsName(value string) string { var result strings.Builder for _, r := range norm.NFD.String(strings.ToLower(value)) { @@ -121,6 +125,7 @@ func matchingGoodReadsBook(document Document, results []biblioreads.SearchResult } var best biblioreads.SearchResult bestScore := 0 + bestHasRating := false for _, result := range results { if result.BookURL == "" { continue @@ -132,20 +137,39 @@ func matchingGoodReadsBook(document Document, results []biblioreads.SearchResult if normalizeGoodReadsName(result.Title) == fullTitle { score = 3 } - } else if resultTitle == "" || (goodReadsMainTitle(result.Title) != title && mainTitle != resultTitle) { + } else if resultTitle == "" || (goodReadsMainTitle(result.Title) != title && mainTitle != resultTitle && + goodReadsTitle(goodReadsParentheticalTitle.ReplaceAllString(result.Title, "")) != title) { continue } author := strings.ReplaceAll(normalizeGoodReadsName(result.Author), " ", "") if author == "" { continue } - if _, matches := authors[author]; matches && score > bestScore { - best, bestScore = result, score + if _, matches := authors[author]; matches { + hasRating := hasGoodReadsRating(result.Rating) + prefer := bestScore == 0 + if !prefer && score == bestScore && result.ID > 0 && best.ID > 0 && result.ID != best.ID { + prefer = result.ID < best.ID + } else if !prefer { + if hasRating != bestHasRating { + prefer = hasRating + } else { + prefer = score > bestScore + } + } + if prefer { + best, bestScore, bestHasRating = result, score, hasRating + } } } return best, bestScore > 0 } +func hasGoodReadsRating(rating string) bool { + value, err := strconv.ParseFloat(rating, 64) + return err == nil && value > 0 && value <= 5 && !math.IsNaN(value) && !math.IsInf(value, 0) +} + // goodReadsQuery returns the Goodreads search string for document: its title // followed by its first author, if any. func goodReadsQuery(document Document) string { @@ -180,7 +204,7 @@ func (b *BleveIndexer) documentsNeedingGoodReads(size int) ([]Document, uint64, // EnrichGoodReads looks up on Goodreads every document not processed yet (see // Document.GoodReadsEnriched), storing the book URL on the document and the -// author URL on the matching author. Lookups are throttled (see Config.GoodReadsEnrichInterval). +// author URL on the matching author. The source handles per-instance throttling. // It stops at the first error that means Goodreads can't be reached right // now, leaving the remaining documents for the next run. func (b *BleveIndexer) EnrichGoodReads(batchSize int) error { @@ -209,10 +233,8 @@ func (b *BleveIndexer) EnrichGoodReads(batchSize int) error { if len(chunk) == 0 { break } - for _, document := range chunk { - if _, err := b.enrichGoodReads(context.Background(), document, true); err != nil { - return err - } + if err := b.enrichGoodReadsBatch(chunk); err != nil { + return err } } @@ -220,14 +242,48 @@ func (b *BleveIndexer) EnrichGoodReads(batchSize int) error { return nil } -// GoodReadsBook returns the stored book URL, or searches a document not looked -// up yet and persists its match. A fresh search also returns the rating, which -// is not stored. Returns an empty result if there's no match. +func (b *BleveIndexer) enrichGoodReadsBatch(documents []Document) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + jobs := make(chan Document, len(documents)) + for _, document := range documents { + jobs <- document + } + close(jobs) + var wg sync.WaitGroup + errs := make(chan error, 1) + for range min(goodReadsEnrichWorkers, len(documents)) { + wg.Add(1) + go func() { + defer wg.Done() + for document := range jobs { + if _, err := b.enrichGoodReads(ctx, document); err != nil { + select { + case errs <- err: + cancel() + default: + } + return + } + } + }() + } + wg.Wait() + select { + case err := <-errs: + return err + default: + return nil + } +} + +// GoodReadsBook returns the stored book URL and rating, or searches a document +// not looked up yet and persists its match. Returns an empty result if there's no match. func (b *BleveIndexer) GoodReadsBook(ctx context.Context, document Document) (biblioreads.SearchResult, error) { if document.GoodReadsBookURL != "" || document.GoodReadsEnriched || b.goodReadsSource == nil { - return biblioreads.SearchResult{BookURL: document.GoodReadsBookURL}, nil + return biblioreads.SearchResult{BookURL: document.GoodReadsBookURL, Rating: document.GoodReadsRating}, nil } - return b.enrichGoodReads(ctx, document, false) + return b.enrichGoodReads(ctx, document) } // StoreGoodReadsBookIDs persists identifiers fetched for bookURL, provided the @@ -259,10 +315,49 @@ func (b *BleveIndexer) StoreGoodReadsBookIDs(document Document, bookURL, resourc return nil } +// StoreGoodReadsRating persists the current rating for the same document and +// Goodreads match that were read before fetching the book details. +func (b *BleveIndexer) StoreGoodReadsRating(document Document, bookURL, rating string) error { + if bookURL == "" { + return fmt.Errorf("invalid GoodReads book URL for document %s", document.ID) + } + if rating == "" { + return nil + } + b.documentsMu.Lock() + defer b.documentsMu.Unlock() + current, found, err := b.documentByIndexIDLocked(document.ID) + if err != nil { + return fmt.Errorf("error reading document %s for GoodReads rating: %w", document.ID, err) + } + if !found { + return ErrDocumentNotFound + } + if goodReadsQuery(current) != goodReadsQuery(document) || current.GoodReadsBookURL != bookURL { + return fmt.Errorf("GoodReads match changed for document %s", document.ID) + } + ratingValue := parseGoodReadsRating(rating, document.ID) + if current.GoodReadsRating == rating && current.GoodReadsRatingValue == ratingValue { + return nil + } + current.GoodReadsRating = rating + current.GoodReadsRatingValue = ratingValue + if err := b.documentsIdx.Index(current.ID, current); err != nil { + return fmt.Errorf("error storing GoodReads rating for document %s: %w", document.ID, err) + } + return nil +} + // enrichGoodReads searches document on Goodreads, unless it already has a // book URL, and stores matching URLs on the document and author respectively. // Unavailability or persistence errors leave the document unenriched for retry. -func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document, throttle bool) (biblioreads.SearchResult, error) { +func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) (biblioreads.SearchResult, error) { + select { + case b.goodReadsSlots <- struct{}{}: + defer func() { <-b.goodReadsSlots }() + case <-ctx.Done(): + return biblioreads.SearchResult{}, ctx.Err() + } q := goodReadsQuery(document) result := biblioreads.SearchResult{BookURL: document.GoodReadsBookURL} if result.BookURL == "" && document.Title != "" { @@ -273,9 +368,6 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document, t queries = append(queries, title) } for _, query := range queries { - if throttle { - b.waitGoodReadsTurn() - } res, err := b.goodReadsSource.SearchBooks(ctx, query) switch { case errors.Is(err, biblioreads.ErrUnavailable), errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): @@ -310,15 +402,18 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document, t // Looked up concurrently (e.g. by the background enrichment and a detail // page request at the same time): keep the stored result. if current.GoodReadsEnriched { - if current.GoodReadsBookURL != result.BookURL { - result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL} - } + result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: current.GoodReadsRating} return result, nil } if current.GoodReadsBookURL != "" && current.GoodReadsBookURL != result.BookURL { - result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL} + result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: current.GoodReadsRating} } current.GoodReadsBookURL = result.BookURL + if result.Rating == "" { + result.Rating = current.GoodReadsRating + } + current.GoodReadsRating = result.Rating + current.GoodReadsRatingValue = parseGoodReadsRating(result.Rating, current.ID) current.GoodReadsEnriched = true if err := b.storeGoodReadsAuthorURL(current, result.AuthorURL, result.Author); err != nil { return result, fmt.Errorf("error storing GoodReads author URL for document %s: %w", current.ID, err) @@ -329,16 +424,16 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document, t return result, nil } -// waitGoodReadsTurn blocks until at least goodReadsInterval has passed since -// the previous GoodReads lookup, shared by EnrichGoodReads and the per-file -// worker. -func (b *BleveIndexer) waitGoodReadsTurn() { - b.goodReadsThrottleMu.Lock() - defer b.goodReadsThrottleMu.Unlock() - if wait := time.Until(b.goodReadsLastRequest.Add(b.goodReadsInterval)); wait > 0 { - time.Sleep(wait) +func parseGoodReadsRating(rating, documentID string) float64 { + if rating == "" { + return 0 } - b.goodReadsLastRequest = time.Now() + value, err := strconv.ParseFloat(rating, 64) + if err != nil || value < 0 || value > 5 || math.IsNaN(value) || math.IsInf(value, 0) { + log.Printf("Invalid GoodReads rating for document %s: %q", documentID, rating) + return 0 + } + return value } // runGoodReadsEnrichWorker looks up documents queued by indexFile for the @@ -353,7 +448,7 @@ func (b *BleveIndexer) runGoodReadsEnrichWorker() { if document.ID == "" || document.GoodReadsEnriched { continue } - if _, err := b.enrichGoodReads(context.Background(), document, true); err != nil { + if _, err := b.enrichGoodReads(context.Background(), document); err != nil { log.Println(err) } } @@ -403,14 +498,21 @@ func (b *BleveIndexer) preserveGoodReadsLocked(documents []Document) { stored := make(map[string]Document, len(searchResult.Hits)) for _, hit := range searchResult.Hits { bookURL, _ := hit.Fields["GoodReadsBookURL"].(string) + rating, _ := hit.Fields["GoodReadsRating"].(string) + ratingValue, _ := hit.Fields["GoodReadsRatingValue"].(float64) + if ratingValue == 0 { + ratingValue = parseGoodReadsRating(rating, hit.ID) + } resourceID, _ := hit.Fields["GoodReadsResourceID"].(string) legacyBookID, _ := hit.Fields["GoodReadsLegacyBookID"].(float64) enriched, _ := hit.Fields["GoodReadsEnriched"].(bool) - stored[hit.ID] = Document{GoodReadsBookURL: bookURL, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} + stored[hit.ID] = Document{GoodReadsBookURL: bookURL, GoodReadsRating: rating, GoodReadsRatingValue: ratingValue, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} } for i := range documents { if s, ok := stored[documents[i].ID]; ok { documents[i].GoodReadsBookURL = s.GoodReadsBookURL + documents[i].GoodReadsRating = s.GoodReadsRating + documents[i].GoodReadsRatingValue = s.GoodReadsRatingValue documents[i].GoodReadsResourceID = s.GoodReadsResourceID documents[i].GoodReadsLegacyBookID = s.GoodReadsLegacyBookID documents[i].GoodReadsEnriched = s.GoodReadsEnriched diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go index ae78b727..aeb2c3f0 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/index/bleve_goodreads_write_test.go @@ -78,7 +78,7 @@ const ( func newGoodReadsTestIndexer(t *testing.T, source GoodReadsSource) *BleveIndexer { t.Helper() - idx, _ := newTestTextRankIndexer(t, Config{GoodReadsSource: source, GoodReadsEnrichInterval: time.Millisecond}) + idx, _ := newTestTextRankIndexer(t, Config{GoodReadsSource: source}) appFS := afero.NewMemMapFs() if err := afero.WriteFile(appFS, "lib/book.pdf", []byte(""), 0644); err != nil { t.Fatal(err) @@ -107,6 +107,9 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { if doc.GoodReadsBookURL != wantGoodReadsBookURL { t.Errorf("unexpected book URL %q", doc.GoodReadsBookURL) } + if doc.GoodReadsRating != "3.82" { + t.Errorf("unexpected Goodreads rating %q", doc.GoodReadsRating) + } assertNoDocumentAuthorURL(t, idx.documentsIdx) if !doc.GoodReadsEnriched { t.Error("expected document to be marked as GoodReads-enriched") @@ -289,7 +292,7 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != wantGoodReadsBookURL { + if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != wantGoodReadsBookURL || doc.GoodReadsRating != "3.82" || doc.GoodReadsRatingValue != 3.82 { t.Errorf("expected GoodReads fields to survive a TextRank write, got %+v", doc) } if doc.GoodReadsResourceID != "work-id" || doc.GoodReadsLegacyBookID != 12207444 { @@ -308,7 +311,7 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { t.Fatal(err) } idx.documentsMu.Unlock() - if _, err := idx.enrichGoodReads(context.Background(), stale, true); err != nil { + if _, err := idx.enrichGoodReads(context.Background(), stale); err != nil { t.Fatal(err) } if doc, _ = idx.documentByIndexID(goodReadsTestID); !slices.Equal(doc.TextRankWords, []string{"guinevere"}) { @@ -393,8 +396,8 @@ func TestGoodReadsBook(t *testing.T) { t.Errorf("expected the link to be persisted, got %+v", stored) } - if got, _ := idx.GoodReadsBook(context.Background(), stored); got.BookURL != wantGoodReadsBookURL || got.Rating != "" { - t.Errorf("expected stored URL without a persisted rating, got %+v", got) + if got, _ := idx.GoodReadsBook(context.Background(), stored); got.BookURL != wantGoodReadsBookURL || got.Rating != "3.82" { + t.Errorf("expected stored URL and rating, got %+v", got) } req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{stored.ID})) req.Fields = []string{"*"} @@ -402,7 +405,7 @@ func TestGoodReadsBook(t *testing.T) { if err != nil || len(result.Hits) != 1 { t.Fatalf("reading stored Goodreads fields: %v", err) } - for _, field := range []string{"Rating", "rating", "GoodReadsRating"} { + for _, field := range []string{"Rating", "rating"} { if _, exists := result.Hits[0].Fields[field]; exists { t.Errorf("rating must not be stored in document field %s", field) } @@ -415,6 +418,24 @@ func TestGoodReadsBook(t *testing.T) { } } +func TestParseGoodReadsRating(t *testing.T) { + for _, tt := range []struct { + rating string + want float64 + }{ + {rating: "0.00", want: 0}, + {rating: "3.82", want: 3.82}, + {rating: "5.01", want: 0}, + {rating: "unavailable", want: 0}, + } { + t.Run(tt.rating, func(t *testing.T) { + if got := parseGoodReadsRating(tt.rating, goodReadsTestID); got != tt.want { + t.Errorf("parseGoodReadsRating(%q) = %v, want %v", tt.rating, got, tt.want) + } + }) + } +} + func TestGoodReadsBookReturnsErrorWhenUnavailable(t *testing.T) { source := &fakeGoodReadsSource{err: biblioreads.ErrUnavailable} idx := newGoodReadsTestIndexer(t, source) @@ -458,6 +479,14 @@ func TestMatchingGoodReadsBook(t *testing.T) { {name: "document omnibus range", title: correct.Title + " (#1-#3)", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, {name: "standalone volume retained", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1)", Author: correct.Author, BookURL: correct.BookURL}}}, {name: "exact preferred", results: []biblioreads.SearchResult{{Title: correct.Title + " (Middle Earth, #1)", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: correct.BookURL}, + {name: "lower Goodreads ID preferred over rating for equal titles", results: []biblioreads.SearchResult{ + {ID: 1, Title: correct.Title, Author: correct.Author, BookURL: "/book/show/1-unrated", Rating: "0.00"}, + {ID: 2, Title: correct.Title, Author: correct.Author, BookURL: "/book/show/2-rated", Rating: "4.05"}, + }, want: "/book/show/1-unrated"}, + {name: "unrated exact match retained when no rated match exists", results: []biblioreads.SearchResult{ + {Title: correct.Title, Author: correct.Author, BookURL: "/book/show/1-unrated", Rating: "0.00"}, + {Title: correct.Title, Author: correct.Author, BookURL: "/book/show/2"}, + }, want: "/book/show/1-unrated"}, {name: "colon subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, {name: "period subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + ". Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, {name: "dash subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + " — Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, @@ -468,7 +497,7 @@ func TestMatchingGoodReadsBook(t *testing.T) { {name: "series preferred over subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: "/book/show/2"}, {Title: correct.Title + " (#1-#3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, {name: "arbitrary prefix rejected", results: []biblioreads.SearchResult{{Title: correct.Title + " y otras historias", Author: correct.Author, BookURL: correct.BookURL}}}, {name: "volume retained", results: []biblioreads.SearchResult{{Title: correct.Title + " 2", Author: correct.Author, BookURL: correct.BookURL}}}, - {name: "arbitrary parenthetical retained", results: []biblioreads.SearchResult{{Title: correct.Title + " (A parody)", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "trailing parenthetical accepted as weaker match", results: []biblioreads.SearchResult{{Title: correct.Title + " (An edition)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, {name: "missing result author", results: []biblioreads.SearchResult{{Title: correct.Title, BookURL: correct.BookURL}}}, {name: "missing result title", results: []biblioreads.SearchResult{{Author: correct.Author, BookURL: correct.BookURL}}}, {name: "missing URL", results: []biblioreads.SearchResult{{Title: correct.Title, Author: correct.Author}}}, @@ -493,6 +522,52 @@ func TestMatchingGoodReadsBook(t *testing.T) { } } +func TestMatchingGoodReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { + const olderBookURL = "/book/show/1368314.El_Cercano_Oriente" + const newerBookURL = "/book/show/243175949-el-cercano-oriente" + document := Document{Metadata: metadata.Metadata{ + Title: "El cercano Oriente", Authors: []string{"Isaac Asimov"}, + }} + older := biblioreads.SearchResult{ + ID: 1368314, Title: "El Cercano Oriente", Author: "Isaac Asimov", + BookURL: olderBookURL, Rating: "4.10", + } + newer := biblioreads.SearchResult{ + ID: 243175949, Title: "El Cercano Oriente", Author: "Isaac Asimov", + BookURL: newerBookURL, Rating: "4.26", + } + + for _, results := range [][]biblioreads.SearchResult{{newer, older}, {older, newer}} { + match, ok := matchingGoodReadsBook(document, results) + if !ok || match.BookURL != olderBookURL { + t.Fatalf("expected lower Goodreads ID %q, got %+v, %v", olderBookURL, match, ok) + } + } +} + +func TestMatchingGoodReadsBookPrefersRatedParentheticalTitle(t *testing.T) { + doc := Document{Metadata: metadata.Metadata{Title: "La promesa", Authors: []string{"Pearl S. Buck"}}} + rated := biblioreads.SearchResult{ + Title: "La promesa (The Promise)", Author: "Pearl S. Buck", + BookURL: "/book/show/7377104-la-promesa", Rating: "4.04", + } + unrated := biblioreads.SearchResult{ + Title: "La promesa", Author: "Pearl S. Buck", + BookURL: "/book/show/210898619-la-promesa", Rating: "0.00", + } + for _, results := range [][]biblioreads.SearchResult{{rated, unrated}, {unrated, rated}} { + match, ok := matchingGoodReadsBook(doc, results) + if !ok || match.BookURL != rated.BookURL || match.Rating != rated.Rating { + t.Fatalf("expected rated translated-title entry, got %+v, %v", match, ok) + } + } + rated.Author = "Another author" + match, ok := matchingGoodReadsBook(doc, []biblioreads.SearchResult{rated, unrated}) + if !ok || match.BookURL != unrated.BookURL { + t.Fatalf("rated entry by another author must not win, got %+v, %v", match, ok) + } +} + func TestGoodReadsBookMatchesDolnickSubtitle(t *testing.T) { const title = "La escritura de los dioses" const author = "Edward Dolnick" diff --git a/internal/index/document.go b/internal/index/document.go index e99d769d..70cf61a7 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -20,16 +20,18 @@ type SearchFields struct { // author and its own series - deliberately not its subjects, see // similarToQuery), ranked by score and pruned to those scoring close // enough to the best match (see runSimilarityQuery). - SimilarTo string - PubDateFrom date.Date - PubDateTo date.Date - EstReadTimeFrom float64 - EstReadTimeTo float64 - WordsPerMinute float64 - PagesFrom float64 - PagesTo float64 - IllustratedOnly bool - SortBy []string + SimilarTo string + PubDateFrom date.Date + PubDateTo date.Date + EstReadTimeFrom float64 + EstReadTimeTo float64 + WordsPerMinute float64 + PagesFrom float64 + PagesTo float64 + GoodReadsRatingFrom float64 + GoodReadsRatingTo float64 + IllustratedOnly bool + SortBy []string } type Document struct { @@ -80,6 +82,10 @@ type Document struct { // It is stored but not indexed, and filled by EnrichGoodReads. // Author URLs are stored on Author instead. GoodReadsBookURL string + // GoodReadsRating is the score returned with the Goodreads book search result. + // The string is retained for display; GoodReadsRatingValue is indexed for filtering. + GoodReadsRating string + GoodReadsRatingValue float64 // GoodReadsResourceID identifies the work; GoodReadsLegacyBookID identifies // the edition. Both are stored only, populated when book details are fetched, // and used together by BiblioReads to retrieve reviews. diff --git a/internal/index/goodreads_concurrency_test.go b/internal/index/goodreads_concurrency_test.go new file mode 100644 index 00000000..b4443492 --- /dev/null +++ b/internal/index/goodreads_concurrency_test.go @@ -0,0 +1,65 @@ +package index + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/metadata" +) + +type blockingGoodReadsSource struct { + started chan struct{} + release chan struct{} +} + +func (s *blockingGoodReadsSource) SearchBooks(ctx context.Context, _ string) (biblioreads.SearchBooksResponse, error) { + s.started <- struct{}{} + select { + case <-s.release: + return biblioreads.SearchBooksResponse{Result: goodReadsTestResults}, nil + case <-ctx.Done(): + return biblioreads.SearchBooksResponse{}, ctx.Err() + } +} + +func TestGoodReadsEnrichmentUsesBoundedConcurrency(t *testing.T) { + source := &blockingGoodReadsSource{ + started: make(chan struct{}, 8), + release: make(chan struct{}), + } + idx := newGoodReadsTestIndexer(t, source) + var documents []Document + for i := range 8 { + doc := Document{ + ID: fmt.Sprintf("book-%d.pdf", i), + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + } + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + documents = append(documents, doc) + } + done := make(chan error, 1) + go func() { done <- idx.enrichGoodReadsBatch(documents) }() + defer func() { + close(source.release) + if err := <-done; err != nil { + t.Errorf("enrichment failed: %v", err) + } + }() + for range goodReadsEnrichWorkers { + select { + case <-source.started: + case <-time.After(time.Second): + t.Fatal("expected concurrent Goodreads lookups") + } + } + select { + case <-source.started: + t.Fatal("enrichment exceeded its concurrency bound") + case <-time.After(30 * time.Millisecond): + } +} diff --git a/internal/index/goodreads_fields_test.go b/internal/index/goodreads_fields_test.go index ddc9be5c..ed965130 100644 --- a/internal/index/goodreads_fields_test.go +++ b/internal/index/goodreads_fields_test.go @@ -40,6 +40,8 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { }, AuthorsSlugs: []string{"alice-borchardt"}, GoodReadsBookURL: bookURL, + GoodReadsRating: "4.25", + GoodReadsRatingValue: 4.25, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: legacyBookID, } @@ -54,12 +56,15 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { if got.GoodReadsBookURL != bookURL { t.Errorf("language %q: expected stored book URL, got %q", lang, got.GoodReadsBookURL) } + if got.GoodReadsRating != "4.25" { + t.Errorf("language %q: expected stored rating, got %q", lang, got.GoodReadsRating) + } if got.GoodReadsResourceID != resourceID || got.GoodReadsLegacyBookID != legacyBookID { t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, got.GoodReadsResourceID, got.GoodReadsLegacyBookID) } } - for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155", "zyxwvresource", resourceID} { + for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155", "zyxwvresource", resourceID, "4.25"} { res, err := idx.Search(index.SearchFields{Keywords: keywords}, 1, 10) if err != nil { t.Fatal(err) @@ -68,9 +73,18 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { t.Errorf("expected no results searching keywords %q, got %d", keywords, res.TotalHits()) } } + filtered, err := idx.Search(index.SearchFields{GoodReadsRatingFrom: 4.2, GoodReadsRatingTo: 4.3}, 1, 10) + if err != nil || filtered.TotalHits() != 2 { + t.Errorf("expected both documents in the Goodreads score range, got %d results and error %v", filtered.TotalHits(), err) + } + filtered, err = idx.Search(index.SearchFields{GoodReadsRatingFrom: 4.3}, 1, 10) + if err != nil || filtered.TotalHits() != 0 { + t.Errorf("expected no documents above the Goodreads score, got %d results and error %v", filtered.TotalHits(), err) + } queries := map[string]bleve.SearchRequest{ "book URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsBookURL", bookURL)), + "book rating field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsRating", "4.25")), "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsAuthorURL", authorURL)), "resource ID field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsResourceID", resourceID)), "legacy ID numeric field": *bleve.NewSearchRequest(legacyBookIDQuery(legacyBookID)), diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index 3e4c0925..ebfbdd01 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "strconv" "strings" "time" @@ -25,6 +26,7 @@ type BiblioReadsClient interface { type GoodReadsIndex interface { GoodReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) StoreGoodReadsBookIDs(document index.Document, bookURL, resourceID string, legacyBookID int) error + StoreGoodReadsRating(document index.Document, bookURL, rating string) error } type BiblioReads struct { @@ -36,6 +38,11 @@ func NewBiblioReads(client BiblioReadsClient, idx GoodReadsIndex) *BiblioReads { return &BiblioReads{client: client, index: idx} } +// DocumentLink returns the indexed book link without making a network request. +func (b *BiblioReads) DocumentLink(document index.Document) servicelinks.Link { + return (servicelinks.Goodreads{}).Document(document) +} + func bookID(bookURL string) (string, error) { id, ok := strings.CutPrefix(bookURL, goodReadsBookPrefix) if !ok || id == "" { @@ -53,6 +60,13 @@ func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblio if err != nil { return book, fmt.Errorf("retrieving Goodreads book %s: %w", id, err) } + if book.Rating != "" && document.GoodReadsRatingValue > 0 && isZeroRating(book.Rating) { + book.Rating = document.GoodReadsRating + } else if book.Rating != "" { + if err := b.index.StoreGoodReadsRating(document, document.GoodReadsBookURL, book.Rating); err != nil { + log.Printf("Error saving Goodreads rating for %s: %s", document.ID, err) + } + } if book.ResourceID != "" && book.LegacyBookID > 0 { if err := b.index.StoreGoodReadsBookIDs(document, document.GoodReadsBookURL, book.ResourceID, book.LegacyBookID); err != nil { log.Printf("Error saving Goodreads identifiers for %s: %s", document.ID, err) @@ -61,6 +75,11 @@ func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblio return book, nil } +func isZeroRating(rating string) bool { + value, err := strconv.ParseFloat(rating, 64) + return err == nil && value == 0 +} + func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score, error) { score := Score{Source: "Goodreads"} if document.GoodReadsBookURL == "" { diff --git a/internal/reviews/model.go b/internal/reviews/model.go index 435b23c1..6f042ed4 100644 --- a/internal/reviews/model.go +++ b/internal/reviews/model.go @@ -1,7 +1,11 @@ // Package reviews provides service-independent review data and service adapters. package reviews -import "time" +import ( + "strconv" + "strings" + "time" +) // Score has an empty URL for an unmatched document and an empty Value for an unavailable score. type Score struct { @@ -11,10 +15,23 @@ type Score struct { URL string } +// HasScore reports whether the value is scored or has valuations despite a zero value. +func (s Score) HasScore() bool { + if s.Value == "" { + return false + } + value, err := strconv.ParseFloat(s.Value, 64) + if err != nil || value != 0 { + return true + } + count, err := strconv.ParseFloat(strings.ReplaceAll(strings.TrimSpace(s.Count), ",", ""), 64) + return err == nil && count > 0 +} + type Review struct { Reviewer string - // Rating is on a five-point scale; zero means unrated. - Rating int + // Rating is on a five-point scale and may be fractional; zero means unrated. + Rating float64 // Text may contain untrusted HTML, which must be sanitized before rendering. Text string CreatedAt time.Time diff --git a/internal/webserver/controller/document/detail.go b/internal/webserver/controller/document/detail.go index 9be75178..4b7651d0 100644 --- a/internal/webserver/controller/document/detail.go +++ b/internal/webserver/controller/document/detail.go @@ -74,9 +74,19 @@ func (d *Controller) Detail(c fiber.Ctx) error { "IllustratorSummaries": illustratorSummaries, "Debug": c.RequestCtx().QueryArgs().Has("debug"), "ReviewsEnabled": d.reviewProvider != nil, + "DocumentServiceLink": d.documentServiceLink(document), }, "layout") } +func (d *Controller) documentServiceLink(document index.Document) servicelinks.Link { + if provider, ok := d.reviewProvider.(interface { + DocumentLink(index.Document) servicelinks.Link + }); ok { + return provider.DocumentLink(document) + } + return servicelinks.Link{} +} + // authorAndIllustratorSummaries resolves the (up to 2) authors and (up to 2) // illustrators shown on a document's detail page, so their summary partials can // render eagerly when the author is already enriched, skipping the async diff --git a/internal/webserver/controller/document/reviews.go b/internal/webserver/controller/document/reviews.go index a46cb118..85990d8d 100644 --- a/internal/webserver/controller/document/reviews.go +++ b/internal/webserver/controller/document/reviews.go @@ -4,17 +4,20 @@ import ( "context" "html/template" "log" + "math" "time" "github.com/gofiber/fiber/v3" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" + "golang.org/x/exp/slices" ) type documentReview struct { reviews.Review HTML template.HTML CreatedOn string + StarFills [5]int } // Reviews loads the first page of user reviews when the detail page's bottom is visible. @@ -34,13 +37,29 @@ func (d *Controller) Reviews(c fiber.Ctx) error { ctx, cancel := context.WithTimeout(c.Context(), reviewsTimeout) defer cancel() lang := c.Locals("Lang").(string) - page, err := d.reviewProvider.Reviews(ctx, document, lang) + reviewLanguage := c.Query("language") + supportedLanguages := c.Locals("SupportedLanguages").([]string) + if reviewLanguage == "" { + reviewLanguage = lang + } + if !slices.Contains(supportedLanguages, reviewLanguage) { + return fiber.ErrBadRequest + } + page, err := d.reviewProvider.Reviews(ctx, document, reviewLanguage) if err != nil { log.Printf("Error retrieving reviews for %s: %s", document.ID, err) return fiber.ErrInternalServerError } if !page.Available { - return c.SendString("") + link := d.documentServiceLink(document) + if link.URL == "" { + return c.SendString("") + } + return c.Render("partials/document-reviews", fiber.Map{ + "Lang": lang, "Slug": document.Slug, "ReviewLanguage": reviewLanguage, + "SupportedLanguages": supportedLanguages, "Source": link.Source, + "BookURL": link.URL, "Unavailable": true, + }) } renderedReviews := make([]documentReview, 0, len(page.Reviews)) for _, review := range page.Reviews { @@ -48,15 +67,19 @@ func (d *Controller) Reviews(c fiber.Ctx) error { if !review.CreatedAt.IsZero() { createdOn = review.CreatedAt.Format(time.DateOnly) } - renderedReviews = append(renderedReviews, documentReview{ + rendered := documentReview{ Review: review, HTML: template.HTML(metadata.SanitizeDescription(review.Text)), CreatedOn: createdOn, - }) + } + for point := range rendered.StarFills { + rendered.StarFills[point] = int(math.Round(min(1, max(0, review.Rating-float64(point))) * 100)) + } + renderedReviews = append(renderedReviews, rendered) } return c.Render("partials/document-reviews", fiber.Map{ - "Reviews": renderedReviews, - "Source": page.Source, - "BookURL": page.URL, + "Lang": lang, "Slug": document.Slug, "ReviewLanguage": reviewLanguage, + "SupportedLanguages": supportedLanguages, "Reviews": renderedReviews, + "Source": page.Source, "BookURL": page.URL, }) } diff --git a/internal/webserver/controller/document/score.go b/internal/webserver/controller/document/score.go index 812d5dc3..b4c53e54 100644 --- a/internal/webserver/controller/document/score.go +++ b/internal/webserver/controller/document/score.go @@ -30,14 +30,15 @@ func (d *Controller) Score(c fiber.Ctx) error { log.Printf("Error retrieving score for %s: %s", document.ID, err) return c.SendString("") } - if score.URL != "" && score.Value == "" { - return c.SendString("") - } return c.Render("partials/document-score", fiber.Map{ - "Source": score.Source, - "Rating": score.Value, - "RatingCount": score.Count, - "URL": score.URL, - "DocumentSlug": document.Slug, + "Lang": c.Locals("Lang"), + "ReviewLanguage": c.Locals("Lang"), + "SupportedLanguages": c.Locals("SupportedLanguages"), + "Source": score.Source, + "Rating": score.Value, + "RatingCount": score.Count, + "Scored": score.HasScore(), + "URL": score.URL, + "DocumentSlug": document.Slug, }) } diff --git a/internal/webserver/controller/search/parse.go b/internal/webserver/controller/search/parse.go index 6ea798f2..e2d94d2d 100644 --- a/internal/webserver/controller/search/parse.go +++ b/internal/webserver/controller/search/parse.go @@ -12,20 +12,22 @@ import ( // ParseDocumentSearchQuery reads document search/filter query params (search, // language, subjects, similar, sort-by, pub-date-from/to, est-read-time-from/to, -// pages-from/to, illustrated-only) into an index.SearchFields. +// pages-from/to, Goodreads score, illustrated-only) into an index.SearchFields. func ParseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.SearchFields, error) { searchFields := index.SearchFields{ - Keywords: c.Query("search"), - Language: c.Query("language"), - Subjects: c.Query("subjects"), - SimilarTo: similarToSlug(c), - SortBy: parseDocumentSortBy(c), - EstReadTimeFrom: fiber.Query[float64](c, "est-read-time-from", 0), - EstReadTimeTo: fiber.Query[float64](c, "est-read-time-to", 0), - WordsPerMinute: wordsPerMinute, - PagesFrom: fiber.Query[float64](c, "pages-from", 0), - PagesTo: fiber.Query[float64](c, "pages-to", 0), - IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", + Keywords: c.Query("search"), + Language: c.Query("language"), + Subjects: c.Query("subjects"), + SimilarTo: similarToSlug(c), + SortBy: parseDocumentSortBy(c), + EstReadTimeFrom: fiber.Query[float64](c, "est-read-time-from", 0), + EstReadTimeTo: fiber.Query[float64](c, "est-read-time-to", 0), + WordsPerMinute: wordsPerMinute, + PagesFrom: fiber.Query[float64](c, "pages-from", 0), + PagesTo: fiber.Query[float64](c, "pages-to", 0), + GoodReadsRatingFrom: fiber.Query[float64](c, "goodreads-score-from", 0), + GoodReadsRatingTo: fiber.Query[float64](c, "goodreads-score-to", 0), + IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", } if c.Query("pub-date-from") != "" { @@ -56,6 +58,10 @@ func ParseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.Search searchFields.PagesFrom, searchFields.PagesTo = searchFields.PagesTo, searchFields.PagesFrom } + if searchFields.GoodReadsRatingTo != 0 && searchFields.GoodReadsRatingFrom > searchFields.GoodReadsRatingTo { + searchFields.GoodReadsRatingFrom, searchFields.GoodReadsRatingTo = searchFields.GoodReadsRatingTo, searchFields.GoodReadsRatingFrom + } + return searchFields, nil } diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 5b888ba3..6eb48fd3 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -3,6 +3,7 @@ package webserver_test import ( "context" "errors" + "fmt" "io" "net/http" "strings" @@ -39,6 +40,61 @@ type fakeBookSource struct { reviewCalls []reviewCall } +func TestDocumentReviewsCanSelectSupportedLanguage(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "4", ResourceID: "work-id", LegacyBookID: 1, + }} + app, _ := newGoodReadsTestApp(t, source) + req := mustGetRequest(t, "/documents/linked/reviews?language=fr") + req.Header.Set("Accept-Language", "es") + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if len(source.reviewCalls) != 1 || source.reviewCalls[0].languageCode != "fr" { + t.Fatalf("expected Goodreads reviews in French, got %+v", source.reviewCalls) + } + if page.Find("h2").Text() != "Reseñas de usuarios de Goodreads" { + t.Fatalf("expected page language to remain Spanish while selecting French reviews, got %q", page.Find("h2").Text()) + } + selectBox := page.Find(`#review-language-linked`) + if selectBox.Length() != 1 || selectBox.Find(`option[value="fr"][selected]`).Length() != 1 { + html, _ := selectBox.Html() + t.Fatalf("expected the review-language selector to retain the selected language: %q", html) + } + if indicator, _ := selectBox.Attr("hx-indicator"); indicator != "#document-reviews-linked" || + page.Find(`#document-reviews-linked .document-reviews-loading > div`).Length() != 2 { + t.Fatal("expected a review skeleton indicator for language changes") + } + for _, language := range webserver.SupportedLanguages() { + if selectBox.Find(`option[value="`+language+`"]`).Length() != 1 { + t.Errorf("expected supported language %q in the review selector", language) + } + } +} + +func TestDocumentReviewsRejectUnsupportedLanguage(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "4", ResourceID: "work-id", LegacyBookID: 1, + }} + app, _ := newGoodReadsTestApp(t, source) + response, err := app.Test(mustGetRequest(t, "/documents/linked/reviews?language=zz")) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusBadRequest, t) + if len(source.reviewCalls) != 0 { + t.Fatalf("unsupported language must not be sent to Goodreads, got %+v", source.reviewCalls) + } +} + type fakeReviewProvider struct { score reviews.Score page reviews.Page @@ -121,7 +177,7 @@ func TestDocumentAlternativeReviewProvider(t *testing.T) { if err != nil { t.Fatal(err) } - link := page.Find(`a[href="https://example.org/books/book-1"]`) + link := page.Find(`li a[href="https://example.org/books/book-1"]`) if link.Text() != "4.5" || !strings.Contains(body, "Another service rating") || strings.Contains(body, "goodreads.com") { t.Fatalf("score must use the selected provider's labels, value and URL: %s", body) } @@ -145,6 +201,42 @@ func TestDocumentAlternativeReviewProvider(t *testing.T) { } } +func TestDocumentScoreMarksZeroWithoutValuationsAsNotScored(t *testing.T) { + for _, tt := range []struct { + name string + count string + wantScore string + wantScored bool + }{ + {name: "zero valuations", count: "0", wantScore: "Not scored"}, + {name: "valuations exist", count: "650", wantScore: "0.00", wantScored: true}, + } { + t.Run(tt.name, func(t *testing.T) { + provider := &fakeReviewProvider{ + score: reviews.Score{ + Source: "Goodreads", Value: "0.00", Count: tt.count, + URL: "https://www.goodreads.com/book/show/1", + }, + } + app, _ := newReviewProviderTestApp(t, nil, provider) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unmatched/score"))) + if err != nil { + t.Fatal(err) + } + row := page.Find("li") + if tt.wantScored { + if got := strings.TrimSpace(row.Find("a").Text()); got != tt.wantScore { + t.Fatalf("expected score %q, got %q", tt.wantScore, got) + } + return + } + if got := strings.TrimSpace(row.Text()); got != tt.wantScore || row.Find("a").Length() != 0 { + t.Fatalf("expected unlinked %q label, got %q", tt.wantScore, got) + } + }) + } +} + type reviewCall struct { resourceID string legacyBookID int @@ -214,9 +306,11 @@ func newServiceProvidersTestApp(t *testing.T, source *fakeBookSource, provider d for _, doc := range []index.Document{ { ID: "linked.epub", Slug: "linked", - Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, - GoodReadsBookURL: "/book/show/12207444-la-reina-dragon", - GoodReadsEnriched: true, + Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, + GoodReadsBookURL: "/book/show/12207444-la-reina-dragon", + GoodReadsRating: "3.82", + GoodReadsRatingValue: 3.82, + GoodReadsEnriched: true, }, { ID: "unlinked.epub", Slug: "unlinked", @@ -242,6 +336,41 @@ func newServiceProvidersTestApp(t *testing.T, source *fakeBookSource, provider d return app, idx } +func TestSearchResultsShowStoredGoodReadsRating(t *testing.T) { + app, _ := newGoodReadsTestApp(t, nil) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) + if err != nil { + t.Fatal(err) + } + rating := page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if rating.Length() != 1 || strings.TrimSpace(rating.Text()) != "3.82" { + t.Fatalf("expected stored Goodreads rating in search result, got %q", rating.Text()) + } + if page.Find(`li:has(a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]) i.bi-star-fill`).Length() != 1 { + t.Fatal("expected a star icon beside the Goodreads score") + } +} + +func TestDocumentDetailShowsStoredZeroRatingAsNotScored(t *testing.T) { + source := &fakeBookSource{results: []biblioreads.SearchResult{{ + Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", Rating: "0.00", + }}} + app, _ := newGoodReadsTestApp(t, source) + getBody(t, app, "/documents/unlinked/goodreads") + + body := getBody(t, app, "/search?type=documents&search=Unlinked") + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + row := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { + return strings.TrimSpace(item.Text()) == "Not scored" && item.Find("i.bi-star-fill").Length() == 1 + }) + if row.Length() == 0 || row.Find("a").Length() != 0 { + t.Fatalf("expected stored zero rating to render as Not scored, got %q in %s", row.Text(), body) + } +} + func getBody(t *testing.T, app *fiber.App, path string) string { t.Helper() req := mustGetRequest(t, path) @@ -317,6 +446,56 @@ func TestDocumentDetailGoodReadsRating(t *testing.T) { } } +func TestDocumentDetailUpdatesStoredGoodReadsRating(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "4.12", ResourceID: "work-id", LegacyBookID: 12207444, + }} + app, idx := newGoodReadsTestApp(t, source) + + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + if err != nil { + t.Fatal(err) + } + link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if got := strings.TrimSpace(link.Text()); got != "4.12" { + t.Fatalf("expected latest Goodreads rating 4.12, got %q", got) + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if stored.GoodReadsRating != "4.12" || stored.GoodReadsRatingValue != 4.12 { + t.Fatalf("expected latest rating and searchable value to be stored, got %+v", stored) + } + results, err := idx.Search(index.SearchFields{GoodReadsRatingFrom: 4.1, GoodReadsRatingTo: 4.2}, 1, 10) + if err != nil || results.TotalHits() != 1 { + t.Fatalf("expected document to match updated rating range, got %d results and error %v", results.TotalHits(), err) + } +} + +func TestDocumentDetailKeepsStoredGoodReadsRatingWhenScraperReturnsZero(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "0.00", RatingCount: "650", ResourceID: "work-id", LegacyBookID: 12207444, + }} + app, idx := newGoodReadsTestApp(t, source) + + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + if err != nil { + t.Fatal(err) + } + link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if got := strings.TrimSpace(link.Text()); got != "3.82" { + t.Fatalf("expected known Goodreads rating to be retained, got %q", got) + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if stored.GoodReadsRating != "3.82" || stored.GoodReadsRatingValue != 3.82 { + t.Fatalf("zero scraper rating must not replace indexed Goodreads score, got %+v", stored) + } +} + func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { source := &fakeBookSource{ book: biblioreads.BookResponse{Rating: "4.20", RatingCount: "10", ResourceID: "work-id", LegacyBookID: 1}, @@ -332,7 +511,7 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { if err != nil { t.Fatal(err) } - if got := strings.TrimSpace(page.Find(`a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.10" { + if got := strings.TrimSpace(page.Find(`li a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.10" { t.Errorf("expected rating link with 4.10, got %q", got) } if len(source.queries) != 1 || source.queries[0] != "Unlinked Jane Doe" { @@ -367,7 +546,7 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { if err != nil { t.Fatal(err) } - if got := strings.TrimSpace(page.Find(`a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.20" { + if got := strings.TrimSpace(page.Find(`li a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.20" { t.Errorf("expected later request to use book-scraper rating, got %q", got) } if len(source.queries) != 1 { @@ -392,7 +571,7 @@ func TestDocumentDetailGoodReadsSkipsUnmatchedDocuments(t *testing.T) { score := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { return item.Find("i.bi-star-fill").Length() == 1 }) - if strings.TrimSpace(score.Text()) != "No score" || score.Find("a").Length() != 0 { + if strings.TrimSpace(score.Text()) != "Not scored" || score.Find("a").Length() != 0 { t.Errorf("%s: expected unlinked no-score row, got %q", path, score.Text()) } } @@ -403,8 +582,7 @@ func TestDocumentDetailGoodReadsSkipsUnmatchedDocuments(t *testing.T) { func TestDocumentDetailGoodReadsRatingRendersNothingWhenUnavailable(t *testing.T) { for name, source := range map[string]*fakeBookSource{ - "error": {err: errors.New("unavailable")}, - "no rating": {}, + "error": {err: errors.New("unavailable")}, } { t.Run(name, func(t *testing.T) { app, _ := newGoodReadsTestApp(t, source) @@ -418,8 +596,12 @@ func TestDocumentDetailGoodReadsRatingRendersNothingWhenUnavailable(t *testing.T func TestDocumentDetailGoodReadsIdentifiersWithoutRating(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}} app, idx := newGoodReadsTestApp(t, source) - if body := strings.TrimSpace(getBody(t, app, "/documents/linked/goodreads")); body != "" { - t.Fatalf("expected no rating, got %q", body) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(page.Find("li").Text()) != "Not scored" || page.Find("h2").Text() != "Goodreads user reviews" { + t.Fatal("expected no score and a reviews heading for a matched book without a rating") } stored, err := idx.Document("linked") if err != nil { @@ -493,9 +675,18 @@ func TestDocumentDetailGoodReadsSearchDoesNotScrape(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.TrimSpace(page.Find("li").Text()) != "No score" || page.Find("a").Length() != 0 { + if strings.TrimSpace(page.Find("li").Text()) != "Not scored" || page.Find("a").Length() != 0 { t.Errorf("expected no-score row without a link, got %q", body) } + } else if tt.enriched && len(tt.results) > 0 { + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if page.Find("#document-reviews-unlinked h2").Text() != "Goodreads user reviews" || + page.Find("#document-reviews-unlinked[hx-get]").Length() != 0 { + t.Fatal("a match without a rating must show the heading without fetching reviews") + } } else if body != "" { t.Errorf("expected no rating from search, got %q", body) } @@ -517,11 +708,11 @@ func TestDocumentDetailGoodReadsNoScoreTranslations(t *testing.T) { lang string text string }{ - {"en", "No score"}, - {"es", "Sin puntuación"}, - {"de", "Keine Bewertung"}, - {"fr", "Aucune note"}, - {"ru", "Нет оценки"}, + {"en", "Not scored"}, + {"es", "Sin puntuar"}, + {"de", "Nicht bewertet"}, + {"fr", "Non noté"}, + {"ru", "Не оценено"}, } { for _, path := range []string{"/documents/unmatched/score", "/documents/unmatched/goodreads"} { t.Run(locale.lang+path, func(t *testing.T) { @@ -565,6 +756,9 @@ func TestDocumentDetailGoodReadsReviewsLazyPlaceholder(t *testing.T) { if target.Length() != 1 || target.Is("[hx-get]") { t.Fatal("expected an inactive reviews target before the rating loads") } + if target.Find("h2").Text() != "Goodreads user reviews" { + t.Fatal("expected the reviews heading before the score loads") + } if !target.Is(".document-detail-content > :last-child") { t.Fatal("reviews must appear at the bottom of the document content") } @@ -576,6 +770,9 @@ func TestDocumentDetailGoodReadsReviewsLazyPlaceholder(t *testing.T) { if placeholder.Length() != 1 { t.Fatal("expected successful rating to enable reviews on intersection only") } + if placeholder.Find("h2").Text() != "Goodreads user reviews" || placeholder.Find(`[aria-hidden="true"] h2`).Length() != 0 { + t.Fatal("the reviews heading must remain visible and accessible while reviews load") + } skeletons := placeholder.Find(".border-top") if skeletons.Length() != 2 || placeholder.Find(`div[aria-hidden="true"]`).Length() != 1 { t.Fatal("expected two decorative review-shaped skeletons") @@ -593,6 +790,43 @@ func TestDocumentDetailGoodReadsReviewsLazyPlaceholder(t *testing.T) { } } +func TestDocumentDetailGoodReadsReviewsHeader(t *testing.T) { + for _, tt := range []struct { + name string + source *fakeBookSource + slug string + want int + }{ + {name: "matched", source: &fakeBookSource{}, slug: "linked", want: 1}, + {name: "zero score", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "0.00"}}, slug: "linked", want: 1}, + {name: "unavailable", source: &fakeBookSource{err: biblioreads.ErrUnavailable}, slug: "linked", want: 1}, + {name: "disabled", slug: "linked"}, + {name: "unmatched", source: &fakeBookSource{}, slug: "unmatched"}, + {name: "not looked up", source: &fakeBookSource{}, slug: "unlinked"}, + } { + t.Run(tt.name, func(t *testing.T) { + app, _ := newGoodReadsTestApp(t, tt.source) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/"+tt.slug))) + if err != nil { + t.Fatal(err) + } + header := page.Find(`#document-reviews-` + tt.slug + ` h2 a`) + if header.Length() != tt.want { + t.Fatalf("expected %d reviews headings, got %d", tt.want, header.Length()) + } + if tt.want == 1 { + url, _ := header.Attr("href") + if header.Text() != "Goodreads user reviews" || url != "https://www.goodreads.com/book/show/12207444-la-reina-dragon" { + t.Fatal("expected a linked Goodreads reviews heading") + } + } + if tt.source != nil && (len(tt.source.ids) != 0 || len(tt.source.reviewCalls) != 0 || len(tt.source.queries) != 0) { + t.Fatal("rendering the heading must not make any API calls") + } + }) + } +} + func TestDocumentDetailGoodReadsReviews(t *testing.T) { source := &fakeBookSource{ book: biblioreads.BookResponse{Rating: "4.20", ResourceID: "work-id", LegacyBookID: 12207444}, @@ -651,6 +885,97 @@ func TestDocumentDetailGoodReadsReviews(t *testing.T) { } } +func TestDocumentReviewFractionalStars(t *testing.T) { + for _, tt := range []struct { + rating float64 + full int + empty int + width int + }{ + {rating: 0}, + {rating: 0.25, empty: 4, width: 25}, + {rating: 3.5, full: 3, empty: 1, width: 50}, + {rating: 4.25, full: 4, width: 25}, + {rating: 4.75, full: 4, width: 75}, + {rating: 4, full: 4, empty: 1}, + {rating: 5, full: 5}, + } { + t.Run(fmt.Sprint(tt.rating), func(t *testing.T) { + source := &fakeBookSource{ + book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}, + reviews: biblioreads.ReviewsResponse{Reviews: []biblioreads.Review{ + {Reviewer: biblioreads.Reviewer{Name: "Reader"}, Rating: tt.rating}, + }}, + } + app, _ := newGoodReadsTestApp(t, source) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews"))) + if err != nil { + t.Fatal(err) + } + stars := page.Find(`article span[role="img"]`) + if tt.rating == 0 { + if stars.Length() != 0 { + t.Fatal("unrated reviews must not display stars") + } + if page.Find("article header span.text-muted").Text() != "Not scored" { + t.Fatal("unrated reviews must display Not scored") + } + return + } + if page.Find("article header span.text-muted").Length() != 0 { + t.Fatal("scored reviews must not display the unrated label") + } + if label, _ := stars.Attr("aria-label"); label != fmt.Sprintf("%g/5", tt.rating) { + t.Fatalf("expected exact accessible rating, got %q", label) + } + if stars.ChildrenFiltered("i.bi-star-fill").Length() != tt.full || + stars.ChildrenFiltered("i.bi-star").Length() != tt.empty || + stars.Children().Length() != 5 { + t.Fatalf("unexpected full/empty star counts: %s", stars.Text()) + } + partial := stars.Find(".review-star-fill") + if tt.width == 0 { + if partial.Length() != 0 { + t.Fatal("whole ratings must not include a partial star") + } + } else if style, _ := partial.Attr("style"); style != fmt.Sprintf("width: %d%%", tt.width) || + stars.Find(`.review-star[aria-hidden="true"]`).Length() != 1 { + t.Fatalf("expected a %d%% decorative star fill, got %q", tt.width, style) + } + }) + } +} + +func TestDocumentReviewNotScoredTranslations(t *testing.T) { + source := &fakeBookSource{ + book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}, + reviews: biblioreads.ReviewsResponse{Reviews: []biblioreads.Review{ + {Reviewer: biblioreads.Reviewer{Name: "Reader"}, Text: "A review without a rating"}, + }}, + } + app, _ := newGoodReadsTestApp(t, source) + for _, tt := range []struct { + lang string + text string + }{ + {"en", "Not scored"}, + {"de", "Nicht bewertet"}, + {"es", "Sin puntuar"}, + {"fr", "Non noté"}, + {"ru", "Не оценено"}, + } { + t.Run(tt.lang, func(t *testing.T) { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews?l="+tt.lang))) + if err != nil { + t.Fatal(err) + } + if got := page.Find("article header span.text-muted").Text(); got != tt.text { + t.Fatalf("expected %q, got %q", tt.text, got) + } + }) + } +} + func TestDocumentDetailGoodReadsReviewsAvailability(t *testing.T) { for _, tt := range []struct { name string @@ -663,7 +988,7 @@ func TestDocumentDetailGoodReadsReviewsAvailability(t *testing.T) { }{ {name: "disabled", slug: "linked", status: http.StatusOK}, {name: "unmatched", source: &fakeBookSource{}, slug: "unmatched", status: http.StatusOK}, - {name: "no score", source: &fakeBookSource{book: biblioreads.BookResponse{ResourceID: "work", LegacyBookID: 1}}, slug: "linked", status: http.StatusOK}, + {name: "no score", source: &fakeBookSource{book: biblioreads.BookResponse{ResourceID: "work", LegacyBookID: 1}}, slug: "linked", status: http.StatusOK, text: "Goodreads user reviews"}, {name: "book unavailable", source: &fakeBookSource{err: errors.New("unavailable")}, slug: "linked", status: http.StatusInternalServerError}, {name: "missing identifiers", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "4"}}, slug: "linked", status: http.StatusInternalServerError}, {name: "reviews unavailable", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}, reviewErr: errors.New("unavailable")}, slug: "linked", status: http.StatusInternalServerError, calls: 1}, diff --git a/internal/webserver/embedded/css/display.css b/internal/webserver/embedded/css/display.css index c09a4ba0..7de2cd82 100644 --- a/internal/webserver/embedded/css/display.css +++ b/internal/webserver/embedded/css/display.css @@ -352,6 +352,31 @@ main .form-control:focus { line-clamp: 5; } +.document-reviews-loading { + display: none; +} + +.document-reviews.htmx-request .document-reviews-content { + display: none; +} + +.document-reviews.htmx-request .document-reviews-loading { + display: block; +} + +.review-star { + position: relative; + display: inline-block; +} + +.review-star-fill { + position: absolute; + top: 0; + left: 0; + overflow: hidden; + white-space: nowrap; +} + #searchbox-container .input-group .form-control:focus { box-shadow: none !important; } diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index 8f3bbf6a..df1d1231 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -17,6 +17,8 @@ "Illustrated": "Illustriert" "Illustrations": "Illustrationen" "Only illustrated": "Nur illustrierte" +"Goodreads score": "Goodreads-Bewertung" +"Clear Goodreads score": "Goodreads-Bewertung löschen" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Lesezeit berechnet mit einer Geschwindigkeit von %s Wörtern pro Minute. Sie können die Lesegeschwindigkeit in Ihrem Profil ändern." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Lesezeit berechnet mit einer Geschwindigkeit von %s Wörtern pro Minute. Melden Sie sich an, um diesen Wert anzupassen." "This filter only applies to EPUB documents": "Dieser Filter gilt nur für EPUB-Dokumente" @@ -415,4 +417,5 @@ "Show less": "Weniger anzeigen" "No user reviews available in the current language": "Keine Nutzerrezensionen in der aktuellen Sprache verfügbar" "No score": "Keine Bewertung" +"Not scored": "Nicht bewertet" "%s ratings": "%s Bewertungen" diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index 900f026c..cfa42fc3 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -17,6 +17,8 @@ "Illustrated": "Ilustrado" "Illustrations": "Ilustraciones" "Only illustrated": "Solo ilustrados" +"Goodreads score": "Puntuación de Goodreads" +"Clear Goodreads score": "Borrar puntuación de Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Tiempo de lectura calculado en base a una velocidad de %s palabras por minuto. Puedes cambiarla en tu perfil." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Tiempo de lectura calculado en base a una velocidad de %s palabras por minuto. Inicia sesión para personalizar este valor." "This filter only applies to EPUB documents": "Este filtro solo se aplica a documentos EPUB" @@ -411,4 +413,5 @@ "Show less": "Mostrar menos" "No user reviews available in the current language": "No hay reseñas de usuarios disponibles en el idioma actual" "No score": "Sin puntuación" +"Not scored": "Sin puntuar" "%s ratings": "%s valoraciones" diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index cd701672..691de1e4 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -17,6 +17,8 @@ "Illustrated": "Illustré" "Illustrations": "Illustrations" "Only illustrated": "Seulement illustrés" +"Goodreads score": "Note Goodreads" +"Clear Goodreads score": "Effacer la note Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Temps de lecture calculé avec une vitesse de %s mots par minute. Vous pouvez modifier votre vitesse de lecture dans votre profil." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Temps de lecture calculé avec une vitesse de %s mots par minute. Connectez-vous pour personnaliser cette valeur." "This filter only applies to EPUB documents": "Ce filtre s'applique uniquement aux documents EPUB" @@ -413,4 +415,5 @@ "Show less": "Afficher moins" "No user reviews available in the current language": "Aucun avis utilisateur disponible dans la langue actuelle" "No score": "Aucune note" +"Not scored": "Non noté" "%s ratings": "%s notes" diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index 18a59f9e..0f30d58c 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -17,6 +17,8 @@ "Illustrated": "Иллюстрированное" "Illustrations": "Иллюстрации" "Only illustrated": "Только с иллюстрациями" +"Goodreads score": "Оценка Goodreads" +"Clear Goodreads score": "Очистить оценку Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Время чтения рассчитано при скорости %s слов в минуту. Изменить скорость чтения можно в профиле." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Время чтения рассчитано при скорости %s слов в минуту. Войдите, чтобы изменить это значение." "This filter only applies to EPUB documents": "Этот фильтр применяется только к документам EPUB" @@ -407,4 +409,5 @@ "Show less": "Показать меньше" "No user reviews available in the current language": "Нет отзывов пользователей на текущем языке" "No score": "Нет оценки" +"Not scored": "Не оценено" "%s ratings": "Оценок: %s" diff --git a/internal/webserver/embedded/views/document/detail.html b/internal/webserver/embedded/views/document/detail.html index 62459519..1e22afdf 100644 --- a/internal/webserver/embedded/views/document/detail.html +++ b/internal/webserver/embedded/views/document/detail.html @@ -120,7 +120,11 @@

    Words

    {{end}} {{if .ReviewsEnabled}} -
    +
    + {{if .DocumentServiceLink.URL}} + {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .DocumentServiceLink.Source "BookURL" .DocumentServiceLink.URL "Slug" .Document.Slug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .Lang}} + {{end}} +
    {{end}} diff --git a/internal/webserver/embedded/views/partials/docs-list-item.html b/internal/webserver/embedded/views/partials/docs-list-item.html index 94beb0ed..b4ad96f3 100644 --- a/internal/webserver/embedded/views/partials/docs-list-item.html +++ b/internal/webserver/embedded/views/partials/docs-list-item.html @@ -71,7 +71,7 @@

    {{end}} - {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount}} + {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount "ShowGoodReadsScore" true}} {{ if .Document.Subjects }}
      diff --git a/internal/webserver/embedded/views/partials/document-metadata-filters.html b/internal/webserver/embedded/views/partials/document-metadata-filters.html index 6ea67bad..ce1efb55 100644 --- a/internal/webserver/embedded/views/partials/document-metadata-filters.html +++ b/internal/webserver/embedded/views/partials/document-metadata-filters.html @@ -19,6 +19,32 @@ {{end}} +
      + {{t .Lang "Goodreads score"}} +
      +
      +
      + {{$goodReadsRatingFrom := ""}} + {{if .SearchFields}} + {{$goodReadsRatingFrom = .SearchFields.GoodReadsRatingFrom}} + {{end}} +
      + + +
      + {{$goodReadsRatingTo := ""}} + {{if .SearchFields}} + {{$goodReadsRatingTo = .SearchFields.GoodReadsRatingTo}} + {{end}} +
      + + +
      + +
      +
      +
      +
      {{t .Lang "Subjects"}}
      diff --git a/internal/webserver/embedded/views/partials/document-metadata.html b/internal/webserver/embedded/views/partials/document-metadata.html index 9426e7da..88f2a549 100644 --- a/internal/webserver/embedded/views/partials/document-metadata.html +++ b/internal/webserver/embedded/views/partials/document-metadata.html @@ -15,6 +15,9 @@ {{ if .ReviewsEnabled }}
      • {{ end }} + {{if and .ShowGoodReadsScore .Document.GoodReadsRating .Document.GoodReadsBookURL}} + {{template "partials/document-score" dict "Lang" .Lang "Source" "Goodreads" "Rating" .Document.GoodReadsRating "Scored" (gt .Document.GoodReadsRatingValue 0.0) "URL" (printf "https://www.goodreads.com%s" .Document.GoodReadsBookURL)}} + {{end}} {{ if and .IllustratedMinAmount (ge .Document.Illustrations .IllustratedMinAmount) }}
    • {{t .Lang "Illustrated"}}
      • {{ end }} diff --git a/internal/webserver/embedded/views/partials/document-reviews-header.html b/internal/webserver/embedded/views/partials/document-reviews-header.html new file mode 100644 index 00000000..30aa41f9 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-reviews-header.html @@ -0,0 +1,14 @@ +
      +

      {{t .Lang "%s user reviews" .Source}}

      + {{if gt (len .SupportedLanguages) 1}} + + + {{end}} +
      diff --git a/internal/webserver/embedded/views/partials/document-reviews.html b/internal/webserver/embedded/views/partials/document-reviews.html index a2c04858..b8b4242c 100644 --- a/internal/webserver/embedded/views/partials/document-reviews.html +++ b/internal/webserver/embedded/views/partials/document-reviews.html @@ -1,11 +1,13 @@ -
      -

      {{t .Lang "%s user reviews" .Source}}

      +
      + {{template "partials/document-reviews-header" .}} +
      + {{if not .Unavailable}} {{range $i, $review := .Reviews}}
      +
      diff --git a/internal/webserver/embedded/views/partials/document-score.html b/internal/webserver/embedded/views/partials/document-score.html index e679451f..0aa15f71 100644 --- a/internal/webserver/embedded/views/partials/document-score.html +++ b/internal/webserver/embedded/views/partials/document-score.html @@ -1,8 +1,9 @@ -
    • {{if .Rating}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}{{else}}{{t .Lang "No score"}}{{end}}
      • -{{if and .Rating .DocumentSlug}} -
      +
    • {{if .Scored}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}{{else}}{{t .Lang "Not scored"}}{{end}}
      • +{{if and .URL .DocumentSlug}} +
      + {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .Source "BookURL" .URL "Slug" .DocumentSlug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .ReviewLanguage}} + {{if .Rating}} {{end}} diff --git a/internal/webserver/model/highlight_repository.go b/internal/webserver/model/highlight_repository.go index d7339a15..58204213 100644 --- a/internal/webserver/model/highlight_repository.go +++ b/internal/webserver/model/highlight_repository.go @@ -141,6 +141,8 @@ func hasSearchFields(sf index.SearchFields) bool { sf.EstReadTimeTo != 0 || sf.PagesFrom != 0 || sf.PagesTo != 0 || + sf.GoodReadsRatingFrom != 0 || + sf.GoodReadsRatingTo != 0 || sf.IllustratedOnly } @@ -191,6 +193,14 @@ func matchesSearchFields(doc AugmentedDocument, sf index.SearchFields, illustrat return false } + if sf.GoodReadsRatingFrom > 0 || sf.GoodReadsRatingTo > 0 { + if doc.GoodReadsRatingValue <= 0 || + (sf.GoodReadsRatingFrom > 0 && doc.GoodReadsRatingValue < sf.GoodReadsRatingFrom) || + (sf.GoodReadsRatingTo > 0 && doc.GoodReadsRatingValue > sf.GoodReadsRatingTo) { + return false + } + } + if sf.IllustratedOnly && illustratedMinAmount > 0 && doc.Illustrations < illustratedMinAmount { return false } diff --git a/internal/webserver/model/highlight_search_test.go b/internal/webserver/model/highlight_search_test.go new file mode 100644 index 00000000..e77e127d --- /dev/null +++ b/internal/webserver/model/highlight_search_test.go @@ -0,0 +1,32 @@ +package model + +import ( + "testing" + + "github.com/svera/coreander/v5/internal/index" +) + +func TestMatchesSearchFieldsGoodReadsRatingRange(t *testing.T) { + tests := []struct { + name string + score float64 + from float64 + to float64 + want bool + }{ + {name: "inside range", score: 4.25, from: 4.2, to: 4.3, want: true}, + {name: "below minimum", score: 4.1, from: 4.2, to: 4.3}, + {name: "above maximum", score: 4.4, from: 4.2, to: 4.3}, + {name: "unrated", from: 4.2, to: 4.3}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + doc := AugmentedDocument{Document: index.Document{GoodReadsRatingValue: tt.score}} + fields := index.SearchFields{GoodReadsRatingFrom: tt.from, GoodReadsRatingTo: tt.to} + if got := matchesSearchFields(doc, fields, 0); got != tt.want { + t.Errorf("matchesSearchFields() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index 15ee46fc..5b0e58da 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -145,6 +145,40 @@ func TestUnifiedSearch(t *testing.T) { } } +func TestGoodReadsScoreFilter(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), webserver.Config{}) + + response, err := app.Test(mustGetRequest(t, "/search?type=documents&goodreads-score-from=4.2&goodreads-score-to=4.8")) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("Expected status %d, received %d", http.StatusOK, response.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + for _, field := range []struct { + name string + value string + }{ + {name: "goodreads-score-from", value: "4.2"}, + {name: "goodreads-score-to", value: "4.8"}, + } { + input := doc.Find(`input[name="` + field.name + `"]`) + if input.Length() == 0 { + t.Errorf("Expected Goodreads score filter %q", field.name) + continue + } + if got, _ := input.First().Attr("value"); got != field.value { + t.Errorf("Expected %s value %q, got %q", field.name, field.value, got) + } + } +} + // TestSimilarDocumentsFullPage checks that navigating to a document's // /similar route (the "Similar documents" > "See all" link) renders the same // full search UI as a regular search, scoped to that document, with any diff --git a/internal/webserver/service_links_test.go b/internal/webserver/service_links_test.go index 8c018d29..024ddec0 100644 --- a/internal/webserver/service_links_test.go +++ b/internal/webserver/service_links_test.go @@ -45,12 +45,12 @@ func TestAlternativeServiceLinks(t *testing.T) { t.Fatalf("%s: unexpected Goodreads branding", path) } } - for _, path := range []string{"/documents/linked/score", "/documents/linked/reviews"} { + for path, wantLinks := range map[string]int{"/documents/linked/score": 2, "/documents/linked/reviews": 1} { page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) if err != nil { t.Fatal(err) } - if page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`).Length() != 1 { + if page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`).Length() != wantLinks { t.Fatalf("%s: BiblioReads must retain its Goodreads document URL regardless of the author link provider", path) } } From 669eed550033872843794815a45d39671d8f2098 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Sat, 31 Oct 2026 11:30:32 +0100 Subject: [PATCH 11/50] Simplify --- internal/datasource/biblioreads/client.go | 16 +- internal/index/bleve_goodreads_write.go | 133 ++++++---- internal/index/bleve_goodreads_write_test.go | 244 +++++++++++++++++- internal/reviews/biblioreads.go | 15 +- .../controller/document/controller.go | 1 + .../webserver/controller/document/score.go | 11 +- .../controller/document/update_goodreads.go | 35 +++ .../webserver/document_goodreads_auth_test.go | 58 +++++ internal/webserver/document_goodreads_test.go | 148 +++++++++++ .../webserver/embedded/translations/de.yml | 4 + .../webserver/embedded/translations/es.yml | 4 + .../webserver/embedded/translations/fr.yml | 4 + .../webserver/embedded/translations/ru.yml | 4 + .../partials/document-goodreads-form.html | 10 + .../partials/document-reviews-skeleton.html | 16 ++ .../views/partials/document-reviews.html | 17 +- .../views/partials/document-score.html | 27 +- internal/webserver/routes.go | 1 + 18 files changed, 634 insertions(+), 114 deletions(-) create mode 100644 internal/webserver/controller/document/update_goodreads.go create mode 100644 internal/webserver/document_goodreads_auth_test.go create mode 100644 internal/webserver/embedded/views/partials/document-goodreads-form.html create mode 100644 internal/webserver/embedded/views/partials/document-reviews-skeleton.html diff --git a/internal/datasource/biblioreads/client.go b/internal/datasource/biblioreads/client.go index 03443653..f03801c5 100644 --- a/internal/datasource/biblioreads/client.go +++ b/internal/datasource/biblioreads/client.go @@ -11,7 +11,6 @@ import ( "math/rand/v2" "net/http" "net/url" - "sort" "strings" "sync" "time" @@ -327,22 +326,19 @@ func (c *Client) acquireInstance(ctx context.Context, tried map[string]bool) (st } c.mu.Lock() start := rand.IntN(len(c.instances)) - candidates := make([]string, 0, len(c.instances)) + instance := "" now := time.Now() for i := range c.instances { - instance := c.instances[(start+i)%len(c.instances)] - if !tried[instance] && !now.Before(c.cooldownUntil[instance]) { - candidates = append(candidates, instance) + candidate := c.instances[(start+i)%len(c.instances)] + if !tried[candidate] && !now.Before(c.cooldownUntil[candidate]) && + (instance == "" || c.nextRequest[candidate].Before(c.nextRequest[instance])) { + instance = candidate } } - sort.SliceStable(candidates, func(i, j int) bool { - return c.nextRequest[candidates[i]].Before(c.nextRequest[candidates[j]]) - }) - if len(candidates) == 0 { + if instance == "" { c.mu.Unlock() return "", fmt.Errorf("%w: no untried instance is outside its cooldown", ErrUnavailable) } - instance := candidates[0] wait := time.Until(c.nextRequest[instance]) if wait <= 0 { c.nextRequest[instance] = time.Now().Add(c.requestInterval) diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go index 5d611179..5244ec55 100644 --- a/internal/index/bleve_goodreads_write.go +++ b/internal/index/bleve_goodreads_write.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "math" + "net/url" "regexp" "strconv" "strings" @@ -38,6 +39,10 @@ var goodReadsSubtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+` var goodReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) +var goodReadsBookPath = regexp.MustCompile(`^/book/show/[1-9][0-9]*(?:[.-][A-Za-z0-9._~-]+)?/?$`) + +var ErrInvalidGoodReadsBookURL = errors.New("invalid Goodreads book URL") + func normalizeGoodReadsName(value string) string { var result strings.Builder for _, r := range norm.NFD.String(strings.ToLower(value)) { @@ -112,6 +117,38 @@ func goodReadsMainTitle(value string) string { return normalizeGoodReadsName(value) } +func goodReadsTitleMatchScore(candidate, title, fullTitle, mainTitle string) int { + candidateTitle := goodReadsTitle(candidate) + if candidateTitle == "" { + return 0 + } + if candidateTitle == title { + if normalizeGoodReadsName(candidate) == fullTitle { + return 3 + } + return 2 + } + if goodReadsMainTitle(candidate) == title || mainTitle == candidateTitle || + goodReadsTitle(goodReadsParentheticalTitle.ReplaceAllString(candidate, "")) == title { + return 1 + } + return 0 +} + +func preferGoodReadsBook(candidate, best biblioreads.SearchResult, score, bestScore int) bool { + if bestScore == 0 { + return true + } + if score == bestScore && candidate.ID > 0 && best.ID > 0 && candidate.ID != best.ID { + return candidate.ID < best.ID + } + hasRating, bestHasRating := hasGoodReadsRating(candidate.Rating), hasGoodReadsRating(best.Rating) + if hasRating != bestHasRating { + return hasRating + } + return score > bestScore +} + func matchingGoodReadsBook(document Document, results []biblioreads.SearchResult) (biblioreads.SearchResult, bool) { title := goodReadsTitle(document.Title) if title == "" { @@ -125,41 +162,20 @@ func matchingGoodReadsBook(document Document, results []biblioreads.SearchResult } var best biblioreads.SearchResult bestScore := 0 - bestHasRating := false for _, result := range results { if result.BookURL == "" { continue } - resultTitle := goodReadsTitle(result.Title) - score := 1 - if resultTitle == title { - score = 2 - if normalizeGoodReadsName(result.Title) == fullTitle { - score = 3 - } - } else if resultTitle == "" || (goodReadsMainTitle(result.Title) != title && mainTitle != resultTitle && - goodReadsTitle(goodReadsParentheticalTitle.ReplaceAllString(result.Title, "")) != title) { + score := goodReadsTitleMatchScore(result.Title, title, fullTitle, mainTitle) + if score == 0 { continue } author := strings.ReplaceAll(normalizeGoodReadsName(result.Author), " ", "") if author == "" { continue } - if _, matches := authors[author]; matches { - hasRating := hasGoodReadsRating(result.Rating) - prefer := bestScore == 0 - if !prefer && score == bestScore && result.ID > 0 && best.ID > 0 && result.ID != best.ID { - prefer = result.ID < best.ID - } else if !prefer { - if hasRating != bestHasRating { - prefer = hasRating - } else { - prefer = score > bestScore - } - } - if prefer { - best, bestScore, bestHasRating = result, score, hasRating - } + if _, matches := authors[author]; matches && preferGoodReadsBook(result, best, score, bestScore) { + best, bestScore = result, score } } return best, bestScore > 0 @@ -286,49 +302,57 @@ func (b *BleveIndexer) GoodReadsBook(ctx context.Context, document Document) (bi return b.enrichGoodReads(ctx, document) } -// StoreGoodReadsBookIDs persists identifiers fetched for bookURL, provided the -// document still has the same metadata and Goodreads match as when it was read. -func (b *BleveIndexer) StoreGoodReadsBookIDs(document Document, bookURL, resourceID string, legacyBookID int) error { - if bookURL == "" || resourceID == "" || legacyBookID <= 0 { - return fmt.Errorf("invalid GoodReads identifiers for document %s", document.ID) +// UpdateGoodReadsBookURL saves an administrator's correction without retaining +// the previous book's rating or review identifiers. +func (b *BleveIndexer) UpdateGoodReadsBookURL(documentID, rawURL string) error { + bookURL, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || len(rawURL) > 2048 || bookURL.User != nil || + (bookURL.Scheme != "https" && bookURL.Scheme != "http") || + (bookURL.Host != "www.goodreads.com" && bookURL.Host != "goodreads.com") || + !goodReadsBookPath.MatchString(bookURL.Path) { + return ErrInvalidGoodReadsBookURL } + path := strings.TrimSuffix(bookURL.Path, "/") + b.documentsMu.Lock() defer b.documentsMu.Unlock() - current, found, err := b.documentByIndexIDLocked(document.ID) + current, found, err := b.documentByIndexIDLocked(documentID) if err != nil { - return fmt.Errorf("error reading document %s for GoodReads identifiers: %w", document.ID, err) + return fmt.Errorf("reading document %s for Goodreads correction: %w", documentID, err) } if !found { return ErrDocumentNotFound } - if goodReadsQuery(current) != goodReadsQuery(document) || current.GoodReadsBookURL != bookURL { - return fmt.Errorf("GoodReads match changed for document %s", document.ID) - } - if current.GoodReadsResourceID == resourceID && current.GoodReadsLegacyBookID == legacyBookID { + if current.GoodReadsBookURL == path { return nil } - current.GoodReadsResourceID = resourceID - current.GoodReadsLegacyBookID = legacyBookID + current.GoodReadsBookURL = path + current.GoodReadsRating = "" + current.GoodReadsRatingValue = 0 + current.GoodReadsResourceID = "" + current.GoodReadsLegacyBookID = 0 + current.GoodReadsEnriched = true if err := b.documentsIdx.Index(current.ID, current); err != nil { - return fmt.Errorf("error storing GoodReads identifiers for document %s: %w", document.ID, err) + return fmt.Errorf("saving Goodreads correction for document %s: %w", documentID, err) } return nil } -// StoreGoodReadsRating persists the current rating for the same document and -// Goodreads match that were read before fetching the book details. -func (b *BleveIndexer) StoreGoodReadsRating(document Document, bookURL, rating string) error { +// StoreGoodReadsDetails persists a nonempty rating and complete review identifiers, +// provided the document still has the metadata and match read before the fetch. +func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, book biblioreads.BookResponse) error { if bookURL == "" { return fmt.Errorf("invalid GoodReads book URL for document %s", document.ID) } - if rating == "" { + hasIDs := book.ResourceID != "" && book.LegacyBookID > 0 + if book.Rating == "" && !hasIDs { return nil } b.documentsMu.Lock() defer b.documentsMu.Unlock() current, found, err := b.documentByIndexIDLocked(document.ID) if err != nil { - return fmt.Errorf("error reading document %s for GoodReads rating: %w", document.ID, err) + return fmt.Errorf("error reading document %s for GoodReads details: %w", document.ID, err) } if !found { return ErrDocumentNotFound @@ -336,14 +360,25 @@ func (b *BleveIndexer) StoreGoodReadsRating(document Document, bookURL, rating s if goodReadsQuery(current) != goodReadsQuery(document) || current.GoodReadsBookURL != bookURL { return fmt.Errorf("GoodReads match changed for document %s", document.ID) } - ratingValue := parseGoodReadsRating(rating, document.ID) - if current.GoodReadsRating == rating && current.GoodReadsRatingValue == ratingValue { + changed := false + if book.Rating != "" { + ratingValue := parseGoodReadsRating(book.Rating, document.ID) + if current.GoodReadsRating != book.Rating || current.GoodReadsRatingValue != ratingValue { + current.GoodReadsRating = book.Rating + current.GoodReadsRatingValue = ratingValue + changed = true + } + } + if hasIDs && (current.GoodReadsResourceID != book.ResourceID || current.GoodReadsLegacyBookID != book.LegacyBookID) { + current.GoodReadsResourceID = book.ResourceID + current.GoodReadsLegacyBookID = book.LegacyBookID + changed = true + } + if !changed { return nil } - current.GoodReadsRating = rating - current.GoodReadsRatingValue = ratingValue if err := b.documentsIdx.Index(current.ID, current); err != nil { - return fmt.Errorf("error storing GoodReads rating for document %s: %w", document.ID, err) + return fmt.Errorf("error storing GoodReads details for document %s: %w", document.ID, err) } return nil } diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go index aeb2c3f0..7ef8517c 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/index/bleve_goodreads_write_test.go @@ -282,7 +282,7 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { t.Fatal(err) } stale.TextRankEnriched = false - if err := idx.StoreGoodReadsBookIDs(stale, wantGoodReadsBookURL, "work-id", 12207444); err != nil { + if err := idx.StoreGoodReadsDetails(stale, wantGoodReadsBookURL, biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}); err != nil { t.Fatal(err) } stale.TextRankWords = []string{"dragon"} @@ -451,6 +451,59 @@ func TestGoodReadsBookReturnsErrorWhenUnavailable(t *testing.T) { } } +func TestGoodReadsTitleMatchScore(t *testing.T) { + for _, tt := range []struct { + name string + title string + candidate string + want int + }{ + {name: "normalized exact", title: "La promesa", candidate: "LA PROMESA!", want: 3}, + {name: "series suffix", title: "La promesa", candidate: "La promesa (Series, #1)", want: 2}, + {name: "subtitle", title: "La promesa", candidate: "La promesa: Una historia", want: 1}, + {name: "document subtitle", title: "La promesa: Una historia", candidate: "La promesa", want: 1}, + {name: "parenthetical", title: "La promesa", candidate: "La promesa (The Promise)", want: 1}, + {name: "different subtitles", title: "La promesa: Una historia", candidate: "La promesa: Otra historia"}, + {name: "standalone volume", title: "La promesa", candidate: "La promesa (#1)"}, + {name: "unrelated", title: "La promesa", candidate: "Otro libro"}, + {name: "empty candidate", title: "La promesa"}, + } { + t.Run(tt.name, func(t *testing.T) { + got := goodReadsTitleMatchScore(tt.candidate, goodReadsTitle(tt.title), + normalizeGoodReadsName(tt.title), goodReadsMainTitle(tt.title)) + if got != tt.want { + t.Fatalf("title match score = %d, want %d", got, tt.want) + } + }) + } +} + +func TestPreferGoodReadsBook(t *testing.T) { + for _, tt := range []struct { + name string + candidate, best biblioreads.SearchResult + score, bestScore int + want bool + }{ + {name: "first match", score: 1, want: true}, + {name: "lower ID before rating for equal strength", candidate: biblioreads.SearchResult{ID: 1}, best: biblioreads.SearchResult{ID: 2, Rating: "4"}, score: 3, bestScore: 3, want: true}, + {name: "higher ID loses despite rating", candidate: biblioreads.SearchResult{ID: 2, Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 3, bestScore: 3}, + {name: "rating before strength for different strength", candidate: biblioreads.SearchResult{ID: 2, Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 1, bestScore: 3, want: true}, + {name: "stronger unrated loses to rated", best: biblioreads.SearchResult{Rating: "4"}, score: 3, bestScore: 1}, + {name: "strength with same rating status", score: 3, bestScore: 1, want: true}, + {name: "weaker match loses with same rating status", score: 1, bestScore: 3}, + {name: "unknown ID falls back to rating", candidate: biblioreads.SearchResult{Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 3, bestScore: 3, want: true}, + {name: "same ID falls back to rating", candidate: biblioreads.SearchResult{ID: 1, Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 3, bestScore: 3, want: true}, + {name: "complete tie retains first match", candidate: biblioreads.SearchResult{ID: 1, Rating: "4"}, best: biblioreads.SearchResult{ID: 1, Rating: "3"}, score: 3, bestScore: 3}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := preferGoodReadsBook(tt.candidate, tt.best, tt.score, tt.bestScore); got != tt.want { + t.Fatalf("candidate preference = %v, want %v", got, tt.want) + } + }) + } +} + func TestMatchingGoodReadsBook(t *testing.T) { correct := biblioreads.SearchResult{ Title: "El señor de los anillos", Author: "J.R.R. Tolkien", @@ -717,8 +770,8 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { } } -func TestStoreGoodReadsBookIDs(t *testing.T) { - for _, scenario := range []string{"linked", "newly linked", "changed metadata", "changed match", "deleted", "invalid identifiers"} { +func TestStoreGoodReadsDetails(t *testing.T) { + for _, scenario := range []string{"linked", "newly linked", "changed metadata", "changed match", "deleted", "partial identifiers"} { t.Run(scenario, func(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) doc := Document{ @@ -738,7 +791,7 @@ func TestStoreGoodReadsBookIDs(t *testing.T) { doc.Title = "Another title" case "changed match": doc.GoodReadsBookURL = "/book/show/2-other" - case "invalid identifiers": + case "partial identifiers": resourceID = "" } if scenario != "deleted" { @@ -746,10 +799,14 @@ func TestStoreGoodReadsBookIDs(t *testing.T) { t.Fatal(err) } } - err := idx.StoreGoodReadsBookIDs(stale, wantGoodReadsBookURL, resourceID, 12207444) - wantError := scenario != "linked" && scenario != "newly linked" + + book := biblioreads.BookResponse{Rating: "4.12", ResourceID: resourceID, LegacyBookID: 12207444} + writes := &goodReadsWriteCountingIndex{countedGoodReadsIndex: idx.documentsIdx} + idx.documentsIdx = writes + err := idx.StoreGoodReadsDetails(stale, wantGoodReadsBookURL, book) + wantError := scenario != "linked" && scenario != "newly linked" && scenario != "partial identifiers" if (err != nil) != wantError { - t.Fatalf("StoreGoodReadsBookIDs error = %v, want error %v", err, wantError) + t.Fatalf("StoreGoodReadsDetails error = %v, want error %v", err, wantError) } if scenario == "deleted" { if !errors.Is(err, ErrDocumentNotFound) { @@ -762,20 +819,185 @@ func TestStoreGoodReadsBookIDs(t *testing.T) { t.Fatal(err) } if wantError { - if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { - t.Fatalf("unexpected identifiers persisted: %+v", stored) + if writes.count != 0 { + t.Fatal("stale details must not cause any index writes") + } + if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || stored.GoodReadsRating != "" { + t.Fatalf("unexpected details persisted: %+v", stored) } return } - if stored.GoodReadsResourceID != resourceID || stored.GoodReadsLegacyBookID != 12207444 { + wantLegacyBookID := 12207444 + if scenario == "partial identifiers" { + wantLegacyBookID = 0 + } + if stored.GoodReadsResourceID != resourceID || stored.GoodReadsLegacyBookID != wantLegacyBookID { t.Fatalf("identifiers not persisted: %+v", stored) } + if stored.GoodReadsRating != book.Rating || stored.GoodReadsRatingValue != 4.12 { + t.Fatalf("rating not persisted with identifiers: %+v", stored) + } if !stored.GoodReadsEnriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("existing enrichment was lost: %+v", stored) } - if err := idx.StoreGoodReadsBookIDs(stale, wantGoodReadsBookURL, resourceID, 12207444); err != nil { + if err := idx.StoreGoodReadsDetails(stale, wantGoodReadsBookURL, book); err != nil { t.Fatalf("repeated store failed: %v", err) } + if writes.count != 1 { + t.Fatalf("expected one combined write and no write for unchanged details, got %d", writes.count) + } + }) + } +} + +type countedGoodReadsIndex interface { + bleve.Index +} + +type goodReadsWriteCountingIndex struct { + countedGoodReadsIndex + count int +} + +func (i *goodReadsWriteCountingIndex) Index(id string, data any) error { + i.count++ + return i.countedGoodReadsIndex.Index(id, data) +} + +func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { + for _, tt := range []struct { + name string + book biblioreads.BookResponse + wantRating string + wantWork string + wantID int + }{ + {name: "rating only", book: biblioreads.BookResponse{Rating: "4.12"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "identifiers only", book: biblioreads.BookResponse{ResourceID: "new-work", LegacyBookID: 2}, wantRating: "3.82", wantWork: "new-work", wantID: 2}, + {name: "missing work", book: biblioreads.BookResponse{Rating: "4.12", LegacyBookID: 2}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "missing edition", book: biblioreads.BookResponse{Rating: "4.12", ResourceID: "new-work"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "empty response", wantRating: "3.82", wantWork: "old-work", wantID: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + idx := newGoodReadsTestIndexer(t, nil) + doc := Document{ + ID: goodReadsTestID, GoodReadsBookURL: wantGoodReadsBookURL, + GoodReadsRating: "3.82", GoodReadsRatingValue: 3.82, + GoodReadsResourceID: "old-work", GoodReadsLegacyBookID: 1, + } + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, tt.book); err != nil { + t.Fatal(err) + } + stored, err := idx.documentByIndexID(doc.ID) + if err != nil { + t.Fatal(err) + } + if stored.GoodReadsRating != tt.wantRating || stored.GoodReadsRatingValue != parseGoodReadsRating(tt.wantRating, doc.ID) || + stored.GoodReadsResourceID != tt.wantWork || stored.GoodReadsLegacyBookID != tt.wantID { + t.Fatalf("partial details changed fields that were not supplied: %+v", stored) + } + }) + } +} + +func TestUpdateGoodReadsBookURL(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + {name: "numeric ID", url: "https://www.goodreads.com/book/show/1368314", want: "/book/show/1368314"}, + {name: "dotted slug", url: "https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente", want: "/book/show/1368314.El_Cercano_Oriente"}, + {name: "hyphenated slug", url: "https://www.goodreads.com/book/show/243175949-el-cercano-oriente", want: "/book/show/243175949-el-cercano-oriente"}, + {name: "normalize shared URL", url: " http://goodreads.com/book/show/1368314/?from_search=true#reviews ", want: "/book/show/1368314"}, + {name: "same URL", url: "https://www.goodreads.com" + wantGoodReadsBookURL, want: wantGoodReadsBookURL}, + {name: "empty"}, + {name: "relative", url: "/book/show/1368314"}, + {name: "other domain", url: "https://example.com/book/show/1368314"}, + {name: "deceptive domain", url: "https://www.goodreads.com.example.com/book/show/1368314"}, + {name: "credentials", url: "https://reader@www.goodreads.com/book/show/1368314"}, + {name: "port", url: "https://www.goodreads.com:4000/book/show/1368314"}, + {name: "wrong scheme", url: "ftp://www.goodreads.com/book/show/1368314"}, + {name: "author", url: "https://www.goodreads.com/author/show/16667.Isaac_Asimov"}, + {name: "non-numeric ID", url: "https://www.goodreads.com/book/show/not-a-book"}, + {name: "zero ID", url: "https://www.goodreads.com/book/show/0"}, + {name: "extra path", url: "https://www.goodreads.com/book/show/1368314/other"}, + {name: "encoded path separator", url: "https://www.goodreads.com/book/show/1368314%2Fother"}, + {name: "malformed URL", url: "https://%zz/book/show/1368314"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := &fakeGoodReadsSource{results: goodReadsTestResults} + idx := newGoodReadsTestIndexer(t, source) + doc := Document{ + ID: goodReadsTestID, Slug: "la-reina-dragon", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + TextRankWords: []string{"dragon"}, + GoodReadsBookURL: wantGoodReadsBookURL, + GoodReadsRating: "3.82", + GoodReadsRatingValue: 3.82, + GoodReadsResourceID: "old-work", + GoodReadsLegacyBookID: 12207444, + GoodReadsEnriched: true, + } + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + err := idx.UpdateGoodReadsBookURL(doc.ID, tt.url) + if tt.want == "" { + if !errors.Is(err, ErrInvalidGoodReadsBookURL) { + t.Fatalf("expected invalid URL error, got %v", err) + } + } else if err != nil { + t.Fatal(err) + } + stored, err := idx.documentByIndexID(doc.ID) + if err != nil { + t.Fatal(err) + } + if tt.want == "" || tt.want == doc.GoodReadsBookURL { + if stored.GoodReadsBookURL != doc.GoodReadsBookURL || + stored.GoodReadsRating != doc.GoodReadsRating || + stored.GoodReadsResourceID != doc.GoodReadsResourceID || + stored.GoodReadsLegacyBookID != doc.GoodReadsLegacyBookID { + t.Fatalf("invalid or unchanged URL modified stored data: %+v", stored) + } + return + } + if stored.GoodReadsBookURL != tt.want || !stored.GoodReadsEnriched || + stored.GoodReadsRating != "" || stored.GoodReadsRatingValue != 0 || + stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { + t.Fatalf("correction must clear old rating and review identifiers: %+v", stored) + } + if stored.Title != doc.Title || stored.Slug != doc.Slug || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { + t.Fatalf("correction changed unrelated document data: %+v", stored) + } + if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, biblioreads.BookResponse{Rating: "4.5"}); err == nil { + t.Fatal("stale rating write must not overwrite corrected book") + } + if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, biblioreads.BookResponse{ResourceID: "old-work", LegacyBookID: 12207444}); err == nil { + t.Fatal("stale review identifiers must not overwrite corrected book") + } + got, err := idx.enrichGoodReads(context.Background(), doc) + if err != nil || got.BookURL != tt.want { + t.Fatalf("in-flight enrichment must keep the correction: %+v, %v", got, err) + } + if err := idx.EnrichGoodReads(10); err != nil { + t.Fatal(err) + } + if len(source.Queries()) != 0 { + t.Fatal("corrected book must not be automatically rematched") + } }) } } + +func TestUpdateGoodReadsBookURLMissingDocument(t *testing.T) { + idx := newGoodReadsTestIndexer(t, nil) + if err := idx.UpdateGoodReadsBookURL("missing", "https://www.goodreads.com/book/show/1368314"); !errors.Is(err, ErrDocumentNotFound) { + t.Fatalf("expected missing-document error, got %v", err) + } +} diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index ebfbdd01..21b41f29 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -25,8 +25,7 @@ type BiblioReadsClient interface { type GoodReadsIndex interface { GoodReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) - StoreGoodReadsBookIDs(document index.Document, bookURL, resourceID string, legacyBookID int) error - StoreGoodReadsRating(document index.Document, bookURL, rating string) error + StoreGoodReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error } type BiblioReads struct { @@ -60,17 +59,13 @@ func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblio if err != nil { return book, fmt.Errorf("retrieving Goodreads book %s: %w", id, err) } + details := book if book.Rating != "" && document.GoodReadsRatingValue > 0 && isZeroRating(book.Rating) { book.Rating = document.GoodReadsRating - } else if book.Rating != "" { - if err := b.index.StoreGoodReadsRating(document, document.GoodReadsBookURL, book.Rating); err != nil { - log.Printf("Error saving Goodreads rating for %s: %s", document.ID, err) - } + details.Rating = "" } - if book.ResourceID != "" && book.LegacyBookID > 0 { - if err := b.index.StoreGoodReadsBookIDs(document, document.GoodReadsBookURL, book.ResourceID, book.LegacyBookID); err != nil { - log.Printf("Error saving Goodreads identifiers for %s: %s", document.ID, err) - } + if err := b.index.StoreGoodReadsDetails(document, document.GoodReadsBookURL, details); err != nil { + log.Printf("Error saving Goodreads details for %s: %s", document.ID, err) } return book, nil } diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index 33ab92fc..55fa43e6 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -44,6 +44,7 @@ type IdxReaderWriter interface { SameSeries(slug string, quantity int) (result.Paginated[[]index.Document], error) NewFile(fileName string, contents []byte) (string, error) DeleteDocument(slug string) error + UpdateGoodReadsBookURL(documentID, bookURL string) error Documents(slugs []string) (map[string]index.Document, error) Languages() ([]string, error) Subjects() (map[string][]string, error) diff --git a/internal/webserver/controller/document/score.go b/internal/webserver/controller/document/score.go index b4c53e54..cb48f907 100644 --- a/internal/webserver/controller/document/score.go +++ b/internal/webserver/controller/document/score.go @@ -6,6 +6,7 @@ import ( "time" "github.com/gofiber/fiber/v3" + "github.com/svera/coreander/v5/internal/webserver/model" ) const reviewsTimeout = 30 * time.Second @@ -26,9 +27,15 @@ func (d *Controller) Score(c fiber.Ctx) error { ctx, cancel := context.WithTimeout(c.Context(), reviewsTimeout) defer cancel() score, err := d.reviewProvider.Score(ctx, document) + session, _ := c.Locals("Session").(model.Session) + canEditGoodReads := session.Role == model.RoleAdmin && score.Source == "Goodreads" if err != nil { log.Printf("Error retrieving score for %s: %s", document.ID, err) - return c.SendString("") + if !canEditGoodReads { + return c.SendString("") + } + score.Value, score.Count = "", "" + score.URL = d.documentServiceLink(document).URL } return c.Render("partials/document-score", fiber.Map{ "Lang": c.Locals("Lang"), @@ -40,5 +47,7 @@ func (d *Controller) Score(c fiber.Ctx) error { "Scored": score.HasScore(), "URL": score.URL, "DocumentSlug": document.Slug, + "CanEditGoodReads": canEditGoodReads, + "ScoreUnavailable": err != nil, }) } diff --git a/internal/webserver/controller/document/update_goodreads.go b/internal/webserver/controller/document/update_goodreads.go new file mode 100644 index 00000000..17c01d38 --- /dev/null +++ b/internal/webserver/controller/document/update_goodreads.go @@ -0,0 +1,35 @@ +package document + +import ( + "errors" + "log" + + "github.com/gofiber/fiber/v3" + "github.com/svera/coreander/v5/internal/index" +) + +func (d *Controller) UpdateGoodReads(c fiber.Ctx) error { + document, err := d.idx.Document(c.Params("slug")) + if err != nil { + log.Printf("Error reading document for Goodreads correction: %s", err) + return fiber.ErrInternalServerError + } + if document.Slug == "" { + return fiber.ErrNotFound + } + bookURL := c.FormValue("bookURL") + if err := d.idx.UpdateGoodReadsBookURL(document.ID, bookURL); err != nil { + if errors.Is(err, index.ErrInvalidGoodReadsBookURL) { + return c.Status(fiber.StatusUnprocessableEntity).Render("partials/document-goodreads-form", fiber.Map{ + "Lang": c.Locals("Lang"), "Slug": document.Slug, "BookURL": bookURL, "InvalidURL": true, + }) + } + if errors.Is(err, index.ErrDocumentNotFound) { + return fiber.ErrNotFound + } + log.Printf("Error correcting Goodreads link for %s: %s", document.ID, err) + return fiber.ErrInternalServerError + } + c.Set("HX-Refresh", "true") + return c.SendStatus(fiber.StatusNoContent) +} diff --git a/internal/webserver/document_goodreads_auth_test.go b/internal/webserver/document_goodreads_auth_test.go new file mode 100644 index 00000000..b66a7ba7 --- /dev/null +++ b/internal/webserver/document_goodreads_auth_test.go @@ -0,0 +1,58 @@ +package webserver_test + +import ( + "net/http" + "net/url" + "testing" + + "github.com/svera/coreander/v5/internal/webserver" + "github.com/svera/coreander/v5/internal/webserver/infrastructure" + "github.com/svera/coreander/v5/internal/webserver/model" +) + +func TestDocumentUpdateGoodReadsRequiresAdmin(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs(testLibraryDir), webserver.Config{}) + t.Cleanup(func() { _ = app.Shutdown() }) + users := &model.UserRepository{DB: db} + if err := users.Create(&model.User{ + Uuid: "regular-goodreads-user", Name: "Regular user", Username: "regular-goodreads", + Email: "regular@example.com", Password: model.Hash("regular-password"), + Role: model.RoleRegular, WordsPerMinute: 250, + }); err != nil { + t.Fatal(err) + } + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + regularCookie, err := login(app, "regular@example.com", "regular-password", t) + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + cookie *http.Cookie + status int + }{ + {name: "anonymous", cookie: &http.Cookie{}, status: http.StatusForbidden}, + {name: "regular user", cookie: regularCookie, status: http.StatusForbidden}, + {name: "administrator", cookie: adminCookie, status: http.StatusNoContent}, + } { + t.Run(tt.name, func(t *testing.T) { + response, err := putRequest(url.Values{ + "bookURL": {"https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente"}, + }, tt.cookie, app, "/documents/john-doe-test-epub/goodreads", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, tt.status, t) + }) + } +} diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 6eb48fd3..4a935be6 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "sync" "testing" @@ -446,6 +447,153 @@ func TestDocumentDetailGoodReadsRating(t *testing.T) { } } +func TestDocumentGoodReadsEditor(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{Rating: "3.82"}} + app, _ := newGoodReadsTestApp(t, source) + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + cookie *http.Cookie + want int + }{ + {name: "anonymous", cookie: &http.Cookie{}}, + {name: "administrator", cookie: adminCookie, want: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + response, err := getRequest(tt.cookie, app, "/documents/linked/score", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + control := page.Find(`li a[data-bs-toggle="collapse"][href="#edit-goodreads-linked"]`) + if control.Length() != tt.want { + t.Fatalf("expected %d edit links, got %d", tt.want, control.Length()) + } + if tt.want == 0 { + if page.Find(`input[name="bookURL"]`).Length() != 0 { + t.Fatal("anonymous readers must not receive the editor") + } + return + } + if control.Text() != "Edit Goodreads link" || + page.Find(`#edit-goodreads-linked.collapse:not(.show)`).Length() != 1 || + page.Find(`form[hx-put="/documents/linked/goodreads"][hx-target="this"]`).Length() != 1 || + page.Find(`label[for="goodreads-book-url-linked"]`).Length() != 1 { + t.Fatal("expected accessible, initially collapsed Goodreads editor beside the score") + } + if value, _ := page.Find(`input[name="bookURL"]`).Attr("value"); value != "https://www.goodreads.com/book/show/12207444-la-reina-dragon" { + t.Fatalf("editor must be prefilled with the current link, got %q", value) + } + }) + } + + source.err = biblioreads.ErrUnavailable + response, err := getRequest(adminCookie, app, "/documents/linked/score", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(page.Find("li").Text(), "Unable to load Goodreads score") || + page.Find(`input[name="bookURL"]`).Length() != 1 { + t.Fatal("an unavailable score must not prevent administrators from correcting the link") + } +} + +func TestDocumentUpdateGoodReads(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "3.82", ResourceID: "old-work", LegacyBookID: 12207444, + }} + app, idx := newGoodReadsTestApp(t, source) + getBody(t, app, "/documents/linked/score") + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + response, err := putRequest(url.Values{ + "bookURL": {"https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente?from_search=true"}, + }, adminCookie, app, "/documents/linked/goodreads", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusNoContent, t) + if response.Header.Get("HX-Refresh") != "true" { + t.Fatal("successful correction must reload the detail page") + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if stored.GoodReadsBookURL != "/book/show/1368314.El_Cercano_Oriente" || + stored.GoodReadsRating != "" || stored.GoodReadsRatingValue != 0 || + stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || + !stored.GoodReadsEnriched { + t.Fatalf("expected correction persisted without stale rating/review identifiers: %+v", stored) + } + source.book = biblioreads.BookResponse{Rating: "4.2", ResourceID: "new-work", LegacyBookID: 1368314} + body := getBody(t, app, "/documents/linked/score") + if !strings.Contains(body, `href="https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente"`) || + source.ids[len(source.ids)-1] != "1368314.El_Cercano_Oriente" { + t.Fatal("subsequent scores must use the corrected book") + } + getBody(t, app, "/documents/linked/reviews") + if len(source.reviewCalls) != 1 || source.reviewCalls[0].resourceID != "new-work" || + source.reviewCalls[0].legacyBookID != 1368314 { + t.Fatalf("subsequent reviews must use the corrected book: %+v", source.reviewCalls) + } +} + +func TestDocumentUpdateGoodReadsValidation(t *testing.T) { + app, idx := newGoodReadsTestApp(t, nil) + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"", "https://example.org/book/show/123", "https://www.goodreads.com/author/show/16667"} { + response, err := putRequest(url.Values{"bookURL": {value}}, adminCookie, app, "/documents/linked/goodreads", t) + if err != nil { + t.Fatal(err) + } + mustReturnStatus(response, http.StatusUnprocessableEntity, t) + page, err := goquery.NewDocumentFromReader(response.Body) + response.Body.Close() + if err != nil { + t.Fatal(err) + } + if page.Find(`form#goodreads-link-form-linked input[aria-invalid="true"]`).Length() != 1 || + page.Find(`[role="alert"]`).Text() != "Enter a valid Goodreads book URL." { + t.Fatal("invalid URLs must render inline validation feedback") + } + if got, _ := page.Find(`input[name="bookURL"]`).Attr("value"); got != value { + t.Fatalf("invalid input should be retained for correction, got %q", got) + } + stored, err := idx.Document("linked") + if err != nil || stored.GoodReadsBookURL != "/book/show/12207444-la-reina-dragon" || stored.GoodReadsRating != "3.82" { + t.Fatalf("invalid input must not change the stored match: %+v, %v", stored, err) + } + } + response, err := putRequest(url.Values{ + "bookURL": {"https://www.goodreads.com/book/show/1368314"}, + }, adminCookie, app, "/documents/missing/goodreads", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusNotFound, t) +} + func TestDocumentDetailUpdatesStoredGoodReadsRating(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "4.12", ResourceID: "work-id", LegacyBookID: 12207444, diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index df1d1231..ca48d562 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -408,6 +408,10 @@ "This document is no longer available. Its offline copy has been removed.": "Dieses Dokument ist nicht mehr verfügbar. Seine Offline-Kopie wurde entfernt." "Reload document": "Dokument neu laden" "Goodreads rating": "Goodreads-Bewertung" +"Edit Goodreads link": "Goodreads-Link bearbeiten" +"Goodreads book URL": "Goodreads-Buch-URL" +"Enter a valid Goodreads book URL.": "Geben Sie eine gültige Goodreads-Buch-URL ein." +"Unable to load Goodreads score": "Goodreads-Bewertung konnte nicht geladen werden" "%s rating": "%s-Bewertung" "%s user reviews": "%s-Nutzerrezensionen" "Go to %s author page": "Zur Autorenseite auf %s" diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index cfa42fc3..8819a150 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -404,6 +404,10 @@ "This document is no longer available. Its offline copy has been removed.": "Este documento ya no está disponible. Se ha eliminado su copia sin conexión." "Reload document": "Recargar documento" "Goodreads rating": "Puntuación en Goodreads" +"Edit Goodreads link": "Editar enlace de Goodreads" +"Goodreads book URL": "URL del libro en Goodreads" +"Enter a valid Goodreads book URL.": "Introduce una URL válida de un libro en Goodreads." +"Unable to load Goodreads score": "No se pudo cargar la puntuación de Goodreads" "%s rating": "Puntuación en %s" "%s user reviews": "Reseñas de usuarios de %s" "Go to %s author page": "Ir a la página del autor en %s" diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index 691de1e4..81038ae5 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -406,6 +406,10 @@ "This document is no longer available. Its offline copy has been removed.": "Ce document n'est plus disponible. Sa copie hors connexion a été supprimée." "Reload document": "Recharger le document" "Goodreads rating": "Note Goodreads" +"Edit Goodreads link": "Modifier le lien Goodreads" +"Goodreads book URL": "URL du livre sur Goodreads" +"Enter a valid Goodreads book URL.": "Saisissez une URL de livre Goodreads valide." +"Unable to load Goodreads score": "Impossible de charger la note Goodreads" "%s rating": "Note %s" "%s user reviews": "Avis des utilisateurs de %s" "Go to %s author page": "Voir la page de l'auteur sur %s" diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index 0f30d58c..8072e96c 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -400,6 +400,10 @@ "This document is no longer available. Its offline copy has been removed.": "Этот документ больше недоступен. Его копия для чтения без подключения к сети удалена." "Reload document": "Перезагрузить документ" "Goodreads rating": "Рейтинг Goodreads" +"Edit Goodreads link": "Изменить ссылку Goodreads" +"Goodreads book URL": "URL книги на Goodreads" +"Enter a valid Goodreads book URL.": "Введите корректный URL книги на Goodreads." +"Unable to load Goodreads score": "Не удалось загрузить рейтинг Goodreads" "%s rating": "Рейтинг %s" "%s user reviews": "Отзывы пользователей %s" "Go to %s author page": "Перейти на страницу автора в %s" diff --git a/internal/webserver/embedded/views/partials/document-goodreads-form.html b/internal/webserver/embedded/views/partials/document-goodreads-form.html new file mode 100644 index 00000000..dcce5b66 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-goodreads-form.html @@ -0,0 +1,10 @@ + diff --git a/internal/webserver/embedded/views/partials/document-reviews-skeleton.html b/internal/webserver/embedded/views/partials/document-reviews-skeleton.html new file mode 100644 index 00000000..b1c14a15 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-reviews-skeleton.html @@ -0,0 +1,16 @@ +{{range 2}} +
      +
      +
      +
      + {{range 5}}{{end}} + +
      +
      +
      +
      +
      +
      +
      +
      +{{end}} diff --git a/internal/webserver/embedded/views/partials/document-reviews.html b/internal/webserver/embedded/views/partials/document-reviews.html index b8b4242c..fc8827a8 100644 --- a/internal/webserver/embedded/views/partials/document-reviews.html +++ b/internal/webserver/embedded/views/partials/document-reviews.html @@ -25,21 +25,6 @@ {{end}}
      diff --git a/internal/webserver/embedded/views/partials/document-score.html b/internal/webserver/embedded/views/partials/document-score.html index 0aa15f71..209bde66 100644 --- a/internal/webserver/embedded/views/partials/document-score.html +++ b/internal/webserver/embedded/views/partials/document-score.html @@ -1,25 +1,18 @@ -
    • {{if .Scored}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}{{else}}{{t .Lang "Not scored"}}{{end}}
      • +
    • + {{if .ScoreUnavailable}}{{t .Lang "Unable to load Goodreads score"}}{{else if .Scored}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}{{else}}{{t .Lang "Not scored"}}{{end}} + {{if .CanEditGoodReads}} + +
      + {{template "partials/document-goodreads-form" dict "Lang" .Lang "Slug" .DocumentSlug "BookURL" .URL}} +
      + {{end}} +
      • {{if and .URL .DocumentSlug}}
      {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .Source "BookURL" .URL "Slug" .DocumentSlug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .ReviewLanguage}} {{if .Rating}} {{end}}
      diff --git a/internal/webserver/routes.go b/internal/webserver/routes.go index 4aaa1086..0e4b7a65 100644 --- a/internal/webserver/routes.go +++ b/internal/webserver/routes.go @@ -98,6 +98,7 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se app.Get("/upload", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UploadForm) docsGroup.Post("/", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Upload) docsGroup.Delete("/:slug", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Delete) + docsGroup.Put("/:slug/goodreads", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UpdateGoodReads) // Authentication requirement is configurable for all routes below this middleware app.Use(configurableAuthentication) From 7b5c9e13f7c3bd967960db35eda13c1f8f6eb8e3 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Sun, 1 Nov 2026 20:06:19 +0100 Subject: [PATCH 12/50] Fix title matching --- internal/datasource/biblioreads/cache_test.go | 165 +++++ internal/datasource/biblioreads/client.go | 2 +- .../datasource/biblioreads/client_test.go | 636 ++---------------- .../datasource/biblioreads/failover_test.go | 396 +++++++++++ .../biblioreads/fixtures/search-books.json | 3 +- internal/datasource/biblioreads/model.go | 15 +- internal/index/author.go | 23 +- internal/index/bleve.go | 2 - internal/index/bleve_document_read.go | 41 +- internal/index/bleve_goodreads_write.go | 126 +++- internal/index/bleve_goodreads_write_test.go | 165 ++++- internal/index/document.go | 4 +- internal/index/goodreads_fields_test.go | 132 +++- internal/reviews/biblioreads.go | 2 +- .../webserver/controller/author/documents.go | 2 + internal/webserver/controller/search/parse.go | 4 + .../webserver/controller/search/parse_test.go | 40 ++ .../webserver/controller/search/search.go | 2 + .../webserver/controller/series/documents.go | 6 + .../controller/series/documents_test.go | 36 + internal/webserver/document_goodreads_test.go | 49 +- .../webserver/embedded/translations/de.yml | 2 + .../webserver/embedded/translations/es.yml | 2 + .../webserver/embedded/translations/fr.yml | 2 + .../webserver/embedded/translations/ru.yml | 2 + .../views/partials/document-metadata.html | 4 +- internal/webserver/search_test.go | 46 ++ 27 files changed, 1205 insertions(+), 704 deletions(-) create mode 100644 internal/datasource/biblioreads/cache_test.go create mode 100644 internal/datasource/biblioreads/failover_test.go create mode 100644 internal/webserver/controller/search/parse_test.go create mode 100644 internal/webserver/controller/series/documents_test.go diff --git a/internal/datasource/biblioreads/cache_test.go b/internal/datasource/biblioreads/cache_test.go new file mode 100644 index 00000000..a8547aa7 --- /dev/null +++ b/internal/datasource/biblioreads/cache_test.go @@ -0,0 +1,165 @@ +package biblioreads_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +func TestIncompleteBookResponsesAreNotCached(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92"}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + for range 2 { + book, err := client.Book(context.Background(), "5898652") + if err != nil || book.Rating != "3.92" { + t.Fatalf("expected available score even without review identifiers, got %+v, %v", book, err) + } + } + if hits.Load() != 2 { + t.Fatal("incomplete book details must not be cached") + } +} + +func TestReviewsEmptyResponsesAreNotCached(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"totalCount":0,"reviews":[]}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + for range 2 { + response, err := client.Reviews(context.Background(), "work", 1, "", "") + if err != nil || len(response.Reviews) != 0 { + t.Fatalf("expected valid empty result, got %+v, %v", response, err) + } + } + if hits.Load() != 2 { + t.Fatal("empty results must not hide reviews behind the response cache") + } +} + +func TestReviewsLanguageCache(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + var payload struct { + LanguageCode string `json:"languageCode"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Errorf("decode request: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + if payload.LanguageCode != "es" && payload.LanguageCode != "en" { + t.Errorf("unexpected language %q", payload.LanguageCode) + w.WriteHeader(http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(biblioreads.ReviewsResponse{ + Version: "BiblioReads v4.4.0", Status: "Received", StatusCode: http.StatusOK, TotalCount: 1, + Reviews: []biblioreads.Review{{Text: payload.LanguageCode}}, + }) + })) + defer server.Close() + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + for _, language := range []string{"es", "en", "es", "en"} { + response, err := client.Reviews(context.Background(), "work", 1, "", language) + if err != nil { + t.Fatal(err) + } + if len(response.Reviews) != 1 || response.Reviews[0].Text != language { + t.Fatalf("expected %s reviews, got %+v", language, response.Reviews) + } + } + if hits.Load() != 2 { + t.Fatalf("expected one cached response per language, got %d requests", hits.Load()) + } +} + +func TestCache(t *testing.T) { + t.Run("successful responses are cached per request", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92","resourceID":"work-id","legacyBookID":5898652,"title":"La Reina Dragon","genres":["Fantasy"],"totalCount":1,"reviews":[{"id":"review-1"}]}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + + first, err := client.Book(context.Background(), "1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + first.Genres[0] = "modified" + + second, err := client.Book(context.Background(), "1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hits.Load() != 1 { + t.Errorf("expected cached response to be used, got %d requests", hits.Load()) + } + if second.Title != "La Reina Dragon" || second.Genres[0] != "Fantasy" { + t.Errorf("expected an unmodified copy of the cached response, got %+v", second) + } + + if _, err := client.Book(context.Background(), "2"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.SearchBooks(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "", ""); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "token", ""); err != nil { + t.Fatalf("unexpected error: %v", err) + } + for range 2 { + if _, err := client.Author(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if hits.Load() != 6 { + t.Errorf("expected different requests not to share cache entries, got %d requests", hits.Load()) + } + }) + + t.Run("entries expire after the TTL", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, validBookResponse, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(100*time.Millisecond)) + + for range 3 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if hits.Load() != 1 { + t.Fatalf("expected 1 request before expiry, got %d", hits.Load()) + } + + time.Sleep(150 * time.Millisecond) + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hits.Load() != 2 { + t.Errorf("expected expired entry to be refreshed, got %d requests", hits.Load()) + } + }) + + t.Run("caching can be disabled", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, validBookResponse, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(0)) + + for range 3 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if hits.Load() != 3 { + t.Errorf("expected 3 requests with caching disabled, got %d", hits.Load()) + } + }) +} diff --git a/internal/datasource/biblioreads/client.go b/internal/datasource/biblioreads/client.go index f03801c5..bf6b8b1e 100644 --- a/internal/datasource/biblioreads/client.go +++ b/internal/datasource/biblioreads/client.go @@ -64,7 +64,7 @@ type options struct { type Option func(*options) // WithRequestInterval sets the minimum spacing between request starts to each instance. -// Zero disables spacing; the default is one second. +// Zero disables spacing. func WithRequestInterval(interval time.Duration) Option { return func(o *options) { o.requestInterval = max(interval, 0) diff --git a/internal/datasource/biblioreads/client_test.go b/internal/datasource/biblioreads/client_test.go index d46e97ad..556afdf2 100644 --- a/internal/datasource/biblioreads/client_test.go +++ b/internal/datasource/biblioreads/client_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "net/http" "net/http/httptest" "os" @@ -29,24 +28,45 @@ func newTestClient(opts ...biblioreads.Option) *biblioreads.Client { return biblioreads.NewClient(append([]biblioreads.Option{biblioreads.WithRequestInterval(0)}, opts...)...) } +func newFailoverClient(t *testing.T, handler http.Handler) *biblioreads.Client { + t.Helper() + first := httptest.NewServer(handler) + t.Cleanup(first.Close) + second := httptest.NewServer(handler) + t.Cleanup(second.Close) + return newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) +} + +func readFixture(t *testing.T, name string) []byte { + t.Helper() + contents, err := os.ReadFile("fixtures/" + name) + if err != nil { + t.Fatalf("couldn't read fixture %s: %v", name, err) + } + return contents +} + +func captureRequest(t *testing.T, w http.ResponseWriter, r *http.Request, captured *capturedRequest) bool { + t.Helper() + captured.method = r.Method + captured.path = r.URL.Path + captured.contentType = r.Header.Get("Content-Type") + if err := json.NewDecoder(r.Body).Decode(&captured.payload); err != nil { + t.Errorf("couldn't decode request payload: %v", err) + w.WriteHeader(http.StatusBadRequest) + return false + } + return true +} + func newMockServer(t *testing.T, status int, fixture string, captured *capturedRequest) *httptest.Server { t.Helper() + contents := readFixture(t, fixture) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - captured.method = r.Method - captured.path = r.URL.Path - captured.contentType = r.Header.Get("Content-Type") - if err := json.NewDecoder(r.Body).Decode(&captured.payload); err != nil { - t.Errorf("couldn't decode request payload: %v", err) - } - w.WriteHeader(status) - if fixture == "" { - _, _ = w.Write([]byte(`{"status":"Error - Invalid Query"}`)) + if !captureRequest(t, w, r, captured) { return } - contents, err := os.ReadFile("fixtures/" + fixture) - if err != nil { - t.Fatalf("couldn't read fixture %s: %v", fixture, err) - } + w.WriteHeader(status) _, _ = w.Write(contents) })) t.Cleanup(server.Close) @@ -96,6 +116,7 @@ func TestSearchBooks(t *testing.T) { got.BookURL != "/book/show/12207444-la-reina-dragon" || got.Author != "Alice Borchardt" || got.Rating != "3.82" || + got.RatingCount != "2285" || got.ID != 1 { t.Errorf("unexpected result: %+v", got) } @@ -104,98 +125,6 @@ func TestSearchBooks(t *testing.T) { } } -func TestBookResponseRecovery(t *testing.T) { - for _, tt := range []struct { - name string - status int - body string - }{ - {"JSON server error", http.StatusGatewayTimeout, `{"status":"Error - Book not found"}`}, - {"invalid success", http.StatusOK, `{}`}, - {"application error", http.StatusOK, `{"status":"Error","statusCode":503}`}, - {"missing score", http.StatusOK, `{"status":"Received","statusCode":200,"resourceID":"work-id","legacyBookID":5898652,"title":"Incomplete result"}`}, - {"missing review identifiers", http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92"}`}, - } { - t.Run(tt.name, func(t *testing.T) { - var hits atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if hits.Add(1) == 1 { - w.WriteHeader(tt.status) - _, _ = w.Write([]byte(tt.body)) - return - } - _, _ = w.Write([]byte(validBookResponse)) - }) - first := httptest.NewServer(handler) - defer first.Close() - second := httptest.NewServer(handler) - defer second.Close() - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) - for range 2 { - book, err := client.Book(context.Background(), "5898652-la-cinta-roja") - if err != nil || book.Rating != "3.92" || book.ResourceID != "work-id" || book.LegacyBookID != 5898652 { - t.Fatalf("expected complete book after failover, got %+v, %v", book, err) - } - if book.Title != "" { - t.Fatalf("fields from previous attempts must not survive failover, got title %q", book.Title) - } - } - if hits.Load() != 2 { - t.Fatalf("expected recovered book to be cached, got %d requests", hits.Load()) - } - }) - } -} - -func TestIncompleteBookResponsesAreNotCached(t *testing.T) { - var hits atomic.Int32 - server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92"}`, &hits) - client := newTestClient(biblioreads.WithInstances([]string{server.URL})) - for range 2 { - book, err := client.Book(context.Background(), "5898652") - if err != nil || book.Rating != "3.92" { - t.Fatalf("expected available score even without review identifiers, got %+v, %v", book, err) - } - } - if hits.Load() != 2 { - t.Fatal("incomplete book details must not be cached") - } -} - -func TestIncompleteBookFallbackPrefersScore(t *testing.T) { - scored := `{"status":"Received","statusCode":200,"rating":"3.92","title":"Scored result"}` - unscored := `{"status":"Received","statusCode":200,"resourceID":"work-id","legacyBookID":5898652,"title":"Unscored result"}` - for _, scoredFirst := range []bool{false, true} { - t.Run(fmt.Sprintf("scored first=%t", scoredFirst), func(t *testing.T) { - var hits atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body := unscored - if (hits.Add(1)%2 == 1) == scoredFirst { - body = scored - } - _, _ = w.Write([]byte(body)) - }) - first := httptest.NewServer(handler) - defer first.Close() - second := httptest.NewServer(handler) - defer second.Close() - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) - for range 2 { - book, err := client.Book(context.Background(), "5898652") - if err != nil || book.Rating != "3.92" || book.Title != "Scored result" { - t.Fatalf("expected scored fallback, got %+v, %v", book, err) - } - if book.ResourceID != "" || book.LegacyBookID != 0 { - t.Fatalf("fields from separate incomplete responses must not be combined, got %+v", book) - } - } - if hits.Load() != 4 { - t.Fatalf("expected both instances tried without caching incomplete results, got %d requests", hits.Load()) - } - }) - } -} - func TestBook(t *testing.T) { var captured capturedRequest server := newMockServer(t, http.StatusOK, "book-scraper.json", &captured) @@ -254,10 +183,7 @@ func TestAuthor(t *testing.T) { } assertRequest(t, captured, "/api/author/info", map[string]any{"queryURL": tt.url}) - fixture, err := os.ReadFile("fixtures/author-info.json") - if err != nil { - t.Fatalf("couldn't read author fixture: %v", err) - } + fixture := readFixture(t, "author-info.json") encoded, err := json.Marshal(res) if err != nil { t.Fatalf("couldn't encode author response: %v", err) @@ -341,97 +267,6 @@ func TestReviewsInvalidResponses(t *testing.T) { } } -func TestReviewsFailover(t *testing.T) { - fixture, err := os.ReadFile("fixtures/reviews-scraper.json") - if err != nil { - t.Fatal(err) - } - var hits atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if hits.Add(1) == 1 { - _, _ = w.Write([]byte(`{}`)) - return - } - _, _ = w.Write(fixture) - }) - first := httptest.NewServer(handler) - defer first.Close() - second := httptest.NewServer(handler) - defer second.Close() - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) - response, err := client.Reviews(context.Background(), "work", 1, "", "") - if err != nil || len(response.Reviews) == 0 || hits.Load() != 2 { - t.Fatalf("expected failover to working instance, got %d requests, %+v, %v", hits.Load(), response, err) - } -} - -func TestReviewsEmptyResponseRecovery(t *testing.T) { - fixture, err := os.ReadFile("fixtures/reviews-scraper.json") - if err != nil { - t.Fatal(err) - } - var hits atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if hits.Add(1) == 1 { - _, _ = w.Write([]byte(`{"status":"Received","statusCode":200,"totalCount":0,"reviews":[]}`)) - return - } - _, _ = w.Write(fixture) - }) - first := httptest.NewServer(handler) - defer first.Close() - second := httptest.NewServer(handler) - defer second.Close() - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) - for range 2 { - reviews, err := client.Reviews(context.Background(), "work", 1, "", "") - if err != nil || len(reviews.Reviews) == 0 { - t.Fatalf("expected reviews after empty first response, got %+v, %v", reviews, err) - } - } - if hits.Load() != 2 { - t.Fatalf("expected non-empty reviews to be cached, got %d requests", hits.Load()) - } -} - -func TestReviewsEmptyResponsesAreNotCached(t *testing.T) { - var hits atomic.Int32 - server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"totalCount":0,"reviews":[]}`, &hits) - client := newTestClient(biblioreads.WithInstances([]string{server.URL})) - for range 2 { - response, err := client.Reviews(context.Background(), "work", 1, "", "") - if err != nil || len(response.Reviews) != 0 { - t.Fatalf("expected valid empty result, got %+v, %v", response, err) - } - } - if hits.Load() != 2 { - t.Fatal("empty results must not hide reviews behind the response cache") - } -} - -func TestReviewsEmptyFallbackSurvivesInvalidResponse(t *testing.T) { - var hits atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body := `{"version":"BiblioReads v4.4.0","status":"Received","statusCode":200,"totalCount":0,"reviews":[],"nextPageToken":"empty-page"}` - if hits.Add(1) > 1 { - body = `{"reviews":[{"text":"Unfiltered review"}],"nextPageToken":"invalid-page"}` - } - _, _ = w.Write([]byte(body)) - }) - first := httptest.NewServer(handler) - defer first.Close() - second := httptest.NewServer(handler) - defer second.Close() - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) - response, err := client.Reviews(context.Background(), "work", 1, "", "es") - if err != nil || len(response.Reviews) != 0 || response.NextPageToken != "empty-page" { - t.Fatalf("expected original empty fallback without fields from the invalid response, got %+v, %v", response, err) - } - if hits.Load() != 2 { - t.Fatalf("expected both instances tried, got %d requests", hits.Load()) - } -} - func TestReviewsLanguageVersionCompatibility(t *testing.T) { for _, tt := range []struct { version string @@ -474,78 +309,6 @@ func TestReviewsLanguageVersionCompatibility(t *testing.T) { } } -func TestReviewsLanguageFailsOverFromLegacyInstance(t *testing.T) { - var hits atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - response := biblioreads.ReviewsResponse{ - Status: "Received", StatusCode: http.StatusOK, TotalCount: 1, - Reviews: []biblioreads.Review{{Text: "An unfiltered English review"}}, - } - if hits.Add(1) > 1 { - response.Version = "BiblioReads v4.4.0" - response.Reviews[0].Text = "Una reseña en español" - } - if err := json.NewEncoder(w).Encode(response); err != nil { - t.Errorf("encode response: %v", err) - } - }) - first := httptest.NewServer(handler) - defer first.Close() - second := httptest.NewServer(handler) - defer second.Close() - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) - for range 2 { - response, err := client.Reviews(context.Background(), "work", 1, "", "es") - if err != nil { - t.Fatal(err) - } - if len(response.Reviews) != 1 || response.Reviews[0].Text != "Una reseña en español" { - t.Fatalf("expected Spanish reviews rather than legacy unfiltered results, got %+v", response) - } - } - if hits.Load() != 2 { - t.Fatalf("expected failover followed by a cache hit, got %d requests", hits.Load()) - } -} - -func TestReviewsLanguageCache(t *testing.T) { - var hits atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - hits.Add(1) - var payload struct { - LanguageCode string `json:"languageCode"` - } - if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { - t.Errorf("decode request: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - if payload.LanguageCode != "es" && payload.LanguageCode != "en" { - t.Errorf("unexpected language %q", payload.LanguageCode) - w.WriteHeader(http.StatusBadRequest) - return - } - _ = json.NewEncoder(w).Encode(biblioreads.ReviewsResponse{ - Version: "BiblioReads v4.4.0", Status: "Received", StatusCode: http.StatusOK, TotalCount: 1, - Reviews: []biblioreads.Review{{Text: payload.LanguageCode}}, - }) - })) - defer server.Close() - client := newTestClient(biblioreads.WithInstances([]string{server.URL})) - for _, language := range []string{"es", "en", "es", "en"} { - response, err := client.Reviews(context.Background(), "work", 1, "", language) - if err != nil { - t.Fatal(err) - } - if len(response.Reviews) != 1 || response.Reviews[0].Text != language { - t.Fatalf("expected %s reviews, got %+v", language, response.Reviews) - } - } - if hits.Load() != 2 { - t.Fatalf("expected one cached response per language, got %d requests", hits.Load()) - } -} - func TestReviews(t *testing.T) { t.Run("fractional ratings", func(t *testing.T) { var hits atomic.Int32 @@ -599,12 +362,7 @@ func TestReviews(t *testing.T) { t.Run("next page", func(t *testing.T) { var captured capturedRequest server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - captured.method = r.Method - captured.path = r.URL.Path - captured.contentType = r.Header.Get("Content-Type") - if err := json.NewDecoder(r.Body).Decode(&captured.payload); err != nil { - t.Errorf("decode request: %v", err) - w.WriteHeader(http.StatusBadRequest) + if !captureRequest(t, w, r, &captured) { return } _, _ = w.Write([]byte(`{"version":"BiblioReads v4.4.0","status":"Received","statusCode":200,"totalCount":1,"reviews":[{"text":"Una reseña"}]}`)) @@ -626,8 +384,8 @@ func TestReviews(t *testing.T) { } func TestErrorStatus(t *testing.T) { - var captured capturedRequest - server := newMockServer(t, http.StatusNotFound, "", &captured) + var hits atomic.Int32 + server := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) if _, err := client.Book(context.Background(), "0"); err == nil { @@ -645,319 +403,3 @@ func newCountingServer(t *testing.T, status int, body string, hits *atomic.Int32 t.Cleanup(server.Close) return server } - -func TestFailover(t *testing.T) { - t.Run("unavailable instances are skipped", func(t *testing.T) { - var unavailableHits, rateLimitedHits, htmlHits, workingHits atomic.Int32 - unavailable := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &unavailableHits) - rateLimited := newCountingServer(t, http.StatusTooManyRequests, `{"status":"Too many requests"}`, &rateLimitedHits) - html := newCountingServer(t, http.StatusOK, "Parked domain", &htmlHits) - working := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92","title":"La Reina Dragon"}`, &workingHits) - down := httptest.NewServer(http.NotFoundHandler()) - down.Close() - - instances := []string{down.URL, unavailable.URL, rateLimited.URL, html.URL, working.URL} - for range 20 { - client := newTestClient(biblioreads.WithInstances(instances), biblioreads.WithCacheTTL(0)) - res, err := client.Book(context.Background(), "12207444") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.Title != "La Reina Dragon" { - t.Fatalf("unexpected title %q", res.Title) - } - } - if workingHits.Load() != 20 { - t.Errorf("expected working instance to be hit 20 times, got %d", workingHits.Load()) - } - }) - - t.Run("requests start at different instances", func(t *testing.T) { - var firstHits, secondHits atomic.Int32 - first := newCountingServer(t, http.StatusOK, validBookResponse, &firstHits) - second := newCountingServer(t, http.StatusOK, validBookResponse, &secondHits) - - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) - for range 100 { - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if firstHits.Load() == 0 || secondHits.Load() == 0 { - t.Errorf("expected load to be spread, got %d and %d hits", firstHits.Load(), secondHits.Load()) - } - }) - - t.Run("JSON error responses are not retried", func(t *testing.T) { - var hits atomic.Int32 - first := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) - second := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) - - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) - _, err := client.Book(context.Background(), "0") - if err == nil { - t.Fatal("expected error, got nil") - } - if errors.Is(err, biblioreads.ErrUnavailable) { - t.Errorf("expected a non-ErrUnavailable error, got %v", err) - } - if hits.Load() != 1 { - t.Errorf("expected a single request, got %d", hits.Load()) - } - }) - - t.Run("error when all instances fail", func(t *testing.T) { - var hits atomic.Int32 - first := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) - second := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) - - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) - if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { - t.Fatalf("expected ErrUnavailable, got %v", err) - } - if hits.Load() != 2 { - t.Errorf("expected both instances to be tried, got %d requests", hits.Load()) - } - }) - - t.Run("cancelled context stops failover", func(t *testing.T) { - var hits atomic.Int32 - first := newCountingServer(t, http.StatusOK, `{}`, &hits) - second := newCountingServer(t, http.StatusOK, `{}`, &hits) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) - if _, err := client.Book(ctx, "1"); err == nil { - t.Fatal("expected error, got nil") - } - if hits.Load() != 0 { - t.Errorf("expected no requests, got %d", hits.Load()) - } - }) -} - -func TestCooldown(t *testing.T) { - t.Run("failing instance is skipped on later requests", func(t *testing.T) { - var badHits, goodHits atomic.Int32 - bad := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &badHits) - good := newCountingServer(t, http.StatusOK, validBookResponse, &goodHits) - - client := newTestClient(biblioreads.WithInstances([]string{bad.URL, good.URL}), biblioreads.WithCacheTTL(0)) - for range 20 { - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if badHits.Load() > 1 { - t.Errorf("expected failing instance to be hit at most once, got %d", badHits.Load()) - } - if goodHits.Load() != 20 { - t.Errorf("expected working instance to be hit 20 times, got %d", goodHits.Load()) - } - }) - - t.Run("instance is used again after Retry-After", func(t *testing.T) { - var limitedHits, otherHits atomic.Int32 - limited := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if limitedHits.Add(1) == 1 { - w.Header().Set("Retry-After", "1") - w.WriteHeader(http.StatusTooManyRequests) - return - } - _, _ = w.Write([]byte(validBookResponse)) - })) - t.Cleanup(limited.Close) - other := newCountingServer(t, http.StatusOK, validBookResponse, &otherHits) - - client := newTestClient(biblioreads.WithInstances([]string{limited.URL, other.URL}), biblioreads.WithCacheTTL(0)) - for limitedHits.Load() == 0 { - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - for range 20 { - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if limitedHits.Load() != 1 { - t.Fatalf("expected rate limited instance to be skipped during cooldown, got %d hits", limitedHits.Load()) - } - - time.Sleep(1100 * time.Millisecond) - for range 20 { - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if limitedHits.Load() == 1 { - t.Error("expected rate limited instance to be used again after its cooldown") - } - }) - - t.Run("respects cooldown when all instances are cooling down", func(t *testing.T) { - var hits atomic.Int32 - flaky := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if hits.Add(1) == 1 { - w.Header().Set("Retry-After", "1") - w.WriteHeader(http.StatusTooManyRequests) - return - } - _, _ = w.Write([]byte(validBookResponse)) - })) - t.Cleanup(flaky.Close) - - client := newTestClient(biblioreads.WithInstances([]string{flaky.URL}), biblioreads.WithCacheTTL(0)) - if _, err := client.Book(context.Background(), "1"); err == nil { - t.Fatal("expected error on first request, got nil") - } - if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { - t.Fatalf("expected unavailable during cooldown, got %v", err) - } - time.Sleep(1100 * time.Millisecond) - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("expected cooling down instance to be probed, got %v", err) - } - }) - - t.Run("JSON client error responses don't trigger cooldown", func(t *testing.T) { - var firstHits, secondHits atomic.Int32 - notFound := `{"status":"Error - Book not found"}` - first := newCountingServer(t, http.StatusNotFound, notFound, &firstHits) - second := newCountingServer(t, http.StatusNotFound, notFound, &secondHits) - - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) - for range 20 { - if _, err := client.Book(context.Background(), "0"); err == nil { - t.Fatal("expected error, got nil") - } - } - if firstHits.Load()+secondHits.Load() != 20 { - t.Errorf("expected one request per call, got %d", firstHits.Load()+secondHits.Load()) - } - if firstHits.Load() == 0 || secondHits.Load() == 0 { - t.Errorf("expected no instance to be benched, got %d and %d hits", firstHits.Load(), secondHits.Load()) - } - }) - - t.Run("cancelled requests don't trigger cooldown", func(t *testing.T) { - var firstHits, secondHits atomic.Int32 - first := newCountingServer(t, http.StatusOK, validBookResponse, &firstHits) - second := newCountingServer(t, http.StatusOK, validBookResponse, &secondHits) - - client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - for range 5 { - _, _ = client.Book(ctx, "1") - } - for range 40 { - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if firstHits.Load() == 0 || secondHits.Load() == 0 { - t.Errorf("expected no instance to be benched, got %d and %d hits", firstHits.Load(), secondHits.Load()) - } - }) -} - -func TestCache(t *testing.T) { - t.Run("successful responses are cached per request", func(t *testing.T) { - var hits atomic.Int32 - server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92","resourceID":"work-id","legacyBookID":5898652,"title":"La Reina Dragon","genres":["Fantasy"],"totalCount":1,"reviews":[{"id":"review-1"}]}`, &hits) - client := newTestClient(biblioreads.WithInstances([]string{server.URL})) - - first, err := client.Book(context.Background(), "1") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - first.Genres[0] = "modified" - - second, err := client.Book(context.Background(), "1") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if hits.Load() != 1 { - t.Errorf("expected cached response to be used, got %d requests", hits.Load()) - } - if second.Title != "La Reina Dragon" || second.Genres[0] != "Fantasy" { - t.Errorf("expected an unmodified copy of the cached response, got %+v", second) - } - - if _, err := client.Book(context.Background(), "2"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err := client.SearchBooks(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "", ""); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "token", ""); err != nil { - t.Fatalf("unexpected error: %v", err) - } - for range 2 { - if _, err := client.Author(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if hits.Load() != 6 { - t.Errorf("expected different requests not to share cache entries, got %d requests", hits.Load()) - } - }) - - t.Run("entries expire after the TTL", func(t *testing.T) { - var hits atomic.Int32 - server := newCountingServer(t, http.StatusOK, validBookResponse, &hits) - client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(100*time.Millisecond)) - - for range 3 { - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if hits.Load() != 1 { - t.Fatalf("expected 1 request before expiry, got %d", hits.Load()) - } - - time.Sleep(150 * time.Millisecond) - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if hits.Load() != 2 { - t.Errorf("expected expired entry to be refreshed, got %d requests", hits.Load()) - } - }) - - t.Run("errors put the instance into cooldown", func(t *testing.T) { - var hits atomic.Int32 - server := newCountingServer(t, http.StatusGatewayTimeout, `{"status":"Error - Book not found"}`, &hits) - client := newTestClient(biblioreads.WithInstances([]string{server.URL})) - - for range 3 { - if _, err := client.Book(context.Background(), "0"); err == nil { - t.Fatal("expected error, got nil") - } - } - if hits.Load() != 1 { - t.Errorf("expected subsequent requests to respect cooldown, got %d requests", hits.Load()) - } - }) - - t.Run("caching can be disabled", func(t *testing.T) { - var hits atomic.Int32 - server := newCountingServer(t, http.StatusOK, validBookResponse, &hits) - client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(0)) - - for range 3 { - if _, err := client.Book(context.Background(), "1"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if hits.Load() != 3 { - t.Errorf("expected 3 requests with caching disabled, got %d", hits.Load()) - } - }) -} diff --git a/internal/datasource/biblioreads/failover_test.go b/internal/datasource/biblioreads/failover_test.go new file mode 100644 index 00000000..9bc46a90 --- /dev/null +++ b/internal/datasource/biblioreads/failover_test.go @@ -0,0 +1,396 @@ +package biblioreads_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +func TestBookResponseRecovery(t *testing.T) { + for _, tt := range []struct { + name string + status int + body string + }{ + {"JSON server error", http.StatusGatewayTimeout, `{"status":"Error - Book not found"}`}, + {"invalid success", http.StatusOK, `{}`}, + {"application error", http.StatusOK, `{"status":"Error","statusCode":503}`}, + {"missing score", http.StatusOK, `{"status":"Received","statusCode":200,"resourceID":"work-id","legacyBookID":5898652,"title":"Incomplete result"}`}, + {"missing review identifiers", http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92"}`}, + } { + t.Run(tt.name, func(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + w.WriteHeader(tt.status) + _, _ = w.Write([]byte(tt.body)) + return + } + _, _ = w.Write([]byte(validBookResponse)) + }) + client := newFailoverClient(t, handler) + for range 2 { + book, err := client.Book(context.Background(), "5898652-la-cinta-roja") + if err != nil || book.Rating != "3.92" || book.ResourceID != "work-id" || book.LegacyBookID != 5898652 { + t.Fatalf("expected complete book after failover, got %+v, %v", book, err) + } + if book.Title != "" { + t.Fatalf("fields from previous attempts must not survive failover, got title %q", book.Title) + } + } + if hits.Load() != 2 { + t.Fatalf("expected recovered book to be cached, got %d requests", hits.Load()) + } + }) + } +} + +func TestIncompleteBookFallbackPrefersScore(t *testing.T) { + scored := `{"status":"Received","statusCode":200,"rating":"3.92","title":"Scored result"}` + unscored := `{"status":"Received","statusCode":200,"resourceID":"work-id","legacyBookID":5898652,"title":"Unscored result"}` + for _, scoredFirst := range []bool{false, true} { + t.Run(fmt.Sprintf("scored first=%t", scoredFirst), func(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := unscored + if (hits.Add(1)%2 == 1) == scoredFirst { + body = scored + } + _, _ = w.Write([]byte(body)) + }) + client := newFailoverClient(t, handler) + for range 2 { + book, err := client.Book(context.Background(), "5898652") + if err != nil || book.Rating != "3.92" || book.Title != "Scored result" { + t.Fatalf("expected scored fallback, got %+v, %v", book, err) + } + if book.ResourceID != "" || book.LegacyBookID != 0 { + t.Fatalf("fields from separate incomplete responses must not be combined, got %+v", book) + } + } + if hits.Load() != 4 { + t.Fatalf("expected both instances tried without caching incomplete results, got %d requests", hits.Load()) + } + }) + } +} + +func TestReviewsResponseRecovery(t *testing.T) { + fixture := readFixture(t, "reviews-scraper.json") + for _, tt := range []struct { + name string + body string + }{ + {name: "invalid response", body: `{}`}, + {name: "empty response", body: `{"status":"Received","statusCode":200,"totalCount":0,"reviews":[]}`}, + } { + t.Run(tt.name, func(t *testing.T) { + var hits atomic.Int32 + client := newFailoverClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + _, _ = w.Write([]byte(tt.body)) + return + } + _, _ = w.Write(fixture) + })) + for range 2 { + reviews, err := client.Reviews(context.Background(), "work", 1, "", "") + if err != nil || len(reviews.Reviews) == 0 { + t.Fatalf("expected reviews after failover, got %+v, %v", reviews, err) + } + } + if hits.Load() != 2 { + t.Fatalf("expected failover followed by a cache hit, got %d requests", hits.Load()) + } + }) + } +} + +func TestReviewsEmptyFallbackSurvivesInvalidResponse(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := `{"version":"BiblioReads v4.4.0","status":"Received","statusCode":200,"totalCount":0,"reviews":[],"nextPageToken":"empty-page"}` + if hits.Add(1) > 1 { + body = `{"reviews":[{"text":"Unfiltered review"}],"nextPageToken":"invalid-page"}` + } + _, _ = w.Write([]byte(body)) + }) + client := newFailoverClient(t, handler) + response, err := client.Reviews(context.Background(), "work", 1, "", "es") + if err != nil || len(response.Reviews) != 0 || response.NextPageToken != "empty-page" { + t.Fatalf("expected original empty fallback without fields from the invalid response, got %+v, %v", response, err) + } + if hits.Load() != 2 { + t.Fatalf("expected both instances tried, got %d requests", hits.Load()) + } +} + +func TestReviewsLanguageFailsOverFromLegacyInstance(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response := biblioreads.ReviewsResponse{ + Status: "Received", StatusCode: http.StatusOK, TotalCount: 1, + Reviews: []biblioreads.Review{{Text: "An unfiltered English review"}}, + } + if hits.Add(1) > 1 { + response.Version = "BiblioReads v4.4.0" + response.Reviews[0].Text = "Una reseña en español" + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Errorf("encode response: %v", err) + } + }) + client := newFailoverClient(t, handler) + for range 2 { + response, err := client.Reviews(context.Background(), "work", 1, "", "es") + if err != nil { + t.Fatal(err) + } + if len(response.Reviews) != 1 || response.Reviews[0].Text != "Una reseña en español" { + t.Fatalf("expected Spanish reviews rather than legacy unfiltered results, got %+v", response) + } + } + if hits.Load() != 2 { + t.Fatalf("expected failover followed by a cache hit, got %d requests", hits.Load()) + } +} + +func TestFailover(t *testing.T) { + t.Run("unavailable instances are skipped", func(t *testing.T) { + var unavailableHits, rateLimitedHits, htmlHits, workingHits atomic.Int32 + unavailable := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &unavailableHits) + rateLimited := newCountingServer(t, http.StatusTooManyRequests, `{"status":"Too many requests"}`, &rateLimitedHits) + html := newCountingServer(t, http.StatusOK, "Parked domain", &htmlHits) + working := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92","title":"La Reina Dragon"}`, &workingHits) + down := httptest.NewServer(http.NotFoundHandler()) + down.Close() + + instances := []string{down.URL, unavailable.URL, rateLimited.URL, html.URL, working.URL} + for range 20 { + client := newTestClient(biblioreads.WithInstances(instances), biblioreads.WithCacheTTL(0)) + res, err := client.Book(context.Background(), "12207444") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.Title != "La Reina Dragon" { + t.Fatalf("unexpected title %q", res.Title) + } + } + if workingHits.Load() != 20 { + t.Errorf("expected working instance to be hit 20 times, got %d", workingHits.Load()) + } + }) + + t.Run("requests start at different instances", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + first := newCountingServer(t, http.StatusOK, validBookResponse, &firstHits) + second := newCountingServer(t, http.StatusOK, validBookResponse, &secondHits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + for range 100 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected load to be spread, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) + + t.Run("JSON error responses are not retried", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) + second := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + _, err := client.Book(context.Background(), "0") + if err == nil { + t.Fatal("expected error, got nil") + } + if errors.Is(err, biblioreads.ErrUnavailable) { + t.Errorf("expected a non-ErrUnavailable error, got %v", err) + } + if hits.Load() != 1 { + t.Errorf("expected a single request, got %d", hits.Load()) + } + }) + + t.Run("error when all instances fail", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) + second := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected ErrUnavailable, got %v", err) + } + if hits.Load() != 2 { + t.Errorf("expected both instances to be tried, got %d requests", hits.Load()) + } + }) + + t.Run("cancelled context stops failover", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusOK, `{}`, &hits) + second := newCountingServer(t, http.StatusOK, `{}`, &hits) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(ctx, "1"); err == nil { + t.Fatal("expected error, got nil") + } + if hits.Load() != 0 { + t.Errorf("expected no requests, got %d", hits.Load()) + } + }) +} + +func TestCooldown(t *testing.T) { + t.Run("errors put the instance into cooldown", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusGatewayTimeout, `{"status":"Error - Book not found"}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + + for range 3 { + if _, err := client.Book(context.Background(), "0"); err == nil { + t.Fatal("expected error, got nil") + } + } + if hits.Load() != 1 { + t.Errorf("expected subsequent requests to respect cooldown, got %d requests", hits.Load()) + } + }) + + t.Run("failing instance is skipped on later requests", func(t *testing.T) { + var badHits, goodHits atomic.Int32 + bad := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &badHits) + good := newCountingServer(t, http.StatusOK, validBookResponse, &goodHits) + + client := newTestClient(biblioreads.WithInstances([]string{bad.URL, good.URL}), biblioreads.WithCacheTTL(0)) + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if badHits.Load() > 1 { + t.Errorf("expected failing instance to be hit at most once, got %d", badHits.Load()) + } + if goodHits.Load() != 20 { + t.Errorf("expected working instance to be hit 20 times, got %d", goodHits.Load()) + } + }) + + t.Run("instance is used again after Retry-After", func(t *testing.T) { + var limitedHits, otherHits atomic.Int32 + limited := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if limitedHits.Add(1) == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = w.Write([]byte(validBookResponse)) + })) + t.Cleanup(limited.Close) + other := newCountingServer(t, http.StatusOK, validBookResponse, &otherHits) + + client := newTestClient(biblioreads.WithInstances([]string{limited.URL, other.URL}), biblioreads.WithCacheTTL(0)) + for limitedHits.Load() == 0 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if limitedHits.Load() != 1 { + t.Fatalf("expected rate limited instance to be skipped during cooldown, got %d hits", limitedHits.Load()) + } + + time.Sleep(1100 * time.Millisecond) + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if limitedHits.Load() == 1 { + t.Error("expected rate limited instance to be used again after its cooldown") + } + }) + + t.Run("respects cooldown when all instances are cooling down", func(t *testing.T) { + var hits atomic.Int32 + flaky := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = w.Write([]byte(validBookResponse)) + })) + t.Cleanup(flaky.Close) + + client := newTestClient(biblioreads.WithInstances([]string{flaky.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(context.Background(), "1"); err == nil { + t.Fatal("expected error on first request, got nil") + } + if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected unavailable during cooldown, got %v", err) + } + time.Sleep(1100 * time.Millisecond) + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("expected cooling down instance to be probed, got %v", err) + } + }) + + t.Run("JSON client error responses don't trigger cooldown", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + notFound := `{"status":"Error - Book not found"}` + first := newCountingServer(t, http.StatusNotFound, notFound, &firstHits) + second := newCountingServer(t, http.StatusNotFound, notFound, &secondHits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + for range 20 { + if _, err := client.Book(context.Background(), "0"); err == nil { + t.Fatal("expected error, got nil") + } + } + if firstHits.Load()+secondHits.Load() != 20 { + t.Errorf("expected one request per call, got %d", firstHits.Load()+secondHits.Load()) + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected no instance to be benched, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) + + t.Run("cancelled requests don't trigger cooldown", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + first := newCountingServer(t, http.StatusOK, validBookResponse, &firstHits) + second := newCountingServer(t, http.StatusOK, validBookResponse, &secondHits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for range 5 { + _, _ = client.Book(ctx, "1") + } + for range 40 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected no instance to be benched, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) +} diff --git a/internal/datasource/biblioreads/fixtures/search-books.json b/internal/datasource/biblioreads/fixtures/search-books.json index 3c3de16c..12001a0a 100644 --- a/internal/datasource/biblioreads/fixtures/search-books.json +++ b/internal/datasource/biblioreads/fixtures/search-books.json @@ -12,7 +12,8 @@ "bookURL": "/book/show/12207444-la-reina-dragon", "author": "Alice Borchardt", "authorURL": "/author/show/151155.Alice_Borchardt", - "rating": "3.82" + "rating": "3.82", + "ratingCount": "2285" } ], "lastScraped": "2026-10-28T17:45:31.333Z" diff --git a/internal/datasource/biblioreads/model.go b/internal/datasource/biblioreads/model.go index bc8ad4bd..6f087099 100644 --- a/internal/datasource/biblioreads/model.go +++ b/internal/datasource/biblioreads/model.go @@ -11,13 +11,14 @@ type SearchBooksResponse struct { } type SearchResult struct { - ID int `json:"id"` - Cover string `json:"cover"` - Title string `json:"title"` - BookURL string `json:"bookURL"` - Author string `json:"author"` - AuthorURL string `json:"authorURL"` - Rating string `json:"rating"` + ID int `json:"id"` + Cover string `json:"cover"` + Title string `json:"title"` + BookURL string `json:"bookURL"` + Author string `json:"author"` + AuthorURL string `json:"authorURL"` + Rating string `json:"rating"` + RatingCount string `json:"ratingCount"` } type BookResponse struct { diff --git a/internal/index/author.go b/internal/index/author.go index 7ea162b4..c27c1bed 100644 --- a/internal/index/author.go +++ b/internal/index/author.go @@ -10,18 +10,17 @@ import ( ) type Author struct { - Slug string - Name string - BirthName string - DataSourceID string - RetrievedOn time.Time - WikipediaLink map[string]string - InstanceOf float64 - Description map[string]string - DateOfBirth precisiondate.PrecisionDate - DateOfDeath precisiondate.PrecisionDate - Website string - // GoodReadsAuthorURL is the Goodreads author path returned by BiblioReads. + Slug string + Name string + BirthName string + DataSourceID string + RetrievedOn time.Time + WikipediaLink map[string]string + InstanceOf float64 + Description map[string]string + DateOfBirth precisiondate.PrecisionDate + DateOfDeath precisiondate.PrecisionDate + Website string GoodReadsAuthorURL string DataSourceImage string Gender float64 diff --git a/internal/index/bleve.go b/internal/index/bleve.go index fd3b1be0..ed1c4343 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -465,7 +465,6 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.TypeMapping[lang].AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsRating", storedOnlyFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsRatingValue", numericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) @@ -495,7 +494,6 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsRating", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsRatingValue", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) diff --git a/internal/index/bleve_document_read.go b/internal/index/bleve_document_read.go index afebb639..eaebfa89 100644 --- a/internal/index/bleve_document_read.go +++ b/internal/index/bleve_document_read.go @@ -40,6 +40,27 @@ const titleBoost = 3.0 // document detail page's "related documents" widget. var DefaultDocumentSortBy = []string{"-_score", "Series", "SeriesIndex"} +// goodReadsScoreSort treats a stored zero as missing without changing the index. +type goodReadsScoreSort struct { + search.SortField +} + +func (s *goodReadsScoreSort) Value(hit *search.DocumentMatch) string { + value := s.SortField.Value(hit) + if s.SortField.DecodeValue(value) == "0" { + if s.Desc { + return search.LowTerm + } + return search.HighTerm + } + return value +} + +func (s *goodReadsScoreSort) Copy() search.SearchSort { + copied := *s + return &copied +} + // Search look for documents which match the passed keywords and filters. // Returns a maximum documents, offset by func (b *BleveIndexer) Search(searchFields SearchFields, page, resultsPerPage int) (result.CappedPaginatedResult[[]Document], error) { @@ -402,6 +423,16 @@ func (b *BleveIndexer) searchAndHydrate(query query.Query, size, offset int, sor // with an empty sort order, which falls back to arbitrary (index) order instead of relevance. if len(sortBy) > 0 { searchOptions.SortBy(sortBy) + for i, key := range sortBy { + if strings.TrimPrefix(key, "-") == "GoodReadsRatingValue" { + searchOptions.Sort[i] = &goodReadsScoreSort{SortField: search.SortField{ + Field: "GoodReadsRatingValue", + Desc: strings.HasPrefix(key, "-"), + Type: search.SortFieldAsNumber, + Missing: search.SortFieldMissingLast, + }} + } + } } searchOptions.Fields = []string{"*"} b.documentsMu.RLock() @@ -591,6 +622,14 @@ func compareDocumentsBySortKey(a, b Document, key string) int { c = cmp.Compare(a.Publication.Date, b.Publication.Date) case "Words": c = cmp.Compare(a.Words, b.Words) + case "GoodReadsRatingValue": + if (a.GoodReadsRatingValue == 0) != (b.GoodReadsRatingValue == 0) { + if a.GoodReadsRatingValue == 0 { + return 1 + } + return -1 + } + c = cmp.Compare(a.GoodReadsRatingValue, b.GoodReadsRatingValue) case "Series": c = cmp.Compare(a.Series, b.Series) case "SeriesIndex": @@ -1038,7 +1077,6 @@ func hydrateDocument(match *search.DocumentMatch) Document { } goodReadsBookURL, _ := match.Fields["GoodReadsBookURL"].(string) - goodReadsRating, _ := match.Fields["GoodReadsRating"].(string) goodReadsRatingValue, _ := match.Fields["GoodReadsRatingValue"].(float64) goodReadsResourceID, _ := match.Fields["GoodReadsResourceID"].(string) goodReadsLegacyBookID, _ := match.Fields["GoodReadsLegacyBookID"].(float64) @@ -1071,7 +1109,6 @@ func hydrateDocument(match *search.DocumentMatch) Document { TextRankWords: textRankWords, TextRankEnriched: textRankEnriched, GoodReadsBookURL: goodReadsBookURL, - GoodReadsRating: goodReadsRating, GoodReadsRatingValue: goodReadsRatingValue, GoodReadsResourceID: goodReadsResourceID, GoodReadsLegacyBookID: int(goodReadsLegacyBookID), diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go index 5244ec55..de222ecd 100644 --- a/internal/index/bleve_goodreads_write.go +++ b/internal/index/bleve_goodreads_write.go @@ -30,7 +30,7 @@ type GoodReadsSource interface { SearchBooks(ctx context.Context, query string) (biblioreads.SearchBooksResponse, error) } -var goodReadsFields = []string{"GoodReadsBookURL", "GoodReadsRating", "GoodReadsRatingValue", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} +var goodReadsFields = []string{"GoodReadsBookURL", "GoodReadsRatingValue", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} // Strip named series annotations and omnibus ranges, not standalone volume numbers. var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) @@ -39,7 +39,7 @@ var goodReadsSubtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+` var goodReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) -var goodReadsBookPath = regexp.MustCompile(`^/book/show/[1-9][0-9]*(?:[.-][A-Za-z0-9._~-]+)?/?$`) +var goodReadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) var ErrInvalidGoodReadsBookURL = errors.New("invalid Goodreads book URL") @@ -58,6 +58,32 @@ func normalizeGoodReadsName(value string) string { return strings.Join(strings.Fields(result.String()), " ") } +func goodReadsAuthorNameMatches(name, candidate string) bool { + name = normalizeGoodReadsName(name) + candidate = normalizeGoodReadsName(candidate) + if name == "" || candidate == "" { + return false + } + if strings.ReplaceAll(name, " ", "") == strings.ReplaceAll(candidate, " ", "") { + return true + } + short, long := strings.Fields(name), strings.Fields(candidate) + if len(short) > len(long) { + short, long = long, short + } + if len(short) < 2 || short[0] != long[0] || short[len(short)-1] != long[len(long)-1] { + return false + } + // Allow omitted middle names, but not conflicting ones. + next := 1 + for _, part := range long[1 : len(long)-1] { + if next < len(short)-1 && part == short[next] { + next++ + } + } + return next == len(short)-1 +} + func goodReadsAuthor(document Document, matchedName string) (string, string) { nameKey := strings.ReplaceAll(normalizeGoodReadsName(matchedName), " ", "") if nameKey == "" { @@ -68,7 +94,16 @@ func goodReadsAuthor(document Document, matchedName string) (string, string) { return document.AuthorsSlugs[i], name } } - return "", "" + var slug, authorName string + for i, name := range document.Authors { + if i < len(document.AuthorsSlugs) && goodReadsAuthorNameMatches(name, matchedName) { + if slug != "" { + return "", "" + } + slug, authorName = document.AuthorsSlugs[i], name + } + } + return slug, authorName } // storeGoodReadsAuthorURL requires documentsMu to be held. @@ -117,16 +152,13 @@ func goodReadsMainTitle(value string) string { return normalizeGoodReadsName(value) } -func goodReadsTitleMatchScore(candidate, title, fullTitle, mainTitle string) int { +func goodReadsTitleMatchScore(candidate, title, mainTitle string) int { candidateTitle := goodReadsTitle(candidate) if candidateTitle == "" { return 0 } if candidateTitle == title { - if normalizeGoodReadsName(candidate) == fullTitle { - return 3 - } - return 2 + return 3 } if goodReadsMainTitle(candidate) == title || mainTitle == candidateTitle || goodReadsTitle(goodReadsParentheticalTitle.ReplaceAllString(candidate, "")) == title { @@ -139,8 +171,15 @@ func preferGoodReadsBook(candidate, best biblioreads.SearchResult, score, bestSc if bestScore == 0 { return true } - if score == bestScore && candidate.ID > 0 && best.ID > 0 && candidate.ID != best.ID { - return candidate.ID < best.ID + if score == bestScore { + count, bestCount := goodReadsRatingCount(candidate.RatingCount), goodReadsRatingCount(best.RatingCount) + if count != bestCount { + return count > bestCount + } + id, bestID := goodReadsBookID(candidate.BookURL), goodReadsBookID(best.BookURL) + if id > 0 && bestID > 0 && id != bestID { + return id < bestID + } } hasRating, bestHasRating := hasGoodReadsRating(candidate.Rating), hasGoodReadsRating(best.Rating) if hasRating != bestHasRating { @@ -149,33 +188,54 @@ func preferGoodReadsBook(candidate, best biblioreads.SearchResult, score, bestSc return score > bestScore } +func goodReadsRatingCount(value string) uint64 { + if strings.TrimSpace(value) == "" { + return 0 + } + count, err := strconv.ParseUint(strings.ReplaceAll(strings.TrimSpace(value), ",", ""), 10, 64) + if err != nil { + log.Printf("Invalid GoodReads rating count: %q", value) + return 0 + } + return count +} + +func goodReadsBookID(bookURL string) uint64 { + match := goodReadsBookPath.FindStringSubmatch(bookURL) + if match == nil { + return 0 + } + id, err := strconv.ParseUint(match[1], 10, 64) + if err != nil { + log.Printf("Invalid GoodReads book ID in URL: %q", bookURL) + return 0 + } + return id +} + func matchingGoodReadsBook(document Document, results []biblioreads.SearchResult) (biblioreads.SearchResult, bool) { title := goodReadsTitle(document.Title) if title == "" { return biblioreads.SearchResult{}, false } - fullTitle := normalizeGoodReadsName(document.Title) mainTitle := goodReadsMainTitle(document.Title) - authors := make(map[string]struct{}, len(document.Authors)) - for _, name := range document.Authors { - authors[strings.ReplaceAll(normalizeGoodReadsName(name), " ", "")] = struct{}{} - } var best biblioreads.SearchResult bestScore := 0 for _, result := range results { if result.BookURL == "" { continue } - score := goodReadsTitleMatchScore(result.Title, title, fullTitle, mainTitle) + score := goodReadsTitleMatchScore(result.Title, title, mainTitle) if score == 0 { continue } - author := strings.ReplaceAll(normalizeGoodReadsName(result.Author), " ", "") - if author == "" { - continue - } - if _, matches := authors[author]; matches && preferGoodReadsBook(result, best, score, bestScore) { - best, bestScore = result, score + for _, author := range document.Authors { + if goodReadsAuthorNameMatches(author, result.Author) { + if preferGoodReadsBook(result, best, score, bestScore) { + best, bestScore = result, score + } + break + } } } return best, bestScore > 0 @@ -297,7 +357,7 @@ func (b *BleveIndexer) enrichGoodReadsBatch(documents []Document) error { // not looked up yet and persists its match. Returns an empty result if there's no match. func (b *BleveIndexer) GoodReadsBook(ctx context.Context, document Document) (biblioreads.SearchResult, error) { if document.GoodReadsBookURL != "" || document.GoodReadsEnriched || b.goodReadsSource == nil { - return biblioreads.SearchResult{BookURL: document.GoodReadsBookURL, Rating: document.GoodReadsRating}, nil + return biblioreads.SearchResult{BookURL: document.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", document.GoodReadsRatingValue)}, nil } return b.enrichGoodReads(ctx, document) } @@ -327,7 +387,6 @@ func (b *BleveIndexer) UpdateGoodReadsBookURL(documentID, rawURL string) error { return nil } current.GoodReadsBookURL = path - current.GoodReadsRating = "" current.GoodReadsRatingValue = 0 current.GoodReadsResourceID = "" current.GoodReadsLegacyBookID = 0 @@ -363,8 +422,7 @@ func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, changed := false if book.Rating != "" { ratingValue := parseGoodReadsRating(book.Rating, document.ID) - if current.GoodReadsRating != book.Rating || current.GoodReadsRatingValue != ratingValue { - current.GoodReadsRating = book.Rating + if current.GoodReadsRatingValue != ratingValue { current.GoodReadsRatingValue = ratingValue changed = true } @@ -437,17 +495,16 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) ( // Looked up concurrently (e.g. by the background enrichment and a detail // page request at the same time): keep the stored result. if current.GoodReadsEnriched { - result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: current.GoodReadsRating} + result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", current.GoodReadsRatingValue)} return result, nil } if current.GoodReadsBookURL != "" && current.GoodReadsBookURL != result.BookURL { - result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: current.GoodReadsRating} + result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", current.GoodReadsRatingValue)} } current.GoodReadsBookURL = result.BookURL - if result.Rating == "" { - result.Rating = current.GoodReadsRating + if result.Rating == "" && current.GoodReadsRatingValue > 0 { + result.Rating = fmt.Sprintf("%.2f", current.GoodReadsRatingValue) } - current.GoodReadsRating = result.Rating current.GoodReadsRatingValue = parseGoodReadsRating(result.Rating, current.ID) current.GoodReadsEnriched = true if err := b.storeGoodReadsAuthorURL(current, result.AuthorURL, result.Author); err != nil { @@ -533,20 +590,15 @@ func (b *BleveIndexer) preserveGoodReadsLocked(documents []Document) { stored := make(map[string]Document, len(searchResult.Hits)) for _, hit := range searchResult.Hits { bookURL, _ := hit.Fields["GoodReadsBookURL"].(string) - rating, _ := hit.Fields["GoodReadsRating"].(string) ratingValue, _ := hit.Fields["GoodReadsRatingValue"].(float64) - if ratingValue == 0 { - ratingValue = parseGoodReadsRating(rating, hit.ID) - } resourceID, _ := hit.Fields["GoodReadsResourceID"].(string) legacyBookID, _ := hit.Fields["GoodReadsLegacyBookID"].(float64) enriched, _ := hit.Fields["GoodReadsEnriched"].(bool) - stored[hit.ID] = Document{GoodReadsBookURL: bookURL, GoodReadsRating: rating, GoodReadsRatingValue: ratingValue, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} + stored[hit.ID] = Document{GoodReadsBookURL: bookURL, GoodReadsRatingValue: ratingValue, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} } for i := range documents { if s, ok := stored[documents[i].ID]; ok { documents[i].GoodReadsBookURL = s.GoodReadsBookURL - documents[i].GoodReadsRating = s.GoodReadsRating documents[i].GoodReadsRatingValue = s.GoodReadsRatingValue documents[i].GoodReadsResourceID = s.GoodReadsResourceID documents[i].GoodReadsLegacyBookID = s.GoodReadsLegacyBookID diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go index 7ef8517c..291537b0 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/index/bleve_goodreads_write_test.go @@ -107,8 +107,8 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { if doc.GoodReadsBookURL != wantGoodReadsBookURL { t.Errorf("unexpected book URL %q", doc.GoodReadsBookURL) } - if doc.GoodReadsRating != "3.82" { - t.Errorf("unexpected Goodreads rating %q", doc.GoodReadsRating) + if doc.GoodReadsRatingValue != 3.82 { + t.Errorf("unexpected Goodreads rating %v", doc.GoodReadsRatingValue) } assertNoDocumentAuthorURL(t, idx.documentsIdx) if !doc.GoodReadsEnriched { @@ -292,7 +292,7 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != wantGoodReadsBookURL || doc.GoodReadsRating != "3.82" || doc.GoodReadsRatingValue != 3.82 { + if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != wantGoodReadsBookURL || doc.GoodReadsRatingValue != 3.82 { t.Errorf("expected GoodReads fields to survive a TextRank write, got %+v", doc) } if doc.GoodReadsResourceID != "work-id" || doc.GoodReadsLegacyBookID != 12207444 { @@ -459,7 +459,8 @@ func TestGoodReadsTitleMatchScore(t *testing.T) { want int }{ {name: "normalized exact", title: "La promesa", candidate: "LA PROMESA!", want: 3}, - {name: "series suffix", title: "La promesa", candidate: "La promesa (Series, #1)", want: 2}, + {name: "series suffix", title: "La promesa", candidate: "La promesa (Series, #1)", want: 3}, + {name: "document series suffix", title: "La promesa (Series, #1)", candidate: "La promesa", want: 3}, {name: "subtitle", title: "La promesa", candidate: "La promesa: Una historia", want: 1}, {name: "document subtitle", title: "La promesa: Una historia", candidate: "La promesa", want: 1}, {name: "parenthetical", title: "La promesa", candidate: "La promesa (The Promise)", want: 1}, @@ -469,8 +470,7 @@ func TestGoodReadsTitleMatchScore(t *testing.T) { {name: "empty candidate", title: "La promesa"}, } { t.Run(tt.name, func(t *testing.T) { - got := goodReadsTitleMatchScore(tt.candidate, goodReadsTitle(tt.title), - normalizeGoodReadsName(tt.title), goodReadsMainTitle(tt.title)) + got := goodReadsTitleMatchScore(tt.candidate, goodReadsTitle(tt.title), goodReadsMainTitle(tt.title)) if got != tt.want { t.Fatalf("title match score = %d, want %d", got, tt.want) } @@ -486,8 +486,11 @@ func TestPreferGoodReadsBook(t *testing.T) { want bool }{ {name: "first match", score: 1, want: true}, - {name: "lower ID before rating for equal strength", candidate: biblioreads.SearchResult{ID: 1}, best: biblioreads.SearchResult{ID: 2, Rating: "4"}, score: 3, bestScore: 3, want: true}, - {name: "higher ID loses despite rating", candidate: biblioreads.SearchResult{ID: 2, Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 3, bestScore: 3}, + {name: "lower ID before rating for equal strength", candidate: biblioreads.SearchResult{BookURL: "/book/show/1"}, best: biblioreads.SearchResult{BookURL: "/book/show/2", Rating: "4"}, score: 3, bestScore: 3, want: true}, + {name: "higher ID loses despite rating", candidate: biblioreads.SearchResult{BookURL: "/book/show/2", Rating: "4"}, best: biblioreads.SearchResult{BookURL: "/book/show/1"}, score: 3, bestScore: 3}, + {name: "more ratings before ID", candidate: biblioreads.SearchResult{BookURL: "/book/show/2", RatingCount: "100"}, best: biblioreads.SearchResult{BookURL: "/book/show/1", RatingCount: "3"}, score: 3, bestScore: 3, want: true}, + {name: "fewer ratings loses", candidate: biblioreads.SearchResult{RatingCount: "3"}, best: biblioreads.SearchResult{RatingCount: "100"}, score: 3, bestScore: 3}, + {name: "result rank is not book ID", candidate: biblioreads.SearchResult{ID: 2, BookURL: "/book/show/1"}, best: biblioreads.SearchResult{ID: 1, BookURL: "/book/show/2"}, score: 3, bestScore: 3, want: true}, {name: "rating before strength for different strength", candidate: biblioreads.SearchResult{ID: 2, Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 1, bestScore: 3, want: true}, {name: "stronger unrated loses to rated", best: biblioreads.SearchResult{Rating: "4"}, score: 3, bestScore: 1}, {name: "strength with same rating status", score: 3, bestScore: 1, want: true}, @@ -504,6 +507,89 @@ func TestPreferGoodReadsBook(t *testing.T) { } } +func TestGoodReadsAuthorNameMatches(t *testing.T) { + tests := []struct { + name, candidate string + want bool + }{ + {"Paul Kennedy", "Paul M. Kennedy", true}, + {"Paul Kennedy", "Paul Michael Kennedy", true}, + {"Paul Michael Kennedy", "Paul Michael James Kennedy", true}, + {"J. R. R. Tolkien", "J.R.R. Tolkien", true}, + {"JRR Tolkien", "J.R.R. Tolkien", true}, + {"José García", "JOSE M. GARCIA", true}, + {"Paul M. Kennedy", "Paul A. Kennedy", false}, + {"Paul Michael James Kennedy", "Paul James Michael Kennedy", false}, + {"Paul Kennedy", "Pauline Kennedy", false}, + {"Paul Kennedy", "Paul Kennedys", false}, + {"Paul Kennedy", "John Kennedy", false}, + {"Kennedy", "Paul M. Kennedy", false}, + {"Paul", "Paul M. Kennedy", false}, + {"", "Paul Kennedy", false}, + {" . ", "", false}, + } + for _, tt := range tests { + t.Run(tt.name+"/"+tt.candidate, func(t *testing.T) { + if got := goodReadsAuthorNameMatches(tt.name, tt.candidate); got != tt.want { + t.Errorf("match = %v, want %v", got, tt.want) + } + if got := goodReadsAuthorNameMatches(tt.candidate, tt.name); got != tt.want { + t.Errorf("reverse match = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGoodReadsAuthorPrefersExactAndRejectsAmbiguousPartialMatches(t *testing.T) { + doc := Document{ + Metadata: metadata.Metadata{Authors: []string{"Paul Kennedy", "Paul M. Kennedy"}}, + AuthorsSlugs: []string{"paul-kennedy", "paul-m-kennedy"}, + } + if slug, name := goodReadsAuthor(doc, "Paul M. Kennedy"); slug != "paul-m-kennedy" || name != "Paul M. Kennedy" { + t.Fatalf("expected exact author, got %q, %q", slug, name) + } + doc.Authors = []string{"Paul M. Kennedy", "Paul A. Kennedy"} + if slug, name := goodReadsAuthor(doc, "Paul Kennedy"); slug != "" || name != "" { + t.Fatalf("ambiguous partial match must not select an author, got %q, %q", slug, name) + } +} + +func TestGoodReadsBookMatchesKennedyMiddleName(t *testing.T) { + const title = "Auge y caída de las grandes potencias" + const bookURL = "/book/show/27214" + const authorURL = "/author/show/8390124.Paul_M_Kennedy" + source := &fakeGoodReadsSource{results: []biblioreads.SearchResult{{ + Title: title, Author: "Paul M. Kennedy", BookURL: bookURL, AuthorURL: authorURL, Rating: "4.14", + }}} + idx := newGoodReadsTestIndexer(t, source) + doc := Document{ + ID: goodReadsTestID, AuthorsSlugs: []string{"paul-kennedy", "another-writer"}, + Metadata: metadata.Metadata{Title: title, Authors: []string{"Paul Kennedy", "Another Writer"}}, + } + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + got, err := idx.GoodReadsBook(context.Background(), doc) + if err != nil || got.BookURL != bookURL || got.Rating != "4.14" { + t.Fatalf("expected Kennedy's book and rating, got %+v, %v", got, err) + } + stored, err := idx.documentByIndexID(doc.ID) + if err != nil || !stored.GoodReadsEnriched || stored.GoodReadsBookURL != bookURL || stored.GoodReadsRatingValue != 4.14 { + t.Fatalf("expected persisted book match and rating, got %+v, %v", stored, err) + } + author, err := idx.Author("paul-kennedy", "") + if err != nil || author.GoodReadsAuthorURL != authorURL || author.Name != "Paul Kennedy" { + t.Fatalf("expected author URL with original name preserved, got %+v, %v", author, err) + } + other, err := idx.Author("another-writer", "") + if err != nil || other.GoodReadsAuthorURL != "" { + t.Fatalf("unmatched coauthor must not receive the URL, got %+v, %v", other, err) + } + if want := []string{title + " Paul Kennedy"}; !slices.Equal(source.Queries(), want) { + t.Fatalf("expected combined search to match without fallback, got %q", source.Queries()) + } +} + func TestMatchingGoodReadsBook(t *testing.T) { correct := biblioreads.SearchResult{ Title: "El señor de los anillos", Author: "J.R.R. Tolkien", @@ -531,7 +617,7 @@ func TestMatchingGoodReadsBook(t *testing.T) { {name: "omnibus range with dash", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1–3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, {name: "document omnibus range", title: correct.Title + " (#1-#3)", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, {name: "standalone volume retained", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1)", Author: correct.Author, BookURL: correct.BookURL}}}, - {name: "exact preferred", results: []biblioreads.SearchResult{{Title: correct.Title + " (Middle Earth, #1)", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: correct.BookURL}, + {name: "series suffix ranks equally", results: []biblioreads.SearchResult{{Title: correct.Title + " (Middle Earth, #1)", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: "/book/show/2"}, {name: "lower Goodreads ID preferred over rating for equal titles", results: []biblioreads.SearchResult{ {ID: 1, Title: correct.Title, Author: correct.Author, BookURL: "/book/show/1-unrated", Rating: "0.00"}, {ID: 2, Title: correct.Title, Author: correct.Author, BookURL: "/book/show/2-rated", Rating: "4.05"}, @@ -575,6 +661,47 @@ func TestMatchingGoodReadsBook(t *testing.T) { } } +func TestMatchingGoodReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { + document := Document{Metadata: metadata.Metadata{ + Title: "Los asesinatos de Manhattan", Authors: []string{"Douglas Preston", "Lincoln Child"}, + }} + popular := biblioreads.SearchResult{ + ID: 1, Title: "Los asesinatos de Manhattan (Pendergast, #3)", Author: "Douglas Preston", + BookURL: "/book/show/67005.Los_asesinatos_de_Manhattan", Rating: "4.26", RatingCount: "57,469", + } + obscure := biblioreads.SearchResult{ + ID: 3, Title: document.Title, Author: "Douglas Preston", + BookURL: "/book/show/152308668-los-asesinatos-de-manhattan", Rating: "3.67", RatingCount: "3", + } + for _, results := range [][]biblioreads.SearchResult{{popular, obscure}, {obscure, popular}} { + match, ok := matchingGoodReadsBook(document, results) + if !ok || match.BookURL != popular.BookURL { + t.Fatalf("expected popular series entry, got %+v, %v", match, ok) + } + } + popular.Author = "Another author" + match, ok := matchingGoodReadsBook(document, []biblioreads.SearchResult{popular, obscure}) + if !ok || match.BookURL != obscure.BookURL { + t.Fatalf("rating count must not bypass author matching, got %+v, %v", match, ok) + } +} + +func TestGoodReadsRatingCount(t *testing.T) { + for _, tt := range []struct { + value string + want uint64 + }{ + {"", 0}, {"0", 0}, {"57469", 57469}, {" 57,469 ", 57469}, + {"unknown", 0}, {"-1", 0}, {"3.5", 0}, {"18446744073709551616", 0}, + } { + t.Run(tt.value, func(t *testing.T) { + if got := goodReadsRatingCount(tt.value); got != tt.want { + t.Fatalf("rating count = %d, want %d", got, tt.want) + } + }) + } +} + func TestMatchingGoodReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { const olderBookURL = "/book/show/1368314.El_Cercano_Oriente" const newerBookURL = "/book/show/243175949-el-cercano-oriente" @@ -582,11 +709,11 @@ func TestMatchingGoodReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { Title: "El cercano Oriente", Authors: []string{"Isaac Asimov"}, }} older := biblioreads.SearchResult{ - ID: 1368314, Title: "El Cercano Oriente", Author: "Isaac Asimov", + ID: 2, Title: "El Cercano Oriente", Author: "Isaac Asimov", BookURL: olderBookURL, Rating: "4.10", } newer := biblioreads.SearchResult{ - ID: 243175949, Title: "El Cercano Oriente", Author: "Isaac Asimov", + ID: 1, Title: "El Cercano Oriente", Author: "Isaac Asimov", BookURL: newerBookURL, Rating: "4.26", } @@ -822,7 +949,7 @@ func TestStoreGoodReadsDetails(t *testing.T) { if writes.count != 0 { t.Fatal("stale details must not cause any index writes") } - if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || stored.GoodReadsRating != "" { + if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || stored.GoodReadsRatingValue != 0 { t.Fatalf("unexpected details persisted: %+v", stored) } return @@ -834,12 +961,13 @@ func TestStoreGoodReadsDetails(t *testing.T) { if stored.GoodReadsResourceID != resourceID || stored.GoodReadsLegacyBookID != wantLegacyBookID { t.Fatalf("identifiers not persisted: %+v", stored) } - if stored.GoodReadsRating != book.Rating || stored.GoodReadsRatingValue != 4.12 { + if stored.GoodReadsRatingValue != 4.12 { t.Fatalf("rating not persisted with identifiers: %+v", stored) } if !stored.GoodReadsEnriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("existing enrichment was lost: %+v", stored) } + book.Rating = "4.120" if err := idx.StoreGoodReadsDetails(stale, wantGoodReadsBookURL, book); err != nil { t.Fatalf("repeated store failed: %v", err) } @@ -882,8 +1010,8 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) doc := Document{ ID: goodReadsTestID, GoodReadsBookURL: wantGoodReadsBookURL, - GoodReadsRating: "3.82", GoodReadsRatingValue: 3.82, - GoodReadsResourceID: "old-work", GoodReadsLegacyBookID: 1, + GoodReadsRatingValue: 3.82, + GoodReadsResourceID: "old-work", GoodReadsLegacyBookID: 1, } if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { t.Fatal(err) @@ -895,7 +1023,7 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.GoodReadsRating != tt.wantRating || stored.GoodReadsRatingValue != parseGoodReadsRating(tt.wantRating, doc.ID) || + if stored.GoodReadsRatingValue != parseGoodReadsRating(tt.wantRating, doc.ID) || stored.GoodReadsResourceID != tt.wantWork || stored.GoodReadsLegacyBookID != tt.wantID { t.Fatalf("partial details changed fields that were not supplied: %+v", stored) } @@ -937,7 +1065,6 @@ func TestUpdateGoodReadsBookURL(t *testing.T) { Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, TextRankWords: []string{"dragon"}, GoodReadsBookURL: wantGoodReadsBookURL, - GoodReadsRating: "3.82", GoodReadsRatingValue: 3.82, GoodReadsResourceID: "old-work", GoodReadsLegacyBookID: 12207444, @@ -960,7 +1087,7 @@ func TestUpdateGoodReadsBookURL(t *testing.T) { } if tt.want == "" || tt.want == doc.GoodReadsBookURL { if stored.GoodReadsBookURL != doc.GoodReadsBookURL || - stored.GoodReadsRating != doc.GoodReadsRating || + stored.GoodReadsRatingValue != doc.GoodReadsRatingValue || stored.GoodReadsResourceID != doc.GoodReadsResourceID || stored.GoodReadsLegacyBookID != doc.GoodReadsLegacyBookID { t.Fatalf("invalid or unchanged URL modified stored data: %+v", stored) @@ -968,7 +1095,7 @@ func TestUpdateGoodReadsBookURL(t *testing.T) { return } if stored.GoodReadsBookURL != tt.want || !stored.GoodReadsEnriched || - stored.GoodReadsRating != "" || stored.GoodReadsRatingValue != 0 || + stored.GoodReadsRatingValue != 0 || stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { t.Fatalf("correction must clear old rating and review identifiers: %+v", stored) } diff --git a/internal/index/document.go b/internal/index/document.go index 70cf61a7..022aeb64 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -82,9 +82,7 @@ type Document struct { // It is stored but not indexed, and filled by EnrichGoodReads. // Author URLs are stored on Author instead. GoodReadsBookURL string - // GoodReadsRating is the score returned with the Goodreads book search result. - // The string is retained for display; GoodReadsRatingValue is indexed for filtering. - GoodReadsRating string + // GoodReadsRatingValue is the Goodreads score, stored and indexed for filtering and sorting. GoodReadsRatingValue float64 // GoodReadsResourceID identifies the work; GoodReadsLegacyBookID identifies // the edition. Both are stored only, populated when book details are fetched, diff --git a/internal/index/goodreads_fields_test.go b/internal/index/goodreads_fields_test.go index ed965130..777c86b3 100644 --- a/internal/index/goodreads_fields_test.go +++ b/internal/index/goodreads_fields_test.go @@ -1,6 +1,7 @@ package index_test import ( + "slices" "testing" "github.com/blevesearch/bleve/v2" @@ -10,6 +11,79 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) +func TestGoodReadsScoreSorting(t *testing.T) { + docs := []index.Document{ + {ID: "high", Slug: "high", GoodReadsRatingValue: 4.75}, + {ID: "unrated", Slug: "unrated"}, + {ID: "low", Slug: "low", GoodReadsRatingValue: 3.25}, + } + for i := range docs { + docs[i].Metadata = metadata.Metadata{Title: docs[i].Slug, Authors: []string{"Writer"}, Format: "EPUB", Series: "Series"} + docs[i].AuthorsSlugs = []string{"writer"} + docs[i].SeriesSlug = "series" + docs[i].TextRankPhrases = []string{"atomic bomb", "los alamos", "manhattan project"} + } + reference := index.Document{ + ID: "reference", Slug: "reference", + Metadata: metadata.Metadata{Title: "Reference", Authors: []string{"Other"}, Format: "EPUB"}, + AuthorsSlugs: []string{"other"}, + TextRankPhrases: docs[0].TextRankPhrases, + } + idx := newTextRankTestIndex(t, append(docs, reference)) + t.Cleanup(func() { _ = idx.Close() }) + + for _, scope := range []string{"search", "author", "series", "similar"} { + for _, direction := range []string{"asc", "desc"} { + t.Run(scope+"/"+direction, func(t *testing.T) { + key := "GoodReadsRatingValue" + want := []string{"low", "high"} + if direction == "desc" { + key = "-" + key + slices.Reverse(want) + } + want = append(want, "unrated") + fields := index.SearchFields{SortBy: []string{key}} + switch scope { + case "search": + fields.Keywords = "writer" + case "author": + fields.AuthorSlug = "writer" + case "series": + fields.Keywords = "series" + case "similar": + fields.SimilarTo = "reference" + } + var got []string + for page := 1; page <= 2; page++ { + switch scope { + case "author": + res, err := idx.SearchByAuthor(fields, page, 2) + if err != nil { + t.Fatal(err) + } + got = append(got, slugsOf(res.Hits())...) + case "series": + res, err := idx.SearchBySeries(fields, page, 2) + if err != nil { + t.Fatal(err) + } + got = append(got, slugsOf(res.Hits())...) + default: + res, err := idx.Search(fields, page, 2) + if err != nil { + t.Fatal(err) + } + got = append(got, slugsOf(res.Hits())...) + } + } + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + } + } +} + func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) if err != nil { @@ -40,7 +114,6 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { }, AuthorsSlugs: []string{"alice-borchardt"}, GoodReadsBookURL: bookURL, - GoodReadsRating: "4.25", GoodReadsRatingValue: 4.25, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: legacyBookID, @@ -56,8 +129,8 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { if got.GoodReadsBookURL != bookURL { t.Errorf("language %q: expected stored book URL, got %q", lang, got.GoodReadsBookURL) } - if got.GoodReadsRating != "4.25" { - t.Errorf("language %q: expected stored rating, got %q", lang, got.GoodReadsRating) + if got.GoodReadsRatingValue != 4.25 { + t.Errorf("language %q: expected stored rating, got %v", lang, got.GoodReadsRatingValue) } if got.GoodReadsResourceID != resourceID || got.GoodReadsLegacyBookID != legacyBookID { t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, got.GoodReadsResourceID, got.GoodReadsLegacyBookID) @@ -84,7 +157,6 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { queries := map[string]bleve.SearchRequest{ "book URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsBookURL", bookURL)), - "book rating field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsRating", "4.25")), "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsAuthorURL", authorURL)), "resource ID field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsResourceID", resourceID)), "legacy ID numeric field": *bleve.NewSearchRequest(legacyBookIDQuery(legacyBookID)), @@ -103,6 +175,58 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { } } +func TestGoodReadsScoreSortingMissingFieldsAndTies(t *testing.T) { + documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + idx := index.NewBleve(documents, authors, nil, "", nil, index.Config{}) + t.Cleanup(func() { _ = idx.Close() }) + for _, doc := range []map[string]any{ + {"Slug": "unrated-zero", "GoodReadsRatingValue": 0.0, "SeriesIndex": 2.0}, + {"Slug": "high", "GoodReadsRatingValue": 4.75, "SeriesIndex": 1.0}, + {"Slug": "unrated-missing", "SeriesIndex": 1.0}, + {"Slug": "low-second", "GoodReadsRatingValue": 3.25, "SeriesIndex": 2.0}, + {"Slug": "low-first", "GoodReadsRatingValue": 3.25, "SeriesIndex": 1.0}, + } { + for _, field := range []string{"Title", "Description", "Series", "SeriesSlug", "Format"} { + doc[field] = "" + } + doc["Words"], doc["Pages"] = 0.0, 0.0 + if err := documents.Index(doc["Slug"].(string), doc); err != nil { + t.Fatal(err) + } + } + for _, direction := range []string{"asc", "desc"} { + t.Run(direction, func(t *testing.T) { + key := "GoodReadsRatingValue" + want := []string{"low-first", "low-second", "high", "unrated-missing", "unrated-zero"} + if direction == "desc" { + key = "-" + key + want = []string{"high", "low-first", "low-second", "unrated-missing", "unrated-zero"} + } + var got []string + for page := 1; page <= 3; page++ { + res, err := idx.Search(index.SearchFields{SortBy: []string{key, "SeriesIndex"}}, page, 2) + if err != nil { + t.Fatal(err) + } + if res.TotalHits() != len(want) { + t.Errorf("got %d hits, want %d", res.TotalHits(), len(want)) + } + got = append(got, slugsOf(res.Hits())...) + } + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + } +} + func legacyBookIDQuery(id int) *query.NumericRangeQuery { value := float64(id) q := bleve.NewNumericRangeQuery(&value, &value) diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index 21b41f29..eecb9944 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -61,7 +61,7 @@ func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblio } details := book if book.Rating != "" && document.GoodReadsRatingValue > 0 && isZeroRating(book.Rating) { - book.Rating = document.GoodReadsRating + book.Rating = fmt.Sprintf("%.2f", document.GoodReadsRatingValue) details.Rating = "" } if err := b.index.StoreGoodReadsDetails(document, document.GoodReadsBookURL, details); err != nil { diff --git a/internal/webserver/controller/author/documents.go b/internal/webserver/controller/author/documents.go index 7e80e6b5..a0523e49 100644 --- a/internal/webserver/controller/author/documents.go +++ b/internal/webserver/controller/author/documents.go @@ -85,6 +85,8 @@ func (a *Controller) Documents(c fiber.Ctx) error { {"pub-date-newer-first", "newer"}, {"est-read-time-shorter-first", "shorter"}, {"est-read-time-longer-first", "longer"}, + {"goodreads-score-asc", "Goodreads score (ascending)"}, + {"goodreads-score-desc", "Goodreads score (descending)"}, }, } diff --git a/internal/webserver/controller/search/parse.go b/internal/webserver/controller/search/parse.go index e2d94d2d..4db6f3fc 100644 --- a/internal/webserver/controller/search/parse.go +++ b/internal/webserver/controller/search/parse.go @@ -152,6 +152,10 @@ func parseDocumentSortBy(c fiber.Ctx) []string { return []string{"Words"} case "est-read-time-longer-first": return []string{"-Words"} + case "goodreads-score-asc": + return []string{"GoodReadsRatingValue"} + case "goodreads-score-desc": + return []string{"-GoodReadsRatingValue"} } } return index.DefaultDocumentSortBy diff --git a/internal/webserver/controller/search/parse_test.go b/internal/webserver/controller/search/parse_test.go new file mode 100644 index 00000000..c5a801ed --- /dev/null +++ b/internal/webserver/controller/search/parse_test.go @@ -0,0 +1,40 @@ +package search + +import ( + "net/http/httptest" + "slices" + "testing" + + "github.com/gofiber/fiber/v3" +) + +func TestParseGoodReadsScoreSort(t *testing.T) { + for _, direction := range []string{"asc", "desc"} { + t.Run(direction, func(t *testing.T) { + app := fiber.New() + app.Get("/", func(c fiber.Ctx) error { + fields, err := ParseDocumentSearchQuery(c, 250) + if err != nil { + t.Error(err) + return c.SendStatus(fiber.StatusBadRequest) + } + key := "GoodReadsRatingValue" + if direction == "desc" { + key = "-" + key + } + if !slices.Equal(fields.SortBy, []string{key}) { + t.Errorf("got %v, want [%s]", fields.SortBy, key) + } + return c.SendStatus(fiber.StatusOK) + }) + response, err := app.Test(httptest.NewRequest("GET", "/?sort-by=goodreads-score-"+direction, nil)) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != fiber.StatusOK { + t.Fatalf("unexpected status %d", response.StatusCode) + } + }) + } +} diff --git a/internal/webserver/controller/search/search.go b/internal/webserver/controller/search/search.go index f06735c7..bcdd8259 100644 --- a/internal/webserver/controller/search/search.go +++ b/internal/webserver/controller/search/search.go @@ -222,6 +222,8 @@ func documentSortOptions() []struct { {"pub-date-newer-first", "newer"}, {"est-read-time-shorter-first", "shorter"}, {"est-read-time-longer-first", "longer"}, + {"goodreads-score-asc", "Goodreads score (ascending)"}, + {"goodreads-score-desc", "Goodreads score (descending)"}, } } diff --git a/internal/webserver/controller/series/documents.go b/internal/webserver/controller/series/documents.go index 840cddbe..9c44993b 100644 --- a/internal/webserver/controller/series/documents.go +++ b/internal/webserver/controller/series/documents.go @@ -93,6 +93,8 @@ func (a *Controller) Documents(c fiber.Ctx) error { {"pub-date-newer-first", "newer"}, {"est-read-time-shorter-first", "shorter"}, {"est-read-time-longer-first", "longer"}, + {"goodreads-score-asc", "Goodreads score (ascending)"}, + {"goodreads-score-desc", "Goodreads score (descending)"}, }, } @@ -124,6 +126,10 @@ func (d *Controller) parseSortBy(c fiber.Ctx) []string { return []string{"Words"} case "est-read-time-longer-first": return []string{"-Words"} + case "goodreads-score-asc": + return []string{"GoodReadsRatingValue", "SeriesIndex"} + case "goodreads-score-desc": + return []string{"-GoodReadsRatingValue", "SeriesIndex"} } } return []string{"SeriesIndex"} diff --git a/internal/webserver/controller/series/documents_test.go b/internal/webserver/controller/series/documents_test.go new file mode 100644 index 00000000..d42da08a --- /dev/null +++ b/internal/webserver/controller/series/documents_test.go @@ -0,0 +1,36 @@ +package series + +import ( + "net/http/httptest" + "slices" + "testing" + + "github.com/gofiber/fiber/v3" +) + +func TestParseGoodReadsScoreSort(t *testing.T) { + for _, direction := range []string{"asc", "desc"} { + t.Run(direction, func(t *testing.T) { + app := fiber.New() + app.Get("/", func(c fiber.Ctx) error { + key := "GoodReadsRatingValue" + if direction == "desc" { + key = "-" + key + } + want := []string{key, "SeriesIndex"} + if got := (&Controller{}).parseSortBy(c); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + return c.SendStatus(fiber.StatusOK) + }) + response, err := app.Test(httptest.NewRequest("GET", "/?sort-by=goodreads-score-"+direction, nil)) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != fiber.StatusOK { + t.Fatalf("unexpected status %d", response.StatusCode) + } + }) + } +} diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 4a935be6..49d6c904 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -309,7 +309,6 @@ func newServiceProvidersTestApp(t *testing.T, source *fakeBookSource, provider d ID: "linked.epub", Slug: "linked", Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, GoodReadsBookURL: "/book/show/12207444-la-reina-dragon", - GoodReadsRating: "3.82", GoodReadsRatingValue: 3.82, GoodReadsEnriched: true, }, @@ -338,17 +337,35 @@ func newServiceProvidersTestApp(t *testing.T, source *fakeBookSource, provider d } func TestSearchResultsShowStoredGoodReadsRating(t *testing.T) { - app, _ := newGoodReadsTestApp(t, nil) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) - if err != nil { - t.Fatal(err) - } - rating := page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) - if rating.Length() != 1 || strings.TrimSpace(rating.Text()) != "3.82" { - t.Fatalf("expected stored Goodreads rating in search result, got %q", rating.Text()) - } - if page.Find(`li:has(a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]) i.bi-star-fill`).Length() != 1 { - t.Fatal("expected a star icon beside the Goodreads score") + for _, tt := range []struct { + rating string + want string + }{ + {rating: "3.82", want: "3.82"}, + {rating: "4.1", want: "4.10"}, + {rating: "5", want: "5.00"}, + } { + t.Run(tt.rating, func(t *testing.T) { + app, idx := newGoodReadsTestApp(t, nil) + doc, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, biblioreads.BookResponse{Rating: tt.rating}); err != nil { + t.Fatal(err) + } + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) + if err != nil { + t.Fatal(err) + } + rating := page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if rating.Length() != 1 || strings.TrimSpace(rating.Text()) != tt.want { + t.Fatalf("expected stored Goodreads rating in search result, got %q", rating.Text()) + } + if page.Find(`li:has(a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]) i.bi-star-fill`).Length() != 1 { + t.Fatal("expected a star icon beside the Goodreads score") + } + }) } } @@ -537,7 +554,7 @@ func TestDocumentUpdateGoodReads(t *testing.T) { t.Fatal(err) } if stored.GoodReadsBookURL != "/book/show/1368314.El_Cercano_Oriente" || - stored.GoodReadsRating != "" || stored.GoodReadsRatingValue != 0 || + stored.GoodReadsRatingValue != 0 || stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || !stored.GoodReadsEnriched { t.Fatalf("expected correction persisted without stale rating/review identifiers: %+v", stored) @@ -580,7 +597,7 @@ func TestDocumentUpdateGoodReadsValidation(t *testing.T) { t.Fatalf("invalid input should be retained for correction, got %q", got) } stored, err := idx.Document("linked") - if err != nil || stored.GoodReadsBookURL != "/book/show/12207444-la-reina-dragon" || stored.GoodReadsRating != "3.82" { + if err != nil || stored.GoodReadsBookURL != "/book/show/12207444-la-reina-dragon" || stored.GoodReadsRatingValue != 3.82 { t.Fatalf("invalid input must not change the stored match: %+v, %v", stored, err) } } @@ -612,7 +629,7 @@ func TestDocumentDetailUpdatesStoredGoodReadsRating(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.GoodReadsRating != "4.12" || stored.GoodReadsRatingValue != 4.12 { + if stored.GoodReadsRatingValue != 4.12 { t.Fatalf("expected latest rating and searchable value to be stored, got %+v", stored) } results, err := idx.Search(index.SearchFields{GoodReadsRatingFrom: 4.1, GoodReadsRatingTo: 4.2}, 1, 10) @@ -639,7 +656,7 @@ func TestDocumentDetailKeepsStoredGoodReadsRatingWhenScraperReturnsZero(t *testi if err != nil { t.Fatal(err) } - if stored.GoodReadsRating != "3.82" || stored.GoodReadsRatingValue != 3.82 { + if stored.GoodReadsRatingValue != 3.82 { t.Fatalf("zero scraper rating must not replace indexed Goodreads score, got %+v", stored) } } diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index ca48d562..218b56ba 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -18,6 +18,8 @@ "Illustrations": "Illustrationen" "Only illustrated": "Nur illustrierte" "Goodreads score": "Goodreads-Bewertung" +"Goodreads score (ascending)": "Goodreads-Bewertung (aufsteigend)" +"Goodreads score (descending)": "Goodreads-Bewertung (absteigend)" "Clear Goodreads score": "Goodreads-Bewertung löschen" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Lesezeit berechnet mit einer Geschwindigkeit von %s Wörtern pro Minute. Sie können die Lesegeschwindigkeit in Ihrem Profil ändern." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Lesezeit berechnet mit einer Geschwindigkeit von %s Wörtern pro Minute. Melden Sie sich an, um diesen Wert anzupassen." diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index 8819a150..dbd6cec9 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -18,6 +18,8 @@ "Illustrations": "Ilustraciones" "Only illustrated": "Solo ilustrados" "Goodreads score": "Puntuación de Goodreads" +"Goodreads score (ascending)": "Puntuación de Goodreads (ascendente)" +"Goodreads score (descending)": "Puntuación de Goodreads (descendente)" "Clear Goodreads score": "Borrar puntuación de Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Tiempo de lectura calculado en base a una velocidad de %s palabras por minuto. Puedes cambiarla en tu perfil." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Tiempo de lectura calculado en base a una velocidad de %s palabras por minuto. Inicia sesión para personalizar este valor." diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index 81038ae5..68e5e9e2 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -18,6 +18,8 @@ "Illustrations": "Illustrations" "Only illustrated": "Seulement illustrés" "Goodreads score": "Note Goodreads" +"Goodreads score (ascending)": "Note Goodreads (croissante)" +"Goodreads score (descending)": "Note Goodreads (décroissante)" "Clear Goodreads score": "Effacer la note Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Temps de lecture calculé avec une vitesse de %s mots par minute. Vous pouvez modifier votre vitesse de lecture dans votre profil." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Temps de lecture calculé avec une vitesse de %s mots par minute. Connectez-vous pour personnaliser cette valeur." diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index 8072e96c..703999f0 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -18,6 +18,8 @@ "Illustrations": "Иллюстрации" "Only illustrated": "Только с иллюстрациями" "Goodreads score": "Оценка Goodreads" +"Goodreads score (ascending)": "Оценка Goodreads (по возрастанию)" +"Goodreads score (descending)": "Оценка Goodreads (по убыванию)" "Clear Goodreads score": "Очистить оценку Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Время чтения рассчитано при скорости %s слов в минуту. Изменить скорость чтения можно в профиле." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Время чтения рассчитано при скорости %s слов в минуту. Войдите, чтобы изменить это значение." diff --git a/internal/webserver/embedded/views/partials/document-metadata.html b/internal/webserver/embedded/views/partials/document-metadata.html index 88f2a549..0e23ff2d 100644 --- a/internal/webserver/embedded/views/partials/document-metadata.html +++ b/internal/webserver/embedded/views/partials/document-metadata.html @@ -15,8 +15,8 @@ {{ if .ReviewsEnabled }}
      • {{ end }} - {{if and .ShowGoodReadsScore .Document.GoodReadsRating .Document.GoodReadsBookURL}} - {{template "partials/document-score" dict "Lang" .Lang "Source" "Goodreads" "Rating" .Document.GoodReadsRating "Scored" (gt .Document.GoodReadsRatingValue 0.0) "URL" (printf "https://www.goodreads.com%s" .Document.GoodReadsBookURL)}} + {{if and .ShowGoodReadsScore .Document.GoodReadsBookURL}} + {{template "partials/document-score" dict "Lang" .Lang "Source" "Goodreads" "Rating" (printf "%.2f" .Document.GoodReadsRatingValue) "Scored" (gt .Document.GoodReadsRatingValue 0.0) "URL" (printf "https://www.goodreads.com%s" .Document.GoodReadsBookURL)}} {{end}} {{ if and .IllustratedMinAmount (ge .Document.Illustrations .IllustratedMinAmount) }}
    • {{t .Lang "Illustrated"}}
      • diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index 5b0e58da..1281aa67 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -145,6 +145,52 @@ func TestUnifiedSearch(t *testing.T) { } } +func TestGoodReadsScoreSortOptions(t *testing.T) { + app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, + loadDirInMemoryFs("testdata/library"), webserver.Config{}) + for _, path := range []string{ + "/search?type=documents&", + "/authors/john-doe?", + "/series/the-lord-of-the-rings?", + } { + for _, direction := range []string{"asc", "desc"} { + for _, fragment := range []bool{false, true} { + mode := "/full" + if fragment { + mode = "/fragment" + } + t.Run(path+direction+mode, func(t *testing.T) { + selected := "goodreads-score-" + direction + req := mustGetRequest(t, path+"sort-by="+selected) + if fragment { + req.Header.Set("HX-Request", "true") + } + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("unexpected status %d", response.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"goodreads-score-asc", "goodreads-score-desc"} { + if doc.Find(`#sort-by option[value="`+value+`"]`).Length() != 1 { + t.Errorf("missing sort option %q", value) + } + } + if doc.Find(`#sort-by option[value="`+selected+`"][selected]`).Length() != 1 { + t.Errorf("expected selected option %q", selected) + } + }) + } + } + } +} + func TestGoodReadsScoreFilter(t *testing.T) { db := infrastructure.Connect(":memory:", 250) app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), webserver.Config{}) From d141e84f30073d98c5eca1fe182023fe306fb7f1 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 2 Nov 2026 23:11:57 +0100 Subject: [PATCH 13/50] WIP --- internal/index/bleve.go | 2 + internal/index/bleve_document_read.go | 2 + internal/index/bleve_goodreads_write.go | 22 ++++--- internal/index/bleve_goodreads_write_test.go | 21 ++++++- internal/index/document.go | 1 + internal/index/goodreads_fields_test.go | 6 +- internal/reviews/biblioreads.go | 7 ++- .../controller/document/controller.go | 2 + .../webserver/controller/document/score.go | 23 ++++--- internal/webserver/document_goodreads_test.go | 60 ++++++++++++++++--- .../embedded/js/search-filter-utils.js | 9 +-- .../partials/document-metadata-filters.html | 22 +++++-- .../views/partials/document-metadata.html | 2 +- .../views/partials/document-score.html | 6 +- .../js-tests/search-filter-utils.test.mjs | 44 ++++++++++++++ internal/webserver/search_test.go | 33 +++++++--- 16 files changed, 210 insertions(+), 52 deletions(-) create mode 100644 internal/webserver/js-tests/search-filter-utils.test.mjs diff --git a/internal/index/bleve.go b/internal/index/bleve.go index ed1c4343..9fe0ed61 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -466,6 +466,7 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.TypeMapping[lang].AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsRatingValue", numericFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsRatingCount", storedOnlyFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) @@ -495,6 +496,7 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsRatingValue", numericFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsRatingCount", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) diff --git a/internal/index/bleve_document_read.go b/internal/index/bleve_document_read.go index eaebfa89..3008eabb 100644 --- a/internal/index/bleve_document_read.go +++ b/internal/index/bleve_document_read.go @@ -1078,6 +1078,7 @@ func hydrateDocument(match *search.DocumentMatch) Document { goodReadsBookURL, _ := match.Fields["GoodReadsBookURL"].(string) goodReadsRatingValue, _ := match.Fields["GoodReadsRatingValue"].(float64) + goodReadsRatingCount, _ := match.Fields["GoodReadsRatingCount"].(string) goodReadsResourceID, _ := match.Fields["GoodReadsResourceID"].(string) goodReadsLegacyBookID, _ := match.Fields["GoodReadsLegacyBookID"].(float64) goodReadsEnriched, _ := match.Fields["GoodReadsEnriched"].(bool) @@ -1110,6 +1111,7 @@ func hydrateDocument(match *search.DocumentMatch) Document { TextRankEnriched: textRankEnriched, GoodReadsBookURL: goodReadsBookURL, GoodReadsRatingValue: goodReadsRatingValue, + GoodReadsRatingCount: goodReadsRatingCount, GoodReadsResourceID: goodReadsResourceID, GoodReadsLegacyBookID: int(goodReadsLegacyBookID), GoodReadsEnriched: goodReadsEnriched, diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go index de222ecd..2bfdd75f 100644 --- a/internal/index/bleve_goodreads_write.go +++ b/internal/index/bleve_goodreads_write.go @@ -30,7 +30,7 @@ type GoodReadsSource interface { SearchBooks(ctx context.Context, query string) (biblioreads.SearchBooksResponse, error) } -var goodReadsFields = []string{"GoodReadsBookURL", "GoodReadsRatingValue", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} +var goodReadsFields = []string{"GoodReadsBookURL", "GoodReadsRatingValue", "GoodReadsRatingCount", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} // Strip named series annotations and omnibus ranges, not standalone volume numbers. var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) @@ -357,7 +357,7 @@ func (b *BleveIndexer) enrichGoodReadsBatch(documents []Document) error { // not looked up yet and persists its match. Returns an empty result if there's no match. func (b *BleveIndexer) GoodReadsBook(ctx context.Context, document Document) (biblioreads.SearchResult, error) { if document.GoodReadsBookURL != "" || document.GoodReadsEnriched || b.goodReadsSource == nil { - return biblioreads.SearchResult{BookURL: document.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", document.GoodReadsRatingValue)}, nil + return biblioreads.SearchResult{BookURL: document.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", document.GoodReadsRatingValue), RatingCount: document.GoodReadsRatingCount}, nil } return b.enrichGoodReads(ctx, document) } @@ -388,6 +388,7 @@ func (b *BleveIndexer) UpdateGoodReadsBookURL(documentID, rawURL string) error { } current.GoodReadsBookURL = path current.GoodReadsRatingValue = 0 + current.GoodReadsRatingCount = "" current.GoodReadsResourceID = "" current.GoodReadsLegacyBookID = 0 current.GoodReadsEnriched = true @@ -397,14 +398,14 @@ func (b *BleveIndexer) UpdateGoodReadsBookURL(documentID, rawURL string) error { return nil } -// StoreGoodReadsDetails persists a nonempty rating and complete review identifiers, +// StoreGoodReadsDetails persists a nonempty rating, rating count, and complete review identifiers, // provided the document still has the metadata and match read before the fetch. func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, book biblioreads.BookResponse) error { if bookURL == "" { return fmt.Errorf("invalid GoodReads book URL for document %s", document.ID) } hasIDs := book.ResourceID != "" && book.LegacyBookID > 0 - if book.Rating == "" && !hasIDs { + if book.Rating == "" && book.RatingCount == "" && !hasIDs { return nil } b.documentsMu.Lock() @@ -420,6 +421,10 @@ func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, return fmt.Errorf("GoodReads match changed for document %s", document.ID) } changed := false + if book.RatingCount != "" && current.GoodReadsRatingCount != book.RatingCount { + current.GoodReadsRatingCount = book.RatingCount + changed = true + } if book.Rating != "" { ratingValue := parseGoodReadsRating(book.Rating, document.ID) if current.GoodReadsRatingValue != ratingValue { @@ -495,17 +500,18 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) ( // Looked up concurrently (e.g. by the background enrichment and a detail // page request at the same time): keep the stored result. if current.GoodReadsEnriched { - result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", current.GoodReadsRatingValue)} + result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", current.GoodReadsRatingValue), RatingCount: current.GoodReadsRatingCount} return result, nil } if current.GoodReadsBookURL != "" && current.GoodReadsBookURL != result.BookURL { - result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", current.GoodReadsRatingValue)} + result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", current.GoodReadsRatingValue), RatingCount: current.GoodReadsRatingCount} } current.GoodReadsBookURL = result.BookURL if result.Rating == "" && current.GoodReadsRatingValue > 0 { result.Rating = fmt.Sprintf("%.2f", current.GoodReadsRatingValue) } current.GoodReadsRatingValue = parseGoodReadsRating(result.Rating, current.ID) + current.GoodReadsRatingCount = result.RatingCount current.GoodReadsEnriched = true if err := b.storeGoodReadsAuthorURL(current, result.AuthorURL, result.Author); err != nil { return result, fmt.Errorf("error storing GoodReads author URL for document %s: %w", current.ID, err) @@ -591,15 +597,17 @@ func (b *BleveIndexer) preserveGoodReadsLocked(documents []Document) { for _, hit := range searchResult.Hits { bookURL, _ := hit.Fields["GoodReadsBookURL"].(string) ratingValue, _ := hit.Fields["GoodReadsRatingValue"].(float64) + ratingCount, _ := hit.Fields["GoodReadsRatingCount"].(string) resourceID, _ := hit.Fields["GoodReadsResourceID"].(string) legacyBookID, _ := hit.Fields["GoodReadsLegacyBookID"].(float64) enriched, _ := hit.Fields["GoodReadsEnriched"].(bool) - stored[hit.ID] = Document{GoodReadsBookURL: bookURL, GoodReadsRatingValue: ratingValue, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} + stored[hit.ID] = Document{GoodReadsBookURL: bookURL, GoodReadsRatingValue: ratingValue, GoodReadsRatingCount: ratingCount, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} } for i := range documents { if s, ok := stored[documents[i].ID]; ok { documents[i].GoodReadsBookURL = s.GoodReadsBookURL documents[i].GoodReadsRatingValue = s.GoodReadsRatingValue + documents[i].GoodReadsRatingCount = s.GoodReadsRatingCount documents[i].GoodReadsResourceID = s.GoodReadsResourceID documents[i].GoodReadsLegacyBookID = s.GoodReadsLegacyBookID documents[i].GoodReadsEnriched = s.GoodReadsEnriched diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go index 291537b0..f0d77906 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/index/bleve_goodreads_write_test.go @@ -66,7 +66,7 @@ func (goodReadsTestReader) Cover(string, int) (image.Image, error) { } var goodReadsTestResults = []biblioreads.SearchResult{ - {Title: "La Reina Dragon", Author: "Alice Borchardt", BookURL: "/book/show/12207444-la-reina-dragon", AuthorURL: "/author/show/151155.Alice_Borchardt", Rating: "3.82"}, + {Title: "La Reina Dragon", Author: "Alice Borchardt", BookURL: "/book/show/12207444-la-reina-dragon", AuthorURL: "/author/show/151155.Alice_Borchardt", Rating: "3.82", RatingCount: "2,285"}, {BookURL: "/book/show/2-other", AuthorURL: "/author/show/2.Other"}, } @@ -110,6 +110,9 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { if doc.GoodReadsRatingValue != 3.82 { t.Errorf("unexpected Goodreads rating %v", doc.GoodReadsRatingValue) } + if doc.GoodReadsRatingCount != "2,285" { + t.Errorf("unexpected Goodreads rating count %q", doc.GoodReadsRatingCount) + } assertNoDocumentAuthorURL(t, idx.documentsIdx) if !doc.GoodReadsEnriched { t.Error("expected document to be marked as GoodReads-enriched") @@ -292,7 +295,7 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != wantGoodReadsBookURL || doc.GoodReadsRatingValue != 3.82 { + if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != wantGoodReadsBookURL || doc.GoodReadsRatingValue != 3.82 || doc.GoodReadsRatingCount != "2,285" { t.Errorf("expected GoodReads fields to survive a TextRank write, got %+v", doc) } if doc.GoodReadsResourceID != "work-id" || doc.GoodReadsLegacyBookID != 12207444 { @@ -997,6 +1000,7 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { name string book biblioreads.BookResponse wantRating string + wantCount string wantWork string wantID int }{ @@ -1005,12 +1009,15 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { {name: "missing work", book: biblioreads.BookResponse{Rating: "4.12", LegacyBookID: 2}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, {name: "missing edition", book: biblioreads.BookResponse{Rating: "4.12", ResourceID: "new-work"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, {name: "empty response", wantRating: "3.82", wantWork: "old-work", wantID: 1}, + {name: "count only", book: biblioreads.BookResponse{RatingCount: "650"}, wantRating: "3.82", wantCount: "650", wantWork: "old-work", wantID: 1}, + {name: "zero count", book: biblioreads.BookResponse{RatingCount: "0"}, wantRating: "3.82", wantCount: "0", wantWork: "old-work", wantID: 1}, } { t.Run(tt.name, func(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) doc := Document{ ID: goodReadsTestID, GoodReadsBookURL: wantGoodReadsBookURL, GoodReadsRatingValue: 3.82, + GoodReadsRatingCount: "2,285", GoodReadsResourceID: "old-work", GoodReadsLegacyBookID: 1, } if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { @@ -1027,6 +1034,13 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { stored.GoodReadsResourceID != tt.wantWork || stored.GoodReadsLegacyBookID != tt.wantID { t.Fatalf("partial details changed fields that were not supplied: %+v", stored) } + wantCount := tt.wantCount + if wantCount == "" { + wantCount = doc.GoodReadsRatingCount + } + if stored.GoodReadsRatingCount != wantCount { + t.Fatalf("expected rating count %q, got %q", wantCount, stored.GoodReadsRatingCount) + } }) } } @@ -1066,6 +1080,7 @@ func TestUpdateGoodReadsBookURL(t *testing.T) { TextRankWords: []string{"dragon"}, GoodReadsBookURL: wantGoodReadsBookURL, GoodReadsRatingValue: 3.82, + GoodReadsRatingCount: "2,285", GoodReadsResourceID: "old-work", GoodReadsLegacyBookID: 12207444, GoodReadsEnriched: true, @@ -1088,6 +1103,7 @@ func TestUpdateGoodReadsBookURL(t *testing.T) { if tt.want == "" || tt.want == doc.GoodReadsBookURL { if stored.GoodReadsBookURL != doc.GoodReadsBookURL || stored.GoodReadsRatingValue != doc.GoodReadsRatingValue || + stored.GoodReadsRatingCount != doc.GoodReadsRatingCount || stored.GoodReadsResourceID != doc.GoodReadsResourceID || stored.GoodReadsLegacyBookID != doc.GoodReadsLegacyBookID { t.Fatalf("invalid or unchanged URL modified stored data: %+v", stored) @@ -1096,6 +1112,7 @@ func TestUpdateGoodReadsBookURL(t *testing.T) { } if stored.GoodReadsBookURL != tt.want || !stored.GoodReadsEnriched || stored.GoodReadsRatingValue != 0 || + stored.GoodReadsRatingCount != "" || stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { t.Fatalf("correction must clear old rating and review identifiers: %+v", stored) } diff --git a/internal/index/document.go b/internal/index/document.go index 022aeb64..394d5d10 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -84,6 +84,7 @@ type Document struct { GoodReadsBookURL string // GoodReadsRatingValue is the Goodreads score, stored and indexed for filtering and sorting. GoodReadsRatingValue float64 + GoodReadsRatingCount string // GoodReadsResourceID identifies the work; GoodReadsLegacyBookID identifies // the edition. Both are stored only, populated when book details are fetched, // and used together by BiblioReads to retrieve reviews. diff --git a/internal/index/goodreads_fields_test.go b/internal/index/goodreads_fields_test.go index 777c86b3..0930aed7 100644 --- a/internal/index/goodreads_fields_test.go +++ b/internal/index/goodreads_fields_test.go @@ -115,6 +115,7 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { AuthorsSlugs: []string{"alice-borchardt"}, GoodReadsBookURL: bookURL, GoodReadsRatingValue: 4.25, + GoodReadsRatingCount: "2285", GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: legacyBookID, } @@ -132,12 +133,15 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { if got.GoodReadsRatingValue != 4.25 { t.Errorf("language %q: expected stored rating, got %v", lang, got.GoodReadsRatingValue) } + if got.GoodReadsRatingCount != "2285" { + t.Errorf("language %q: expected stored rating count, got %q", lang, got.GoodReadsRatingCount) + } if got.GoodReadsResourceID != resourceID || got.GoodReadsLegacyBookID != legacyBookID { t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, got.GoodReadsResourceID, got.GoodReadsLegacyBookID) } } - for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155", "zyxwvresource", resourceID, "4.25"} { + for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155", "zyxwvresource", resourceID, "4.25", "2285"} { res, err := idx.Search(index.SearchFields{Keywords: keywords}, 1, 10) if err != nil { t.Fatal(err) diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index eecb9944..ba8a3c14 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -42,6 +42,11 @@ func (b *BiblioReads) DocumentLink(document index.Document) servicelinks.Link { return (servicelinks.Goodreads{}).Document(document) } +// CanEditDocumentLink reports that Goodreads book links can be corrected. +func (b *BiblioReads) CanEditDocumentLink() bool { + return true +} + func bookID(bookURL string) (string, error) { id, ok := strings.CutPrefix(bookURL, goodReadsBookPrefix) if !ok || id == "" { @@ -89,7 +94,7 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score return score, err } document.GoodReadsBookURL = result.BookURL - score.Value, score.URL = result.Rating, (servicelinks.Goodreads{}).Document(document).URL + score.Value, score.Count, score.URL = result.Rating, result.RatingCount, (servicelinks.Goodreads{}).Document(document).URL return score, nil } book, err := b.book(ctx, document) diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index 55fa43e6..bc6d8284 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -20,6 +20,8 @@ type ReviewProvider interface { Score(ctx context.Context, document index.Document) (reviews.Score, error) // Reviews returns reviews in the current UI language (a two-letter ISO code). Reviews(ctx context.Context, document index.Document, language string) (reviews.Page, error) + // CanEditDocumentLink reports whether this provider supports correcting its document link. + CanEditDocumentLink() bool } type LinkProvider interface { diff --git a/internal/webserver/controller/document/score.go b/internal/webserver/controller/document/score.go index cb48f907..a4f394c0 100644 --- a/internal/webserver/controller/document/score.go +++ b/internal/webserver/controller/document/score.go @@ -28,26 +28,23 @@ func (d *Controller) Score(c fiber.Ctx) error { defer cancel() score, err := d.reviewProvider.Score(ctx, document) session, _ := c.Locals("Session").(model.Session) - canEditGoodReads := session.Role == model.RoleAdmin && score.Source == "Goodreads" + canEditDocumentLink := session.Role == model.RoleAdmin && d.reviewProvider.CanEditDocumentLink() if err != nil { log.Printf("Error retrieving score for %s: %s", document.ID, err) - if !canEditGoodReads { + if !canEditDocumentLink { return c.SendString("") } score.Value, score.Count = "", "" score.URL = d.documentServiceLink(document).URL } return c.Render("partials/document-score", fiber.Map{ - "Lang": c.Locals("Lang"), - "ReviewLanguage": c.Locals("Lang"), - "SupportedLanguages": c.Locals("SupportedLanguages"), - "Source": score.Source, - "Rating": score.Value, - "RatingCount": score.Count, - "Scored": score.HasScore(), - "URL": score.URL, - "DocumentSlug": document.Slug, - "CanEditGoodReads": canEditGoodReads, - "ScoreUnavailable": err != nil, + "Source": score.Source, + "Rating": score.Value, + "RatingCount": score.Count, + "Scored": score.HasScore(), + "URL": score.URL, + "DocumentSlug": document.Slug, + "CanEditDocumentLink": canEditDocumentLink, + "ScoreUnavailable": err != nil, }) } diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 49d6c904..73a55cfa 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -97,13 +97,18 @@ func TestDocumentReviewsRejectUnsupportedLanguage(t *testing.T) { } type fakeReviewProvider struct { - score reviews.Score - page reviews.Page - scoreErr error - reviewsErr error - scoreSlugs []string - reviewSlugs []string - reviewLanguages []string + score reviews.Score + page reviews.Page + scoreErr error + reviewsErr error + canEditDocumentLink bool + scoreSlugs []string + reviewSlugs []string + reviewLanguages []string +} + +func (f *fakeReviewProvider) CanEditDocumentLink() bool { + return f.canEditDocumentLink } func (f *fakeReviewProvider) Score(_ context.Context, document index.Document) (reviews.Score, error) { @@ -351,7 +356,7 @@ func TestSearchResultsShowStoredGoodReadsRating(t *testing.T) { if err != nil { t.Fatal(err) } - if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, biblioreads.BookResponse{Rating: tt.rating}); err != nil { + if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, biblioreads.BookResponse{Rating: tt.rating, RatingCount: "2,285"}); err != nil { t.Fatal(err) } page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) @@ -365,10 +370,45 @@ func TestSearchResultsShowStoredGoodReadsRating(t *testing.T) { if page.Find(`li:has(a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]) i.bi-star-fill`).Length() != 1 { t.Fatal("expected a star icon beside the Goodreads score") } + if rating.Parent().Find("small").Text() != "(2,285 ratings)" { + t.Fatal("expected rating count beside the Goodreads score") + } }) } } +func TestSearchResultsShowGoodReadsSearchRatingCount(t *testing.T) { + source := &fakeBookSource{results: []biblioreads.SearchResult{{ + Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", + Rating: "4.20", RatingCount: "1,234", + }}} + app, idx := newGoodReadsTestApp(t, source) + score := getBody(t, app, "/documents/unlinked/score") + if !strings.Contains(score, "(1,234 ratings)") { + t.Fatal("expected the search-derived rating count in the document score") + } + stored, err := idx.Document("unlinked") + if err != nil || stored.GoodReadsRatingCount != "1,234" { + t.Fatalf("expected the search-derived count to be persisted, got %+v, %v", stored, err) + } + for _, path := range []string{ + "/search?type=documents&search=Unlinked", + "/search?type=documents&search=Unlinked&l=es", + } { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) + if err != nil { + t.Fatal(err) + } + rating := page.Find(`a[href="https://www.goodreads.com/book/show/1-unlinked"]`) + if rating.Text() != "4.20" || !strings.Contains(rating.Parent().Find("small").Text(), "1,234") { + t.Fatalf("expected the stored score and rating count in search results for %s", path) + } + } + if len(source.ids) != 0 || len(source.queries) != 1 { + t.Fatalf("expected one book search and no detail fetches, got queries %q and ids %q", source.queries, source.ids) + } +} + func TestDocumentDetailShowsStoredZeroRatingAsNotScored(t *testing.T) { source := &fakeBookSource{results: []biblioreads.SearchResult{{ Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", Rating: "0.00", @@ -500,7 +540,9 @@ func TestDocumentGoodReadsEditor(t *testing.T) { } return } - if control.Text() != "Edit Goodreads link" || + if control.Text() != "" || + control.Find(`i.bi-pencil[alt="Edit Goodreads link"]`).Length() != 1 || + control.Filter(`[title="Edit Goodreads link"][aria-label="Edit Goodreads link"]`).Length() != 1 || page.Find(`#edit-goodreads-linked.collapse:not(.show)`).Length() != 1 || page.Find(`form[hx-put="/documents/linked/goodreads"][hx-target="this"]`).Length() != 1 || page.Find(`label[for="goodreads-book-url-linked"]`).Length() != 1 { diff --git a/internal/webserver/embedded/js/search-filter-utils.js b/internal/webserver/embedded/js/search-filter-utils.js index 6e4e77c9..c56f8a5f 100644 --- a/internal/webserver/embedded/js/search-filter-utils.js +++ b/internal/webserver/embedded/js/search-filter-utils.js @@ -112,7 +112,7 @@ function updateDateClearButtonState(dateControl) { function updateRangeClearButtonState(group) { const clearButton = group.querySelector('.clear-range-control') if (!clearButton) return - clearButton.disabled = !Array.from(group.querySelectorAll('input')).some(hasValue) + clearButton.disabled = !Array.from(group.querySelectorAll('input, select')).some(hasValue) } function resetDateControl(dateControl) { @@ -128,7 +128,7 @@ function resetDateControl(dateControl) { } function resetRangeGroup(group) { - group.querySelectorAll('input').forEach(input => { input.value = '' }) + group.querySelectorAll('input, select').forEach(input => { input.value = '' }) updateRangeClearButtonState(group) } @@ -201,14 +201,15 @@ export function initClearRangeControls(searchFilters) { const group = clearLink.closest('.input-group') if (!group) return - group.querySelectorAll('input').forEach(input => { + group.querySelectorAll('input, select').forEach(input => { input.addEventListener('input', () => updateRangeClearButtonState(group)) + input.addEventListener('change', () => updateRangeClearButtonState(group)) }) clearLink.addEventListener('click', (e) => { e.preventDefault() resetRangeGroup(group) - const firstInput = group.querySelector('input') + const firstInput = group.querySelector('input, select') if (firstInput) firstInput.dispatchEvent(new Event('input', { bubbles: true })) }) diff --git a/internal/webserver/embedded/views/partials/document-metadata-filters.html b/internal/webserver/embedded/views/partials/document-metadata-filters.html index ce1efb55..7664c91a 100644 --- a/internal/webserver/embedded/views/partials/document-metadata-filters.html +++ b/internal/webserver/embedded/views/partials/document-metadata-filters.html @@ -24,20 +24,34 @@
      - {{$goodReadsRatingFrom := ""}} + {{$goodReadsRatingFrom := 0.0}} {{if .SearchFields}} {{$goodReadsRatingFrom = .SearchFields.GoodReadsRatingFrom}} {{end}}
      - +
      - {{$goodReadsRatingTo := ""}} + {{$goodReadsRatingTo := 0.0}} {{if .SearchFields}} {{$goodReadsRatingTo = .SearchFields.GoodReadsRatingTo}} {{end}}
      - +
      diff --git a/internal/webserver/embedded/views/partials/document-metadata.html b/internal/webserver/embedded/views/partials/document-metadata.html index 0e23ff2d..f5349ef2 100644 --- a/internal/webserver/embedded/views/partials/document-metadata.html +++ b/internal/webserver/embedded/views/partials/document-metadata.html @@ -16,7 +16,7 @@
      • {{ end }} {{if and .ShowGoodReadsScore .Document.GoodReadsBookURL}} - {{template "partials/document-score" dict "Lang" .Lang "Source" "Goodreads" "Rating" (printf "%.2f" .Document.GoodReadsRatingValue) "Scored" (gt .Document.GoodReadsRatingValue 0.0) "URL" (printf "https://www.goodreads.com%s" .Document.GoodReadsBookURL)}} + {{template "partials/document-score" dict "Lang" .Lang "Source" "Goodreads" "Rating" (printf "%.2f" .Document.GoodReadsRatingValue) "RatingCount" .Document.GoodReadsRatingCount "Scored" (gt .Document.GoodReadsRatingValue 0.0) "URL" (printf "https://www.goodreads.com%s" .Document.GoodReadsBookURL)}} {{end}} {{ if and .IllustratedMinAmount (ge .Document.Illustrations .IllustratedMinAmount) }}
    • {{t .Lang "Illustrated"}}
      • diff --git a/internal/webserver/embedded/views/partials/document-score.html b/internal/webserver/embedded/views/partials/document-score.html index 209bde66..485f20c6 100644 --- a/internal/webserver/embedded/views/partials/document-score.html +++ b/internal/webserver/embedded/views/partials/document-score.html @@ -1,7 +1,7 @@
    • {{if .ScoreUnavailable}}{{t .Lang "Unable to load Goodreads score"}}{{else if .Scored}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}{{else}}{{t .Lang "Not scored"}}{{end}} - {{if .CanEditGoodReads}} - + {{if .CanEditDocumentLink}} +
      {{template "partials/document-goodreads-form" dict "Lang" .Lang "Slug" .DocumentSlug "BookURL" .URL}}
      @@ -9,7 +9,7 @@
      • {{if and .URL .DocumentSlug}}
      - {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .Source "BookURL" .URL "Slug" .DocumentSlug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .ReviewLanguage}} + {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .Source "BookURL" .URL "Slug" .DocumentSlug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .Lang}} {{if .Rating}}
      - {{template "partials/document-metadata-filters" dict "Lang" .Lang "FilterIdPrefix" "highlight" "SearchFields" .SearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} + {{template "partials/document-metadata-filters" dict "Lang" .Lang "ReviewSourceName" .ReviewSourceName "FilterIdPrefix" "highlight" "SearchFields" .SearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} {{template "partials/search-filters-actions" dict "Lang" .Lang "ShowSearchButton" false}} diff --git a/internal/webserver/embedded/views/partials/completed-list.html b/internal/webserver/embedded/views/partials/completed-list.html index bf1304cd..913cbf88 100644 --- a/internal/webserver/embedded/views/partials/completed-list.html +++ b/internal/webserver/embedded/views/partials/completed-list.html @@ -12,7 +12,7 @@
      - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
      {{end}} diff --git a/internal/webserver/embedded/views/partials/count-and-sort.html b/internal/webserver/embedded/views/partials/count-and-sort.html index 1bfd8ca7..581de826 100644 --- a/internal/webserver/embedded/views/partials/count-and-sort.html +++ b/internal/webserver/embedded/views/partials/count-and-sort.html @@ -8,6 +8,6 @@ {{if gt .Results.TotalHits 0}}
      - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
      {{end}} diff --git a/internal/webserver/embedded/views/partials/docs-list-fragments.html b/internal/webserver/embedded/views/partials/docs-list-fragments.html index 1230fc1a..e5992224 100644 --- a/internal/webserver/embedded/views/partials/docs-list-fragments.html +++ b/internal/webserver/embedded/views/partials/docs-list-fragments.html @@ -1,10 +1,10 @@ {{if and .Author (ne .Author.Name "")}}
      - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}}
      {{else if .SeriesTitle}}
      - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}}
      {{else}}
      {{template "partials/search-type-tabs" .}}
      diff --git a/internal/webserver/embedded/views/partials/document-goodreads-form.html b/internal/webserver/embedded/views/partials/document-goodreads-form.html index dcce5b66..b937442a 100644 --- a/internal/webserver/embedded/views/partials/document-goodreads-form.html +++ b/internal/webserver/embedded/views/partials/document-goodreads-form.html @@ -1,4 +1,4 @@ -
    - {{template "partials/document-search-filters" dict "Lang" .Lang "FilterIdPrefix" "sidebar" "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" true "HideSearch" true}} + {{template "partials/document-search-filters" dict "Lang" .Lang "ReviewSourceName" .ReviewSourceName "FilterIdPrefix" "sidebar" "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" true "HideSearch" true}}
    {{if not .DocumentsOnly}}
    diff --git a/internal/webserver/embedded/views/partials/search-type-tabs.html b/internal/webserver/embedded/views/partials/search-type-tabs.html index 1caa82da..365d3e35 100644 --- a/internal/webserver/embedded/views/partials/search-type-tabs.html +++ b/internal/webserver/embedded/views/partials/search-type-tabs.html @@ -16,7 +16,7 @@
    {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}}
    diff --git a/internal/webserver/embedded/views/partials/searchbox.html b/internal/webserver/embedded/views/partials/searchbox.html index 61becca4..a7497e12 100644 --- a/internal/webserver/embedded/views/partials/searchbox.html +++ b/internal/webserver/embedded/views/partials/searchbox.html @@ -32,7 +32,7 @@
    - {{template "partials/document-search-filters" dict "Lang" .Lang "HomeSearch" true "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} + {{template "partials/document-search-filters" dict "Lang" .Lang "ReviewSourceName" .ReviewSourceName "HomeSearch" true "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}}
    {{template "partials/author-search-filters" dict "Lang" .Lang "HomeSearch" true}} diff --git a/internal/webserver/embedded/views/partials/similar-to-banner.html b/internal/webserver/embedded/views/partials/similar-to-banner.html index 3622aa9f..42615242 100644 --- a/internal/webserver/embedded/views/partials/similar-to-banner.html +++ b/internal/webserver/embedded/views/partials/similar-to-banner.html @@ -9,7 +9,7 @@
    {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}}
    @@ -19,7 +19,7 @@ {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}} diff --git a/internal/webserver/embedded/views/partials/sort.html b/internal/webserver/embedded/views/partials/sort.html index 4bdc09d2..fdc91260 100644 --- a/internal/webserver/embedded/views/partials/sort.html +++ b/internal/webserver/embedded/views/partials/sort.html @@ -4,7 +4,7 @@ diff --git a/internal/webserver/embedded/views/series/list.html b/internal/webserver/embedded/views/series/list.html index 7409ef2b..e9f231d2 100644 --- a/internal/webserver/embedded/views/series/list.html +++ b/internal/webserver/embedded/views/series/list.html @@ -1,6 +1,6 @@

    {{t .Lang "Collection \"%s\"" .Title}}

    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}}
    diff --git a/internal/webserver/model/highlight_repository.go b/internal/webserver/model/highlight_repository.go index 58204213..199ced5f 100644 --- a/internal/webserver/model/highlight_repository.go +++ b/internal/webserver/model/highlight_repository.go @@ -141,8 +141,8 @@ func hasSearchFields(sf index.SearchFields) bool { sf.EstReadTimeTo != 0 || sf.PagesFrom != 0 || sf.PagesTo != 0 || - sf.GoodReadsRatingFrom != 0 || - sf.GoodReadsRatingTo != 0 || + sf.ReviewScoreFrom != 0 || + sf.ReviewScoreTo != 0 || sf.IllustratedOnly } @@ -193,10 +193,10 @@ func matchesSearchFields(doc AugmentedDocument, sf index.SearchFields, illustrat return false } - if sf.GoodReadsRatingFrom > 0 || sf.GoodReadsRatingTo > 0 { + if sf.ReviewScoreFrom > 0 || sf.ReviewScoreTo > 0 { if doc.GoodReadsRatingValue <= 0 || - (sf.GoodReadsRatingFrom > 0 && doc.GoodReadsRatingValue < sf.GoodReadsRatingFrom) || - (sf.GoodReadsRatingTo > 0 && doc.GoodReadsRatingValue > sf.GoodReadsRatingTo) { + (sf.ReviewScoreFrom > 0 && doc.GoodReadsRatingValue < sf.ReviewScoreFrom) || + (sf.ReviewScoreTo > 0 && doc.GoodReadsRatingValue > sf.ReviewScoreTo) { return false } } diff --git a/internal/webserver/model/highlight_search_test.go b/internal/webserver/model/highlight_search_test.go index e77e127d..b8aebd45 100644 --- a/internal/webserver/model/highlight_search_test.go +++ b/internal/webserver/model/highlight_search_test.go @@ -23,7 +23,7 @@ func TestMatchesSearchFieldsGoodReadsRatingRange(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { doc := AugmentedDocument{Document: index.Document{GoodReadsRatingValue: tt.score}} - fields := index.SearchFields{GoodReadsRatingFrom: tt.from, GoodReadsRatingTo: tt.to} + fields := index.SearchFields{ReviewScoreFrom: tt.from, ReviewScoreTo: tt.to} if got := matchesSearchFields(doc, fields, 0); got != tt.want { t.Errorf("matchesSearchFields() = %v, want %v", got, tt.want) } diff --git a/internal/webserver/routes.go b/internal/webserver/routes.go index 0e4b7a65..c2e1a09d 100644 --- a/internal/webserver/routes.go +++ b/internal/webserver/routes.go @@ -52,6 +52,7 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se c.Locals("Version", c.App().Config().AppName) c.Locals("SupportedLanguages", supportedLanguages) c.Locals("Lang", chooseBestLanguage(c)) + c.Locals("ReviewSourceName", controllers.Documents.ReviewSourceName()) q := c.Queries() delete(q, "l") c.Locals("URLPath", c.Path()) @@ -98,7 +99,7 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se app.Get("/upload", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UploadForm) docsGroup.Post("/", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Upload) docsGroup.Delete("/:slug", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Delete) - docsGroup.Put("/:slug/goodreads", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UpdateGoodReads) + docsGroup.Put("/:slug/review-link", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UpdateDocumentLink) // Authentication requirement is configurable for all routes below this middleware app.Use(configurableAuthentication) diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index f2ae13b0..86307ecd 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -160,7 +160,7 @@ func TestGoodReadsScoreSortOptions(t *testing.T) { mode = "/fragment" } t.Run(path+direction+mode, func(t *testing.T) { - selected := "goodreads-score-" + direction + selected := "review-score-" + direction req := mustGetRequest(t, path+"sort-by="+selected) if fragment { req.Header.Set("HX-Request", "true") @@ -177,7 +177,7 @@ func TestGoodReadsScoreSortOptions(t *testing.T) { if err != nil { t.Fatal(err) } - for _, value := range []string{"goodreads-score-asc", "goodreads-score-desc"} { + for _, value := range []string{"review-score-asc", "review-score-desc"} { if doc.Find(`#sort-by option[value="`+value+`"]`).Length() != 1 { t.Errorf("missing sort option %q", value) } @@ -191,11 +191,11 @@ func TestGoodReadsScoreSortOptions(t *testing.T) { } } -func TestGoodReadsScoreFilter(t *testing.T) { +func TestReviewScoreFilter(t *testing.T) { db := infrastructure.Connect(":memory:", 250) app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), webserver.Config{}) - response, err := app.Test(mustGetRequest(t, "/search?type=documents&goodreads-score-from=4&goodreads-score-to=5")) + response, err := app.Test(mustGetRequest(t, "/search?type=documents&review-score-from=4&review-score-to=5")) if err != nil { t.Fatal(err) } @@ -211,8 +211,8 @@ func TestGoodReadsScoreFilter(t *testing.T) { name string value string }{ - {name: "goodreads-score-from", value: "4"}, - {name: "goodreads-score-to", value: "5"}, + {name: "review-score-from", value: "4"}, + {name: "review-score-to", value: "5"}, } { selects := doc.Find(`select[name="` + field.name + `"]`) if selects.Length() == 0 { @@ -238,7 +238,7 @@ func TestGoodReadsScoreFilter(t *testing.T) { t.Errorf("Expected blank and scores 1 to 5 for %s, got %v", field.name, options) } }) - if doc.Find(`input[name="` + field.name + `"]`).Length() != 0 { + if doc.Find(`input[name="`+field.name+`"]`).Length() != 0 { t.Errorf("Expected no numeric input for %s", field.name) } } From cd39940c5ec9c14db51f05d0113d751e7d40ad56 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 2 Nov 2026 23:50:11 +0100 Subject: [PATCH 16/50] WIP --- .../document/update_document_link.go | 2 +- .../webserver/controller/search/parse_test.go | 41 --------------- .../controller/series/documents_test.go | 36 ------------- internal/webserver/document_goodreads_test.go | 50 +++++++++++++++++++ .../webserver/embedded/translations/de.yml | 15 ++---- .../webserver/embedded/translations/es.yml | 15 ++---- .../webserver/embedded/translations/fr.yml | 15 ++---- .../webserver/embedded/translations/ru.yml | 15 ++---- .../partials/document-goodreads-form.html | 4 +- .../views/partials/document-score.html | 6 +-- internal/webserver/series_test.go | 27 +++++++++- internal/webserver/webserver_test.go | 7 ++- 12 files changed, 104 insertions(+), 129 deletions(-) delete mode 100644 internal/webserver/controller/search/parse_test.go delete mode 100644 internal/webserver/controller/series/documents_test.go diff --git a/internal/webserver/controller/document/update_document_link.go b/internal/webserver/controller/document/update_document_link.go index b023740c..8e60048d 100644 --- a/internal/webserver/controller/document/update_document_link.go +++ b/internal/webserver/controller/document/update_document_link.go @@ -24,7 +24,7 @@ func (d *Controller) UpdateDocumentLink(c fiber.Ctx) error { if err := d.reviewProvider.UpdateDocumentLink(document, link); err != nil { if errors.Is(err, reviews.ErrInvalidDocumentLink) { return c.Status(fiber.StatusUnprocessableEntity).Render("partials/document-goodreads-form", fiber.Map{ - "Slug": document.Slug, "BookURL": link, "InvalidURL": true, + "Slug": document.Slug, "BookURL": link, "InvalidURL": true, "Source": d.reviewProvider.Name(), }) } if errors.Is(err, reviews.ErrDocumentNotFound) { diff --git a/internal/webserver/controller/search/parse_test.go b/internal/webserver/controller/search/parse_test.go deleted file mode 100644 index 9536ce05..00000000 --- a/internal/webserver/controller/search/parse_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package search - -import ( - "net/http/httptest" - "slices" - "testing" - - "github.com/gofiber/fiber/v3" - "github.com/svera/coreander/v5/internal/index" -) - -func TestParseReviewScoreSort(t *testing.T) { - for _, direction := range []string{"asc", "desc"} { - t.Run(direction, func(t *testing.T) { - app := fiber.New() - app.Get("/", func(c fiber.Ctx) error { - fields, err := ParseDocumentSearchQuery(c, 250) - if err != nil { - t.Error(err) - return c.SendStatus(fiber.StatusBadRequest) - } - key := index.ScoreValueSortField - if direction == "desc" { - key = "-" + key - } - if !slices.Equal(fields.SortBy, []string{key}) { - t.Errorf("got %v, want [%s]", fields.SortBy, key) - } - return c.SendStatus(fiber.StatusOK) - }) - response, err := app.Test(httptest.NewRequest("GET", "/?sort-by=review-score-"+direction, nil)) - if err != nil { - t.Fatal(err) - } - response.Body.Close() - if response.StatusCode != fiber.StatusOK { - t.Fatalf("unexpected status %d", response.StatusCode) - } - }) - } -} diff --git a/internal/webserver/controller/series/documents_test.go b/internal/webserver/controller/series/documents_test.go deleted file mode 100644 index c3bd89fe..00000000 --- a/internal/webserver/controller/series/documents_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package series - -import ( - "net/http/httptest" - "slices" - "testing" - - "github.com/gofiber/fiber/v3" -) - -func TestParseGoodReadsScoreSort(t *testing.T) { - for _, direction := range []string{"asc", "desc"} { - t.Run(direction, func(t *testing.T) { - app := fiber.New() - app.Get("/", func(c fiber.Ctx) error { - key := "GoodReadsRatingValue" - if direction == "desc" { - key = "-" + key - } - want := []string{key, "SeriesIndex"} - if got := (&Controller{}).parseSortBy(c); !slices.Equal(got, want) { - t.Errorf("got %v, want %v", got, want) - } - return c.SendStatus(fiber.StatusOK) - }) - response, err := app.Test(httptest.NewRequest("GET", "/?sort-by=review-score-"+direction, nil)) - if err != nil { - t.Fatal(err) - } - response.Body.Close() - if response.StatusCode != fiber.StatusOK { - t.Fatalf("unexpected status %d", response.StatusCode) - } - }) - } -} diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 9a03d02f..d8d0e761 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/url" + "slices" "strings" "sync" "testing" @@ -393,6 +394,55 @@ func TestSearchResultsShowStoredGoodReadsRating(t *testing.T) { } } +func TestDocumentSearchSortsByReviewScore(t *testing.T) { + app, idx := newGoodReadsTestApp(t, &fakeBookSource{}) + scores := map[string]string{ + "linked": "3.20", + "unlinked": "4.10", + "unmatched": "4.80", + } + documents, err := idx.Search(index.SearchFields{}, 1, 10) + if err != nil { + t.Fatal(err) + } + for _, doc := range documents.Hits() { + if err := idx.UpdateGoodReadsBookURL(doc.ID, "https://www.goodreads.com/book/show/1-"+doc.Slug); err != nil { + t.Fatalf("set score link for %s: %v", doc.Slug, err) + } + updated, err := idx.Document(doc.Slug) + if err != nil { + t.Fatalf("reload %s: %v", doc.Slug, err) + } + if err := idx.StoreGoodReadsDetails(updated, updated.GoodReadsBookURL, biblioreads.BookResponse{Rating: scores[doc.Slug]}); err != nil { + t.Fatalf("set score for %s: %v", doc.Slug, err) + } + } + + for _, tt := range []struct { + direction string + want []string + }{ + {direction: "asc", want: []string{"linked", "unlinked", "unmatched"}}, + {direction: "desc", want: []string{"unmatched", "unlinked", "linked"}}, + } { + t.Run(tt.direction, func(t *testing.T) { + body := getBody(t, app, "/search?type=documents&sort-by=review-score-"+tt.direction) + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + var got []string + page.Find("#list .list-group-item h2 a").Each(func(_ int, link *goquery.Selection) { + href, _ := link.Attr("href") + got = append(got, strings.TrimPrefix(href, "/documents/")) + }) + if !slices.Equal(got, tt.want) { + t.Fatalf("score sort %s: got %v, want %v", tt.direction, got, tt.want) + } + }) + } +} + func TestSearchResultsShowGoodReadsSearchRatingCount(t *testing.T) { source := &fakeBookSource{results: []biblioreads.SearchResult{{ Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index 694c9f4e..38dd5e5d 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -17,7 +17,6 @@ "Illustrated": "Illustriert" "Illustrations": "Illustrationen" "Only illustrated": "Nur illustrierte" -"Goodreads score": "Goodreads-Bewertung" "%s score": "%s-Bewertung" "Review score": "Rezensionsbewertung" "Clear %s score": "%s-Bewertung löschen" @@ -26,9 +25,6 @@ "%s score (descending)": "%s-Bewertung (absteigend)" "Review score (ascending)": "Rezensionsbewertung (aufsteigend)" "Review score (descending)": "Rezensionsbewertung (absteigend)" -"Goodreads score (ascending)": "Goodreads-Bewertung (aufsteigend)" -"Goodreads score (descending)": "Goodreads-Bewertung (absteigend)" -"Clear Goodreads score": "Goodreads-Bewertung löschen" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Lesezeit berechnet mit einer Geschwindigkeit von %s Wörtern pro Minute. Sie können die Lesegeschwindigkeit in Ihrem Profil ändern." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Lesezeit berechnet mit einer Geschwindigkeit von %s Wörtern pro Minute. Melden Sie sich an, um diesen Wert anzupassen." "This filter only applies to EPUB documents": "Dieser Filter gilt nur für EPUB-Dokumente" @@ -229,7 +225,6 @@ "BC": "v. Chr." "Before Christ": "Vor Christus" "Go to Wikipedia entry": "Zur Wikipedia-Seite" -"Go to Goodreads author page": "Zur Autorenseite auf Goodreads" "Website": "Webseite" "Is not the right author or data is outdated?": "Ist nicht der richtige Autor oder die Daten sind veraltet?" "The Wikidata ID is a unique identifier for the author in the Wikidata database. If you know the correct ID you can update it here.": "Die Wikidata-ID ist ein eindeutiger Identifikator für den Autor in der Wikidata-Datenbank. Wenn Sie die korrekte ID kennen können Sie sie hier aktualisieren." @@ -417,15 +412,13 @@ "A newer version of this document is available.": "Eine neuere Version dieses Dokuments ist verfügbar." "This document is no longer available. Its offline copy has been removed.": "Dieses Dokument ist nicht mehr verfügbar. Seine Offline-Kopie wurde entfernt." "Reload document": "Dokument neu laden" -"Goodreads rating": "Goodreads-Bewertung" -"Edit Goodreads link": "Goodreads-Link bearbeiten" -"Goodreads book URL": "Goodreads-Buch-URL" -"Enter a valid Goodreads book URL.": "Geben Sie eine gültige Goodreads-Buch-URL ein." -"Unable to load Goodreads score": "Goodreads-Bewertung konnte nicht geladen werden" +"Edit %s link": "%s-Link bearbeiten" +"%s book URL": "%s-Buch-URL" +"Enter a valid %s book URL.": "Geben Sie eine gültige %s-Buch-URL ein." +"Unable to load %s score": "%s-Bewertung konnte nicht geladen werden" "%s rating": "%s-Bewertung" "%s user reviews": "%s-Nutzerrezensionen" "Go to %s author page": "Zur Autorenseite auf %s" -"Goodreads user reviews": "Goodreads-Nutzerrezensionen" "Show spoiler": "Spoiler anzeigen" "Show more": "Mehr anzeigen" "Show less": "Weniger anzeigen" diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index 9b36588f..8a17fc25 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -17,7 +17,6 @@ "Illustrated": "Ilustrado" "Illustrations": "Ilustraciones" "Only illustrated": "Solo ilustrados" -"Goodreads score": "Puntuación de Goodreads" "%s score": "Puntuación de %s" "Review score": "Puntuación de reseñas" "Clear %s score": "Borrar puntuación de %s" @@ -26,9 +25,6 @@ "%s score (descending)": "Puntuación de %s (descendente)" "Review score (ascending)": "Puntuación de reseñas (ascendente)" "Review score (descending)": "Puntuación de reseñas (descendente)" -"Goodreads score (ascending)": "Puntuación de Goodreads (ascendente)" -"Goodreads score (descending)": "Puntuación de Goodreads (descendente)" -"Clear Goodreads score": "Borrar puntuación de Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Tiempo de lectura calculado en base a una velocidad de %s palabras por minuto. Puedes cambiarla en tu perfil." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Tiempo de lectura calculado en base a una velocidad de %s palabras por minuto. Inicia sesión para personalizar este valor." "This filter only applies to EPUB documents": "Este filtro solo se aplica a documentos EPUB" @@ -225,7 +221,6 @@ "BC": "a. C." "Before Christ": "Antes de Cristo" "Go to Wikipedia entry": "Ir al artículo en Wikipedia" -"Go to Goodreads author page": "Ir a la página del autor en Goodreads" "Website": "Sitio web" "Is not the right author or data is outdated?": "¿No es la persona correcta o sus datos no están actualizados?" "The Wikidata ID is a unique identifier for the author in the Wikidata database. If you know the correct ID, you can update it here.": "El ID de Wikidata es un identificador único para el autor en la base de datos de Wikidata. Si conoces el ID correcto, puedes actualizarlo aquí." @@ -413,15 +408,13 @@ "A newer version of this document is available.": "Hay una versión más reciente de este documento disponible." "This document is no longer available. Its offline copy has been removed.": "Este documento ya no está disponible. Se ha eliminado su copia sin conexión." "Reload document": "Recargar documento" -"Goodreads rating": "Puntuación en Goodreads" -"Edit Goodreads link": "Editar enlace de Goodreads" -"Goodreads book URL": "URL del libro en Goodreads" -"Enter a valid Goodreads book URL.": "Introduce una URL válida de un libro en Goodreads." -"Unable to load Goodreads score": "No se pudo cargar la puntuación de Goodreads" +"Edit %s link": "Editar enlace de %s" +"%s book URL": "URL del libro de %s" +"Enter a valid %s book URL.": "Introduce una URL válida de un libro de %s." +"Unable to load %s score": "No se pudo cargar la puntuación de %s" "%s rating": "Puntuación en %s" "%s user reviews": "Reseñas de usuarios de %s" "Go to %s author page": "Ir a la página del autor en %s" -"Goodreads user reviews": "Reseñas de usuarios de Goodreads" "Show spoiler": "Mostrar spoiler" "Show more": "Mostrar más" "Show less": "Mostrar menos" diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index a67e7a70..31bb9748 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -17,7 +17,6 @@ "Illustrated": "Illustré" "Illustrations": "Illustrations" "Only illustrated": "Seulement illustrés" -"Goodreads score": "Note Goodreads" "%s score": "Note %s" "Review score": "Note des avis" "Clear %s score": "Effacer la note %s" @@ -26,9 +25,6 @@ "%s score (descending)": "Note %s (décroissante)" "Review score (ascending)": "Note des avis (croissante)" "Review score (descending)": "Note des avis (décroissante)" -"Goodreads score (ascending)": "Note Goodreads (croissante)" -"Goodreads score (descending)": "Note Goodreads (décroissante)" -"Clear Goodreads score": "Effacer la note Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Temps de lecture calculé avec une vitesse de %s mots par minute. Vous pouvez modifier votre vitesse de lecture dans votre profil." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Temps de lecture calculé avec une vitesse de %s mots par minute. Connectez-vous pour personnaliser cette valeur." "This filter only applies to EPUB documents": "Ce filtre s'applique uniquement aux documents EPUB" @@ -226,7 +222,6 @@ "BC": "av. J.-C." "Before Christ": "Avant Jésus-Christ" "Go to Wikipedia entry": "Aller à l'entrée Wikipedia" -"Go to Goodreads author page": "Voir la page de l'auteur sur Goodreads" "Website": "Site" "Is not the right author?": "N'est-ce pas le bon auteur ou les données sont obsolètes?" "The Wikidata ID is a unique identifier for the author in the Wikidata database. If you know the correct ID, you can update it here.": "L'ID Wikidata est un identifiant unique pour l'auteur dans la base de données Wikidata. Si vous connaissez l'ID correct, vous pouvez le mettre à jour ici." @@ -415,15 +410,13 @@ "A newer version of this document is available.": "Une version plus récente de ce document est disponible." "This document is no longer available. Its offline copy has been removed.": "Ce document n'est plus disponible. Sa copie hors connexion a été supprimée." "Reload document": "Recharger le document" -"Goodreads rating": "Note Goodreads" -"Edit Goodreads link": "Modifier le lien Goodreads" -"Goodreads book URL": "URL du livre sur Goodreads" -"Enter a valid Goodreads book URL.": "Saisissez une URL de livre Goodreads valide." -"Unable to load Goodreads score": "Impossible de charger la note Goodreads" +"Edit %s link": "Modifier le lien %s" +"%s book URL": "URL du livre sur %s" +"Enter a valid %s book URL.": "Saisissez une URL de livre %s valide." +"Unable to load %s score": "Impossible de charger la note %s" "%s rating": "Note %s" "%s user reviews": "Avis des utilisateurs de %s" "Go to %s author page": "Voir la page de l'auteur sur %s" -"Goodreads user reviews": "Avis des utilisateurs de Goodreads" "Show spoiler": "Afficher le spoiler" "Show more": "Afficher plus" "Show less": "Afficher moins" diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index 6f16e09b..24d8ad5c 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -17,7 +17,6 @@ "Illustrated": "Иллюстрированное" "Illustrations": "Иллюстрации" "Only illustrated": "Только с иллюстрациями" -"Goodreads score": "Оценка Goodreads" "%s score": "Оценка %s" "Review score": "Оценка отзывов" "Clear %s score": "Очистить оценку %s" @@ -26,9 +25,6 @@ "%s score (descending)": "Оценка %s (по убыванию)" "Review score (ascending)": "Оценка отзывов (по возрастанию)" "Review score (descending)": "Оценка отзывов (по убыванию)" -"Goodreads score (ascending)": "Оценка Goodreads (по возрастанию)" -"Goodreads score (descending)": "Оценка Goodreads (по убыванию)" -"Clear Goodreads score": "Очистить оценку Goodreads" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Время чтения рассчитано при скорости %s слов в минуту. Изменить скорость чтения можно в профиле." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Время чтения рассчитано при скорости %s слов в минуту. Войдите, чтобы изменить это значение." "This filter only applies to EPUB documents": "Этот фильтр применяется только к документам EPUB" @@ -227,7 +223,6 @@ "BC": "до н. э." "Before Christ": "До Рождества Христова" "Go to Wikipedia entry": "Перейти к статье в Википедии" -"Go to Goodreads author page": "Перейти на страницу автора в Goodreads" "Website": "Сайт" "Is not the right author?": "Это не тот автор или данные устарели?" "The Wikidata ID is a unique identifier for the author in the Wikidata database. If you know the correct ID, you can update it here.": "Идентификатор Wikidata — уникальный идентификатор автора в базе данных Wikidata. Если вы знаете правильный ID, укажите его здесь." @@ -409,15 +404,13 @@ "A newer version of this document is available.": "Доступна новая версия этого документа." "This document is no longer available. Its offline copy has been removed.": "Этот документ больше недоступен. Его копия для чтения без подключения к сети удалена." "Reload document": "Перезагрузить документ" -"Goodreads rating": "Рейтинг Goodreads" -"Edit Goodreads link": "Изменить ссылку Goodreads" -"Goodreads book URL": "URL книги на Goodreads" -"Enter a valid Goodreads book URL.": "Введите корректный URL книги на Goodreads." -"Unable to load Goodreads score": "Не удалось загрузить рейтинг Goodreads" +"Edit %s link": "Изменить ссылку %s" +"%s book URL": "URL книги на %s" +"Enter a valid %s book URL.": "Введите корректный URL книги на %s." +"Unable to load %s score": "Не удалось загрузить рейтинг %s" "%s rating": "Рейтинг %s" "%s user reviews": "Отзывы пользователей %s" "Go to %s author page": "Перейти на страницу автора в %s" -"Goodreads user reviews": "Отзывы пользователей Goodreads" "Show spoiler": "Показать спойлер" "Show more": "Показать больше" "Show less": "Показать меньше" diff --git a/internal/webserver/embedded/views/partials/document-goodreads-form.html b/internal/webserver/embedded/views/partials/document-goodreads-form.html index b937442a..35a6f848 100644 --- a/internal/webserver/embedded/views/partials/document-goodreads-form.html +++ b/internal/webserver/embedded/views/partials/document-goodreads-form.html @@ -1,10 +1,10 @@ - +
    {{if .InvalidURL}} - + {{end}} diff --git a/internal/webserver/embedded/views/partials/document-score.html b/internal/webserver/embedded/views/partials/document-score.html index 485f20c6..25ee71da 100644 --- a/internal/webserver/embedded/views/partials/document-score.html +++ b/internal/webserver/embedded/views/partials/document-score.html @@ -1,9 +1,9 @@
  • - {{if .ScoreUnavailable}}{{t .Lang "Unable to load Goodreads score"}}{{else if .Scored}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}{{else}}{{t .Lang "Not scored"}}{{end}} + {{if .ScoreUnavailable}}{{t .Lang "Unable to load %s score" .Source}}{{else if .Scored}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}{{else}}{{t .Lang "Not scored"}}{{end}} {{if .CanEditDocumentLink}} - +
    - {{template "partials/document-goodreads-form" dict "Lang" .Lang "Slug" .DocumentSlug "BookURL" .URL}} + {{template "partials/document-goodreads-form" dict "Lang" .Lang "Slug" .DocumentSlug "BookURL" .URL "Source" .Source}}
    {{end}}
    • diff --git a/internal/webserver/series_test.go b/internal/webserver/series_test.go index cea8191a..38b05e41 100644 --- a/internal/webserver/series_test.go +++ b/internal/webserver/series_test.go @@ -7,6 +7,8 @@ import ( "testing" "github.com/PuerkitoBio/goquery" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/precisiondate" "github.com/svera/coreander/v5/internal/webserver" @@ -70,9 +72,30 @@ func TestSeriesFilters(t *testing.T) { third.Words = 0 catalog[testLibraryDir+"/metadata.pdf"] = third reader := catalogReader{byPath: catalog} - app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, + app, idx := bootstrapAppWithIndexer(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, loadDirInMemoryFs(testLibraryDir), webserver.Config{}, map[string]metadata.Reader{".epub": reader, ".pdf": reader}) + seriesDocuments, err := idx.SearchBySeries(index.SearchFields{Keywords: "the-lord-of-the-rings"}, 1, 10) + if err != nil { + t.Fatal(err) + } + scores := map[string]string{ + "First volume": "3.20", + "Second volume": "4.50", + "Third volume": "4.50", + } + for _, doc := range seriesDocuments.Hits() { + if err := idx.UpdateGoodReadsBookURL(doc.ID, "https://www.goodreads.com/book/show/1-"+doc.Slug); err != nil { + t.Fatalf("set review link for %s: %v", doc.Title, err) + } + updated, err := idx.Document(doc.Slug) + if err != nil { + t.Fatalf("reload %s: %v", doc.Title, err) + } + if err := idx.StoreGoodReadsDetails(updated, updated.GoodReadsBookURL, biblioreads.BookResponse{Rating: scores[doc.Title]}); err != nil { + t.Fatalf("set review score for %s: %v", doc.Title, err) + } + } cases := []struct { name string @@ -81,6 +104,8 @@ func TestSeriesFilters(t *testing.T) { }{ {"default series order", "", []string{"First volume", "Second volume", "Third volume"}}, {"descending series order", "?sort-by=number-desc", []string{"Third volume", "Second volume", "First volume"}}, + {"review score ascending", "?sort-by=review-score-asc", []string{"First volume", "Second volume", "Third volume"}}, + {"review score descending with series-order tie-break", "?sort-by=review-score-desc", []string{"Second volume", "Third volume", "First volume"}}, {"language", "?language=es", []string{"Second volume"}}, {"subjects", "?subjects=fiction", []string{"First volume", "Third volume"}}, {"publishing date", "?pub-date-from=2021-01-01", []string{"Second volume"}}, diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index bf572098..c45df208 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -57,6 +57,11 @@ func TestGET(t *testing.T) { } func bootstrapApp(db *gorm.DB, sender webserver.Sender, appFs afero.Fs, webserverConfig webserver.Config, metadataReaders ...map[string]metadata.Reader) *fiber.App { + app, _ := bootstrapAppWithIndexer(db, sender, appFs, webserverConfig, metadataReaders...) + return app +} + +func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.Fs, webserverConfig webserver.Config, metadataReaders ...map[string]metadata.Reader) (*fiber.App, *index.BleveIndexer) { readers := testMetadataReaders() if len(metadataReaders) > 0 { readers = metadataReaders[0] @@ -92,7 +97,7 @@ func bootstrapApp(db *gorm.DB, sender webserver.Sender, appFs afero.Fs, webserve } controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, servicelinks.Goodreads{}) usersRepository := &model.UserRepository{DB: db} - return webserver.New(webserverConfig, controllers, sender, idx, usersRepository) + return webserver.New(webserverConfig, controllers, sender, idx, usersRepository), idx } func loadFilesInMemoryFs(files []string) afero.Fs { From 0d57615f386aa2d47ff45f2cc853b931af04aa04 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 09:24:34 +0100 Subject: [PATCH 17/50] WIP --- internal/datasource/model/data_source_link.go | 10 +++ internal/datasource/wikidata/service.go | 14 +++++ internal/datasource/wikidata/service_test.go | 18 ++++++ internal/reviews/biblioreads.go | 24 +++++-- internal/reviews/data_source_link_test.go | 27 ++++++++ internal/servicelinks/goodreads.go | 25 -------- internal/servicelinks/goodreads_test.go | 36 ----------- internal/servicelinks/link.go | 9 --- internal/webserver/author_summary_test.go | 32 +++++++--- internal/webserver/controller.go | 6 +- .../webserver/controller/author/controller.go | 63 +++++++++++-------- .../webserver/controller/author/documents.go | 5 +- .../webserver/controller/author/summary.go | 12 ++-- .../webserver/controller/author/update.go | 10 +-- .../controller/document/controller.go | 52 +++++++-------- .../webserver/controller/document/detail.go | 24 ++++--- .../webserver/controller/document/reviews.go | 2 +- .../webserver/controller/document/score.go | 2 +- ...inks_test.go => data_source_links_test.go} | 20 +++--- internal/webserver/document_goodreads_test.go | 11 ++-- .../webserver/embedded/views/author/list.html | 2 +- .../embedded/views/document/detail.html | 8 +-- .../partials/author-data-source-link.html | 7 +++ .../views/partials/author-service-link.html | 6 -- .../partials/author-summary-doc-detail.html | 10 +-- .../views/partials/author-summary.html | 10 +-- internal/webserver/webserver_test.go | 4 +- main.go | 7 +-- 28 files changed, 237 insertions(+), 219 deletions(-) create mode 100644 internal/datasource/model/data_source_link.go create mode 100644 internal/reviews/data_source_link_test.go delete mode 100644 internal/servicelinks/goodreads.go delete mode 100644 internal/servicelinks/goodreads_test.go delete mode 100644 internal/servicelinks/link.go rename internal/webserver/{service_links_test.go => data_source_links_test.go} (69%) create mode 100644 internal/webserver/embedded/views/partials/author-data-source-link.html delete mode 100644 internal/webserver/embedded/views/partials/author-service-link.html diff --git a/internal/datasource/model/data_source_link.go b/internal/datasource/model/data_source_link.go new file mode 100644 index 00000000..fb665585 --- /dev/null +++ b/internal/datasource/model/data_source_link.go @@ -0,0 +1,10 @@ +package model + +type DataSourceLink struct { + Source string + URL string + Icon string + IconClass string + IconInvertible bool + Label string +} diff --git a/internal/datasource/wikidata/service.go b/internal/datasource/wikidata/service.go index 44cab53c..d7fc8389 100644 --- a/internal/datasource/wikidata/service.go +++ b/internal/datasource/wikidata/service.go @@ -14,6 +14,7 @@ import ( gowikidata "github.com/Navid2zp/go-wikidata" "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/precisiondate" ) @@ -42,6 +43,19 @@ type WikidataSource struct { wikidata wikidata } +func (WikidataSource) AuthorLink(author index.Author, language string) model.DataSourceLink { + url := author.WikipediaLink[language] + if url == "" { + return model.DataSourceLink{} + } + return model.DataSourceLink{ + Source: "Wikipedia", + URL: url, + IconClass: "bi-wikipedia", + Label: "Go to Wikipedia entry", + } +} + func NewWikidataSource(w wikidata) WikidataSource { return WikidataSource{w} } diff --git a/internal/datasource/wikidata/service_test.go b/internal/datasource/wikidata/service_test.go index 795712a1..b0de124a 100644 --- a/internal/datasource/wikidata/service_test.go +++ b/internal/datasource/wikidata/service_test.go @@ -6,9 +6,27 @@ import ( gowikidata "github.com/Navid2zp/go-wikidata" "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/precisiondate" ) +func TestWikipediaDataSourceLink(t *testing.T) { + source := NewWikidataSource(Gowikidata{}) + link := source.AuthorLink(index.Author{ + WikipediaLink: map[string]string{ + "en": "https://en.wikipedia.org/wiki/Example", + "es": "https://es.wikipedia.org/wiki/Ejemplo", + }, + }, "es") + if link.Source != "Wikipedia" || link.URL != "https://es.wikipedia.org/wiki/Ejemplo" || + link.IconClass != "bi-wikipedia" || link.Label != "Go to Wikipedia entry" { + t.Fatalf("unexpected Wikipedia data-source link: %+v", link) + } + if missing := source.AuthorLink(index.Author{}, "en"); missing.URL != "" { + t.Fatalf("expected no link for missing Wikipedia URL, got %+v", missing) + } +} + func TestAuthor(t *testing.T) { mockServer := NewMockServer(t, "fixtures") defer mockServer.Close() diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index 494871c3..733870ae 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -10,8 +10,8 @@ import ( "time" "github.com/svera/coreander/v5/internal/datasource/biblioreads" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" - "github.com/svera/coreander/v5/internal/servicelinks" ) const ( @@ -44,8 +44,20 @@ func (b *BiblioReads) Name() string { } // DocumentLink returns the indexed book link without making a network request. -func (b *BiblioReads) DocumentLink(document index.Document) servicelinks.Link { - return (servicelinks.Goodreads{}).Document(document) +func (b *BiblioReads) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { + return goodReadsDataSourceLink(author.GoodReadsAuthorURL, "/author/show/") +} + +func (b *BiblioReads) DocumentLink(document index.Document) datasourcemodel.DataSourceLink { + return goodReadsDataSourceLink(document.GoodReadsBookURL, "/book/show/") +} + +func goodReadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink { + link := datasourcemodel.DataSourceLink{Source: "Goodreads", Icon: "/images/goodreads.svg", IconInvertible: true} + if id, ok := strings.CutPrefix(path, prefix); ok && id != "" { + link.URL = goodReadsBaseURL + path + } + return link } // CanEditDocumentLink reports that Goodreads book links can be corrected. @@ -113,14 +125,14 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score return score, err } document.GoodReadsBookURL = result.BookURL - score.Value, score.Count, score.URL = result.Rating, result.RatingCount, (servicelinks.Goodreads{}).Document(document).URL + score.Value, score.Count, score.URL = result.Rating, result.RatingCount, b.DocumentLink(document).URL return score, nil } book, err := b.book(ctx, document) if err != nil { return score, err } - score.Value, score.Count, score.URL = book.Rating, book.RatingCount, (servicelinks.Goodreads{}).Document(document).URL + score.Value, score.Count, score.URL = book.Rating, book.RatingCount, b.DocumentLink(document).URL return score, nil } @@ -146,7 +158,7 @@ func (b *BiblioReads) Reviews(ctx context.Context, document index.Document, lang if err != nil { return Page{}, fmt.Errorf("retrieving Goodreads reviews for %s: %w", document.ID, err) } - page := Page{Source: "Goodreads", URL: (servicelinks.Goodreads{}).Document(document).URL, Available: true} + page := Page{Source: "Goodreads", URL: b.DocumentLink(document).URL, Available: true} for _, review := range response.Reviews { var createdAt time.Time if review.CreatedAt != "" { diff --git a/internal/reviews/data_source_link_test.go b/internal/reviews/data_source_link_test.go new file mode 100644 index 00000000..7c4f8661 --- /dev/null +++ b/internal/reviews/data_source_link_test.go @@ -0,0 +1,27 @@ +package reviews_test + +import ( + "testing" + + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/reviews" +) + +func TestBiblioReadsDataSourceLinks(t *testing.T) { + provider := reviews.NewBiblioReads(nil, nil) + author := provider.AuthorLink(index.Author{ + GoodReadsAuthorURL: "/author/show/123.Jane_Doe", + }, "en") + if author.URL != "https://www.goodreads.com/author/show/123.Jane_Doe" || + author.Source != "Goodreads" || author.Icon != "/images/goodreads.svg" || !author.IconInvertible { + t.Fatalf("unexpected Goodreads author link: %+v", author) + } + + document := provider.DocumentLink(index.Document{ + GoodReadsBookURL: "/book/show/456.A_Book", + }) + if document.URL != "https://www.goodreads.com/book/show/456.A_Book" || + document.Source != "Goodreads" || document.Icon != "/images/goodreads.svg" || !document.IconInvertible { + t.Fatalf("unexpected Goodreads document link: %+v", document) + } +} diff --git a/internal/servicelinks/goodreads.go b/internal/servicelinks/goodreads.go deleted file mode 100644 index 01e87135..00000000 --- a/internal/servicelinks/goodreads.go +++ /dev/null @@ -1,25 +0,0 @@ -package servicelinks - -import ( - "strings" - - "github.com/svera/coreander/v5/internal/index" -) - -type Goodreads struct{} - -func (Goodreads) Author(author index.Author) Link { - return goodreadsLink(author.GoodReadsAuthorURL, "/author/show/") -} - -func (Goodreads) Document(document index.Document) Link { - return goodreadsLink(document.GoodReadsBookURL, "/book/show/") -} - -func goodreadsLink(path, prefix string) Link { - link := Link{Source: "Goodreads", Icon: "/images/goodreads.svg"} - if id, ok := strings.CutPrefix(path, prefix); ok && id != "" { - link.URL = "https://www.goodreads.com" + path - } - return link -} diff --git a/internal/servicelinks/goodreads_test.go b/internal/servicelinks/goodreads_test.go deleted file mode 100644 index ffaecee0..00000000 --- a/internal/servicelinks/goodreads_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package servicelinks_test - -import ( - "testing" - - "github.com/svera/coreander/v5/internal/index" - "github.com/svera/coreander/v5/internal/servicelinks" -) - -func TestGoodreadsLinks(t *testing.T) { - provider := servicelinks.Goodreads{} - for _, tt := range []struct { - name string - path string - author bool - want string - }{ - {"author", "/author/show/340641.Carmen_Posadas", true, "https://www.goodreads.com/author/show/340641.Carmen_Posadas"}, - {"document", "/book/show/5898652-la-cinta-roja", false, "https://www.goodreads.com/book/show/5898652-la-cinta-roja"}, - {"missing author", "", true, ""}, - {"missing document", "", false, ""}, - {"invalid author", "https://example.org", true, ""}, - {"invalid document", "/author/show/1", false, ""}, - {"empty identifier", "/book/show/", false, ""}, - } { - t.Run(tt.name, func(t *testing.T) { - link := provider.Document(index.Document{GoodReadsBookURL: tt.path}) - if tt.author { - link = provider.Author(index.Author{GoodReadsAuthorURL: tt.path}) - } - if link.URL != tt.want || link.Source != "Goodreads" || link.Icon != "/images/goodreads.svg" { - t.Fatalf("unexpected link: %+v", link) - } - }) - } -} diff --git a/internal/servicelinks/link.go b/internal/servicelinks/link.go deleted file mode 100644 index 29212cd8..00000000 --- a/internal/servicelinks/link.go +++ /dev/null @@ -1,9 +0,0 @@ -// Package servicelinks provides external author and document page links. -package servicelinks - -// Link has an empty URL when the service has no known page for the entity. -type Link struct { - Source string - URL string - Icon string -} diff --git a/internal/webserver/author_summary_test.go b/internal/webserver/author_summary_test.go index f9c3627a..6c5567a9 100644 --- a/internal/webserver/author_summary_test.go +++ b/internal/webserver/author_summary_test.go @@ -128,14 +128,15 @@ func TestAuthorSummaryGoodReadsLink(t *testing.T) { wikipediaURL = "https://en.wikipedia.org/wiki/Alice_Borchardt" ) locales := []struct { - lang string - label string + lang string + label string + wikipediaLabel string }{ - {"en", "Go to Goodreads author page"}, - {"es", "Ir a la página del autor en Goodreads"}, - {"de", "Zur Autorenseite auf Goodreads"}, - {"fr", "Voir la page de l'auteur sur Goodreads"}, - {"ru", "Перейти на страницу автора в Goodreads"}, + {"en", "Go to Goodreads author page", "Go to Wikipedia entry"}, + {"es", "Ir a la página del autor en Goodreads", "Ir al artículo en Wikipedia"}, + {"de", "Zur Autorenseite auf Goodreads", "Zur Wikipedia-Seite"}, + {"fr", "Voir la page de l'auteur sur Goodreads", "Aller à l'entrée Wikipedia"}, + {"ru", "Перейти на страницу автора в Goodreads", "Перейти к статье в Википедии"}, } for _, tc := range []struct { name string @@ -146,7 +147,7 @@ func TestAuthorSummaryGoodReadsLink(t *testing.T) { {"with-wikipedia", true, true, "Q1"}, {"without-goodreads", false, true, "Q1"}, {"without-wikipedia", true, false, "Q1"}, - {"without-wikidata", true, false, ""}, + {"without-source-id", true, false, ""}, } { author := index.Author{ Slug: tc.name, Name: "Alice Borchardt", @@ -158,7 +159,7 @@ func TestAuthorSummaryGoodReadsLink(t *testing.T) { } if tc.wikipedia { for _, locale := range locales { - author.WikipediaLink[locale.lang] = wikipediaURL + author.WikipediaLink[locale.lang] = strings.Replace(wikipediaURL, "en.wikipedia.org", locale.lang+".wikipedia.org", 1) } } if err := idx.IndexAuthor(author); err != nil { @@ -185,6 +186,16 @@ func TestAuthorSummaryGoodReadsLink(t *testing.T) { if err != nil { t.Fatal(err) } + wikipediaLink := page.Find(`a[href="https://` + locale.lang + `.wikipedia.org/wiki/Alice_Borchardt"]`) + if tc.wikipedia && (wikipediaLink.Length() != 1 || strings.TrimSpace(wikipediaLink.Text()) != locale.wikipediaLabel) { + t.Fatalf("expected localized Wikipedia data-source link %q, got %d links with text %q", locale.wikipediaLabel, wikipediaLink.Length(), wikipediaLink.Text()) + } + if tc.wikipedia && wikipediaLink.PrevFiltered("i.bi-wikipedia").Length() != 1 { + t.Fatal("expected the Wikipedia provider's Bootstrap icon") + } + if !tc.wikipedia && page.Find(`a[href$="wikipedia.org/wiki/Alice_Borchardt"]`).Length() != 0 { + t.Fatal("expected no Wikipedia link when there is no link for the current language") + } link := page.Find(`a[href="` + authorURL + `"]`) if !tc.goodreads { if page.Find(`img[src^="/images/goodreads.svg"]`).Length() != 0 { @@ -203,7 +214,8 @@ func TestAuthorSummaryGoodReadsLink(t *testing.T) { t.Error("expected a decorative Goodreads SVG icon before the author link") } if tc.wikipedia { - if href, _ := link.Parent().Prev().Find("a").Attr("href"); href != wikipediaURL { + wantWikipediaURL := strings.Replace(wikipediaURL, "en.wikipedia.org", locale.lang+".wikipedia.org", 1) + if href, _ := link.Parent().Prev().Find("a").Attr("href"); href != wantWikipediaURL { t.Fatalf("expected Goodreads immediately below Wikipedia, previous link is %q", href) } } diff --git a/internal/webserver/controller.go b/internal/webserver/controller.go index 1284ae7d..416ade1a 100644 --- a/internal/webserver/controller.go +++ b/internal/webserver/controller.go @@ -30,7 +30,7 @@ type Controllers struct { Series *series.Controller } -func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender Sender, appFs afero.Fs, dataSource author.DataSource, reviewProvider document.ReviewProvider, linkProvider author.LinkProvider) Controllers { +func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender Sender, appFs afero.Fs, dataSource author.DataSource, reviewProvider document.ReviewProvider, dataSourceLinkProviders ...author.DataSourceLinkProvider) Controllers { usersRepository := &model.UserRepository{DB: db} invitationsRepository := &model.InvitationRepository{DB: db} highlightsRepository := &model.HighlightRepository{DB: db, Idx: idx, IllustratedMinAmount: cfg.IllustratedMinAmount} @@ -102,7 +102,7 @@ func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender S LatestDocsLimit: 6, } - authorsController := author.NewController(highlightsRepository, readingRepository, sender, idx, authorsCfg, dataSource, appFs, imagesFS, linkProvider) + authorsController := author.NewController(highlightsRepository, readingRepository, sender, idx, authorsCfg, dataSource, appFs, imagesFS, dataSourceLinkProviders...) go authorsController.MigrateJPEGsToWebP() return Controllers{ @@ -111,7 +111,7 @@ func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender S Users: user.NewController(usersRepository, invitationsRepository, usersCfg, sender, translator), Completed: completed.NewController(readingRepository, idx), Highlights: highlight.NewController(highlightsRepository, readingRepository, usersRepository, sender, cfg.WordsPerMinute, idx), - Documents: document.NewController(highlightsRepository, usersRepository, readingRepository, annotationsRepository, sender, idx, appFs, documentsCfg, translator, reviewProvider, linkProvider), + Documents: document.NewController(highlightsRepository, usersRepository, readingRepository, annotationsRepository, sender, idx, appFs, documentsCfg, translator, reviewProvider, dataSourceLinkProviders...), Home: home.NewController(highlightsRepository, readingRepository, sender, idx, homeCfg), Authors: authorsController, Search: search.NewController(highlightsRepository, readingRepository, sender, idx, searchCfg), diff --git a/internal/webserver/controller/author/controller.go b/internal/webserver/controller/author/controller.go index e3bd21ce..b05f29ba 100644 --- a/internal/webserver/controller/author/controller.go +++ b/internal/webserver/controller/author/controller.go @@ -4,9 +4,9 @@ import ( "io/fs" "github.com/spf13/afero" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/result" - "github.com/svera/coreander/v5/internal/servicelinks" "github.com/svera/coreander/v5/internal/webserver/model" ) @@ -14,8 +14,8 @@ type Sender interface { From() string } -type LinkProvider interface { - Author(author index.Author) servicelinks.Link +type DataSourceLinkProvider interface { + AuthorLink(author index.Author, language string) datasourcemodel.DataSourceLink } // IdxReader defines a set of author reading operations over an index @@ -45,34 +45,45 @@ type Config struct { } type Controller struct { - hlRepository highlightsRepository - readingRepository readingRepository - idx IdxReader - sender Sender - config Config - dataSource DataSource - appFs afero.Fs - embeddedImagesFS fs.FS - linkProvider LinkProvider + hlRepository highlightsRepository + readingRepository readingRepository + idx IdxReader + sender Sender + config Config + dataSource DataSource + appFs afero.Fs + embeddedImagesFS fs.FS + dataSourceLinkProviders []DataSourceLinkProvider } -func NewController(hlRepository highlightsRepository, readingRepository readingRepository, sender Sender, idx IdxReader, cfg Config, dataSource DataSource, appFs afero.Fs, embeddedImagesFS fs.FS, linkProvider LinkProvider) *Controller { +func NewController(hlRepository highlightsRepository, readingRepository readingRepository, sender Sender, idx IdxReader, cfg Config, dataSource DataSource, appFs afero.Fs, embeddedImagesFS fs.FS, dataSourceLinkProviders ...DataSourceLinkProvider) *Controller { return &Controller{ - hlRepository: hlRepository, - readingRepository: readingRepository, - idx: idx, - sender: sender, - config: cfg, - dataSource: dataSource, - appFs: appFs, - embeddedImagesFS: embeddedImagesFS, - linkProvider: linkProvider, + hlRepository: hlRepository, + readingRepository: readingRepository, + idx: idx, + sender: sender, + config: cfg, + dataSource: dataSource, + appFs: appFs, + embeddedImagesFS: embeddedImagesFS, + dataSourceLinkProviders: dataSourceLinkProviders, } } -func (a *Controller) authorLink(author index.Author) servicelinks.Link { - if a.linkProvider == nil { - return servicelinks.Link{} +func (a *Controller) authorDataSourceLinks(author index.Author, language string) []datasourcemodel.DataSourceLink { + return DataSourceLinks(author, language, a.dataSourceLinkProviders...) +} + +func DataSourceLinks(author index.Author, language string, providers ...DataSourceLinkProvider) []datasourcemodel.DataSourceLink { + var links []datasourcemodel.DataSourceLink + for _, provider := range providers { + if provider == nil { + continue + } + link := provider.AuthorLink(author, language) + if link.URL != "" { + links = append(links, link) + } } - return a.linkProvider.Author(author) + return links } diff --git a/internal/webserver/controller/author/documents.go b/internal/webserver/controller/author/documents.go index 3276ce8d..2104eaef 100644 --- a/internal/webserver/controller/author/documents.go +++ b/internal/webserver/controller/author/documents.go @@ -34,7 +34,8 @@ func (a *Controller) Documents(c fiber.Ctx) error { page = 1 } - author, err := a.idx.Author(authorSlug, c.Locals("Lang").(string)) + lang := c.Locals("Lang").(string) + author, err := a.idx.Author(authorSlug, lang) if err != nil { log.Println(err) } @@ -65,7 +66,7 @@ func (a *Controller) Documents(c fiber.Ctx) error { templateVars := fiber.Map{ "Author": author, - "ServiceLink": a.authorLink(author), + "DataSourceLinks": a.authorDataSourceLinks(author, lang), "ImageVersion": a.getImageVersion(author.Slug), "DocumentSearchFields": searchFields, "DocumentsOnly": true, diff --git a/internal/webserver/controller/author/summary.go b/internal/webserver/controller/author/summary.go index 5fde78ea..ccec6635 100644 --- a/internal/webserver/controller/author/summary.go +++ b/internal/webserver/controller/author/summary.go @@ -47,9 +47,9 @@ func (a *Controller) Summary(c fiber.Ctx) error { if !author.RetrievedOn.IsZero() { templateVars := fiber.Map{ - "Author": author, - "ServiceLink": a.authorLink(author), - "ImageVersion": imageVersion, + "Author": author, + "DataSourceLinks": a.authorDataSourceLinks(author, lang), + "ImageVersion": imageVersion, } if err = c.Render(template, templateVars); err != nil { @@ -76,9 +76,9 @@ func (a *Controller) Summary(c fiber.Ctx) error { } templateVars := fiber.Map{ - "Author": author, - "ServiceLink": a.authorLink(author), - "ImageVersion": imageVersion, + "Author": author, + "DataSourceLinks": a.authorDataSourceLinks(author, lang), + "ImageVersion": imageVersion, } if err = c.Render(template, templateVars); err != nil { diff --git a/internal/webserver/controller/author/update.go b/internal/webserver/controller/author/update.go index 3dd6636b..1627d907 100644 --- a/internal/webserver/controller/author/update.go +++ b/internal/webserver/controller/author/update.go @@ -62,11 +62,11 @@ func (a *Controller) Update(c fiber.Ctx) error { } templateVars := fiber.Map{ - "Author": author, - "ServiceLink": a.authorLink(author), - "ImageVersion": a.getImageVersion(author.Slug), - "Lang": lang, - "Session": session, + "Author": author, + "DataSourceLinks": a.authorDataSourceLinks(author, lang), + "ImageVersion": a.getImageVersion(author.Slug), + "Lang": lang, + "Session": session, } if err = c.Render(template, templateVars); err != nil { diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index db51fcb7..2559bcea 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -10,7 +10,7 @@ import ( "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/result" "github.com/svera/coreander/v5/internal/reviews" - "github.com/svera/coreander/v5/internal/servicelinks" + authorcontroller "github.com/svera/coreander/v5/internal/webserver/controller/author" "github.com/svera/coreander/v5/internal/webserver/model" ) @@ -27,10 +27,6 @@ type ReviewProvider interface { UpdateDocumentLink(document index.Document, link string) error } -type LinkProvider interface { - Author(author index.Author) servicelinks.Link -} - type Sender interface { SendBCC(addresses []string, subject, body string) error SendDocument(address, subject string, file []byte, fileName string) error @@ -97,33 +93,33 @@ type Config struct { } type Controller struct { - hlRepository highlightsRepository - usersRepository usersRepository - readingRepository readingRepository - annotationsRepository annotationsRepository - idx IdxReaderWriter - sender Sender - config Config - appFs afero.Fs - translator i18n.Translator - reviewProvider ReviewProvider - linkProvider LinkProvider + hlRepository highlightsRepository + usersRepository usersRepository + readingRepository readingRepository + annotationsRepository annotationsRepository + idx IdxReaderWriter + sender Sender + config Config + appFs afero.Fs + translator i18n.Translator + reviewProvider ReviewProvider + dataSourceLinkProviders []authorcontroller.DataSourceLinkProvider } // NewController creates a document controller. A nil reviewProvider disables scores and reviews. -func NewController(hlRepository highlightsRepository, usersRepository usersRepository, readingRepository readingRepository, annotationsRepository annotationsRepository, sender Sender, idx IdxReaderWriter, appFs afero.Fs, cfg Config, translator i18n.Translator, reviewProvider ReviewProvider, linkProvider LinkProvider) *Controller { +func NewController(hlRepository highlightsRepository, usersRepository usersRepository, readingRepository readingRepository, annotationsRepository annotationsRepository, sender Sender, idx IdxReaderWriter, appFs afero.Fs, cfg Config, translator i18n.Translator, reviewProvider ReviewProvider, dataSourceLinkProviders ...authorcontroller.DataSourceLinkProvider) *Controller { return &Controller{ - hlRepository: hlRepository, - usersRepository: usersRepository, - readingRepository: readingRepository, - annotationsRepository: annotationsRepository, - idx: idx, - sender: sender, - config: cfg, - appFs: appFs, - translator: translator, - reviewProvider: reviewProvider, - linkProvider: linkProvider, + hlRepository: hlRepository, + usersRepository: usersRepository, + readingRepository: readingRepository, + annotationsRepository: annotationsRepository, + idx: idx, + sender: sender, + config: cfg, + appFs: appFs, + translator: translator, + reviewProvider: reviewProvider, + dataSourceLinkProviders: dataSourceLinkProviders, } } diff --git a/internal/webserver/controller/document/detail.go b/internal/webserver/controller/document/detail.go index 4b7651d0..f6f747a3 100644 --- a/internal/webserver/controller/document/detail.go +++ b/internal/webserver/controller/document/detail.go @@ -6,8 +6,9 @@ import ( "github.com/gofiber/fiber/v3" "github.com/gosimple/slug" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" - "github.com/svera/coreander/v5/internal/servicelinks" + authorcontroller "github.com/svera/coreander/v5/internal/webserver/controller/author" "github.com/svera/coreander/v5/internal/webserver/controller/fsutil" "github.com/svera/coreander/v5/internal/webserver/model" ) @@ -16,9 +17,9 @@ import ( // render eagerly (i.e. without the async /summary htmx round-trip), so it can // only fall back to that round-trip when the author hasn't been enriched yet. type authorSummary struct { - Author index.Author - ImageVersion string - ServiceLink servicelinks.Link + Author index.Author + ImageVersion string + DataSourceLinks []datasourcemodel.DataSourceLink } func (d *Controller) Detail(c fiber.Ctx) error { @@ -74,17 +75,17 @@ func (d *Controller) Detail(c fiber.Ctx) error { "IllustratorSummaries": illustratorSummaries, "Debug": c.RequestCtx().QueryArgs().Has("debug"), "ReviewsEnabled": d.reviewProvider != nil, - "DocumentServiceLink": d.documentServiceLink(document), + "DocumentLink": d.documentLink(document), }, "layout") } -func (d *Controller) documentServiceLink(document index.Document) servicelinks.Link { +func (d *Controller) documentLink(document index.Document) datasourcemodel.DataSourceLink { if provider, ok := d.reviewProvider.(interface { - DocumentLink(index.Document) servicelinks.Link + DocumentLink(index.Document) datasourcemodel.DataSourceLink }); ok { return provider.DocumentLink(document) } - return servicelinks.Link{} + return datasourcemodel.DataSourceLink{} } // authorAndIllustratorSummaries resolves the (up to 2) authors and (up to 2) @@ -130,11 +131,8 @@ func (d *Controller) authorSummaryFor(authorSlug, lang string) authorSummary { log.Println(err) } - var link servicelinks.Link - if d.linkProvider != nil { - link = d.linkProvider.Author(author) - } - return authorSummary{Author: author, ImageVersion: fsutil.AuthorImageVersion(d.appFs, d.config.CacheDir, author.Slug), ServiceLink: link} + links := authorcontroller.DataSourceLinks(author, lang, d.dataSourceLinkProviders...) + return authorSummary{Author: author, ImageVersion: fsutil.AuthorImageVersion(d.appFs, d.config.CacheDir, author.Slug), DataSourceLinks: links} } func (d *Controller) sameSeries(slug string, sessionID int) ([]model.AugmentedDocument, int, error) { diff --git a/internal/webserver/controller/document/reviews.go b/internal/webserver/controller/document/reviews.go index 6055aba2..00073f78 100644 --- a/internal/webserver/controller/document/reviews.go +++ b/internal/webserver/controller/document/reviews.go @@ -51,7 +51,7 @@ func (d *Controller) Reviews(c fiber.Ctx) error { return fiber.ErrInternalServerError } if !page.Available { - link := d.documentServiceLink(document) + link := d.documentLink(document) if link.URL == "" { return c.SendString("") } diff --git a/internal/webserver/controller/document/score.go b/internal/webserver/controller/document/score.go index a4f394c0..c1dee23b 100644 --- a/internal/webserver/controller/document/score.go +++ b/internal/webserver/controller/document/score.go @@ -35,7 +35,7 @@ func (d *Controller) Score(c fiber.Ctx) error { return c.SendString("") } score.Value, score.Count = "", "" - score.URL = d.documentServiceLink(document).URL + score.URL = d.documentLink(document).URL } return c.Render("partials/document-score", fiber.Map{ "Source": score.Source, diff --git a/internal/webserver/service_links_test.go b/internal/webserver/data_source_links_test.go similarity index 69% rename from internal/webserver/service_links_test.go rename to internal/webserver/data_source_links_test.go index 024ddec0..685cac0d 100644 --- a/internal/webserver/service_links_test.go +++ b/internal/webserver/data_source_links_test.go @@ -7,21 +7,21 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/svera/coreander/v5/internal/datasource/biblioreads" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" - "github.com/svera/coreander/v5/internal/servicelinks" ) -type alternativeLinks struct{} +type alternativeDataSourceLinks struct{} -func (alternativeLinks) Author(author index.Author) servicelinks.Link { - return servicelinks.Link{Source: "Another service", URL: "https://example.org/authors/" + author.Slug} +func (alternativeDataSourceLinks) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { + return datasourcemodel.DataSourceLink{Source: "Another source", URL: "https://example.org/authors/" + author.Slug} } -func TestAlternativeServiceLinks(t *testing.T) { +func TestAlternativeDataSourceLinks(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "4", ResourceID: "work", LegacyBookID: 1, }} - app, idx := newServiceProvidersTestApp(t, source, nil, alternativeLinks{}) + app, idx := newDataSourceProvidersTestApp(t, source, nil, alternativeDataSourceLinks{}) for _, slug := range []string{"jane-doe", "john-doe"} { if err := idx.IndexAuthor(index.Author{ Slug: slug, Name: slug, RetrievedOn: time.Now().UTC(), @@ -38,8 +38,8 @@ func TestAlternativeServiceLinks(t *testing.T) { t.Fatal(err) } link := page.Find(`a[href="https://example.org/authors/jane-doe"]`) - if link.Length() != 1 || link.Text() != "Go to Another service author page" { - t.Fatalf("%s: expected injected author service link, got %q", path, link.Text()) + if link.Length() != 1 || link.Text() != "Go to Another source author page" { + t.Fatalf("%s: expected injected author data-source link, got %q", path, link.Text()) } if page.Find(`img[src^="/images/goodreads.svg"]`).Length() != 0 { t.Fatalf("%s: unexpected Goodreads branding", path) @@ -56,8 +56,8 @@ func TestAlternativeServiceLinks(t *testing.T) { } } -func TestServiceLinksWithoutReviewProvider(t *testing.T) { - app, idx := newServiceProvidersTestApp(t, nil, nil, alternativeLinks{}) +func TestDataSourceLinksWithoutReviewProvider(t *testing.T) { + app, idx := newDataSourceProvidersTestApp(t, nil, nil, alternativeDataSourceLinks{}) if err := idx.IndexAuthor(index.Author{Slug: "jane-doe", Name: "Jane Doe", RetrievedOn: time.Now().UTC()}); err != nil { t.Fatal(err) } diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index d8d0e761..69eea0b5 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -18,12 +18,12 @@ import ( "github.com/gofiber/fiber/v3" "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" - "github.com/svera/coreander/v5/internal/servicelinks" "github.com/svera/coreander/v5/internal/webserver" - "github.com/svera/coreander/v5/internal/webserver/controller/author" + authorcontroller "github.com/svera/coreander/v5/internal/webserver/controller/author" "github.com/svera/coreander/v5/internal/webserver/controller/document" "github.com/svera/coreander/v5/internal/webserver/infrastructure" "github.com/svera/coreander/v5/internal/webserver/model" @@ -295,10 +295,10 @@ func newGoodReadsTestApp(t *testing.T, source *fakeBookSource) (*fiber.App, *ind func newReviewProviderTestApp(t *testing.T, source *fakeBookSource, provider document.ReviewProvider) (*fiber.App, *index.BleveIndexer) { t.Helper() - return newServiceProvidersTestApp(t, source, provider, servicelinks.Goodreads{}) + return newDataSourceProvidersTestApp(t, source, provider, nil) } -func newServiceProvidersTestApp(t *testing.T, source *fakeBookSource, provider document.ReviewProvider, links author.LinkProvider) (*fiber.App, *index.BleveIndexer) { +func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provider document.ReviewProvider, dataSourceLinkProvider authorcontroller.DataSourceLinkProvider) (*fiber.App, *index.BleveIndexer) { t.Helper() db := infrastructure.Connect(":memory:", 250) sqlDB, err := db.DB() @@ -324,6 +324,7 @@ func newServiceProvidersTestApp(t *testing.T, source *fakeBookSource, provider d if source != nil { provider = reviews.NewBiblioReads(source, idx) } + goodReadsLinks := reviews.NewBiblioReads(nil, idx) t.Cleanup(func() { _ = idx.Close() }) for _, doc := range []index.Document{ @@ -352,7 +353,7 @@ func newServiceProvidersTestApp(t *testing.T, source *fakeBookSource, provider d cfg := defaultTestConfig() sender := &infrastructure.NoEmail{} - controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, nil, provider, links) + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, wikidata.NewWikidataSource(wikidata.Gowikidata{}), provider, wikidata.NewWikidataSource(wikidata.Gowikidata{}), goodReadsLinks, dataSourceLinkProvider) app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) t.Cleanup(func() { _ = app.Shutdown() }) return app, idx diff --git a/internal/webserver/embedded/views/author/list.html b/internal/webserver/embedded/views/author/list.html index a5e3656c..10f9863b 100644 --- a/internal/webserver/embedded/views/author/list.html +++ b/internal/webserver/embedded/views/author/list.html @@ -1,6 +1,6 @@ {{if ne .Author.Name ""}} {{if not .Author.RetrievedOn.IsZero}} - {{template "partials/author-summary" dict "Lang" .Lang "Author" .Author "Session" .Session "ImageVersion" .ImageVersion "ServiceLink" .ServiceLink}} + {{template "partials/author-summary" dict "Lang" .Lang "Author" .Author "Session" .Session "ImageVersion" .ImageVersion "DataSourceLinks" .DataSourceLinks}} {{else}}
    diff --git a/internal/webserver/embedded/views/document/detail.html b/internal/webserver/embedded/views/document/detail.html index 1e22afdf..d8eac799 100644 --- a/internal/webserver/embedded/views/document/detail.html +++ b/internal/webserver/embedded/views/document/detail.html @@ -60,7 +60,7 @@

    {{.Document.Title}}

    {{range .AuthorSummaries}} {{if not .Author.RetrievedOn.IsZero}} - {{template "partials/author-summary-doc-detail" dict "Lang" $.Lang "Author" .Author "Session" $.Session "ImageVersion" .ImageVersion "ServiceLink" .ServiceLink}} + {{template "partials/author-summary-doc-detail" dict "Lang" $.Lang "Author" .Author "Session" $.Session "ImageVersion" .ImageVersion "DataSourceLinks" .DataSourceLinks}} {{else}} {{template "partials/author-summary-doc-detail-placeholder" dict "AuthorSlug" .Author.Slug}} {{end}} @@ -71,7 +71,7 @@

    {{.Document.Title}}

    {{range $i, $entry := .IllustratorSummaries}} {{if eq $i 0}}
    {{end}} {{if not $entry.Author.RetrievedOn.IsZero}} - {{template "partials/author-summary-doc-detail" dict "Lang" $.Lang "Author" $entry.Author "Session" $.Session "ImageVersion" $entry.ImageVersion "ServiceLink" $entry.ServiceLink}} + {{template "partials/author-summary-doc-detail" dict "Lang" $.Lang "Author" $entry.Author "Session" $.Session "ImageVersion" $entry.ImageVersion "DataSourceLinks" $entry.DataSourceLinks}} {{else}} {{template "partials/author-summary-doc-detail-placeholder" dict "AuthorSlug" $entry.Author.Slug}} {{end}} @@ -121,8 +121,8 @@

    Words

    {{end}} {{if .ReviewsEnabled}}
    - {{if .DocumentServiceLink.URL}} - {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .DocumentServiceLink.Source "BookURL" .DocumentServiceLink.URL "Slug" .Document.Slug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .Lang}} + {{if .DocumentLink.URL}} + {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .DocumentLink.Source "BookURL" .DocumentLink.URL "Slug" .Document.Slug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .Lang}} {{end}}
    {{end}} diff --git a/internal/webserver/embedded/views/partials/author-data-source-link.html b/internal/webserver/embedded/views/partials/author-data-source-link.html new file mode 100644 index 00000000..64026771 --- /dev/null +++ b/internal/webserver/embedded/views/partials/author-data-source-link.html @@ -0,0 +1,7 @@ +{{$lang := .Lang}} +{{range .DataSourceLinks}} +
  • + {{if .IconClass}}{{else if .Icon}}{{end}} + {{if .Label}}{{t $lang .Label}}{{else}}{{t $lang "Go to %s author page" .Source}}{{end}} +
    • +{{end}} diff --git a/internal/webserver/embedded/views/partials/author-service-link.html b/internal/webserver/embedded/views/partials/author-service-link.html deleted file mode 100644 index 9702db05..00000000 --- a/internal/webserver/embedded/views/partials/author-service-link.html +++ /dev/null @@ -1,6 +0,0 @@ -{{if .ServiceLink.URL}} -
  • - {{if .ServiceLink.Icon}}{{end}} - {{t .Lang "Go to %s author page" .ServiceLink.Source}} -
    • -{{end}} diff --git a/internal/webserver/embedded/views/partials/author-summary-doc-detail.html b/internal/webserver/embedded/views/partials/author-summary-doc-detail.html index aab7c133..946679e6 100644 --- a/internal/webserver/embedded/views/partials/author-summary-doc-detail.html +++ b/internal/webserver/embedded/views/partials/author-summary-doc-detail.html @@ -71,7 +71,7 @@

    {{.Author.Name}}

    {{end}} {{end}} - {{if or .Author.Website (index .Author.WikipediaLink .Lang) .ServiceLink.URL}} + {{if or .Author.Website .DataSourceLinks}}
      {{if .Author.Website}}
    • @@ -80,13 +80,7 @@

      {{.Author.Name}}

      • {{end}} - {{if index .Author.WikipediaLink .Lang}} -
    • - - {{t .Lang "Go to Wikipedia entry"}} -
      • - {{end}} - {{template "partials/author-service-link" dict "Lang" .Lang "ServiceLink" .ServiceLink}} + {{template "partials/author-data-source-link" dict "Lang" .Lang "DataSourceLinks" .DataSourceLinks}}
    {{end}}
    diff --git a/internal/webserver/embedded/views/partials/author-summary.html b/internal/webserver/embedded/views/partials/author-summary.html index a039fff3..c13c0fe8 100644 --- a/internal/webserver/embedded/views/partials/author-summary.html +++ b/internal/webserver/embedded/views/partials/author-summary.html @@ -71,7 +71,7 @@

    {{.Author.Name}}

    {{end}} {{end}} - {{if or .Author.Website (index .Author.WikipediaLink .Lang) .ServiceLink.URL}} + {{if or .Author.Website .DataSourceLinks}}
      {{if .Author.Website}}
    • @@ -80,13 +80,7 @@

      {{.Author.Name}}

      • {{end}} - {{if index .Author.WikipediaLink .Lang}} -
    • - - {{t .Lang "Go to Wikipedia entry"}} -
      • - {{end}} - {{template "partials/author-service-link" dict "Lang" .Lang "ServiceLink" .ServiceLink}} + {{template "partials/author-data-source-link" dict "Lang" .Lang "DataSourceLinks" .DataSourceLinks}}
    {{end}}
    diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index c45df208..baecfa59 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -19,7 +19,7 @@ import ( "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" - "github.com/svera/coreander/v5/internal/servicelinks" + "github.com/svera/coreander/v5/internal/reviews" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/infrastructure" "github.com/svera/coreander/v5/internal/webserver/model" @@ -95,7 +95,7 @@ func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.F if err != nil { log.Fatal(err) } - controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, servicelinks.Goodreads{}) + controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, dataSource, reviews.NewBiblioReads(nil, idx)) usersRepository := &model.UserRepository{DB: db} return webserver.New(webserverConfig, controllers, sender, idx, usersRepository), idx } diff --git a/main.go b/main.go index b99ac735..a064c9eb 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,6 @@ import ( "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" - "github.com/svera/coreander/v5/internal/servicelinks" "github.com/svera/coreander/v5/internal/versioncheck" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/controller/document" @@ -206,11 +205,11 @@ func main() { dataSource := wikidata.NewWikidataSource(wikidata.Gowikidata{}) var reviewProvider document.ReviewProvider - linkProvider := servicelinks.Goodreads{} + biblioReads := reviews.NewBiblioReads(biblioreadsClient, idx) if biblioreadsClient != nil { - reviewProvider = reviews.NewBiblioReads(biblioreadsClient, idx) + reviewProvider = biblioReads } - controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, reviewProvider, linkProvider) + controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, reviewProvider, dataSource, biblioReads) usersRepository := &model.UserRepository{DB: db} app := webserver.New(webserverConfig, controllers, sender, idx, usersRepository) if strings.ToLower(input.FQDN) == "localhost" { From 1a9473ba2d4d1cc7a10fed5dcf0633e6e2f5fa09 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 11:37:38 +0100 Subject: [PATCH 18/50] WIP --- internal/index/bleve.go | 14 +- internal/index/bleve_document_read.go | 52 +++-- internal/index/bleve_goodreads_write.go | 77 +++---- internal/index/bleve_goodreads_write_test.go | 211 +++++++----------- internal/index/document.go | 46 ++-- internal/index/goodreads_fields_test.go | 66 +++--- internal/reviews/biblioreads.go | 49 ++-- internal/reviews/biblioreads_test.go | 86 +++++++ internal/reviews/data_source_link_test.go | 2 +- internal/webserver/author_summary_test.go | 58 +++++ internal/webserver/author_update_auth_test.go | 47 ++++ .../controller/document/controller.go | 4 +- .../document/update_document_link.go | 2 +- internal/webserver/controller/search/parse.go | 39 ++-- .../webserver/controller/series/documents.go | 4 +- internal/webserver/data_source_links_test.go | 68 ------ .../webserver/document_goodreads_auth_test.go | 58 ----- internal/webserver/document_goodreads_test.go | 164 ++------------ .../webserver/embedded/views/author/list.html | 2 +- .../embedded/views/highlight/list.html | 4 +- .../views/partials/completed-list.html | 2 +- .../views/partials/count-and-sort.html | 2 +- .../views/partials/docs-list-fragments.html | 4 +- .../views/partials/docs-list-item.html | 2 +- .../partials/document-goodreads-form.html | 10 - .../partials/document-metadata-filters.html | 10 +- .../views/partials/document-metadata.html | 4 +- .../partials/document-review-link-form.html | 10 + .../views/partials/document-score.html | 6 +- .../partials/highlights-list-fragments.html | 2 +- .../embedded/views/partials/navbar.html | 2 +- .../search-filters-sidebar-content.html | 2 +- .../views/partials/search-type-tabs.html | 2 +- .../embedded/views/partials/searchbox.html | 2 +- .../views/partials/similar-to-banner.html | 4 +- .../embedded/views/partials/sort.html | 2 +- .../webserver/embedded/views/series/list.html | 2 +- internal/webserver/highlights_test.go | 46 ++++ .../webserver/model/highlight_repository.go | 17 +- .../webserver/model/highlight_search_test.go | 32 --- internal/webserver/routes.go | 2 +- internal/webserver/search_test.go | 160 ++++++++++++- internal/webserver/series_test.go | 12 +- internal/webserver/webserver_test.go | 22 ++ 44 files changed, 750 insertions(+), 662 deletions(-) create mode 100644 internal/reviews/biblioreads_test.go delete mode 100644 internal/webserver/data_source_links_test.go delete mode 100644 internal/webserver/document_goodreads_auth_test.go delete mode 100644 internal/webserver/embedded/views/partials/document-goodreads-form.html create mode 100644 internal/webserver/embedded/views/partials/document-review-link-form.html delete mode 100644 internal/webserver/model/highlight_search_test.go diff --git a/internal/index/bleve.go b/internal/index/bleve.go index 9fe0ed61..8932cedd 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -30,7 +30,7 @@ import ( // DocumentVersion identifies the mapping used for indexing documents. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. -const DocumentVersion = "v14" +const DocumentVersion = "v16" // AuthorVersion identifies the mapping used for indexing authors. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. @@ -464,9 +464,9 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.TypeMapping[lang].AddFieldMappingsAt("Illustrations", numericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsRatingValue", numericFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsRatingCount", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSourceLink", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSourceScoreValue", numericFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSourceScoreCount", storedOnlyFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) @@ -494,9 +494,9 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("Illustrations", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsBookURL", storedOnlyFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsRatingValue", numericFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsRatingCount", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceLink", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceScoreValue", numericFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceScoreCount", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) diff --git a/internal/index/bleve_document_read.go b/internal/index/bleve_document_read.go index a6cfa50b..c96f404f 100644 --- a/internal/index/bleve_document_read.go +++ b/internal/index/bleve_document_read.go @@ -40,12 +40,12 @@ const titleBoost = 3.0 // document detail page's "related documents" widget. var DefaultDocumentSortBy = []string{"-_score", "Series", "SeriesIndex"} -// goodReadsScoreSort treats a stored zero as missing without changing the index. -type goodReadsScoreSort struct { +// reviewScoreSort treats a stored zero as missing without changing the index. +type reviewScoreSort struct { search.SortField } -func (s *goodReadsScoreSort) Value(hit *search.DocumentMatch) string { +func (s *reviewScoreSort) Value(hit *search.DocumentMatch) string { value := s.SortField.Value(hit) if s.SortField.DecodeValue(value) == "0" { if s.Desc { @@ -56,7 +56,7 @@ func (s *goodReadsScoreSort) Value(hit *search.DocumentMatch) string { return value } -func (s *goodReadsScoreSort) Copy() search.SearchSort { +func (s *reviewScoreSort) Copy() search.SearchSort { copied := *s return &copied } @@ -247,17 +247,23 @@ func (b *BleveIndexer) addFilters(searchFields SearchFields, filtersQuery *query bq.SetBoost(0) filtersQuery.AddQuery(bq) } - if searchFields.ReviewScoreFrom > 0 || searchFields.ReviewScoreTo > 0 { + if searchFields.DataSourceScoreNone { + unscored := 0.0 + ratingQuery := newInclusiveNumericRangeQuery(&unscored, &unscored) + ratingQuery.SetField("DataSourceScoreValue") + ratingQuery.SetBoost(0) + filtersQuery.AddQuery(ratingQuery) + } else if searchFields.DataSourceScoreFrom > 0 || searchFields.DataSourceScoreTo > 0 { min := math.SmallestNonzeroFloat64 var max *float64 - if searchFields.ReviewScoreFrom > 0 { - min = searchFields.ReviewScoreFrom + if searchFields.DataSourceScoreFrom > 0 { + min = searchFields.DataSourceScoreFrom } - if searchFields.ReviewScoreTo > 0 { - max = &searchFields.ReviewScoreTo + if searchFields.DataSourceScoreTo > 0 { + max = &searchFields.DataSourceScoreTo } ratingQuery := newInclusiveNumericRangeQuery(&min, max) - ratingQuery.SetField("GoodReadsRatingValue") + ratingQuery.SetField("DataSourceScoreValue") ratingQuery.SetBoost(0) filtersQuery.AddQuery(ratingQuery) } @@ -424,9 +430,9 @@ func (b *BleveIndexer) searchAndHydrate(query query.Query, size, offset int, sor if len(sortBy) > 0 { searchOptions.SortBy(sortBy) for i, key := range sortBy { - if strings.TrimPrefix(key, "-") == "GoodReadsRatingValue" { - searchOptions.Sort[i] = &goodReadsScoreSort{SortField: search.SortField{ - Field: "GoodReadsRatingValue", + if strings.TrimPrefix(key, "-") == "DataSourceScoreValue" { + searchOptions.Sort[i] = &reviewScoreSort{SortField: search.SortField{ + Field: "DataSourceScoreValue", Desc: strings.HasPrefix(key, "-"), Type: search.SortFieldAsNumber, Missing: search.SortFieldMissingLast, @@ -622,14 +628,14 @@ func compareDocumentsBySortKey(a, b Document, key string) int { c = cmp.Compare(a.Publication.Date, b.Publication.Date) case "Words": c = cmp.Compare(a.Words, b.Words) - case "GoodReadsRatingValue": - if (a.GoodReadsRatingValue == 0) != (b.GoodReadsRatingValue == 0) { - if a.GoodReadsRatingValue == 0 { + case "DataSourceScoreValue": + if (a.DataSourceScoreValue == 0) != (b.DataSourceScoreValue == 0) { + if a.DataSourceScoreValue == 0 { return 1 } return -1 } - c = cmp.Compare(a.GoodReadsRatingValue, b.GoodReadsRatingValue) + c = cmp.Compare(a.DataSourceScoreValue, b.DataSourceScoreValue) case "Series": c = cmp.Compare(a.Series, b.Series) case "SeriesIndex": @@ -1076,9 +1082,9 @@ func hydrateDocument(match *search.DocumentMatch) Document { textRankEnriched = match.Fields["TextRankEnriched"].(bool) } - goodReadsBookURL, _ := match.Fields["GoodReadsBookURL"].(string) - goodReadsRatingValue, _ := match.Fields["GoodReadsRatingValue"].(float64) - goodReadsRatingCount, _ := match.Fields["GoodReadsRatingCount"].(string) + reviewLink, _ := match.Fields["DataSourceLink"].(string) + reviewScoreValue, _ := match.Fields["DataSourceScoreValue"].(float64) + reviewScoreCount, _ := match.Fields["DataSourceScoreCount"].(string) goodReadsResourceID, _ := match.Fields["GoodReadsResourceID"].(string) goodReadsLegacyBookID, _ := match.Fields["GoodReadsLegacyBookID"].(float64) goodReadsEnriched, _ := match.Fields["GoodReadsEnriched"].(bool) @@ -1109,9 +1115,9 @@ func hydrateDocument(match *search.DocumentMatch) Document { TextRankPhrases: textRankPhrases, TextRankWords: textRankWords, TextRankEnriched: textRankEnriched, - GoodReadsBookURL: goodReadsBookURL, - GoodReadsRatingValue: goodReadsRatingValue, - GoodReadsRatingCount: goodReadsRatingCount, + DataSourceLink: reviewLink, + DataSourceScoreValue: reviewScoreValue, + DataSourceScoreCount: reviewScoreCount, GoodReadsResourceID: goodReadsResourceID, GoodReadsLegacyBookID: int(goodReadsLegacyBookID), GoodReadsEnriched: goodReadsEnriched, diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go index 2bfdd75f..f7e2ebf8 100644 --- a/internal/index/bleve_goodreads_write.go +++ b/internal/index/bleve_goodreads_write.go @@ -6,7 +6,6 @@ import ( "fmt" "log" "math" - "net/url" "regexp" "strconv" "strings" @@ -30,7 +29,7 @@ type GoodReadsSource interface { SearchBooks(ctx context.Context, query string) (biblioreads.SearchBooksResponse, error) } -var goodReadsFields = []string{"GoodReadsBookURL", "GoodReadsRatingValue", "GoodReadsRatingCount", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} +var goodReadsFields = []string{"DataSourceLink", "DataSourceScoreValue", "DataSourceScoreCount", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} // Strip named series annotations and omnibus ranges, not standalone volume numbers. var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) @@ -41,8 +40,6 @@ var goodReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) var goodReadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) -var ErrInvalidGoodReadsBookURL = errors.New("invalid Goodreads book URL") - func normalizeGoodReadsName(value string) string { var result strings.Builder for _, r := range norm.NFD.String(strings.ToLower(value)) { @@ -356,44 +353,34 @@ func (b *BleveIndexer) enrichGoodReadsBatch(documents []Document) error { // GoodReadsBook returns the stored book URL and rating, or searches a document // not looked up yet and persists its match. Returns an empty result if there's no match. func (b *BleveIndexer) GoodReadsBook(ctx context.Context, document Document) (biblioreads.SearchResult, error) { - if document.GoodReadsBookURL != "" || document.GoodReadsEnriched || b.goodReadsSource == nil { - return biblioreads.SearchResult{BookURL: document.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", document.GoodReadsRatingValue), RatingCount: document.GoodReadsRatingCount}, nil + if document.DataSourceLink != "" || document.GoodReadsEnriched || b.goodReadsSource == nil { + return biblioreads.SearchResult{BookURL: document.DataSourceLink, Rating: fmt.Sprintf("%.2f", document.DataSourceScoreValue), RatingCount: document.DataSourceScoreCount}, nil } return b.enrichGoodReads(ctx, document) } -// UpdateGoodReadsBookURL saves an administrator's correction without retaining -// the previous book's rating or review identifiers. -func (b *BleveIndexer) UpdateGoodReadsBookURL(documentID, rawURL string) error { - bookURL, err := url.Parse(strings.TrimSpace(rawURL)) - if err != nil || len(rawURL) > 2048 || bookURL.User != nil || - (bookURL.Scheme != "https" && bookURL.Scheme != "http") || - (bookURL.Host != "www.goodreads.com" && bookURL.Host != "goodreads.com") || - !goodReadsBookPath.MatchString(bookURL.Path) { - return ErrInvalidGoodReadsBookURL - } - path := strings.TrimSuffix(bookURL.Path, "/") - +// UpdateDataSourceLink saves a provider-normalized link and clears data cached for the previous match. +func (b *BleveIndexer) UpdateDataSourceLink(documentID, reviewLink string) error { b.documentsMu.Lock() defer b.documentsMu.Unlock() current, found, err := b.documentByIndexIDLocked(documentID) if err != nil { - return fmt.Errorf("reading document %s for Goodreads correction: %w", documentID, err) + return fmt.Errorf("reading document %s for review-link correction: %w", documentID, err) } if !found { return ErrDocumentNotFound } - if current.GoodReadsBookURL == path { + if current.DataSourceLink == reviewLink { return nil } - current.GoodReadsBookURL = path - current.GoodReadsRatingValue = 0 - current.GoodReadsRatingCount = "" + current.DataSourceLink = reviewLink + current.DataSourceScoreValue = 0 + current.DataSourceScoreCount = "" current.GoodReadsResourceID = "" current.GoodReadsLegacyBookID = 0 current.GoodReadsEnriched = true if err := b.documentsIdx.Index(current.ID, current); err != nil { - return fmt.Errorf("saving Goodreads correction for document %s: %w", documentID, err) + return fmt.Errorf("saving review-link correction for document %s: %w", documentID, err) } return nil } @@ -417,18 +404,18 @@ func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, if !found { return ErrDocumentNotFound } - if goodReadsQuery(current) != goodReadsQuery(document) || current.GoodReadsBookURL != bookURL { + if goodReadsQuery(current) != goodReadsQuery(document) || current.DataSourceLink != bookURL { return fmt.Errorf("GoodReads match changed for document %s", document.ID) } changed := false - if book.RatingCount != "" && current.GoodReadsRatingCount != book.RatingCount { - current.GoodReadsRatingCount = book.RatingCount + if book.RatingCount != "" && current.DataSourceScoreCount != book.RatingCount { + current.DataSourceScoreCount = book.RatingCount changed = true } if book.Rating != "" { ratingValue := parseGoodReadsRating(book.Rating, document.ID) - if current.GoodReadsRatingValue != ratingValue { - current.GoodReadsRatingValue = ratingValue + if current.DataSourceScoreValue != ratingValue { + current.DataSourceScoreValue = ratingValue changed = true } } @@ -457,7 +444,7 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) ( return biblioreads.SearchResult{}, ctx.Err() } q := goodReadsQuery(document) - result := biblioreads.SearchResult{BookURL: document.GoodReadsBookURL} + result := biblioreads.SearchResult{BookURL: document.DataSourceLink} if result.BookURL == "" && document.Title != "" { ctx, cancel := context.WithTimeout(ctx, goodReadsRequestTimeout) defer cancel() @@ -500,18 +487,18 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) ( // Looked up concurrently (e.g. by the background enrichment and a detail // page request at the same time): keep the stored result. if current.GoodReadsEnriched { - result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", current.GoodReadsRatingValue), RatingCount: current.GoodReadsRatingCount} + result = biblioreads.SearchResult{BookURL: current.DataSourceLink, Rating: fmt.Sprintf("%.2f", current.DataSourceScoreValue), RatingCount: current.DataSourceScoreCount} return result, nil } - if current.GoodReadsBookURL != "" && current.GoodReadsBookURL != result.BookURL { - result = biblioreads.SearchResult{BookURL: current.GoodReadsBookURL, Rating: fmt.Sprintf("%.2f", current.GoodReadsRatingValue), RatingCount: current.GoodReadsRatingCount} + if current.DataSourceLink != "" && current.DataSourceLink != result.BookURL { + result = biblioreads.SearchResult{BookURL: current.DataSourceLink, Rating: fmt.Sprintf("%.2f", current.DataSourceScoreValue), RatingCount: current.DataSourceScoreCount} } - current.GoodReadsBookURL = result.BookURL - if result.Rating == "" && current.GoodReadsRatingValue > 0 { - result.Rating = fmt.Sprintf("%.2f", current.GoodReadsRatingValue) + current.DataSourceLink = result.BookURL + if result.Rating == "" && current.DataSourceScoreValue > 0 { + result.Rating = fmt.Sprintf("%.2f", current.DataSourceScoreValue) } - current.GoodReadsRatingValue = parseGoodReadsRating(result.Rating, current.ID) - current.GoodReadsRatingCount = result.RatingCount + current.DataSourceScoreValue = parseGoodReadsRating(result.Rating, current.ID) + current.DataSourceScoreCount = result.RatingCount current.GoodReadsEnriched = true if err := b.storeGoodReadsAuthorURL(current, result.AuthorURL, result.Author); err != nil { return result, fmt.Errorf("error storing GoodReads author URL for document %s: %w", current.ID, err) @@ -595,19 +582,19 @@ func (b *BleveIndexer) preserveGoodReadsLocked(documents []Document) { } stored := make(map[string]Document, len(searchResult.Hits)) for _, hit := range searchResult.Hits { - bookURL, _ := hit.Fields["GoodReadsBookURL"].(string) - ratingValue, _ := hit.Fields["GoodReadsRatingValue"].(float64) - ratingCount, _ := hit.Fields["GoodReadsRatingCount"].(string) + bookURL, _ := hit.Fields["DataSourceLink"].(string) + ratingValue, _ := hit.Fields["DataSourceScoreValue"].(float64) + ratingCount, _ := hit.Fields["DataSourceScoreCount"].(string) resourceID, _ := hit.Fields["GoodReadsResourceID"].(string) legacyBookID, _ := hit.Fields["GoodReadsLegacyBookID"].(float64) enriched, _ := hit.Fields["GoodReadsEnriched"].(bool) - stored[hit.ID] = Document{GoodReadsBookURL: bookURL, GoodReadsRatingValue: ratingValue, GoodReadsRatingCount: ratingCount, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} + stored[hit.ID] = Document{DataSourceLink: bookURL, DataSourceScoreValue: ratingValue, DataSourceScoreCount: ratingCount, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} } for i := range documents { if s, ok := stored[documents[i].ID]; ok { - documents[i].GoodReadsBookURL = s.GoodReadsBookURL - documents[i].GoodReadsRatingValue = s.GoodReadsRatingValue - documents[i].GoodReadsRatingCount = s.GoodReadsRatingCount + documents[i].DataSourceLink = s.DataSourceLink + documents[i].DataSourceScoreValue = s.DataSourceScoreValue + documents[i].DataSourceScoreCount = s.DataSourceScoreCount documents[i].GoodReadsResourceID = s.GoodReadsResourceID documents[i].GoodReadsLegacyBookID = s.GoodReadsLegacyBookID documents[i].GoodReadsEnriched = s.GoodReadsEnriched diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go index f0d77906..c5308bd0 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/index/bleve_goodreads_write_test.go @@ -71,7 +71,7 @@ var goodReadsTestResults = []biblioreads.SearchResult{ } const ( - wantGoodReadsBookURL = "/book/show/12207444-la-reina-dragon" + wantDataSourceLink = "/book/show/12207444-la-reina-dragon" wantGoodReadsAuthorURL = "/author/show/151155.Alice_Borchardt" goodReadsTestID = "book.pdf" ) @@ -104,14 +104,14 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { if err != nil { t.Fatal(err) } - if doc.GoodReadsBookURL != wantGoodReadsBookURL { - t.Errorf("unexpected book URL %q", doc.GoodReadsBookURL) + if doc.DataSourceLink != wantDataSourceLink { + t.Errorf("unexpected book URL %q", doc.DataSourceLink) } - if doc.GoodReadsRatingValue != 3.82 { - t.Errorf("unexpected Goodreads rating %v", doc.GoodReadsRatingValue) + if doc.DataSourceScoreValue != 3.82 { + t.Errorf("unexpected Goodreads rating %v", doc.DataSourceScoreValue) } - if doc.GoodReadsRatingCount != "2,285" { - t.Errorf("unexpected Goodreads rating count %q", doc.GoodReadsRatingCount) + if doc.DataSourceScoreCount != "2,285" { + t.Errorf("unexpected Goodreads rating count %q", doc.DataSourceScoreCount) } assertNoDocumentAuthorURL(t, idx.documentsIdx) if !doc.GoodReadsEnriched { @@ -157,7 +157,7 @@ func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != "" { + if !doc.GoodReadsEnriched || doc.DataSourceLink != "" { t.Errorf("expected document enriched with no URLs, got %+v", doc) } if want := []string{"La Reina Dragon Alice Borchardt", "La Reina Dragon"}; !slices.Equal(source.Queries(), want) { @@ -170,7 +170,7 @@ func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { func TestEnrichGoodReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) { const authorURL = "/author/show/2" source := &fakeGoodReadsSource{results: []biblioreads.SearchResult{ - {Title: "La Reina Dragon", Author: "Second Author", BookURL: wantGoodReadsBookURL, AuthorURL: authorURL}, + {Title: "La Reina Dragon", Author: "Second Author", BookURL: wantDataSourceLink, AuthorURL: authorURL}, }} idx := newGoodReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { @@ -234,8 +234,8 @@ func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { if err := idx.EnrichGoodReads(10); err != nil { t.Fatal(err) } - if doc, _ = idx.documentByIndexID(goodReadsTestID); doc.GoodReadsBookURL != wantGoodReadsBookURL { - t.Errorf("expected document to be enriched on retry, got %q", doc.GoodReadsBookURL) + if doc, _ = idx.documentByIndexID(goodReadsTestID); doc.DataSourceLink != wantDataSourceLink { + t.Errorf("expected document to be enriched on retry, got %q", doc.DataSourceLink) } } @@ -254,8 +254,8 @@ func TestNewFileSchedulesGoodReadsEnrichment(t *testing.T) { t.Fatal(err) } if doc.GoodReadsEnriched { - if doc.GoodReadsBookURL != wantGoodReadsBookURL { - t.Errorf("unexpected book URL %q", doc.GoodReadsBookURL) + if doc.DataSourceLink != wantDataSourceLink { + t.Errorf("unexpected book URL %q", doc.DataSourceLink) } assertNoDocumentAuthorURL(t, idx.documentsIdx) return @@ -285,7 +285,7 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { t.Fatal(err) } stale.TextRankEnriched = false - if err := idx.StoreGoodReadsDetails(stale, wantGoodReadsBookURL, biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}); err != nil { + if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}); err != nil { t.Fatal(err) } stale.TextRankWords = []string{"dragon"} @@ -295,7 +295,7 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != wantGoodReadsBookURL || doc.GoodReadsRatingValue != 3.82 || doc.GoodReadsRatingCount != "2,285" { + if !doc.GoodReadsEnriched || doc.DataSourceLink != wantDataSourceLink || doc.DataSourceScoreValue != 3.82 || doc.DataSourceScoreCount != "2,285" { t.Errorf("expected GoodReads fields to survive a TextRank write, got %+v", doc) } if doc.GoodReadsResourceID != "work-id" || doc.GoodReadsLegacyBookID != 12207444 { @@ -352,7 +352,7 @@ func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if err != nil { t.Fatal(err) } - doc.GoodReadsBookURL = "/book/show/1-existing" + doc.DataSourceLink = "/book/show/1-existing" if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { t.Fatal(err) } @@ -368,7 +368,7 @@ func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.GoodReadsBookURL != "/book/show/1-existing" { + if !doc.GoodReadsEnriched || doc.DataSourceLink != "/book/show/1-existing" { t.Errorf("expected existing link kept and document marked enriched, got %+v", doc) } } @@ -388,18 +388,18 @@ func TestGoodReadsBook(t *testing.T) { if err != nil { t.Fatal(err) } - if got.BookURL != wantGoodReadsBookURL || got.Rating != "3.82" { + if got.BookURL != wantDataSourceLink || got.Rating != "3.82" { t.Errorf("expected fresh book URL and search rating, got %+v", got) } stored, err := idx.documentByIndexID(goodReadsTestID) if err != nil { t.Fatal(err) } - if !stored.GoodReadsEnriched || stored.GoodReadsBookURL != wantGoodReadsBookURL { + if !stored.GoodReadsEnriched || stored.DataSourceLink != wantDataSourceLink { t.Errorf("expected the link to be persisted, got %+v", stored) } - if got, _ := idx.GoodReadsBook(context.Background(), stored); got.BookURL != wantGoodReadsBookURL || got.Rating != "3.82" { + if got, _ := idx.GoodReadsBook(context.Background(), stored); got.BookURL != wantDataSourceLink || got.Rating != "3.82" { t.Errorf("expected stored URL and rating, got %+v", got) } req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{stored.ID})) @@ -577,7 +577,7 @@ func TestGoodReadsBookMatchesKennedyMiddleName(t *testing.T) { t.Fatalf("expected Kennedy's book and rating, got %+v, %v", got, err) } stored, err := idx.documentByIndexID(doc.ID) - if err != nil || !stored.GoodReadsEnriched || stored.GoodReadsBookURL != bookURL || stored.GoodReadsRatingValue != 4.14 { + if err != nil || !stored.GoodReadsEnriched || stored.DataSourceLink != bookURL || stored.DataSourceScoreValue != 4.14 { t.Fatalf("expected persisted book match and rating, got %+v, %v", stored, err) } author, err := idx.Author("paul-kennedy", "") @@ -689,7 +689,7 @@ func TestMatchingGoodReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { } } -func TestGoodReadsRatingCount(t *testing.T) { +func TestDataSourceScoreCount(t *testing.T) { for _, tt := range []struct { value string want uint64 @@ -772,7 +772,7 @@ func TestGoodReadsBookMatchesDolnickSubtitle(t *testing.T) { t.Fatalf("expected Dolnick's book and score, got %+v, %v", got, err) } stored, err := idx.documentByIndexID(doc.ID) - if err != nil || !stored.GoodReadsEnriched || stored.GoodReadsBookURL != bookURL { + if err != nil || !stored.GoodReadsEnriched || stored.DataSourceLink != bookURL { t.Fatalf("expected subtitle match persisted, got %+v, %v", stored, err) } if want := []string{title + " " + author}; !slices.Equal(source.Queries(), want) { @@ -853,7 +853,7 @@ func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { if err != nil { t.Fatal(err) } - if !stored.GoodReadsEnriched || stored.GoodReadsBookURL != wantURL { + if !stored.GoodReadsEnriched || stored.DataSourceLink != wantURL { t.Fatalf("expected persisted omnibus match, got %+v", stored) } author, err := idx.Author("j-r-r-tolkien", "") @@ -875,7 +875,7 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { if err != nil { t.Fatal(err) } - doc.GoodReadsBookURL = "/book/show/1-wrong" + doc.DataSourceLink = "/book/show/1-wrong" doc.GoodReadsResourceID = "wrong-work" doc.GoodReadsLegacyBookID = 1 doc.GoodReadsEnriched = true @@ -892,8 +892,8 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { if err != nil { t.Fatal(err) } - if doc.GoodReadsBookURL != wantGoodReadsBookURL { - t.Fatalf("expected refreshed book link, got %q", doc.GoodReadsBookURL) + if doc.DataSourceLink != wantDataSourceLink { + t.Fatalf("expected refreshed book link, got %q", doc.DataSourceLink) } if doc.GoodReadsResourceID != "" || doc.GoodReadsLegacyBookID != 0 { t.Fatalf("expected old identifiers to be cleared, got %q and %d", doc.GoodReadsResourceID, doc.GoodReadsLegacyBookID) @@ -907,7 +907,7 @@ func TestStoreGoodReadsDetails(t *testing.T) { doc := Document{ ID: goodReadsTestID, Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, - GoodReadsBookURL: wantGoodReadsBookURL, + DataSourceLink: wantDataSourceLink, GoodReadsEnriched: true, TextRankWords: []string{"dragon"}, } @@ -915,12 +915,12 @@ func TestStoreGoodReadsDetails(t *testing.T) { resourceID := "work-id" switch scenario { case "newly linked": - stale.GoodReadsBookURL = "" + stale.DataSourceLink = "" stale.GoodReadsEnriched = false case "changed metadata": doc.Title = "Another title" case "changed match": - doc.GoodReadsBookURL = "/book/show/2-other" + doc.DataSourceLink = "/book/show/2-other" case "partial identifiers": resourceID = "" } @@ -933,7 +933,7 @@ func TestStoreGoodReadsDetails(t *testing.T) { book := biblioreads.BookResponse{Rating: "4.12", ResourceID: resourceID, LegacyBookID: 12207444} writes := &goodReadsWriteCountingIndex{countedGoodReadsIndex: idx.documentsIdx} idx.documentsIdx = writes - err := idx.StoreGoodReadsDetails(stale, wantGoodReadsBookURL, book) + err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, book) wantError := scenario != "linked" && scenario != "newly linked" && scenario != "partial identifiers" if (err != nil) != wantError { t.Fatalf("StoreGoodReadsDetails error = %v, want error %v", err, wantError) @@ -952,7 +952,7 @@ func TestStoreGoodReadsDetails(t *testing.T) { if writes.count != 0 { t.Fatal("stale details must not cause any index writes") } - if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || stored.GoodReadsRatingValue != 0 { + if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || stored.DataSourceScoreValue != 0 { t.Fatalf("unexpected details persisted: %+v", stored) } return @@ -964,14 +964,14 @@ func TestStoreGoodReadsDetails(t *testing.T) { if stored.GoodReadsResourceID != resourceID || stored.GoodReadsLegacyBookID != wantLegacyBookID { t.Fatalf("identifiers not persisted: %+v", stored) } - if stored.GoodReadsRatingValue != 4.12 { + if stored.DataSourceScoreValue != 4.12 { t.Fatalf("rating not persisted with identifiers: %+v", stored) } if !stored.GoodReadsEnriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("existing enrichment was lost: %+v", stored) } book.Rating = "4.120" - if err := idx.StoreGoodReadsDetails(stale, wantGoodReadsBookURL, book); err != nil { + if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, book); err != nil { t.Fatalf("repeated store failed: %v", err) } if writes.count != 1 { @@ -1015,133 +1015,76 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { t.Run(tt.name, func(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) doc := Document{ - ID: goodReadsTestID, GoodReadsBookURL: wantGoodReadsBookURL, - GoodReadsRatingValue: 3.82, - GoodReadsRatingCount: "2,285", + ID: goodReadsTestID, DataSourceLink: wantDataSourceLink, + DataSourceScoreValue: 3.82, + DataSourceScoreCount: "2,285", GoodReadsResourceID: "old-work", GoodReadsLegacyBookID: 1, } if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { t.Fatal(err) } - if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, tt.book); err != nil { + if err := idx.StoreGoodReadsDetails(doc, doc.DataSourceLink, tt.book); err != nil { t.Fatal(err) } stored, err := idx.documentByIndexID(doc.ID) if err != nil { t.Fatal(err) } - if stored.GoodReadsRatingValue != parseGoodReadsRating(tt.wantRating, doc.ID) || + if stored.DataSourceScoreValue != parseGoodReadsRating(tt.wantRating, doc.ID) || stored.GoodReadsResourceID != tt.wantWork || stored.GoodReadsLegacyBookID != tt.wantID { t.Fatalf("partial details changed fields that were not supplied: %+v", stored) } wantCount := tt.wantCount if wantCount == "" { - wantCount = doc.GoodReadsRatingCount + wantCount = doc.DataSourceScoreCount } - if stored.GoodReadsRatingCount != wantCount { - t.Fatalf("expected rating count %q, got %q", wantCount, stored.GoodReadsRatingCount) + if stored.DataSourceScoreCount != wantCount { + t.Fatalf("expected rating count %q, got %q", wantCount, stored.DataSourceScoreCount) } }) } } -func TestUpdateGoodReadsBookURL(t *testing.T) { - tests := []struct { - name string - url string - want string - }{ - {name: "numeric ID", url: "https://www.goodreads.com/book/show/1368314", want: "/book/show/1368314"}, - {name: "dotted slug", url: "https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente", want: "/book/show/1368314.El_Cercano_Oriente"}, - {name: "hyphenated slug", url: "https://www.goodreads.com/book/show/243175949-el-cercano-oriente", want: "/book/show/243175949-el-cercano-oriente"}, - {name: "normalize shared URL", url: " http://goodreads.com/book/show/1368314/?from_search=true#reviews ", want: "/book/show/1368314"}, - {name: "same URL", url: "https://www.goodreads.com" + wantGoodReadsBookURL, want: wantGoodReadsBookURL}, - {name: "empty"}, - {name: "relative", url: "/book/show/1368314"}, - {name: "other domain", url: "https://example.com/book/show/1368314"}, - {name: "deceptive domain", url: "https://www.goodreads.com.example.com/book/show/1368314"}, - {name: "credentials", url: "https://reader@www.goodreads.com/book/show/1368314"}, - {name: "port", url: "https://www.goodreads.com:4000/book/show/1368314"}, - {name: "wrong scheme", url: "ftp://www.goodreads.com/book/show/1368314"}, - {name: "author", url: "https://www.goodreads.com/author/show/16667.Isaac_Asimov"}, - {name: "non-numeric ID", url: "https://www.goodreads.com/book/show/not-a-book"}, - {name: "zero ID", url: "https://www.goodreads.com/book/show/0"}, - {name: "extra path", url: "https://www.goodreads.com/book/show/1368314/other"}, - {name: "encoded path separator", url: "https://www.goodreads.com/book/show/1368314%2Fother"}, - {name: "malformed URL", url: "https://%zz/book/show/1368314"}, +func TestUpdateDataSourceLink(t *testing.T) { + idx := newGoodReadsTestIndexer(t, nil) + doc := Document{ + ID: goodReadsTestID, Slug: "la-reina-dragon", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + TextRankWords: []string{"dragon"}, + DataSourceLink: wantDataSourceLink, + DataSourceScoreValue: 3.82, + DataSourceScoreCount: "2,285", + GoodReadsResourceID: "old-work", + GoodReadsLegacyBookID: 12207444, + GoodReadsEnriched: true, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - source := &fakeGoodReadsSource{results: goodReadsTestResults} - idx := newGoodReadsTestIndexer(t, source) - doc := Document{ - ID: goodReadsTestID, Slug: "la-reina-dragon", - Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, - TextRankWords: []string{"dragon"}, - GoodReadsBookURL: wantGoodReadsBookURL, - GoodReadsRatingValue: 3.82, - GoodReadsRatingCount: "2,285", - GoodReadsResourceID: "old-work", - GoodReadsLegacyBookID: 12207444, - GoodReadsEnriched: true, - } - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { - t.Fatal(err) - } - err := idx.UpdateGoodReadsBookURL(doc.ID, tt.url) - if tt.want == "" { - if !errors.Is(err, ErrInvalidGoodReadsBookURL) { - t.Fatalf("expected invalid URL error, got %v", err) - } - } else if err != nil { - t.Fatal(err) - } - stored, err := idx.documentByIndexID(doc.ID) - if err != nil { - t.Fatal(err) - } - if tt.want == "" || tt.want == doc.GoodReadsBookURL { - if stored.GoodReadsBookURL != doc.GoodReadsBookURL || - stored.GoodReadsRatingValue != doc.GoodReadsRatingValue || - stored.GoodReadsRatingCount != doc.GoodReadsRatingCount || - stored.GoodReadsResourceID != doc.GoodReadsResourceID || - stored.GoodReadsLegacyBookID != doc.GoodReadsLegacyBookID { - t.Fatalf("invalid or unchanged URL modified stored data: %+v", stored) - } - return - } - if stored.GoodReadsBookURL != tt.want || !stored.GoodReadsEnriched || - stored.GoodReadsRatingValue != 0 || - stored.GoodReadsRatingCount != "" || - stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { - t.Fatalf("correction must clear old rating and review identifiers: %+v", stored) - } - if stored.Title != doc.Title || stored.Slug != doc.Slug || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { - t.Fatalf("correction changed unrelated document data: %+v", stored) - } - if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, biblioreads.BookResponse{Rating: "4.5"}); err == nil { - t.Fatal("stale rating write must not overwrite corrected book") - } - if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, biblioreads.BookResponse{ResourceID: "old-work", LegacyBookID: 12207444}); err == nil { - t.Fatal("stale review identifiers must not overwrite corrected book") - } - got, err := idx.enrichGoodReads(context.Background(), doc) - if err != nil || got.BookURL != tt.want { - t.Fatalf("in-flight enrichment must keep the correction: %+v, %v", got, err) - } - if err := idx.EnrichGoodReads(10); err != nil { - t.Fatal(err) - } - if len(source.Queries()) != 0 { - t.Fatal("corrected book must not be automatically rematched") - } - }) + if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + const newLink = "provider-specific-document-reference" + if err := idx.UpdateDataSourceLink(doc.ID, newLink); err != nil { + t.Fatal(err) + } + stored, err := idx.documentByIndexID(doc.ID) + if err != nil { + t.Fatal(err) + } + if stored.DataSourceLink != newLink || !stored.GoodReadsEnriched || + stored.DataSourceScoreValue != 0 || stored.DataSourceScoreCount != "" || + stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { + t.Fatalf("updating review link must clear cached score and provider lookup data: %+v", stored) + } + if stored.Title != doc.Title || stored.Slug != doc.Slug || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { + t.Fatalf("review-link update changed unrelated document data: %+v", stored) + } + if err := idx.StoreGoodReadsDetails(doc, doc.DataSourceLink, biblioreads.BookResponse{Rating: "4.5"}); err == nil { + t.Fatal("stale score write must not overwrite the updated link") } } -func TestUpdateGoodReadsBookURLMissingDocument(t *testing.T) { +func TestUpdateDataSourceLinkMissingDocument(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) - if err := idx.UpdateGoodReadsBookURL("missing", "https://www.goodreads.com/book/show/1368314"); !errors.Is(err, ErrDocumentNotFound) { + if err := idx.UpdateDataSourceLink("missing", "provider-specific-document-reference"); !errors.Is(err, ErrDocumentNotFound) { t.Fatalf("expected missing-document error, got %v", err) } } diff --git a/internal/index/document.go b/internal/index/document.go index ba9acc02..3117c786 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -7,8 +7,8 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) -// ScoreValueSortField is the index sort key for the current review score. -const ScoreValueSortField = "GoodReadsRatingValue" +// DataSourceScoreValueSortField is the index sort key for the configured data-source score. +const DataSourceScoreValueSortField = "DataSourceScoreValue" type SearchFields struct { Keywords string @@ -23,18 +23,19 @@ type SearchFields struct { // author and its own series - deliberately not its subjects, see // similarToQuery), ranked by score and pruned to those scoring close // enough to the best match (see runSimilarityQuery). - SimilarTo string - PubDateFrom date.Date - PubDateTo date.Date - EstReadTimeFrom float64 - EstReadTimeTo float64 - WordsPerMinute float64 - PagesFrom float64 - PagesTo float64 - ReviewScoreFrom float64 - ReviewScoreTo float64 - IllustratedOnly bool - SortBy []string + SimilarTo string + PubDateFrom date.Date + PubDateTo date.Date + EstReadTimeFrom float64 + EstReadTimeTo float64 + WordsPerMinute float64 + PagesFrom float64 + PagesTo float64 + DataSourceScoreFrom float64 + DataSourceScoreTo float64 + DataSourceScoreNone bool + IllustratedOnly bool + SortBy []string } type Document struct { @@ -81,13 +82,12 @@ type Document struct { // in the background. See documentsNeedingTextRank, which drives that pass // off this field so it survives (and resumes correctly across) restarts. TextRankEnriched bool - // GoodReadsBookURL is the Goodreads book path returned by BiblioReads. - // It is stored but not indexed, and filled by EnrichGoodReads. - // Author URLs are stored on Author instead. - GoodReadsBookURL string - // GoodReadsRatingValue is the Goodreads score, stored and indexed for filtering and sorting. - GoodReadsRatingValue float64 - GoodReadsRatingCount string + // DataSourceLink is the configured data source's document link. + // It is stored but not indexed. + DataSourceLink string + // DataSourceScoreValue is the configured data source's score, stored and indexed for filtering and sorting. + DataSourceScoreValue float64 + DataSourceScoreCount string // GoodReadsResourceID identifies the work; GoodReadsLegacyBookID identifies // the edition. Both are stored only, populated when book details are fetched, // and used together by BiblioReads to retrieve reviews. @@ -98,6 +98,10 @@ type Document struct { GoodReadsEnriched bool } +func (d Document) DataSourceScore() float64 { + return d.DataSourceScoreValue +} + func (d Document) SupportsAnnotations() bool { return d.Format == "EPUB" } diff --git a/internal/index/goodreads_fields_test.go b/internal/index/goodreads_fields_test.go index b6af86c1..240fb2f7 100644 --- a/internal/index/goodreads_fields_test.go +++ b/internal/index/goodreads_fields_test.go @@ -11,11 +11,11 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) -func TestGoodReadsScoreSorting(t *testing.T) { +func TestReviewScoreSorting(t *testing.T) { docs := []index.Document{ - {ID: "high", Slug: "high", GoodReadsRatingValue: 4.75}, + {ID: "high", Slug: "high", DataSourceScoreValue: 4.75}, {ID: "unrated", Slug: "unrated"}, - {ID: "low", Slug: "low", GoodReadsRatingValue: 3.25}, + {ID: "low", Slug: "low", DataSourceScoreValue: 3.25}, } for i := range docs { docs[i].Metadata = metadata.Metadata{Title: docs[i].Slug, Authors: []string{"Writer"}, Format: "EPUB", Series: "Series"} @@ -35,7 +35,7 @@ func TestGoodReadsScoreSorting(t *testing.T) { for _, scope := range []string{"search", "author", "series", "similar"} { for _, direction := range []string{"asc", "desc"} { t.Run(scope+"/"+direction, func(t *testing.T) { - key := "GoodReadsRatingValue" + key := "DataSourceScoreValue" want := []string{"low", "high"} if direction == "desc" { key = "-" + key @@ -84,7 +84,7 @@ func TestGoodReadsScoreSorting(t *testing.T) { } } -func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { +func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) { documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) if err != nil { t.Fatal(err) @@ -113,9 +113,9 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { Format: "EPUB", }, AuthorsSlugs: []string{"alice-borchardt"}, - GoodReadsBookURL: bookURL, - GoodReadsRatingValue: 4.25, - GoodReadsRatingCount: "2285", + DataSourceLink: bookURL, + DataSourceScoreValue: 4.25, + DataSourceScoreCount: "2285", GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: legacyBookID, } @@ -127,19 +127,29 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { if err != nil { t.Fatal(err) } - if got.GoodReadsBookURL != bookURL { - t.Errorf("language %q: expected stored book URL, got %q", lang, got.GoodReadsBookURL) + if got.DataSourceLink != bookURL { + t.Errorf("language %q: expected stored book URL, got %q", lang, got.DataSourceLink) } - if got.GoodReadsRatingValue != 4.25 { - t.Errorf("language %q: expected stored rating, got %v", lang, got.GoodReadsRatingValue) + if got.DataSourceScoreValue != 4.25 { + t.Errorf("language %q: expected stored rating, got %v", lang, got.DataSourceScoreValue) } - if got.GoodReadsRatingCount != "2285" { - t.Errorf("language %q: expected stored rating count, got %q", lang, got.GoodReadsRatingCount) + if got.DataSourceScoreCount != "2285" { + t.Errorf("language %q: expected stored rating count, got %q", lang, got.DataSourceScoreCount) } if got.GoodReadsResourceID != resourceID || got.GoodReadsLegacyBookID != legacyBookID { t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, got.GoodReadsResourceID, got.GoodReadsLegacyBookID) } } + if err := documentsIndexMem.Index("unscored", index.Document{ + ID: "unscored", + Slug: "unscored", + Metadata: metadata.Metadata{ + Title: "Unscored document", + Format: "EPUB", + }, + }); err != nil { + t.Fatal(err) + } for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155", "zyxwvresource", resourceID, "4.25", "2285"} { res, err := idx.Search(index.SearchFields{Keywords: keywords}, 1, 10) @@ -150,23 +160,27 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { t.Errorf("expected no results searching keywords %q, got %d", keywords, res.TotalHits()) } } - filtered, err := idx.Search(index.SearchFields{ReviewScoreFrom: 4.2, ReviewScoreTo: 4.3}, 1, 10) + filtered, err := idx.Search(index.SearchFields{DataSourceScoreFrom: 4.2, DataSourceScoreTo: 4.3}, 1, 10) if err != nil || filtered.TotalHits() != 2 { - t.Errorf("expected both documents in the Goodreads score range, got %d results and error %v", filtered.TotalHits(), err) + t.Errorf("expected both documents in the review score range, got %d results and error %v", filtered.TotalHits(), err) } - filtered, err = idx.Search(index.SearchFields{ReviewScoreFrom: 4.3}, 1, 10) + filtered, err = idx.Search(index.SearchFields{DataSourceScoreFrom: 4.3}, 1, 10) if err != nil || filtered.TotalHits() != 0 { - t.Errorf("expected no documents above the Goodreads score, got %d results and error %v", filtered.TotalHits(), err) + t.Errorf("expected no documents above the review score, got %d results and error %v", filtered.TotalHits(), err) + } + filtered, err = idx.Search(index.SearchFields{DataSourceScoreNone: true}, 1, 10) + if err != nil || filtered.TotalHits() != 1 || filtered.Hits()[0].DataSourceScoreValue != 0 { + t.Errorf("expected only the unscored document, got %d results and error %v", filtered.TotalHits(), err) } queries := map[string]bleve.SearchRequest{ - "book URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsBookURL", bookURL)), + "book URL field term": *bleve.NewSearchRequest(fieldTermQuery("DataSourceLink", bookURL)), "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsAuthorURL", authorURL)), "resource ID field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsResourceID", resourceID)), "legacy ID numeric field": *bleve.NewSearchRequest(legacyBookIDQuery(legacyBookID)), "_all match": *bleve.NewSearchRequest(bleve.NewMatchQuery("zyxwvbook")), "_all term": *bleve.NewSearchRequest(bleve.NewTermQuery(bookURL)), - "query string field": *bleve.NewSearchRequest(bleve.NewQueryStringQuery(`GoodReadsBookURL:"` + bookURL + `"`)), + "query string field": *bleve.NewSearchRequest(bleve.NewQueryStringQuery(`DataSourceLink:"` + bookURL + `"`)), } for name, req := range queries { res, err := documentsIndexMem.Search(&req) @@ -179,7 +193,7 @@ func TestGoodReadsFieldsAreStoredButNotSearchable(t *testing.T) { } } -func TestGoodReadsScoreSortingMissingFieldsAndTies(t *testing.T) { +func TestReviewScoreSortingMissingFieldsAndTies(t *testing.T) { documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) if err != nil { t.Fatal(err) @@ -191,11 +205,11 @@ func TestGoodReadsScoreSortingMissingFieldsAndTies(t *testing.T) { idx := index.NewBleve(documents, authors, nil, "", nil, index.Config{}) t.Cleanup(func() { _ = idx.Close() }) for _, doc := range []map[string]any{ - {"Slug": "unrated-zero", "GoodReadsRatingValue": 0.0, "SeriesIndex": 2.0}, - {"Slug": "high", "GoodReadsRatingValue": 4.75, "SeriesIndex": 1.0}, + {"Slug": "unrated-zero", "DataSourceScoreValue": 0.0, "SeriesIndex": 2.0}, + {"Slug": "high", "DataSourceScoreValue": 4.75, "SeriesIndex": 1.0}, {"Slug": "unrated-missing", "SeriesIndex": 1.0}, - {"Slug": "low-second", "GoodReadsRatingValue": 3.25, "SeriesIndex": 2.0}, - {"Slug": "low-first", "GoodReadsRatingValue": 3.25, "SeriesIndex": 1.0}, + {"Slug": "low-second", "DataSourceScoreValue": 3.25, "SeriesIndex": 2.0}, + {"Slug": "low-first", "DataSourceScoreValue": 3.25, "SeriesIndex": 1.0}, } { for _, field := range []string{"Title", "Description", "Series", "SeriesSlug", "Format"} { doc[field] = "" @@ -207,7 +221,7 @@ func TestGoodReadsScoreSortingMissingFieldsAndTies(t *testing.T) { } for _, direction := range []string{"asc", "desc"} { t.Run(direction, func(t *testing.T) { - key := "GoodReadsRatingValue" + key := "DataSourceScoreValue" want := []string{"low-first", "low-second", "high", "unrated-missing", "unrated-zero"} if direction == "desc" { key = "-" + key diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index 733870ae..2322cfd1 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "log" + "net/url" + "regexp" "strconv" "strings" "time" @@ -19,6 +21,8 @@ const ( goodReadsBookPrefix = "/book/show/" ) +var goodReadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) + type BiblioReadsClient interface { Book(ctx context.Context, id string) (biblioreads.BookResponse, error) Reviews(ctx context.Context, resourceID string, legacyBookID int, nextPageToken, languageCode string) (biblioreads.ReviewsResponse, error) @@ -27,7 +31,7 @@ type BiblioReadsClient interface { type GoodReadsIndex interface { GoodReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) StoreGoodReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error - UpdateGoodReadsBookURL(documentID, bookURL string) error + UpdateDataSourceLink(documentID, bookURL string) error } type BiblioReads struct { @@ -49,7 +53,7 @@ func (b *BiblioReads) AuthorLink(author index.Author, _ string) datasourcemodel. } func (b *BiblioReads) DocumentLink(document index.Document) datasourcemodel.DataSourceLink { - return goodReadsDataSourceLink(document.GoodReadsBookURL, "/book/show/") + return goodReadsDataSourceLink(document.DataSourceLink, "/book/show/") } func goodReadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink { @@ -67,15 +71,28 @@ func (b *BiblioReads) CanEditDocumentLink() bool { // UpdateDocumentLink saves a corrected Goodreads book link and clears stale score details. func (b *BiblioReads) UpdateDocumentLink(document index.Document, link string) error { - err := b.index.UpdateGoodReadsBookURL(document.ID, link) - switch { - case errors.Is(err, index.ErrInvalidGoodReadsBookURL): - return fmt.Errorf("%w: %v", ErrInvalidDocumentLink, err) - case errors.Is(err, index.ErrDocumentNotFound): - return ErrDocumentNotFound - default: + path, err := normalizeGoodReadsBookURL(link) + if err != nil { + return fmt.Errorf("%w: invalid Goodreads book URL", ErrInvalidDocumentLink) + } + if err := b.index.UpdateDataSourceLink(document.ID, path); err != nil { + if errors.Is(err, index.ErrDocumentNotFound) { + return ErrDocumentNotFound + } return err } + return nil +} + +func normalizeGoodReadsBookURL(rawURL string) (string, error) { + bookURL, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || len(rawURL) > 2048 || bookURL.User != nil || + (bookURL.Scheme != "https" && bookURL.Scheme != "http") || + (bookURL.Host != "www.goodreads.com" && bookURL.Host != "goodreads.com") || + !goodReadsBookPath.MatchString(bookURL.Path) { + return "", fmt.Errorf("invalid Goodreads book URL") + } + return strings.TrimSuffix(bookURL.Path, "/"), nil } func bookID(bookURL string) (string, error) { @@ -87,7 +104,7 @@ func bookID(bookURL string) (string, error) { } func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblioreads.BookResponse, error) { - id, err := bookID(document.GoodReadsBookURL) + id, err := bookID(document.DataSourceLink) if err != nil { return biblioreads.BookResponse{}, err } @@ -96,11 +113,11 @@ func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblio return book, fmt.Errorf("retrieving Goodreads book %s: %w", id, err) } details := book - if book.Rating != "" && document.GoodReadsRatingValue > 0 && isZeroRating(book.Rating) { - book.Rating = fmt.Sprintf("%.2f", document.GoodReadsRatingValue) + if book.Rating != "" && document.DataSourceScoreValue > 0 && isZeroRating(book.Rating) { + book.Rating = fmt.Sprintf("%.2f", document.DataSourceScoreValue) details.Rating = "" } - if err := b.index.StoreGoodReadsDetails(document, document.GoodReadsBookURL, details); err != nil { + if err := b.index.StoreGoodReadsDetails(document, document.DataSourceLink, details); err != nil { log.Printf("Error saving Goodreads details for %s: %s", document.ID, err) } return book, nil @@ -113,7 +130,7 @@ func isZeroRating(rating string) bool { func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score, error) { score := Score{Source: "Goodreads"} - if document.GoodReadsBookURL == "" { + if document.DataSourceLink == "" { result, err := b.index.GoodReadsBook(ctx, document) if err != nil { return score, err @@ -124,7 +141,7 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score if _, err := bookID(result.BookURL); err != nil { return score, err } - document.GoodReadsBookURL = result.BookURL + document.DataSourceLink = result.BookURL score.Value, score.Count, score.URL = result.Rating, result.RatingCount, b.DocumentLink(document).URL return score, nil } @@ -137,7 +154,7 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score } func (b *BiblioReads) Reviews(ctx context.Context, document index.Document, language string) (Page, error) { - if document.GoodReadsBookURL == "" { + if document.DataSourceLink == "" { return Page{}, nil } book, err := b.book(ctx, document) diff --git a/internal/reviews/biblioreads_test.go b/internal/reviews/biblioreads_test.go new file mode 100644 index 00000000..0a0acb08 --- /dev/null +++ b/internal/reviews/biblioreads_test.go @@ -0,0 +1,86 @@ +package reviews_test + +import ( + "context" + "errors" + "testing" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/reviews" +) + +type reviewLinkIndex struct { + link string + err error +} + +func (i *reviewLinkIndex) GoodReadsBook(context.Context, index.Document) (biblioreads.SearchResult, error) { + return biblioreads.SearchResult{}, nil +} + +func (i *reviewLinkIndex) StoreGoodReadsDetails(index.Document, string, biblioreads.BookResponse) error { + return nil +} + +func (i *reviewLinkIndex) UpdateDataSourceLink(_ string, link string) error { + i.link = link + return i.err +} + +func TestBiblioReadsUpdateDocumentLink(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + {name: "numeric ID", url: "https://www.goodreads.com/book/show/1368314", want: "/book/show/1368314"}, + {name: "dotted slug", url: "https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente", want: "/book/show/1368314.El_Cercano_Oriente"}, + {name: "hyphenated slug", url: "https://www.goodreads.com/book/show/243175949-el-cercano-oriente", want: "/book/show/243175949-el-cercano-oriente"}, + {name: "normalize shared URL", url: " http://goodreads.com/book/show/1368314/?from_search=true#reviews ", want: "/book/show/1368314"}, + {name: "empty"}, + {name: "relative", url: "/book/show/1368314"}, + {name: "other domain", url: "https://example.com/book/show/1368314"}, + {name: "deceptive domain", url: "https://www.goodreads.com.example.com/book/show/1368314"}, + {name: "credentials", url: "https://reader@www.goodreads.com/book/show/1368314"}, + {name: "port", url: "https://www.goodreads.com:4000/book/show/1368314"}, + {name: "wrong scheme", url: "ftp://www.goodreads.com/book/show/1368314"}, + {name: "author", url: "https://www.goodreads.com/author/show/16667.Isaac_Asimov"}, + {name: "non-numeric ID", url: "https://www.goodreads.com/book/show/not-a-book"}, + {name: "zero ID", url: "https://www.goodreads.com/book/show/0"}, + {name: "extra path", url: "https://www.goodreads.com/book/show/1368314/other"}, + {name: "encoded path separator", url: "https://www.goodreads.com/book/show/1368314%2Fother"}, + {name: "malformed URL", url: "https://%zz/book/show/1368314"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + idx := &reviewLinkIndex{} + provider := reviews.NewBiblioReads(nil, idx) + err := provider.UpdateDocumentLink(index.Document{ID: "doc-id"}, tt.url) + if tt.want == "" { + if !errors.Is(err, reviews.ErrInvalidDocumentLink) { + t.Fatalf("expected invalid-link error, got %v", err) + } + if idx.link != "" { + t.Fatalf("invalid link was passed to index: %q", idx.link) + } + return + } + if err != nil { + t.Fatal(err) + } + if idx.link != tt.want { + t.Fatalf("normalized link = %q, want %q", idx.link, tt.want) + } + }) + } +} + +func TestBiblioReadsUpdateDocumentLinkMissingDocument(t *testing.T) { + idx := &reviewLinkIndex{err: index.ErrDocumentNotFound} + provider := reviews.NewBiblioReads(nil, idx) + err := provider.UpdateDocumentLink(index.Document{ID: "missing"}, "https://www.goodreads.com/book/show/1368314") + if !errors.Is(err, reviews.ErrDocumentNotFound) { + t.Fatalf("expected missing-document error, got %v", err) + } +} diff --git a/internal/reviews/data_source_link_test.go b/internal/reviews/data_source_link_test.go index 7c4f8661..609381f0 100644 --- a/internal/reviews/data_source_link_test.go +++ b/internal/reviews/data_source_link_test.go @@ -18,7 +18,7 @@ func TestBiblioReadsDataSourceLinks(t *testing.T) { } document := provider.DocumentLink(index.Document{ - GoodReadsBookURL: "/book/show/456.A_Book", + DataSourceLink: "/book/show/456.A_Book", }) if document.URL != "https://www.goodreads.com/book/show/456.A_Book" || document.Source != "Goodreads" || document.Icon != "/images/goodreads.svg" || !document.IconInvertible { diff --git a/internal/webserver/author_summary_test.go b/internal/webserver/author_summary_test.go index 6c5567a9..41576bde 100644 --- a/internal/webserver/author_summary_test.go +++ b/internal/webserver/author_summary_test.go @@ -13,6 +13,8 @@ import ( gowikidata "github.com/Navid2zp/go-wikidata" "github.com/PuerkitoBio/goquery" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/webserver" @@ -225,6 +227,62 @@ func TestAuthorSummaryGoodReadsLink(t *testing.T) { } } +type alternativeDataSourceLinks struct{} + +func (alternativeDataSourceLinks) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { + return datasourcemodel.DataSourceLink{Source: "Another source", URL: "https://example.org/authors/" + author.Slug} +} + +func TestAlternativeDataSourceLinks(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "4", ResourceID: "work", LegacyBookID: 1, + }} + app, idx := newDataSourceProvidersTestApp(t, source, nil, alternativeDataSourceLinks{}) + for _, slug := range []string{"jane-doe", "john-doe"} { + if err := idx.IndexAuthor(index.Author{ + Slug: slug, Name: slug, RetrievedOn: time.Now().UTC(), + }); err != nil { + t.Fatal(err) + } + } + for _, path := range []string{ + "/authors/jane-doe", "/authors/jane-doe/summary", + "/authors/jane-doe/summary?style=clear", "/documents/unlinked", + } { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) + if err != nil { + t.Fatal(err) + } + link := page.Find(`a[href="https://example.org/authors/jane-doe"]`) + if link.Length() != 1 || link.Text() != "Go to Another source author page" { + t.Fatalf("%s: expected injected author data-source link, got %q", path, link.Text()) + } + if page.Find(`img[src^="/images/goodreads.svg"]`).Length() != 0 { + t.Fatalf("%s: unexpected Goodreads branding", path) + } + } + for path, wantLinks := range map[string]int{"/documents/linked/score": 2, "/documents/linked/reviews": 1} { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) + if err != nil { + t.Fatal(err) + } + if page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`).Length() != wantLinks { + t.Fatalf("%s: BiblioReads must retain its Goodreads document URL regardless of the author link provider", path) + } + } +} + +func TestDataSourceLinksWithoutReviewProvider(t *testing.T) { + app, idx := newDataSourceProvidersTestApp(t, nil, nil, alternativeDataSourceLinks{}) + if err := idx.IndexAuthor(index.Author{Slug: "jane-doe", Name: "Jane Doe", RetrievedOn: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + body := getBody(t, app, "/authors/jane-doe/summary") + if !strings.Contains(body, "https://example.org/authors/jane-doe") { + t.Fatal("links must not require a review provider") + } +} + func TestAuthorSummary(t *testing.T) { db := infrastructure.Connect(":memory:", 250) smtpMock := &infrastructure.SMTPMock{} diff --git a/internal/webserver/author_update_auth_test.go b/internal/webserver/author_update_auth_test.go index def10854..dcda24e1 100644 --- a/internal/webserver/author_update_auth_test.go +++ b/internal/webserver/author_update_auth_test.go @@ -59,3 +59,50 @@ func TestAuthorUpdateRequiresAdmin(t *testing.T) { }) } } + +func TestDocumentUpdateDataSourceLinkRequiresAdmin(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs(testLibraryDir), webserver.Config{}) + t.Cleanup(func() { _ = app.Shutdown() }) + users := &model.UserRepository{DB: db} + if err := users.Create(&model.User{ + Uuid: "regular-review-link-user", Name: "Regular user", Username: "regular-review-link", + Email: "regular@example.com", Password: model.Hash("regular-password"), + Role: model.RoleRegular, WordsPerMinute: 250, + }); err != nil { + t.Fatal(err) + } + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + regularCookie, err := login(app, "regular@example.com", "regular-password", t) + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + cookie *http.Cookie + status int + }{ + {name: "anonymous", cookie: &http.Cookie{}, status: http.StatusForbidden}, + {name: "regular user", cookie: regularCookie, status: http.StatusForbidden}, + {name: "administrator without an editable review provider", cookie: adminCookie, status: http.StatusNotFound}, + } { + t.Run(tt.name, func(t *testing.T) { + response, err := putRequest(url.Values{ + "bookURL": {"https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente"}, + }, tt.cookie, app, "/documents/john-doe-test-epub/review-link", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, tt.status, t) + }) + } +} diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index 2559bcea..e10d187b 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -123,8 +123,8 @@ func NewController(hlRepository highlightsRepository, usersRepository usersRepos } } -// ReviewSourceName returns the display name for this controller's review provider. -func (d *Controller) ReviewSourceName() string { +// DataSourceName returns the source name used for review-score labels. +func (d *Controller) DataSourceName() string { if d.reviewProvider == nil { return "" } diff --git a/internal/webserver/controller/document/update_document_link.go b/internal/webserver/controller/document/update_document_link.go index 8e60048d..5197389f 100644 --- a/internal/webserver/controller/document/update_document_link.go +++ b/internal/webserver/controller/document/update_document_link.go @@ -23,7 +23,7 @@ func (d *Controller) UpdateDocumentLink(c fiber.Ctx) error { link := c.FormValue("bookURL") if err := d.reviewProvider.UpdateDocumentLink(document, link); err != nil { if errors.Is(err, reviews.ErrInvalidDocumentLink) { - return c.Status(fiber.StatusUnprocessableEntity).Render("partials/document-goodreads-form", fiber.Map{ + return c.Status(fiber.StatusUnprocessableEntity).Render("partials/document-review-link-form", fiber.Map{ "Slug": document.Slug, "BookURL": link, "InvalidURL": true, "Source": d.reviewProvider.Name(), }) } diff --git a/internal/webserver/controller/search/parse.go b/internal/webserver/controller/search/parse.go index 2ef35b74..5da1bfd9 100644 --- a/internal/webserver/controller/search/parse.go +++ b/internal/webserver/controller/search/parse.go @@ -15,19 +15,24 @@ import ( // pages-from/to, review score, illustrated-only) into an index.SearchFields. func ParseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.SearchFields, error) { searchFields := index.SearchFields{ - Keywords: c.Query("search"), - Language: c.Query("language"), - Subjects: c.Query("subjects"), - SimilarTo: similarToSlug(c), - SortBy: parseDocumentSortBy(c), - EstReadTimeFrom: fiber.Query[float64](c, "est-read-time-from", 0), - EstReadTimeTo: fiber.Query[float64](c, "est-read-time-to", 0), - WordsPerMinute: wordsPerMinute, - PagesFrom: fiber.Query[float64](c, "pages-from", 0), - PagesTo: fiber.Query[float64](c, "pages-to", 0), - ReviewScoreFrom: fiber.Query[float64](c, "review-score-from", 0), - ReviewScoreTo: fiber.Query[float64](c, "review-score-to", 0), - IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", + Keywords: c.Query("search"), + Language: c.Query("language"), + Subjects: c.Query("subjects"), + SimilarTo: similarToSlug(c), + SortBy: parseDocumentSortBy(c), + EstReadTimeFrom: fiber.Query[float64](c, "est-read-time-from", 0), + EstReadTimeTo: fiber.Query[float64](c, "est-read-time-to", 0), + WordsPerMinute: wordsPerMinute, + PagesFrom: fiber.Query[float64](c, "pages-from", 0), + PagesTo: fiber.Query[float64](c, "pages-to", 0), + DataSourceScoreFrom: fiber.Query[float64](c, "review-score-from", 0), + DataSourceScoreTo: fiber.Query[float64](c, "review-score-to", 0), + DataSourceScoreNone: c.Query("review-score-from") == "none" || c.Query("review-score-to") == "none", + IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", + } + if searchFields.DataSourceScoreNone { + searchFields.DataSourceScoreFrom = 0 + searchFields.DataSourceScoreTo = 0 } if c.Query("pub-date-from") != "" { @@ -58,8 +63,8 @@ func ParseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.Search searchFields.PagesFrom, searchFields.PagesTo = searchFields.PagesTo, searchFields.PagesFrom } - if searchFields.ReviewScoreTo != 0 && searchFields.ReviewScoreFrom > searchFields.ReviewScoreTo { - searchFields.ReviewScoreFrom, searchFields.ReviewScoreTo = searchFields.ReviewScoreTo, searchFields.ReviewScoreFrom + if !searchFields.DataSourceScoreNone && searchFields.DataSourceScoreTo != 0 && searchFields.DataSourceScoreFrom > searchFields.DataSourceScoreTo { + searchFields.DataSourceScoreFrom, searchFields.DataSourceScoreTo = searchFields.DataSourceScoreTo, searchFields.DataSourceScoreFrom } return searchFields, nil @@ -153,9 +158,9 @@ func parseDocumentSortBy(c fiber.Ctx) []string { case "est-read-time-longer-first": return []string{"-Words"} case "review-score-asc": - return []string{index.ScoreValueSortField} + return []string{index.DataSourceScoreValueSortField} case "review-score-desc": - return []string{"-" + index.ScoreValueSortField} + return []string{"-" + index.DataSourceScoreValueSortField} } } return index.DefaultDocumentSortBy diff --git a/internal/webserver/controller/series/documents.go b/internal/webserver/controller/series/documents.go index a20138da..a2986bf3 100644 --- a/internal/webserver/controller/series/documents.go +++ b/internal/webserver/controller/series/documents.go @@ -127,9 +127,9 @@ func (d *Controller) parseSortBy(c fiber.Ctx) []string { case "est-read-time-longer-first": return []string{"-Words"} case "review-score-asc": - return []string{index.ScoreValueSortField, "SeriesIndex"} + return []string{index.DataSourceScoreValueSortField, "SeriesIndex"} case "review-score-desc": - return []string{"-" + index.ScoreValueSortField, "SeriesIndex"} + return []string{"-" + index.DataSourceScoreValueSortField, "SeriesIndex"} } } return []string{"SeriesIndex"} diff --git a/internal/webserver/data_source_links_test.go b/internal/webserver/data_source_links_test.go deleted file mode 100644 index 685cac0d..00000000 --- a/internal/webserver/data_source_links_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package webserver_test - -import ( - "strings" - "testing" - "time" - - "github.com/PuerkitoBio/goquery" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" - datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" - "github.com/svera/coreander/v5/internal/index" -) - -type alternativeDataSourceLinks struct{} - -func (alternativeDataSourceLinks) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { - return datasourcemodel.DataSourceLink{Source: "Another source", URL: "https://example.org/authors/" + author.Slug} -} - -func TestAlternativeDataSourceLinks(t *testing.T) { - source := &fakeBookSource{book: biblioreads.BookResponse{ - Rating: "4", ResourceID: "work", LegacyBookID: 1, - }} - app, idx := newDataSourceProvidersTestApp(t, source, nil, alternativeDataSourceLinks{}) - for _, slug := range []string{"jane-doe", "john-doe"} { - if err := idx.IndexAuthor(index.Author{ - Slug: slug, Name: slug, RetrievedOn: time.Now().UTC(), - }); err != nil { - t.Fatal(err) - } - } - for _, path := range []string{ - "/authors/jane-doe", "/authors/jane-doe/summary", - "/authors/jane-doe/summary?style=clear", "/documents/unlinked", - } { - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) - if err != nil { - t.Fatal(err) - } - link := page.Find(`a[href="https://example.org/authors/jane-doe"]`) - if link.Length() != 1 || link.Text() != "Go to Another source author page" { - t.Fatalf("%s: expected injected author data-source link, got %q", path, link.Text()) - } - if page.Find(`img[src^="/images/goodreads.svg"]`).Length() != 0 { - t.Fatalf("%s: unexpected Goodreads branding", path) - } - } - for path, wantLinks := range map[string]int{"/documents/linked/score": 2, "/documents/linked/reviews": 1} { - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) - if err != nil { - t.Fatal(err) - } - if page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`).Length() != wantLinks { - t.Fatalf("%s: BiblioReads must retain its Goodreads document URL regardless of the author link provider", path) - } - } -} - -func TestDataSourceLinksWithoutReviewProvider(t *testing.T) { - app, idx := newDataSourceProvidersTestApp(t, nil, nil, alternativeDataSourceLinks{}) - if err := idx.IndexAuthor(index.Author{Slug: "jane-doe", Name: "Jane Doe", RetrievedOn: time.Now().UTC()}); err != nil { - t.Fatal(err) - } - body := getBody(t, app, "/authors/jane-doe/summary") - if !strings.Contains(body, "https://example.org/authors/jane-doe") { - t.Fatal("links must not require a review provider") - } -} diff --git a/internal/webserver/document_goodreads_auth_test.go b/internal/webserver/document_goodreads_auth_test.go deleted file mode 100644 index aeb406bc..00000000 --- a/internal/webserver/document_goodreads_auth_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package webserver_test - -import ( - "net/http" - "net/url" - "testing" - - "github.com/svera/coreander/v5/internal/webserver" - "github.com/svera/coreander/v5/internal/webserver/infrastructure" - "github.com/svera/coreander/v5/internal/webserver/model" -) - -func TestDocumentUpdateGoodReadsRequiresAdmin(t *testing.T) { - db := infrastructure.Connect(":memory:", 250) - sqlDB, err := db.DB() - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sqlDB.Close() }) - app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs(testLibraryDir), webserver.Config{}) - t.Cleanup(func() { _ = app.Shutdown() }) - users := &model.UserRepository{DB: db} - if err := users.Create(&model.User{ - Uuid: "regular-goodreads-user", Name: "Regular user", Username: "regular-goodreads", - Email: "regular@example.com", Password: model.Hash("regular-password"), - Role: model.RoleRegular, WordsPerMinute: 250, - }); err != nil { - t.Fatal(err) - } - adminCookie, err := login(app, "admin@example.com", "admin", t) - if err != nil { - t.Fatal(err) - } - regularCookie, err := login(app, "regular@example.com", "regular-password", t) - if err != nil { - t.Fatal(err) - } - for _, tt := range []struct { - name string - cookie *http.Cookie - status int - }{ - {name: "anonymous", cookie: &http.Cookie{}, status: http.StatusForbidden}, - {name: "regular user", cookie: regularCookie, status: http.StatusForbidden}, - {name: "administrator without an editable review provider", cookie: adminCookie, status: http.StatusNotFound}, - } { - t.Run(tt.name, func(t *testing.T) { - response, err := putRequest(url.Values{ - "bookURL": {"https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente"}, - }, tt.cookie, app, "/documents/john-doe-test-epub/review-link", t) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - mustReturnStatus(response, tt.status, t) - }) - } -} diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 69eea0b5..0aebffea 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -7,7 +7,6 @@ import ( "io" "net/http" "net/url" - "slices" "strings" "sync" "testing" @@ -331,8 +330,8 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide { ID: "linked.epub", Slug: "linked", Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, - GoodReadsBookURL: "/book/show/12207444-la-reina-dragon", - GoodReadsRatingValue: 3.82, + DataSourceLink: "/book/show/12207444-la-reina-dragon", + DataSourceScoreValue: 3.82, GoodReadsEnriched: true, }, { @@ -359,143 +358,6 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide return app, idx } -func TestSearchResultsShowStoredGoodReadsRating(t *testing.T) { - for _, tt := range []struct { - rating string - want string - }{ - {rating: "3.82", want: "3.82"}, - {rating: "4.1", want: "4.10"}, - {rating: "5", want: "5.00"}, - } { - t.Run(tt.rating, func(t *testing.T) { - app, idx := newGoodReadsTestApp(t, nil) - doc, err := idx.Document("linked") - if err != nil { - t.Fatal(err) - } - if err := idx.StoreGoodReadsDetails(doc, doc.GoodReadsBookURL, biblioreads.BookResponse{Rating: tt.rating, RatingCount: "2,285"}); err != nil { - t.Fatal(err) - } - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) - if err != nil { - t.Fatal(err) - } - rating := page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) - if rating.Length() != 1 || strings.TrimSpace(rating.Text()) != tt.want { - t.Fatalf("expected stored Goodreads rating in search result, got %q", rating.Text()) - } - if page.Find(`li:has(a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]) i.bi-star-fill`).Length() != 1 { - t.Fatal("expected a star icon beside the Goodreads score") - } - if rating.Parent().Find("small").Text() != "(2,285 ratings)" { - t.Fatal("expected rating count beside the Goodreads score") - } - }) - } -} - -func TestDocumentSearchSortsByReviewScore(t *testing.T) { - app, idx := newGoodReadsTestApp(t, &fakeBookSource{}) - scores := map[string]string{ - "linked": "3.20", - "unlinked": "4.10", - "unmatched": "4.80", - } - documents, err := idx.Search(index.SearchFields{}, 1, 10) - if err != nil { - t.Fatal(err) - } - for _, doc := range documents.Hits() { - if err := idx.UpdateGoodReadsBookURL(doc.ID, "https://www.goodreads.com/book/show/1-"+doc.Slug); err != nil { - t.Fatalf("set score link for %s: %v", doc.Slug, err) - } - updated, err := idx.Document(doc.Slug) - if err != nil { - t.Fatalf("reload %s: %v", doc.Slug, err) - } - if err := idx.StoreGoodReadsDetails(updated, updated.GoodReadsBookURL, biblioreads.BookResponse{Rating: scores[doc.Slug]}); err != nil { - t.Fatalf("set score for %s: %v", doc.Slug, err) - } - } - - for _, tt := range []struct { - direction string - want []string - }{ - {direction: "asc", want: []string{"linked", "unlinked", "unmatched"}}, - {direction: "desc", want: []string{"unmatched", "unlinked", "linked"}}, - } { - t.Run(tt.direction, func(t *testing.T) { - body := getBody(t, app, "/search?type=documents&sort-by=review-score-"+tt.direction) - page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - var got []string - page.Find("#list .list-group-item h2 a").Each(func(_ int, link *goquery.Selection) { - href, _ := link.Attr("href") - got = append(got, strings.TrimPrefix(href, "/documents/")) - }) - if !slices.Equal(got, tt.want) { - t.Fatalf("score sort %s: got %v, want %v", tt.direction, got, tt.want) - } - }) - } -} - -func TestSearchResultsShowGoodReadsSearchRatingCount(t *testing.T) { - source := &fakeBookSource{results: []biblioreads.SearchResult{{ - Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", - Rating: "4.20", RatingCount: "1,234", - }}} - app, idx := newGoodReadsTestApp(t, source) - score := getBody(t, app, "/documents/unlinked/score") - if !strings.Contains(score, "(1,234 ratings)") { - t.Fatal("expected the search-derived rating count in the document score") - } - stored, err := idx.Document("unlinked") - if err != nil || stored.GoodReadsRatingCount != "1,234" { - t.Fatalf("expected the search-derived count to be persisted, got %+v, %v", stored, err) - } - for _, path := range []string{ - "/search?type=documents&search=Unlinked", - "/search?type=documents&search=Unlinked&l=es", - } { - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) - if err != nil { - t.Fatal(err) - } - rating := page.Find(`a[href="https://www.goodreads.com/book/show/1-unlinked"]`) - if rating.Text() != "4.20" || !strings.Contains(rating.Parent().Find("small").Text(), "1,234") { - t.Fatalf("expected the stored score and rating count in search results for %s", path) - } - } - if len(source.ids) != 0 || len(source.queries) != 1 { - t.Fatalf("expected one book search and no detail fetches, got queries %q and ids %q", source.queries, source.ids) - } -} - -func TestDocumentDetailShowsStoredZeroRatingAsNotScored(t *testing.T) { - source := &fakeBookSource{results: []biblioreads.SearchResult{{ - Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", Rating: "0.00", - }}} - app, _ := newGoodReadsTestApp(t, source) - getBody(t, app, "/documents/unlinked/goodreads") - - body := getBody(t, app, "/search?type=documents&search=Unlinked") - page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - row := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { - return strings.TrimSpace(item.Text()) == "Not scored" && item.Find("i.bi-star-fill").Length() == 1 - }) - if row.Length() == 0 || row.Find("a").Length() != 0 { - t.Fatalf("expected stored zero rating to render as Not scored, got %q in %s", row.Text(), body) - } -} - func getBody(t *testing.T, app *fiber.App, path string) string { t.Helper() req := mustGetRequest(t, path) @@ -597,7 +459,7 @@ func TestDocumentGoodReadsEditor(t *testing.T) { if err != nil { t.Fatal(err) } - control := page.Find(`li a[data-bs-toggle="collapse"][href="#edit-goodreads-linked"]`) + control := page.Find(`li a[data-bs-toggle="collapse"][href="#edit-review-link-linked"]`) if control.Length() != tt.want { t.Fatalf("expected %d edit links, got %d", tt.want, control.Length()) } @@ -610,9 +472,9 @@ func TestDocumentGoodReadsEditor(t *testing.T) { if control.Text() != "" || control.Find(`i.bi-pencil[alt="Edit Goodreads link"]`).Length() != 1 || control.Filter(`[title="Edit Goodreads link"][aria-label="Edit Goodreads link"]`).Length() != 1 || - page.Find(`#edit-goodreads-linked.collapse:not(.show)`).Length() != 1 || + page.Find(`#edit-review-link-linked.collapse:not(.show)`).Length() != 1 || page.Find(`form[hx-put="/documents/linked/review-link"][hx-target="this"]`).Length() != 1 || - page.Find(`label[for="goodreads-book-url-linked"]`).Length() != 1 { + page.Find(`label[for="document-book-url-linked"]`).Length() != 1 { t.Fatal("expected accessible, initially collapsed Goodreads editor beside the score") } if value, _ := page.Find(`input[name="bookURL"]`).Attr("value"); value != "https://www.goodreads.com/book/show/12207444-la-reina-dragon" { @@ -662,8 +524,8 @@ func TestDocumentUpdateGoodReads(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.GoodReadsBookURL != "/book/show/1368314.El_Cercano_Oriente" || - stored.GoodReadsRatingValue != 0 || + if stored.DataSourceLink != "/book/show/1368314.El_Cercano_Oriente" || + stored.DataSourceScoreValue != 0 || stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || !stored.GoodReadsEnriched { t.Fatalf("expected correction persisted without stale rating/review identifiers: %+v", stored) @@ -698,7 +560,7 @@ func TestDocumentUpdateGoodReadsValidation(t *testing.T) { if err != nil { t.Fatal(err) } - if page.Find(`form#goodreads-link-form-linked input[aria-invalid="true"]`).Length() != 1 || + if page.Find(`form#review-link-form-linked input[aria-invalid="true"]`).Length() != 1 || page.Find(`[role="alert"]`).Text() != "Enter a valid Goodreads book URL." { t.Fatal("invalid URLs must render inline validation feedback") } @@ -706,7 +568,7 @@ func TestDocumentUpdateGoodReadsValidation(t *testing.T) { t.Fatalf("invalid input should be retained for correction, got %q", got) } stored, err := idx.Document("linked") - if err != nil || stored.GoodReadsBookURL != "/book/show/12207444-la-reina-dragon" || stored.GoodReadsRatingValue != 3.82 { + if err != nil || stored.DataSourceLink != "/book/show/12207444-la-reina-dragon" || stored.DataSourceScoreValue != 3.82 { t.Fatalf("invalid input must not change the stored match: %+v, %v", stored, err) } } @@ -738,10 +600,10 @@ func TestDocumentDetailUpdatesStoredGoodReadsRating(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.GoodReadsRatingValue != 4.12 { + if stored.DataSourceScoreValue != 4.12 { t.Fatalf("expected latest rating and searchable value to be stored, got %+v", stored) } - results, err := idx.Search(index.SearchFields{ReviewScoreFrom: 4.1, ReviewScoreTo: 4.2}, 1, 10) + results, err := idx.Search(index.SearchFields{DataSourceScoreFrom: 4.1, DataSourceScoreTo: 4.2}, 1, 10) if err != nil || results.TotalHits() != 1 { t.Fatalf("expected document to match updated rating range, got %d results and error %v", results.TotalHits(), err) } @@ -765,7 +627,7 @@ func TestDocumentDetailKeepsStoredGoodReadsRatingWhenScraperReturnsZero(t *testi if err != nil { t.Fatal(err) } - if stored.GoodReadsRatingValue != 3.82 { + if stored.DataSourceScoreValue != 3.82 { t.Fatalf("zero scraper rating must not replace indexed Goodreads score, got %+v", stored) } } @@ -802,7 +664,7 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.GoodReadsBookURL != "/book/show/1-unlinked" || !stored.GoodReadsEnriched { + if stored.DataSourceLink != "/book/show/1-unlinked" || !stored.GoodReadsEnriched { t.Errorf("expected the found link to be stored, got %+v", stored) } author, err := idx.Author("jane-doe", "") diff --git a/internal/webserver/embedded/views/author/list.html b/internal/webserver/embedded/views/author/list.html index 10f9863b..d715aea1 100644 --- a/internal/webserver/embedded/views/author/list.html +++ b/internal/webserver/embedded/views/author/list.html @@ -22,7 +22,7 @@

    {{.Author.Name}}

    {{end}}
    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}}
    diff --git a/internal/webserver/embedded/views/highlight/list.html b/internal/webserver/embedded/views/highlight/list.html index f1d8ad79..c54cb88e 100644 --- a/internal/webserver/embedded/views/highlight/list.html +++ b/internal/webserver/embedded/views/highlight/list.html @@ -1,7 +1,7 @@

    {{t .Lang "Highlights"}}

    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "highlight"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "highlight"}}
    @@ -28,7 +28,7 @@
    {{t .L
    - {{template "partials/document-metadata-filters" dict "Lang" .Lang "ReviewSourceName" .ReviewSourceName "FilterIdPrefix" "highlight" "SearchFields" .SearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} + {{template "partials/document-metadata-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "FilterIdPrefix" "highlight" "SearchFields" .SearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} {{template "partials/search-filters-actions" dict "Lang" .Lang "ShowSearchButton" false}}
    diff --git a/internal/webserver/embedded/views/partials/completed-list.html b/internal/webserver/embedded/views/partials/completed-list.html index 913cbf88..b63e483c 100644 --- a/internal/webserver/embedded/views/partials/completed-list.html +++ b/internal/webserver/embedded/views/partials/completed-list.html @@ -12,7 +12,7 @@
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}} diff --git a/internal/webserver/embedded/views/partials/count-and-sort.html b/internal/webserver/embedded/views/partials/count-and-sort.html index 581de826..544e211f 100644 --- a/internal/webserver/embedded/views/partials/count-and-sort.html +++ b/internal/webserver/embedded/views/partials/count-and-sort.html @@ -8,6 +8,6 @@ {{if gt .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}} diff --git a/internal/webserver/embedded/views/partials/docs-list-fragments.html b/internal/webserver/embedded/views/partials/docs-list-fragments.html index e5992224..a396824a 100644 --- a/internal/webserver/embedded/views/partials/docs-list-fragments.html +++ b/internal/webserver/embedded/views/partials/docs-list-fragments.html @@ -1,10 +1,10 @@ {{if and .Author (ne .Author.Name "")}}
    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}}
    {{else if .SeriesTitle}}
    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}}
    {{else}}
    {{template "partials/search-type-tabs" .}}
    diff --git a/internal/webserver/embedded/views/partials/docs-list-item.html b/internal/webserver/embedded/views/partials/docs-list-item.html index b4ad96f3..70dd7a84 100644 --- a/internal/webserver/embedded/views/partials/docs-list-item.html +++ b/internal/webserver/embedded/views/partials/docs-list-item.html @@ -71,7 +71,7 @@

    {{end}} - {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount "ShowGoodReadsScore" true}} + {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount "ShowScore" true}} {{ if .Document.Subjects }}
      diff --git a/internal/webserver/embedded/views/partials/document-goodreads-form.html b/internal/webserver/embedded/views/partials/document-goodreads-form.html deleted file mode 100644 index 35a6f848..00000000 --- a/internal/webserver/embedded/views/partials/document-goodreads-form.html +++ /dev/null @@ -1,10 +0,0 @@ - diff --git a/internal/webserver/embedded/views/partials/document-metadata-filters.html b/internal/webserver/embedded/views/partials/document-metadata-filters.html index 7fc5801d..88bb63c9 100644 --- a/internal/webserver/embedded/views/partials/document-metadata-filters.html +++ b/internal/webserver/embedded/views/partials/document-metadata-filters.html @@ -20,17 +20,18 @@ {{end}}
      - {{if .ReviewSourceName}}{{t .Lang "%s score" .ReviewSourceName}}{{else}}{{t .Lang "Review score"}}{{end}} + {{if .DataSourceName}}{{t .Lang "%s score" .DataSourceName}}{{else}}{{t .Lang "Review score"}}{{end}}
      {{$reviewScoreFrom := 0.0}} {{if .SearchFields}} - {{$reviewScoreFrom = .SearchFields.ReviewScoreFrom}} + {{$reviewScoreFrom = .SearchFields.DataSourceScoreFrom}} {{end}}
      + @@ -54,7 +56,7 @@
      - +
      diff --git a/internal/webserver/embedded/views/partials/document-metadata.html b/internal/webserver/embedded/views/partials/document-metadata.html index f5349ef2..1ff5532a 100644 --- a/internal/webserver/embedded/views/partials/document-metadata.html +++ b/internal/webserver/embedded/views/partials/document-metadata.html @@ -15,8 +15,8 @@ {{ if .ReviewsEnabled }}
      • {{ end }} - {{if and .ShowGoodReadsScore .Document.GoodReadsBookURL}} - {{template "partials/document-score" dict "Lang" .Lang "Source" "Goodreads" "Rating" (printf "%.2f" .Document.GoodReadsRatingValue) "RatingCount" .Document.GoodReadsRatingCount "Scored" (gt .Document.GoodReadsRatingValue 0.0) "URL" (printf "https://www.goodreads.com%s" .Document.GoodReadsBookURL)}} + {{if and .ShowScore .Document.DataSourceLink}} + {{template "partials/document-score" dict "Lang" .Lang "Source" "Goodreads" "Rating" (printf "%.2f" .Document.DataSourceScoreValue) "RatingCount" .Document.DataSourceScoreCount "Scored" (gt .Document.DataSourceScoreValue 0.0) "URL" (printf "https://www.goodreads.com%s" .Document.DataSourceLink)}} {{end}} {{ if and .IllustratedMinAmount (ge .Document.Illustrations .IllustratedMinAmount) }}
    • {{t .Lang "Illustrated"}}
      • diff --git a/internal/webserver/embedded/views/partials/document-review-link-form.html b/internal/webserver/embedded/views/partials/document-review-link-form.html new file mode 100644 index 00000000..c70a4fd1 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-review-link-form.html @@ -0,0 +1,10 @@ + diff --git a/internal/webserver/embedded/views/partials/document-score.html b/internal/webserver/embedded/views/partials/document-score.html index 25ee71da..c7c3ed68 100644 --- a/internal/webserver/embedded/views/partials/document-score.html +++ b/internal/webserver/embedded/views/partials/document-score.html @@ -1,9 +1,9 @@
    • {{if .ScoreUnavailable}}{{t .Lang "Unable to load %s score" .Source}}{{else if .Scored}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" .RatingCount}}){{end}}{{else}}{{t .Lang "Not scored"}}{{end}} {{if .CanEditDocumentLink}} - -
      - {{template "partials/document-goodreads-form" dict "Lang" .Lang "Slug" .DocumentSlug "BookURL" .URL "Source" .Source}} + + {{end}}
      • diff --git a/internal/webserver/embedded/views/partials/highlights-list-fragments.html b/internal/webserver/embedded/views/partials/highlights-list-fragments.html index bd4c14dc..c87b6a25 100644 --- a/internal/webserver/embedded/views/partials/highlights-list-fragments.html +++ b/internal/webserver/embedded/views/partials/highlights-list-fragments.html @@ -1,4 +1,4 @@
      - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "highlight"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "highlight"}}
      {{template "partials/highlights-list" .}}
      diff --git a/internal/webserver/embedded/views/partials/navbar.html b/internal/webserver/embedded/views/partials/navbar.html index b7f22379..4f09bbcd 100644 --- a/internal/webserver/embedded/views/partials/navbar.html +++ b/internal/webserver/embedded/views/partials/navbar.html @@ -217,7 +217,7 @@
      {{end}}
      - {{template "partials/document-search-filters" dict "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" .SearchPage "SimilarToActive" .SimilarToActive "SimilarToDocument" .SimilarToDocument}} + {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" .SearchPage "SimilarToActive" .SimilarToActive "SimilarToDocument" .SimilarToDocument}}
      {{template "partials/author-search-filters" dict "Lang" .Lang "SearchFields" .AuthorSearchFields "SelectedGender" .SelectedGender "AuthorsSearchPage" .SearchPage}} diff --git a/internal/webserver/embedded/views/partials/search-filters-sidebar-content.html b/internal/webserver/embedded/views/partials/search-filters-sidebar-content.html index 8e54f876..529de1b8 100644 --- a/internal/webserver/embedded/views/partials/search-filters-sidebar-content.html +++ b/internal/webserver/embedded/views/partials/search-filters-sidebar-content.html @@ -19,7 +19,7 @@
    - {{template "partials/document-search-filters" dict "Lang" .Lang "ReviewSourceName" .ReviewSourceName "FilterIdPrefix" "sidebar" "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" true "HideSearch" true}} + {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "FilterIdPrefix" "sidebar" "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" true "HideSearch" true}}
    {{if not .DocumentsOnly}}
    diff --git a/internal/webserver/embedded/views/partials/search-type-tabs.html b/internal/webserver/embedded/views/partials/search-type-tabs.html index 365d3e35..cf833dad 100644 --- a/internal/webserver/embedded/views/partials/search-type-tabs.html +++ b/internal/webserver/embedded/views/partials/search-type-tabs.html @@ -16,7 +16,7 @@
    {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}}
    diff --git a/internal/webserver/embedded/views/partials/searchbox.html b/internal/webserver/embedded/views/partials/searchbox.html index a7497e12..4d50be5e 100644 --- a/internal/webserver/embedded/views/partials/searchbox.html +++ b/internal/webserver/embedded/views/partials/searchbox.html @@ -32,7 +32,7 @@
    - {{template "partials/document-search-filters" dict "Lang" .Lang "ReviewSourceName" .ReviewSourceName "HomeSearch" true "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} + {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "HomeSearch" true "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}}
    {{template "partials/author-search-filters" dict "Lang" .Lang "HomeSearch" true}} diff --git a/internal/webserver/embedded/views/partials/similar-to-banner.html b/internal/webserver/embedded/views/partials/similar-to-banner.html index 42615242..cc5c2205 100644 --- a/internal/webserver/embedded/views/partials/similar-to-banner.html +++ b/internal/webserver/embedded/views/partials/similar-to-banner.html @@ -9,7 +9,7 @@
    {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}}
    @@ -19,7 +19,7 @@ {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "ReviewSourceName" .ReviewSourceName "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}} diff --git a/internal/webserver/embedded/views/partials/sort.html b/internal/webserver/embedded/views/partials/sort.html index fdc91260..0bd2862e 100644 --- a/internal/webserver/embedded/views/partials/sort.html +++ b/internal/webserver/embedded/views/partials/sort.html @@ -4,7 +4,7 @@ diff --git a/internal/webserver/embedded/views/series/list.html b/internal/webserver/embedded/views/series/list.html index e9f231d2..cdea5070 100644 --- a/internal/webserver/embedded/views/series/list.html +++ b/internal/webserver/embedded/views/series/list.html @@ -1,6 +1,6 @@

    {{t .Lang "Collection \"%s\"" .Title}}

    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "ReviewSourceName" .ReviewSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}}
    diff --git a/internal/webserver/highlights_test.go b/internal/webserver/highlights_test.go index aa7bbf52..b65c9778 100644 --- a/internal/webserver/highlights_test.go +++ b/internal/webserver/highlights_test.go @@ -162,6 +162,52 @@ func TestHighlights(t *testing.T) { }) } +func TestHighlightsReviewScoreFilter(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadFilesInMemoryFs([]string{"testdata/library/metadata.epub"}), webserver.Config{}) + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + response, err := highlight(adminCookie, app, "john-doe-test-epub", fiber.MethodPost, t) + if err != nil { + t.Fatalf("Unexpected error highlighting document: %v", err) + } + response.Body.Close() + if response.StatusCode != fiber.StatusOK { + t.Fatalf("Expected status %d, received %d", fiber.StatusOK, response.StatusCode) + } + + for _, test := range []struct { + name string + query string + expectedCount int + }{ + {name: "no score", query: "review-score-from=none", expectedCount: 1}, + {name: "numeric score range", query: "review-score-from=1", expectedCount: 0}, + } { + t.Run(test.name, func(t *testing.T) { + response, err := getRequest(adminCookie, app, "/highlights?"+test.query, t) + if err != nil { + t.Fatalf("Unexpected error fetching highlights: %v", err) + } + defer response.Body.Close() + if response.StatusCode != fiber.StatusOK { + t.Fatalf("Expected status %d, received %d", fiber.StatusOK, response.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if got := doc.Find("#list .list-group-item").Length(); got != test.expectedCount { + t.Errorf("Expected %d highlighted documents, got %d", test.expectedCount, got) + } + }) + } +} + func highlight(cookie *http.Cookie, app *fiber.App, slug string, method string, t *testing.T) (*http.Response, error) { t.Helper() req, err := http.NewRequest(method, fmt.Sprintf("/highlights/%s", slug), nil) diff --git a/internal/webserver/model/highlight_repository.go b/internal/webserver/model/highlight_repository.go index 199ced5f..6e8eab5d 100644 --- a/internal/webserver/model/highlight_repository.go +++ b/internal/webserver/model/highlight_repository.go @@ -141,8 +141,9 @@ func hasSearchFields(sf index.SearchFields) bool { sf.EstReadTimeTo != 0 || sf.PagesFrom != 0 || sf.PagesTo != 0 || - sf.ReviewScoreFrom != 0 || - sf.ReviewScoreTo != 0 || + sf.DataSourceScoreFrom != 0 || + sf.DataSourceScoreTo != 0 || + sf.DataSourceScoreNone || sf.IllustratedOnly } @@ -193,10 +194,14 @@ func matchesSearchFields(doc AugmentedDocument, sf index.SearchFields, illustrat return false } - if sf.ReviewScoreFrom > 0 || sf.ReviewScoreTo > 0 { - if doc.GoodReadsRatingValue <= 0 || - (sf.ReviewScoreFrom > 0 && doc.GoodReadsRatingValue < sf.ReviewScoreFrom) || - (sf.ReviewScoreTo > 0 && doc.GoodReadsRatingValue > sf.ReviewScoreTo) { + if sf.DataSourceScoreNone || sf.DataSourceScoreFrom > 0 || sf.DataSourceScoreTo > 0 { + score := doc.DataSourceScore() + if sf.DataSourceScoreNone && score != 0 { + return false + } + if !sf.DataSourceScoreNone && (score <= 0 || + (sf.DataSourceScoreFrom > 0 && score < sf.DataSourceScoreFrom) || + (sf.DataSourceScoreTo > 0 && score > sf.DataSourceScoreTo)) { return false } } diff --git a/internal/webserver/model/highlight_search_test.go b/internal/webserver/model/highlight_search_test.go deleted file mode 100644 index b8aebd45..00000000 --- a/internal/webserver/model/highlight_search_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package model - -import ( - "testing" - - "github.com/svera/coreander/v5/internal/index" -) - -func TestMatchesSearchFieldsGoodReadsRatingRange(t *testing.T) { - tests := []struct { - name string - score float64 - from float64 - to float64 - want bool - }{ - {name: "inside range", score: 4.25, from: 4.2, to: 4.3, want: true}, - {name: "below minimum", score: 4.1, from: 4.2, to: 4.3}, - {name: "above maximum", score: 4.4, from: 4.2, to: 4.3}, - {name: "unrated", from: 4.2, to: 4.3}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - doc := AugmentedDocument{Document: index.Document{GoodReadsRatingValue: tt.score}} - fields := index.SearchFields{ReviewScoreFrom: tt.from, ReviewScoreTo: tt.to} - if got := matchesSearchFields(doc, fields, 0); got != tt.want { - t.Errorf("matchesSearchFields() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/internal/webserver/routes.go b/internal/webserver/routes.go index c2e1a09d..d6161971 100644 --- a/internal/webserver/routes.go +++ b/internal/webserver/routes.go @@ -52,7 +52,7 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se c.Locals("Version", c.App().Config().AppName) c.Locals("SupportedLanguages", supportedLanguages) c.Locals("Lang", chooseBestLanguage(c)) - c.Locals("ReviewSourceName", controllers.Documents.ReviewSourceName()) + c.Locals("DataSourceName", controllers.Documents.DataSourceName()) q := c.Queries() delete(q, "l") c.Locals("URLPath", c.Path()) diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index 86307ecd..a709ceda 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -2,12 +2,15 @@ package webserver_test import ( "net/http" + "slices" "strings" "testing" "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v3" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/infrastructure" ) @@ -226,21 +229,170 @@ func TestReviewScoreFilter(t *testing.T) { options := selectBox.Find("option").Map(func(i int, option *goquery.Selection) string { value, _ := option.Attr("value") wantLabel := "" - if i > 0 { - wantLabel = value + " " + strings.Repeat("\u2605", i) + switch value { + case "none": + if strings.TrimSpace(option.Text()) == "" { + t.Errorf("Expected a label for %s no-score option", field.name) + } + wantLabel = option.Text() + case "1", "2", "3", "4", "5": + wantLabel = value + " " + strings.Repeat("\u2605", i-1) } if option.Text() != wantLabel { t.Errorf("Expected %s option %q label %q, got %q", field.name, value, wantLabel, option.Text()) } return value }) - if strings.Join(options, ",") != ",1,2,3,4,5" { - t.Errorf("Expected blank and scores 1 to 5 for %s, got %v", field.name, options) + if strings.Join(options, ",") != ",none,1,2,3,4,5" { + t.Errorf("Expected blank, no-score, and scores 1 to 5 for %s, got %v", field.name, options) } }) if doc.Find(`input[name="`+field.name+`"]`).Length() != 0 { t.Errorf("Expected no numeric input for %s", field.name) } + + } + + response, err = app.Test(mustGetRequest(t, "/search?type=documents&review-score-from=none")) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + doc, err = goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"review-score-from", "review-score-to"} { + selected, _ := doc.Find(`select[name="` + name + `"] option[selected]`).Attr("value") + if selected != "none" { + t.Errorf("Expected no-score option selected for %s, got %q", name, selected) + } + } +} + +func TestSearchResultsShowStoredDataSourceScore(t *testing.T) { + for _, tt := range []struct { + rating string + want string + }{ + {rating: "3.82", want: "3.82"}, + {rating: "4.1", want: "4.10"}, + {rating: "5", want: "5.00"}, + } { + t.Run(tt.rating, func(t *testing.T) { + app, idx := newGoodReadsTestApp(t, nil) + doc, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + setDataSourceScore(t, idx, doc, tt.rating, "2,285") + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) + if err != nil { + t.Fatal(err) + } + rating := page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if rating.Length() != 1 || strings.TrimSpace(rating.Text()) != tt.want { + t.Fatalf("expected stored review score in search result, got %q", rating.Text()) + } + if page.Find(`li:has(a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]) i.bi-star-fill`).Length() != 1 { + t.Fatal("expected a star icon beside the review score") + } + if rating.Parent().Find("small").Text() != "(2,285 ratings)" { + t.Fatal("expected rating count beside the review score") + } + }) + } +} + +func TestSearchResultsSortByDataSourceScore(t *testing.T) { + app, idx := newGoodReadsTestApp(t, &fakeBookSource{}) + scores := map[string]string{ + "linked": "3.20", + "unlinked": "4.10", + "unmatched": "4.80", + } + documents, err := idx.Search(index.SearchFields{}, 1, 10) + if err != nil { + t.Fatal(err) + } + for _, doc := range documents.Hits() { + setDataSourceScore(t, idx, doc, scores[doc.Slug], "") + } + + for _, tt := range []struct { + direction string + want []string + }{ + {direction: "asc", want: []string{"linked", "unlinked", "unmatched"}}, + {direction: "desc", want: []string{"unmatched", "unlinked", "linked"}}, + } { + t.Run(tt.direction, func(t *testing.T) { + body := getBody(t, app, "/search?type=documents&sort-by=review-score-"+tt.direction) + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + var got []string + page.Find("#list .list-group-item h2 a").Each(func(_ int, link *goquery.Selection) { + href, _ := link.Attr("href") + got = append(got, strings.TrimPrefix(href, "/documents/")) + }) + if !slices.Equal(got, tt.want) { + t.Fatalf("score sort %s: got %v, want %v", tt.direction, got, tt.want) + } + }) + } +} + +func TestSearchResultsShowDataSourceScoreFromSearch(t *testing.T) { + source := &fakeBookSource{results: []biblioreads.SearchResult{{ + Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", + Rating: "4.20", RatingCount: "1,234", + }}} + app, idx := newGoodReadsTestApp(t, source) + score := getBody(t, app, "/documents/unlinked/score") + if !strings.Contains(score, "(1,234 ratings)") { + t.Fatal("expected the search-derived rating count in the document score") + } + stored, err := idx.Document("unlinked") + if err != nil || stored.DataSourceScoreCount != "1,234" { + t.Fatalf("expected the search-derived count to be persisted, got %+v, %v", stored, err) + } + for _, path := range []string{ + "/search?type=documents&search=Unlinked", + "/search?type=documents&search=Unlinked&l=es", + } { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) + if err != nil { + t.Fatal(err) + } + rating := page.Find(`a[href="https://www.goodreads.com/book/show/1-unlinked"]`) + if rating.Text() != "4.20" || !strings.Contains(rating.Parent().Find("small").Text(), "1,234") { + t.Fatalf("expected the stored score and rating count in search results for %s", path) + } + } + if len(source.ids) != 0 || len(source.queries) != 1 { + t.Fatalf("expected one book search and no detail fetches, got queries %q and ids %q", source.queries, source.ids) + } +} + +func TestSearchResultsShowStoredZeroRatingAsNotScored(t *testing.T) { + source := &fakeBookSource{results: []biblioreads.SearchResult{{ + Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", Rating: "0.00", + }}} + app, _ := newGoodReadsTestApp(t, source) + getBody(t, app, "/documents/unlinked/goodreads") + + body := getBody(t, app, "/search?type=documents&search=Unlinked") + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + row := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { + return strings.TrimSpace(item.Text()) == "Not scored" && item.Find("i.bi-star-fill").Length() == 1 + }) + if row.Length() == 0 || row.Find("a").Length() != 0 { + t.Fatalf("expected stored zero rating to render as Not scored, got %q in %s", row.Text(), body) } } diff --git a/internal/webserver/series_test.go b/internal/webserver/series_test.go index 38b05e41..0167e201 100644 --- a/internal/webserver/series_test.go +++ b/internal/webserver/series_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/PuerkitoBio/goquery" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/precisiondate" @@ -85,16 +84,7 @@ func TestSeriesFilters(t *testing.T) { "Third volume": "4.50", } for _, doc := range seriesDocuments.Hits() { - if err := idx.UpdateGoodReadsBookURL(doc.ID, "https://www.goodreads.com/book/show/1-"+doc.Slug); err != nil { - t.Fatalf("set review link for %s: %v", doc.Title, err) - } - updated, err := idx.Document(doc.Slug) - if err != nil { - t.Fatalf("reload %s: %v", doc.Title, err) - } - if err := idx.StoreGoodReadsDetails(updated, updated.GoodReadsBookURL, biblioreads.BookResponse{Rating: scores[doc.Title]}); err != nil { - t.Fatalf("set review score for %s: %v", doc.Title, err) - } + setDataSourceScore(t, idx, doc, scores[doc.Title], "") } cases := []struct { diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index baecfa59..f5cd5d02 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -16,6 +16,7 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/gofiber/fiber/v3" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" @@ -100,6 +101,27 @@ func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.F return webserver.New(webserverConfig, controllers, sender, idx, usersRepository), idx } +// setReviewScore seeds a document's review score through the current provider-backed index storage. +func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Document, score, count string) { + t.Helper() + bookURL := doc.DataSourceLink + if bookURL == "" { + bookURL = "https://www.goodreads.com/book/show/1-" + doc.Slug + if err := idx.UpdateDataSourceLink(doc.ID, bookURL); err != nil { + t.Fatalf("set review link for %s: %v", doc.Slug, err) + } + } + updated, err := idx.Document(doc.Slug) + if err != nil { + t.Fatalf("reload %s: %v", doc.Slug, err) + } + if err := idx.StoreGoodReadsDetails(updated, updated.DataSourceLink, biblioreads.BookResponse{ + Rating: score, RatingCount: count, + }); err != nil { + t.Fatalf("set review score for %s: %v", doc.Slug, err) + } +} + func loadFilesInMemoryFs(files []string) afero.Fs { contents := make(map[string][]byte) From 30e37a39e1ab1621caa794c0954aa2fa8a30bae9 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 15:16:40 +0100 Subject: [PATCH 19/50] WIP --- internal/webserver/author_summary_test.go | 67 +++++----- internal/webserver/document_goodreads_test.go | 32 ++--- .../partials/document-metadata-filters.html | 27 +--- .../document-score-filter-control.html | 16 +++ internal/webserver/routes.go | 1 - internal/webserver/search_test.go | 126 +++++++----------- ...pdate_auth_test.go => update_auth_test.go} | 0 internal/webserver/webserver_test.go | 35 +++++ 8 files changed, 152 insertions(+), 152 deletions(-) create mode 100644 internal/webserver/embedded/views/partials/document-score-filter-control.html rename internal/webserver/{author_update_auth_test.go => update_auth_test.go} (100%) diff --git a/internal/webserver/author_summary_test.go b/internal/webserver/author_summary_test.go index 41576bde..f5ec6b29 100644 --- a/internal/webserver/author_summary_test.go +++ b/internal/webserver/author_summary_test.go @@ -122,11 +122,22 @@ func TestClearAuthorRemovesImage(t *testing.T) { } } -func TestAuthorSummaryGoodReadsLink(t *testing.T) { - app, idx := newGoodReadsTestApp(t, nil) +type translatedDataSourceLinks struct{} + +func (translatedDataSourceLinks) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { + if author.DataSourceID == "" { + return datasourcemodel.DataSourceLink{} + } + return datasourcemodel.DataSourceLink{ + Source: "Example catalog", URL: "https://example.org/authors/" + author.Slug, + IconClass: "bi-bookmark", + } +} + +func TestAuthorSummaryDataSourceLinkTranslations(t *testing.T) { + app, idx := newDataSourceProvidersTestApp(t, nil, nil, translatedDataSourceLinks{}) const ( - authorPath = "/author/show/151155.Alice_Borchardt" - authorURL = "https://www.goodreads.com" + authorPath + authorURL = "https://example.org/authors/" wikipediaURL = "https://en.wikipedia.org/wiki/Alice_Borchardt" ) locales := []struct { @@ -134,31 +145,27 @@ func TestAuthorSummaryGoodReadsLink(t *testing.T) { label string wikipediaLabel string }{ - {"en", "Go to Goodreads author page", "Go to Wikipedia entry"}, - {"es", "Ir a la página del autor en Goodreads", "Ir al artículo en Wikipedia"}, - {"de", "Zur Autorenseite auf Goodreads", "Zur Wikipedia-Seite"}, - {"fr", "Voir la page de l'auteur sur Goodreads", "Aller à l'entrée Wikipedia"}, - {"ru", "Перейти на страницу автора в Goodreads", "Перейти к статье в Википедии"}, + {"en", "Go to Example catalog author page", "Go to Wikipedia entry"}, + {"es", "Ir a la página del autor en Example catalog", "Ir al artículo en Wikipedia"}, + {"de", "Zur Autorenseite auf Example catalog", "Zur Wikipedia-Seite"}, + {"fr", "Voir la page de l'auteur sur Example catalog", "Aller à l'entrée Wikipedia"}, + {"ru", "Перейти на страницу автора в Example catalog", "Перейти к статье в Википедии"}, } for _, tc := range []struct { - name string - goodreads bool - wikipedia bool - sourceID string + name string + dataSourceID string + hasDataSource bool + wikipedia bool }{ - {"with-wikipedia", true, true, "Q1"}, - {"without-goodreads", false, true, "Q1"}, - {"without-wikipedia", true, false, "Q1"}, - {"without-source-id", true, false, ""}, + {name: "with-wikipedia", dataSourceID: "Q1", hasDataSource: true, wikipedia: true}, + {name: "without-wikipedia", dataSourceID: "Q1", hasDataSource: true}, + {name: "without-data-source-id"}, } { author := index.Author{ Slug: tc.name, Name: "Alice Borchardt", - DataSourceID: tc.sourceID, RetrievedOn: time.Now().UTC(), + DataSourceID: tc.dataSourceID, RetrievedOn: time.Now().UTC(), WikipediaLink: make(map[string]string), } - if tc.goodreads { - author.GoodReadsAuthorURL = authorPath - } if tc.wikipedia { for _, locale := range locales { author.WikipediaLink[locale.lang] = strings.Replace(wikipediaURL, "en.wikipedia.org", locale.lang+".wikipedia.org", 1) @@ -198,27 +205,23 @@ func TestAuthorSummaryGoodReadsLink(t *testing.T) { if !tc.wikipedia && page.Find(`a[href$="wikipedia.org/wiki/Alice_Borchardt"]`).Length() != 0 { t.Fatal("expected no Wikipedia link when there is no link for the current language") } - link := page.Find(`a[href="` + authorURL + `"]`) - if !tc.goodreads { - if page.Find(`img[src^="/images/goodreads.svg"]`).Length() != 0 { - t.Error("expected no Goodreads icon without an author URL") - } + link := page.Find(`a[href="` + authorURL + tc.name + `"]`) + if !tc.hasDataSource { if link.Length() != 0 { - t.Fatal("expected no Goodreads link without an author URL") + t.Fatal("expected no data-source link without a provider URL") } return } if link.Length() != 1 || strings.TrimSpace(link.Text()) != locale.label { - t.Fatalf("expected one localized Goodreads link %q, got %d links with text %q", locale.label, link.Length(), link.Text()) + t.Fatalf("expected one localized data-source link %q, got %d links with text %q", locale.label, link.Length(), link.Text()) } - icon := link.PrevFiltered(`img[src^="/images/goodreads.svg"]`) - if icon.Length() != 1 || icon.AttrOr("alt", "missing") != "" || icon.AttrOr("width", "") != "16" || icon.AttrOr("height", "") != "16" { - t.Error("expected a decorative Goodreads SVG icon before the author link") + if link.PrevFiltered("i.bi-bookmark").Length() != 1 { + t.Error("expected the provider's Bootstrap icon before the data-source link") } if tc.wikipedia { wantWikipediaURL := strings.Replace(wikipediaURL, "en.wikipedia.org", locale.lang+".wikipedia.org", 1) if href, _ := link.Parent().Prev().Find("a").Attr("href"); href != wantWikipediaURL { - t.Fatalf("expected Goodreads immediately below Wikipedia, previous link is %q", href) + t.Fatalf("expected data-source link immediately below Wikipedia, previous link is %q", href) } } }) diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 0aebffea..0aa0a214 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -193,7 +193,7 @@ func TestDocumentAlternativeReviewProvider(t *testing.T) { if goodReadsPlaceholders(t, detail).Length() != 1 || len(provider.scoreSlugs) != 0 || len(provider.reviewSlugs) != 0 { t.Fatal("detail must defer lookups to the selected provider") } - for _, path := range []string{"/documents/unmatched/score", "/documents/unmatched/goodreads"} { + for _, path := range []string{"/documents/unmatched/score"} { body := getBody(t, app, path) page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) if err != nil { @@ -215,7 +215,7 @@ func TestDocumentAlternativeReviewProvider(t *testing.T) { page.Find("i.bi-star-fill").Length() != 5 || page.Find("script").Length() != 0 { t.Fatalf("reviews must render normalized provider data safely: %s", body) } - if len(provider.scoreSlugs) != 2 || len(provider.reviewSlugs) != 1 || provider.reviewSlugs[0] != "unmatched" { + if len(provider.scoreSlugs) != 1 || len(provider.reviewSlugs) != 1 || provider.reviewSlugs[0] != "unmatched" { t.Fatal("expected calls through the provider interface") } if len(provider.reviewLanguages) != 1 || provider.reviewLanguages[0] != "en" { @@ -404,7 +404,7 @@ func TestDocumentDetailGoodReadsRating(t *testing.T) { t.Error("expected the placeholder inside the document metadata list") } - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) if err != nil { t.Fatal(err) } @@ -588,7 +588,7 @@ func TestDocumentDetailUpdatesStoredGoodReadsRating(t *testing.T) { }} app, idx := newGoodReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) if err != nil { t.Fatal(err) } @@ -615,7 +615,7 @@ func TestDocumentDetailKeepsStoredGoodReadsRatingWhenScraperReturnsZero(t *testi }} app, idx := newGoodReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) if err != nil { t.Fatal(err) } @@ -643,7 +643,7 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { t.Fatalf("expected a placeholder for a document not looked up yet, got %d", got) } - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unlinked/goodreads"))) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unlinked/score"))) if err != nil { t.Fatal(err) } @@ -678,7 +678,7 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { if err := idx.EnrichGoodReads(10); err != nil { t.Fatal(err) } - page, err = goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unlinked/goodreads"))) + page, err = goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unlinked/score"))) if err != nil { t.Fatal(err) } @@ -699,7 +699,7 @@ func TestDocumentDetailGoodReadsSkipsUnmatchedDocuments(t *testing.T) { if got := goodReadsPlaceholders(t, getBody(t, app, "/documents/unmatched")).Length(); got != 1 { t.Errorf("expected an async score placeholder, got %d", got) } - for _, path := range []string{"/documents/unmatched/score", "/documents/unmatched/goodreads"} { + for _, path := range []string{"/documents/unmatched/score"} { page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) if err != nil { t.Fatal(err) @@ -722,7 +722,7 @@ func TestDocumentDetailGoodReadsRatingRendersNothingWhenUnavailable(t *testing.T } { t.Run(name, func(t *testing.T) { app, _ := newGoodReadsTestApp(t, source) - if body := strings.TrimSpace(getBody(t, app, "/documents/linked/goodreads")); body != "" { + if body := strings.TrimSpace(getBody(t, app, "/documents/linked/score")); body != "" { t.Errorf("expected empty response, got %q", body) } }) @@ -732,7 +732,7 @@ func TestDocumentDetailGoodReadsRatingRendersNothingWhenUnavailable(t *testing.T func TestDocumentDetailGoodReadsIdentifiersWithoutRating(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}} app, idx := newGoodReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) if err != nil { t.Fatal(err) } @@ -753,7 +753,7 @@ func TestDocumentDetailGoodReadsIdentifiersWithoutRating(t *testing.T) { {LegacyBookID: 2}, } { source.book = book - getBody(t, app, "/documents/linked/goodreads") + getBody(t, app, "/documents/linked/score") stored, err = idx.Document("linked") if err != nil { t.Fatal(err) @@ -770,7 +770,7 @@ func TestDocumentDetailGoodReadsDisabled(t *testing.T) { if got := goodReadsPlaceholders(t, getBody(t, app, "/documents/"+slug)).Length(); got != 0 { t.Errorf("%s: expected no placeholder when GoodReads is disabled, got %d", slug, got) } - if body := strings.TrimSpace(getBody(t, app, "/documents/"+slug+"/goodreads")); body != "" { + if body := strings.TrimSpace(getBody(t, app, "/documents/"+slug+"/score")); body != "" { t.Errorf("%s: expected empty response, got %q", slug, body) } if body := getBody(t, app, "/documents/"+slug); strings.Contains(body, "bi-star-fill") { @@ -805,7 +805,7 @@ func TestDocumentDetailGoodReadsSearchDoesNotScrape(t *testing.T) { book: biblioreads.BookResponse{Rating: "5.00"}, } app, idx := newGoodReadsTestApp(t, source) - body := strings.TrimSpace(getBody(t, app, "/documents/unlinked/goodreads")) + body := strings.TrimSpace(getBody(t, app, "/documents/unlinked/score")) if tt.noScore { page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) if err != nil { @@ -850,7 +850,7 @@ func TestDocumentDetailGoodReadsNoScoreTranslations(t *testing.T) { {"fr", "Non noté"}, {"ru", "Не оценено"}, } { - for _, path := range []string{"/documents/unmatched/score", "/documents/unmatched/goodreads"} { + for _, path := range []string{"/documents/unmatched/score"} { t.Run(locale.lang+path, func(t *testing.T) { req := mustGetRequest(t, path) req.Header.Set("Accept-Language", locale.lang) @@ -898,7 +898,7 @@ func TestDocumentDetailGoodReadsReviewsLazyPlaceholder(t *testing.T) { if !target.Is(".document-detail-content > :last-child") { t.Fatal("reviews must appear at the bottom of the document content") } - page, err = goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/goodreads"))) + page, err = goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) if err != nil { t.Fatal(err) } @@ -1168,7 +1168,7 @@ func TestDocumentDetailGoodReadsReviewsAfterSearch(t *testing.T) { book: biblioreads.BookResponse{Rating: "4", ResourceID: "work-id", LegacyBookID: 1}, } app, _ := newGoodReadsTestApp(t, source) - getBody(t, app, "/documents/unlinked/goodreads") + getBody(t, app, "/documents/unlinked/score") if len(source.ids) != 0 || len(source.reviewCalls) != 0 { t.Fatal("search rating must not scrape the book or load reviews") } diff --git a/internal/webserver/embedded/views/partials/document-metadata-filters.html b/internal/webserver/embedded/views/partials/document-metadata-filters.html index 88bb63c9..22f64d67 100644 --- a/internal/webserver/embedded/views/partials/document-metadata-filters.html +++ b/internal/webserver/embedded/views/partials/document-metadata-filters.html @@ -28,34 +28,13 @@ {{if .SearchFields}} {{$reviewScoreFrom = .SearchFields.DataSourceScoreFrom}} {{end}} -
    - - -
    {{$reviewScoreTo := 0.0}} {{if .SearchFields}} {{$reviewScoreTo = .SearchFields.DataSourceScoreTo}} {{end}} -
    - - -
    + {{$noScoreSelected := and .SearchFields .SearchFields.DataSourceScoreNone}} + {{template "partials/document-score-filter-control" dict "Lang" .Lang "IdPrefix" $idPrefix "Name" "review-score-from" "Label" "From" "Score" $reviewScoreFrom "NoScoreSelected" $noScoreSelected}} + {{template "partials/document-score-filter-control" dict "Lang" .Lang "IdPrefix" $idPrefix "Name" "review-score-to" "Label" "To" "Score" $reviewScoreTo "NoScoreSelected" $noScoreSelected "RoundedEnd" true}}
    diff --git a/internal/webserver/embedded/views/partials/document-score-filter-control.html b/internal/webserver/embedded/views/partials/document-score-filter-control.html new file mode 100644 index 00000000..6bd1a423 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-score-filter-control.html @@ -0,0 +1,16 @@ +{{$id := .Name}} +{{if .IdPrefix}} + {{$id = printf "%s-%s" .IdPrefix .Name}} +{{end}} +
    + + +
    diff --git a/internal/webserver/routes.go b/internal/webserver/routes.go index d6161971..4cadc0bc 100644 --- a/internal/webserver/routes.go +++ b/internal/webserver/routes.go @@ -128,7 +128,6 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se return controllers.Search.SearchDocuments(c) }) docsGroup.Get("/:slug/score", controllers.Documents.Score) - docsGroup.Get("/:slug/goodreads", controllers.Documents.Score) docsGroup.Get("/:slug/reviews", controllers.Documents.Reviews) docsGroup.Get("/:slug/read", controllers.Documents.Reader) docsGroup.Get("/:slug/annotations", alwaysRequireAuthentication, controllers.Annotations.List) diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index a709ceda..cfc5ed1c 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -9,8 +9,8 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v3" "github.com/spf13/afero" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/infrastructure" ) @@ -148,7 +148,7 @@ func TestUnifiedSearch(t *testing.T) { } } -func TestGoodReadsScoreSortOptions(t *testing.T) { +func TestDataSourceScoreSortOptions(t *testing.T) { app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), webserver.Config{}) for _, path := range []string{ @@ -219,7 +219,7 @@ func TestReviewScoreFilter(t *testing.T) { } { selects := doc.Find(`select[name="` + field.name + `"]`) if selects.Length() == 0 { - t.Errorf("Expected Goodreads score filter %q", field.name) + t.Errorf("Expected data-source score filter %q", field.name) continue } selects.Each(func(_ int, selectBox *goquery.Selection) { @@ -272,53 +272,54 @@ func TestReviewScoreFilter(t *testing.T) { func TestSearchResultsShowStoredDataSourceScore(t *testing.T) { for _, tt := range []struct { - rating string - want string + score float64 + want string }{ - {rating: "3.82", want: "3.82"}, - {rating: "4.1", want: "4.10"}, - {rating: "5", want: "5.00"}, + {score: 3.82, want: "3.82"}, + {score: 4.1, want: "4.10"}, + {score: 5, want: "5.00"}, } { - t.Run(tt.rating, func(t *testing.T) { - app, idx := newGoodReadsTestApp(t, nil) - doc, err := idx.Document("linked") - if err != nil { - t.Fatal(err) - } - setDataSourceScore(t, idx, doc, tt.rating, "2,285") + t.Run(tt.want, func(t *testing.T) { + app, _ := newSearchResultsTestApp(t, []index.Document{{ + ID: "linked.epub", Slug: "linked", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, + DataSourceLink: "/provider/items/linked", + DataSourceScoreValue: tt.score, + DataSourceScoreCount: "2,285", + }}) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) if err != nil { t.Fatal(err) } - rating := page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) - if rating.Length() != 1 || strings.TrimSpace(rating.Text()) != tt.want { - t.Fatalf("expected stored review score in search result, got %q", rating.Text()) - } - if page.Find(`li:has(a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]) i.bi-star-fill`).Length() != 1 { - t.Fatal("expected a star icon beside the review score") - } - if rating.Parent().Find("small").Text() != "(2,285 ratings)" { - t.Fatal("expected rating count beside the review score") + score := page.Find("#list .list-group-item li").FilterFunction(func(_ int, item *goquery.Selection) bool { + text := strings.TrimSpace(item.Text()) + return strings.Contains(text, tt.want) && strings.Contains(text, "2,285") + }) + if score.Length() != 1 || score.Find("i.bi-star-fill").Length() != 1 { + t.Fatalf("expected generic data-source score and count in search result, got %q", score.Text()) } }) } } func TestSearchResultsSortByDataSourceScore(t *testing.T) { - app, idx := newGoodReadsTestApp(t, &fakeBookSource{}) - scores := map[string]string{ - "linked": "3.20", - "unlinked": "4.10", - "unmatched": "4.80", - } - documents, err := idx.Search(index.SearchFields{}, 1, 10) - if err != nil { - t.Fatal(err) - } - for _, doc := range documents.Hits() { - setDataSourceScore(t, idx, doc, scores[doc.Slug], "") - } - + app, _ := newSearchResultsTestApp(t, []index.Document{ + { + ID: "linked.epub", Slug: "linked", + Metadata: metadata.Metadata{Title: "Linked", Format: "EPUB"}, + DataSourceLink: "/provider/items/linked", DataSourceScoreValue: 3.2, + }, + { + ID: "unlinked.epub", Slug: "unlinked", + Metadata: metadata.Metadata{Title: "Unlinked", Format: "EPUB"}, + DataSourceLink: "/provider/items/unlinked", DataSourceScoreValue: 4.1, + }, + { + ID: "unmatched.epub", Slug: "unmatched", + Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, + DataSourceLink: "/provider/items/unmatched", DataSourceScoreValue: 4.8, + }, + }) for _, tt := range []struct { direction string want []string @@ -344,55 +345,22 @@ func TestSearchResultsSortByDataSourceScore(t *testing.T) { } } -func TestSearchResultsShowDataSourceScoreFromSearch(t *testing.T) { - source := &fakeBookSource{results: []biblioreads.SearchResult{{ - Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", - Rating: "4.20", RatingCount: "1,234", - }}} - app, idx := newGoodReadsTestApp(t, source) - score := getBody(t, app, "/documents/unlinked/score") - if !strings.Contains(score, "(1,234 ratings)") { - t.Fatal("expected the search-derived rating count in the document score") - } - stored, err := idx.Document("unlinked") - if err != nil || stored.DataSourceScoreCount != "1,234" { - t.Fatalf("expected the search-derived count to be persisted, got %+v, %v", stored, err) - } - for _, path := range []string{ - "/search?type=documents&search=Unlinked", - "/search?type=documents&search=Unlinked&l=es", - } { - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) - if err != nil { - t.Fatal(err) - } - rating := page.Find(`a[href="https://www.goodreads.com/book/show/1-unlinked"]`) - if rating.Text() != "4.20" || !strings.Contains(rating.Parent().Find("small").Text(), "1,234") { - t.Fatalf("expected the stored score and rating count in search results for %s", path) - } - } - if len(source.ids) != 0 || len(source.queries) != 1 { - t.Fatalf("expected one book search and no detail fetches, got queries %q and ids %q", source.queries, source.ids) - } -} - func TestSearchResultsShowStoredZeroRatingAsNotScored(t *testing.T) { - source := &fakeBookSource{results: []biblioreads.SearchResult{{ - Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", Rating: "0.00", - }}} - app, _ := newGoodReadsTestApp(t, source) - getBody(t, app, "/documents/unlinked/goodreads") - - body := getBody(t, app, "/search?type=documents&search=Unlinked") + app, _ := newSearchResultsTestApp(t, []index.Document{{ + ID: "unscored.epub", Slug: "unscored", + Metadata: metadata.Metadata{Title: "Unscored", Format: "EPUB"}, + DataSourceLink: "/provider/items/unscored", + }}) + body := getBody(t, app, "/search?type=documents&search=Unscored") page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) if err != nil { t.Fatal(err) } - row := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { + row := page.Find("#list .list-group-item li").FilterFunction(func(_ int, item *goquery.Selection) bool { return strings.TrimSpace(item.Text()) == "Not scored" && item.Find("i.bi-star-fill").Length() == 1 }) if row.Length() == 0 || row.Find("a").Length() != 0 { - t.Fatalf("expected stored zero rating to render as Not scored, got %q in %s", row.Text(), body) + t.Fatalf("expected generic zero score to render as Not scored, got %q", row.Text()) } } diff --git a/internal/webserver/author_update_auth_test.go b/internal/webserver/update_auth_test.go similarity index 100% rename from internal/webserver/author_update_auth_test.go rename to internal/webserver/update_auth_test.go diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index f5cd5d02..498b3cfa 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -101,6 +101,41 @@ func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.F return webserver.New(webserverConfig, controllers, sender, idx, usersRepository), idx } +func newSearchResultsTestApp(t *testing.T, documents []index.Document) (*fiber.App, *index.BleveIndexer) { + t.Helper() + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + + documentIndex, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authorsIndex, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + appFS := afero.NewMemMapFs() + cfg := defaultTestConfig() + idx := index.NewBleve(documentIndex, authorsIndex, appFS, testLibraryDir, nil, index.Config{}) + t.Cleanup(func() { _ = idx.Close() }) + for _, doc := range documents { + if err := documentIndex.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + } + + sender := &infrastructure.NoEmail{} + dataSource := wikidata.NewWikidataSource(wikidata.Gowikidata{}) + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, dataSource, nil, dataSource) + app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) + t.Cleanup(func() { _ = app.Shutdown() }) + return app, idx +} + // setReviewScore seeds a document's review score through the current provider-backed index storage. func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Document, score, count string) { t.Helper() From 2ae6d69047150e9578c84a4e5faf2094515992b0 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 15:59:44 +0100 Subject: [PATCH 20/50] WIP --- index_migration_test.go | 2 +- internal/datasource/wikidata/service.go | 2 +- internal/datasource/wikidata/service_test.go | 6 +- internal/index/author.go | 30 +++-- internal/index/authors_enrich_test.go | 52 ++++++++- internal/index/bleve.go | 3 +- internal/index/bleve_author_migrate.go | 21 +++- internal/index/bleve_author_migrate_test.go | 105 ++++++++++++++++-- internal/index/bleve_author_read.go | 29 ++--- internal/index/bleve_author_write.go | 55 +++++++-- internal/index/bleve_document_write.go | 2 +- internal/index/bleve_goodreads_write.go | 9 +- internal/index/bleve_goodreads_write_test.go | 29 +++-- internal/index/goodreads_fields_test.go | 24 ++-- internal/index/rebuild_authors_test.go | 8 +- internal/reviews/biblioreads.go | 5 +- internal/reviews/biblioreads_test.go | 29 +++++ internal/reviews/data_source_link_test.go | 27 ----- internal/webserver/author_summary_test.go | 8 +- .../controller/document/controller.go | 46 ++++---- .../webserver/controller/document/detail.go | 6 +- .../webserver/controller/document/reviews.go | 2 +- .../webserver/controller/document/score.go | 2 +- internal/webserver/document_goodreads_test.go | 2 +- .../partials/author-summary-doc-detail.html | 6 +- .../views/partials/author-summary.html | 6 +- 26 files changed, 360 insertions(+), 156 deletions(-) delete mode 100644 internal/reviews/data_source_link_test.go diff --git a/index_migration_test.go b/index_migration_test.go index 1ac7ad1d..d7454f7c 100644 --- a/index_migration_test.go +++ b/index_migration_test.go @@ -44,7 +44,7 @@ func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { t.Fatal(err) } legacyMapping := index.CreateAuthorsMapping().(*mapping.IndexMappingImpl) - delete(legacyMapping.DefaultMapping.Properties, "GoodReadsAuthorURL") + delete(legacyMapping.DefaultMapping.Properties, "DataSourceLinks") authors, err := bleve.New(homeDir+authorsIndexPath, legacyMapping) if err != nil { t.Fatal(err) diff --git a/internal/datasource/wikidata/service.go b/internal/datasource/wikidata/service.go index d7fc8389..65b63e81 100644 --- a/internal/datasource/wikidata/service.go +++ b/internal/datasource/wikidata/service.go @@ -44,7 +44,7 @@ type WikidataSource struct { } func (WikidataSource) AuthorLink(author index.Author, language string) model.DataSourceLink { - url := author.WikipediaLink[language] + url := author.DataSourceLinks["wikipedia:"+language] if url == "" { return model.DataSourceLink{} } diff --git a/internal/datasource/wikidata/service_test.go b/internal/datasource/wikidata/service_test.go index b0de124a..2924399e 100644 --- a/internal/datasource/wikidata/service_test.go +++ b/internal/datasource/wikidata/service_test.go @@ -13,9 +13,9 @@ import ( func TestWikipediaDataSourceLink(t *testing.T) { source := NewWikidataSource(Gowikidata{}) link := source.AuthorLink(index.Author{ - WikipediaLink: map[string]string{ - "en": "https://en.wikipedia.org/wiki/Example", - "es": "https://es.wikipedia.org/wiki/Ejemplo", + DataSourceLinks: map[string]string{ + "wikipedia:en": "https://en.wikipedia.org/wiki/Example", + "wikipedia:es": "https://es.wikipedia.org/wiki/Ejemplo", }, }, "es") if link.Source != "Wikipedia" || link.URL != "https://es.wikipedia.org/wiki/Ejemplo" || diff --git a/internal/index/author.go b/internal/index/author.go index c27c1bed..14ccf7be 100644 --- a/internal/index/author.go +++ b/internal/index/author.go @@ -10,22 +10,20 @@ import ( ) type Author struct { - Slug string - Name string - BirthName string - DataSourceID string - RetrievedOn time.Time - WikipediaLink map[string]string - InstanceOf float64 - Description map[string]string - DateOfBirth precisiondate.PrecisionDate - DateOfDeath precisiondate.PrecisionDate - Website string - GoodReadsAuthorURL string - DataSourceImage string - Gender float64 - Pseudonyms []string - DocumentCount uint64 + Slug string + Name string + BirthName string + DataSourceID string + RetrievedOn time.Time + InstanceOf float64 + Description map[string]string + DateOfBirth precisiondate.PrecisionDate + DateOfDeath precisiondate.PrecisionDate + DataSourceLinks map[string]string + DataSourceImage string + Gender float64 + Pseudonyms []string + DocumentCount uint64 } func (a Author) YearOfBirthAbs() int { diff --git a/internal/index/authors_enrich_test.go b/internal/index/authors_enrich_test.go index 52199f68..be3e975a 100644 --- a/internal/index/authors_enrich_test.go +++ b/internal/index/authors_enrich_test.go @@ -76,7 +76,9 @@ func (m *mockAuthorDataSource) RetrieveAuthors(candidates map[string][]string, _ } type stubAuthor struct { - sourceID string + sourceID string + website string + wikipedia map[string]string } func (a stubAuthor) BirthName() string { return "" } @@ -86,12 +88,56 @@ func (a stubAuthor) Gender() float64 { return 0 } func (a stubAuthor) DateOfBirth() precisiondate.PrecisionDate { return precisiondate.PrecisionDate{} } func (a stubAuthor) DateOfDeath() precisiondate.PrecisionDate { return precisiondate.PrecisionDate{} } func (a stubAuthor) Image() string { return "" } -func (a stubAuthor) Website() string { return "" } -func (a stubAuthor) WikipediaLink(string) string { return "" } +func (a stubAuthor) Website() string { return a.website } +func (a stubAuthor) WikipediaLink(lang string) string { return a.wikipedia[lang] } func (a stubAuthor) SourceID() string { return a.sourceID } func (a stubAuthor) RetrievedOn() time.Time { return time.Now().UTC() } func (a stubAuthor) Pseudonyms() []string { return nil } +func TestAuthorMetadataRefreshUpdatesLinks(t *testing.T) { + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + defer authors.Close() + idx := index.NewBleve(nil, authors, nil, "", nil, index.Config{}) + author := index.Author{ + Slug: "writer", Name: "Writer", + DataSourceLinks: map[string]string{ + "website": "https://old.example", + "wikipedia:en": "https://en.wikipedia.org/wiki/Old", + "wikipedia:es": "https://es.wikipedia.org/wiki/Old", + "goodreads": "/author/show/1.Writer", + "catalog": "https://catalog.example/writer", + }, + } + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + for _, source := range []stubAuthor{ + {sourceID: "Q1", website: "https://new.example", wikipedia: map[string]string{"en": "https://en.wikipedia.org/wiki/New"}}, + {sourceID: "Q1"}, + } { + index.CombineWithDataSource(&author, source, []string{"en", "es"}) + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + got, err := idx.Author("writer", "en") + if err != nil { + t.Fatal(err) + } + if got.DataSourceLinks["website"] != source.website || + got.DataSourceLinks["wikipedia:en"] != source.wikipedia["en"] || + got.DataSourceLinks["wikipedia:es"] != "" { + t.Fatalf("metadata links were not refreshed: %+v", got.DataSourceLinks) + } + if got.DataSourceLinks["goodreads"] != "/author/show/1.Writer" || + got.DataSourceLinks["catalog"] != "https://catalog.example/writer" { + t.Fatalf("metadata refresh lost independent provider links: %+v", got.DataSourceLinks) + } + } +} + func TestAuthorsWithoutInfo(t *testing.T) { authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) if err != nil { diff --git a/internal/index/bleve.go b/internal/index/bleve.go index 8932cedd..b02f0df8 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -543,8 +543,7 @@ func CreateAuthorsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("RetrievedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceID", keywordFieldMappingNotIndexable) indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceImage", keywordFieldMappingNotIndexable) - indexMapping.DefaultMapping.AddFieldMappingsAt("Website", keywordFieldMappingNotIndexable) - indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsAuthorURL", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceLinks", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DateOfBirth.Date", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DateOfBirth.Precision", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DateOfDeath.Date", numericFieldMapping) diff --git a/internal/index/bleve_author_migrate.go b/internal/index/bleve_author_migrate.go index b07d4c9d..cef2ce2d 100644 --- a/internal/index/bleve_author_migrate.go +++ b/internal/index/bleve_author_migrate.go @@ -6,6 +6,7 @@ import ( "log" "os" "path/filepath" + "strings" "github.com/blevesearch/bleve/v2" ) @@ -105,7 +106,25 @@ func copyAuthors(source, destination bleve.Index, batchSize int) error { } batch := destination.NewBatch() for _, hit := range result.Hits { - if err := batch.Index(hit.ID, hydrateAuthor(hit)); err != nil { + author := hydrateAuthor(hit) + for field, value := range hit.Fields { + source := "" + switch { + case field == "Website": + source = "website" + case strings.HasPrefix(field, "WikipediaLink."): + source = "wikipedia:" + strings.TrimPrefix(field, "WikipediaLink.") + } + if link, ok := value.(string); source != "" && ok && link != "" { + if author.DataSourceLinks == nil { + author.DataSourceLinks = make(map[string]string) + } + if author.DataSourceLinks[source] == "" { + author.DataSourceLinks[source] = link + } + } + } + if err := indexAuthor(batch, hit.ID, author); err != nil { return err } } diff --git a/internal/index/bleve_author_migrate_test.go b/internal/index/bleve_author_migrate_test.go index 8894af54..89e16fb7 100644 --- a/internal/index/bleve_author_migrate_test.go +++ b/internal/index/bleve_author_migrate_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" "time" @@ -16,7 +17,7 @@ import ( func legacyAuthorsIndex(t *testing.T, path string) bleve.Index { t.Helper() m := CreateAuthorsMapping().(*mapping.IndexMappingImpl) - delete(m.DefaultMapping.Properties, "GoodReadsAuthorURL") + delete(m.DefaultMapping.Properties, "DataSourceLinks") idx, err := bleve.New(path, m) if err != nil { t.Fatal(err) @@ -34,15 +35,25 @@ func TestOpenAuthorsIndexMigratesStoredMetadata(t *testing.T) { want := Author{ Slug: "alice-borchardt", Name: "Alice Borchardt", BirthName: "Alice O'Brien", DataSourceID: "Q1", RetrievedOn: retrieved, - WikipediaLink: map[string]string{"en": "https://en.wikipedia.org/wiki/Alice_Borchardt", "es": "https://es.wikipedia.org/wiki/Alice_Borchardt"}, - Description: map[string]string{"en": "Writer", "es": "Escritora"}, InstanceOf: 5, - DateOfBirth: precisiondate.PrecisionDate{Date: date.New(1939, 10, 6), Precision: precisiondate.PrecisionDay}, - DateOfDeath: precisiondate.PrecisionDate{Date: date.New(2007, 7, 24), Precision: precisiondate.PrecisionDay}, - Website: "https://example.org", DataSourceImage: "image.jpg", Gender: 6581072, + DataSourceLinks: map[string]string{ + "wikipedia:en": "https://en.wikipedia.org/wiki/Alice_Borchardt", + "wikipedia:es": "https://es.wikipedia.org/wiki/Alice_Borchardt", + "website": "https://example.org", + }, + Description: map[string]string{"en": "Writer", "es": "Escritora"}, InstanceOf: 5, + DateOfBirth: precisiondate.PrecisionDate{Date: date.New(1939, 10, 6), Precision: precisiondate.PrecisionDay}, + DateOfDeath: precisiondate.PrecisionDate{Date: date.New(2007, 7, 24), Precision: precisiondate.PrecisionDay}, + DataSourceImage: "image.jpg", Gender: 6581072, Pseudonyms: []string{"Alice", "A. Borchardt"}, DocumentCount: 7, } for _, author := range []Author{want, {Slug: "other", Name: "Other", RetrievedOn: retrieved}, {Slug: "third", Name: "Third", RetrievedOn: retrieved}} { - if err := legacy.Index(author.Slug, author); err != nil { + if err := legacy.Index(author.Slug, struct { + Author + Website string + WikipediaLink map[string]string + }{author, author.DataSourceLinks["website"], map[string]string{ + "en": author.DataSourceLinks["wikipedia:en"], "es": author.DataSourceLinks["wikipedia:es"], + }}); err != nil { t.Fatal(err) } } @@ -79,7 +90,7 @@ func TestOpenAuthorsIndexMigratesStoredMetadata(t *testing.T) { t.Fatalf("expected original index removed after successful migration, got %v", err) } - got.GoodReadsAuthorURL = wantGoodReadsAuthorURL + got.DataSourceLinks["goodreads"] = wantGoodReadsAuthorURL if err := idx.IndexAuthor(got); err != nil { t.Fatal(err) } @@ -97,7 +108,7 @@ func TestOpenAuthorsIndexMigratesStoredMetadata(t *testing.T) { if err != nil || len(result.Hits) != 1 { t.Fatalf("reading reopened author: %v", err) } - want.GoodReadsAuthorURL = wantGoodReadsAuthorURL + want.DataSourceLinks["goodreads"] = wantGoodReadsAuthorURL if got := hydrateAuthor(result.Hits[0]); !reflect.DeepEqual(got, want) { t.Fatalf("migrated mapping or URL did not survive reopening: %#v", got) } @@ -154,6 +165,7 @@ func TestOpenAuthorsIndexMigrationFailurePreservesOriginal(t *testing.T) { if err := legacy.Index("saved", Author{Slug: "saved", Name: "Saved Author", DataSourceID: "Q1"}); err != nil { t.Fatal(err) } + if err := legacy.Close(); err != nil { t.Fatal(err) } @@ -172,3 +184,78 @@ func TestOpenAuthorsIndexMigrationFailurePreservesOriginal(t *testing.T) { t.Fatalf("original metadata lost after migration failure: %d, %v", count, err) } } + +func TestOpenAuthorsIndexMigratesLegacyAuthorLinks(t *testing.T) { + path := filepath.Join(t.TempDir(), "authors_index") + m := CreateAuthorsMapping().(*mapping.IndexMappingImpl) + delete(m.DefaultMapping.Properties, "DataSourceLinks") + legacyURL := bleve.NewKeywordFieldMapping() + legacyURL.Index = false + legacyURL.IncludeInAll = false + legacyURL.DocValues = false + m.DefaultMapping.AddFieldMappingsAt("Website", legacyURL) + legacy, err := bleve.New(path, m) + if err != nil { + t.Fatal(err) + } + fields := map[string]any{ + "Slug": "alice", "Name": "Alice", + "DataSourceID": "Q1", "DocumentCount": 7, + "Website": "https://example.org", + "WikipediaLink": map[string]string{ + "en": "https://en.wikipedia.org/wiki/Alice", + "es": "https://es.wikipedia.org/wiki/Alice", + }, + } + if err := legacy.Index("alice", fields); err != nil { + t.Fatal(err) + } + if err := legacy.SetInternal(internalVersion, []byte("2")); err != nil { + t.Fatal(err) + } + if err := legacy.Close(); err != nil { + t.Fatal(err) + } + upgraded, err := OpenAuthorsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + idx := NewBleve(nil, upgraded, nil, "", nil, Config{}) + author, err := idx.Author("alice", "") + if err != nil || author.DataSourceID != "Q1" || author.DocumentCount != 7 || + author.DataSourceLinks["website"] != "https://example.org" || + author.DataSourceLinks["wikipedia:en"] != "https://en.wikipedia.org/wiki/Alice" || + author.DataSourceLinks["wikipedia:es"] != "https://es.wikipedia.org/wiki/Alice" { + t.Fatalf("migration lost legacy links or metadata: %+v, %v", author, err) + } + author.DataSourceLinks["other"] = "https://example.org/alice" + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + if err := upgraded.Close(); err != nil { + t.Fatal(err) + } + reopened, err := OpenAuthorsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{"alice"})) + req.Fields = []string{"*"} + result, err := reopened.Search(req) + if err != nil || len(result.Hits) != 1 { + t.Fatalf("reading reopened author: %v", err) + } + got := hydrateAuthor(result.Hits[0]) + if !reflect.DeepEqual(got, author) { + t.Fatalf("links or metadata changed after reopening:\ngot: %#v\nwant: %#v", got, author) + } + for field := range result.Hits[0].Fields { + if field == "Website" || strings.HasPrefix(field, "WikipediaLink.") { + t.Fatalf("migrated author still stores legacy field %s", field) + } + } + if _, err := os.Stat(path + ".v2-backup"); !os.IsNotExist(err) { + t.Fatalf("migration backup was not removed: %v", err) + } +} diff --git a/internal/index/bleve_author_read.go b/internal/index/bleve_author_read.go index f3007082..6f049541 100644 --- a/internal/index/bleve_author_read.go +++ b/internal/index/bleve_author_read.go @@ -1,6 +1,8 @@ package index import ( + "encoding/json" + "log" "strings" "time" @@ -39,9 +41,6 @@ func (b *BleveIndexer) Author(slug, lang string) (Author, error) { author := hydrateAuthor(searchResult.Hits[0]) // Override language-specific fields if requested language is available - if value, ok := searchResult.Hits[0].Fields["WikipediaLink."+lang].(string); ok { - author.WikipediaLink[lang] = value - } if value, ok := searchResult.Hits[0].Fields["Description."+lang].(string); ok { author.Description[lang] = value } @@ -116,13 +115,6 @@ func hydrateAuthorFromFields(fields map[string]any, docID string) Author { } } - website := "" - if val, ok := fields["Website"]; ok && val != nil { - if str, ok := val.(string); ok { - website = str - } - } - dataSourceImage := "" if val, ok := fields["DataSourceImage"]; ok && val != nil { if str, ok := val.(string); ok { @@ -157,27 +149,22 @@ func hydrateAuthorFromFields(fields map[string]any, docID string) Author { Slug: slug, DataSourceID: dataSourceID, RetrievedOn: retrievedOn, - WikipediaLink: make(map[string]string), InstanceOf: instanceOf, Description: make(map[string]string), DateOfBirth: dateOfBirth, DateOfDeath: dateOfDeath, - Website: website, DataSourceImage: dataSourceImage, Gender: gender, Pseudonyms: slicer(fields["Pseudonyms"]), DocumentCount: documentCount, } - author.GoodReadsAuthorURL, _ = fields["GoodReadsAuthorURL"].(string) - - // Extract Wikipedia links and descriptions for all languages - for key, value := range fields { - if strings.HasPrefix(key, "WikipediaLink.") { - lang := strings.TrimPrefix(key, "WikipediaLink.") - if str, ok := value.(string); ok { - author.WikipediaLink[lang] = str - } + if links, ok := fields["DataSourceLinks"].(string); ok { + if err := json.Unmarshal([]byte(links), &author.DataSourceLinks); err != nil { + log.Printf("Error reading data source links for author %s: %s", docID, err) } + } + // Extract descriptions for all languages. + for key, value := range fields { if strings.HasPrefix(key, "Description.") { lang := strings.TrimPrefix(key, "Description.") if str, ok := value.(string); ok { diff --git a/internal/index/bleve_author_write.go b/internal/index/bleve_author_write.go index e8cc3760..45e94790 100644 --- a/internal/index/bleve_author_write.go +++ b/internal/index/bleve_author_write.go @@ -1,14 +1,30 @@ package index import ( + "encoding/json" "fmt" "log" + "maps" + "strings" "time" "github.com/blevesearch/bleve/v2" datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" ) +// indexAuthor stores links as one stored-only field so new sources need no mapping changes. +func indexAuthor(destination interface{ Index(string, any) error }, id string, author Author) error { + links, err := json.Marshal(author.DataSourceLinks) + if err != nil { + return err + } + author.DataSourceLinks = nil + return destination.Index(id, struct { + Author + DataSourceLinks string + }{author, string(links)}) +} + func (b *BleveIndexer) incrementAuthorCounts(names, slugs []string) error { for i, name := range names { if i < len(slugs) { @@ -37,7 +53,7 @@ func (b *BleveIndexer) incrementAuthorCount(name, authorSlug string) error { } else { existing.DocumentCount++ } - return b.authorsIdx.Index(authorSlug, existing) + return indexAuthor(b.authorsIdx, authorSlug, existing) } // RebuildAuthorsFromDocuments recalculates DocumentCount for every author from the documents @@ -79,7 +95,7 @@ func (b *BleveIndexer) RebuildAuthorsFromDocuments(batchSize int) error { author = Author{Name: names[authorSlug], Slug: authorSlug} } author.DocumentCount = count - if err := batch.Index(authorSlug, author); err != nil { + if err := indexAuthor(batch, authorSlug, author); err != nil { return err } if batch.Size() >= batchSize { @@ -157,11 +173,20 @@ func (b *BleveIndexer) IndexAuthor(author Author) error { if err != nil { return err } - // Goodreads enrichment runs independently of Wikidata and UI edits. - if current.GoodReadsAuthorURL != "" { - author.GoodReadsAuthorURL = current.GoodReadsAuthorURL + // Link enrichment runs independently of metadata enrichment and UI edits. + links := maps.Clone(author.DataSourceLinks) + if len(current.DataSourceLinks) > 0 { + if links == nil { + links = make(map[string]string) + } + for source, link := range current.DataSourceLinks { + if source != "website" && !strings.HasPrefix(source, "wikipedia:") { + links[source] = link + } + } } - if err := b.authorsIdx.Index(author.Slug, author); err != nil { + author.DataSourceLinks = links + if err := indexAuthor(b.authorsIdx, author.Slug, author); err != nil { return err } return nil @@ -207,12 +232,22 @@ func CombineWithDataSource(author *Author, authorDataSource datasourcemodel.Auth author.DataSourceID = authorDataSource.SourceID() author.BirthName = authorDataSource.BirthName() author.RetrievedOn = authorDataSource.RetrievedOn() - author.WikipediaLink = make(map[string]string) + author.DataSourceLinks = maps.Clone(author.DataSourceLinks) + if author.DataSourceLinks == nil { + author.DataSourceLinks = make(map[string]string) + } + for source := range author.DataSourceLinks { + if source == "website" || strings.HasPrefix(source, "wikipedia:") { + delete(author.DataSourceLinks, source) + } + } author.InstanceOf = authorDataSource.InstanceOf() author.Description = make(map[string]string) author.DateOfBirth = authorDataSource.DateOfBirth() author.DateOfDeath = authorDataSource.DateOfDeath() - author.Website = authorDataSource.Website() + if website := authorDataSource.Website(); website != "" { + author.DataSourceLinks["website"] = website + } author.DataSourceImage = authorDataSource.Image() author.Gender = authorDataSource.Gender() author.Pseudonyms = make([]string, 0, len(authorDataSource.Pseudonyms())) @@ -224,7 +259,9 @@ func CombineWithDataSource(author *Author, authorDataSource datasourcemodel.Auth } for _, lang := range supportedLanguages { - author.WikipediaLink[lang] = authorDataSource.WikipediaLink(lang) + if link := authorDataSource.WikipediaLink(lang); link != "" { + author.DataSourceLinks["wikipedia:"+lang] = link + } author.Description[lang] = authorDataSource.Description(lang) } } diff --git a/internal/index/bleve_document_write.go b/internal/index/bleve_document_write.go index fa594df4..225959d4 100644 --- a/internal/index/bleve_document_write.go +++ b/internal/index/bleve_document_write.go @@ -263,7 +263,7 @@ func (b *BleveIndexer) deleteDocumentFromIndex(document Document) error { err = b.authorsIdx.Delete(authorSlug) } else { author.DocumentCount-- - err = b.authorsIdx.Index(authorSlug, author) + err = indexAuthor(b.authorsIdx, authorSlug, author) } b.authorsMu.Unlock() if err != nil { diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go index f7e2ebf8..5ef4ff42 100644 --- a/internal/index/bleve_goodreads_write.go +++ b/internal/index/bleve_goodreads_write.go @@ -130,11 +130,14 @@ func (b *BleveIndexer) storeGoodReadsAuthorURL(document Document, authorURL, mat author.DocumentCount += result.Total } } - if author.GoodReadsAuthorURL != "" { + if author.DataSourceLinks["goodreads"] != "" { return nil } - author.GoodReadsAuthorURL = authorURL - return b.authorsIdx.Index(authorSlug, author) + if author.DataSourceLinks == nil { + author.DataSourceLinks = make(map[string]string) + } + author.DataSourceLinks["goodreads"] = authorURL + return indexAuthor(b.authorsIdx, authorSlug, author) } func goodReadsTitle(value string) string { diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go index c5308bd0..a7518c3c 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/index/bleve_goodreads_write_test.go @@ -118,11 +118,11 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { t.Error("expected document to be marked as GoodReads-enriched") } author, err := idx.Author("alice-borchardt", "") - if err != nil || author.GoodReadsAuthorURL != wantGoodReadsAuthorURL { + if err != nil || author.DataSourceLinks["goodreads"] != wantGoodReadsAuthorURL { t.Fatalf("expected matching author URL stored, got %+v, %v", author, err) } other, err := idx.Author("second-author", "") - if err != nil || other.GoodReadsAuthorURL != "" { + if err != nil || other.DataSourceLinks["goodreads"] != "" { t.Fatalf("unmatched coauthor must not receive the URL, got %+v, %v", other, err) } if want := []string{"La Reina Dragon Alice Borchardt"}; !slices.Equal(source.Queries(), want) { @@ -180,31 +180,42 @@ func TestEnrichGoodReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) if err != nil { t.Fatal(err) } + stale.DataSourceLinks = map[string]string{"other": "https://example.org/second-author"} + if err := idx.IndexAuthor(stale); err != nil { + t.Fatal(err) + } if err := idx.EnrichGoodReads(10); err != nil { t.Fatal(err) } author, err := idx.Author("second-author", "") - if err != nil || author.GoodReadsAuthorURL != authorURL { + if err != nil || author.DataSourceLinks["goodreads"] != authorURL || + author.DataSourceLinks["other"] != "https://example.org/second-author" { t.Fatalf("expected second author URL stored, got %+v, %v", author, err) } first, err := idx.Author("alice-borchardt", "") - if err != nil || first.GoodReadsAuthorURL != "" { + if err != nil || first.DataSourceLinks["goodreads"] != "" { t.Fatalf("first author must not receive coauthor URL, got %+v, %v", first, err) } stale.DataSourceID = "Q2" stale.RetrievedOn = time.Now().UTC() + stale.DataSourceLinks["goodreads"] = "/author/show/999.Stale" if err := idx.IndexAuthor(stale); err != nil { t.Fatal(err) } + if stale.DataSourceLinks["goodreads"] != "/author/show/999.Stale" { + t.Fatal("indexing an author mutated the caller's links") + } author, err = idx.Author("second-author", "") - if err != nil || author.GoodReadsAuthorURL != authorURL || author.DataSourceID != "Q2" || author.DocumentCount != 1 { + if err != nil || author.DataSourceLinks["goodreads"] != authorURL || + author.DataSourceLinks["other"] != "https://example.org/second-author" || + author.DataSourceID != "Q2" || author.DocumentCount != 1 { t.Fatalf("stale author edit lost URL or metadata: %+v, %v", author, err) } if err := idx.incrementAuthorCount(author.Name, author.Slug); err != nil { t.Fatal(err) } author, err = idx.Author("second-author", "") - if err != nil || author.GoodReadsAuthorURL != authorURL || author.DocumentCount != 2 { + if err != nil || author.DataSourceLinks["goodreads"] != authorURL || author.DocumentCount != 2 { t.Fatalf("incrementing document count lost author URL: %+v, %v", author, err) } } @@ -581,11 +592,11 @@ func TestGoodReadsBookMatchesKennedyMiddleName(t *testing.T) { t.Fatalf("expected persisted book match and rating, got %+v, %v", stored, err) } author, err := idx.Author("paul-kennedy", "") - if err != nil || author.GoodReadsAuthorURL != authorURL || author.Name != "Paul Kennedy" { + if err != nil || author.DataSourceLinks["goodreads"] != authorURL || author.Name != "Paul Kennedy" { t.Fatalf("expected author URL with original name preserved, got %+v, %v", author, err) } other, err := idx.Author("another-writer", "") - if err != nil || other.GoodReadsAuthorURL != "" { + if err != nil || other.DataSourceLinks["goodreads"] != "" { t.Fatalf("unmatched coauthor must not receive the URL, got %+v, %v", other, err) } if want := []string{title + " Paul Kennedy"}; !slices.Equal(source.Queries(), want) { @@ -857,7 +868,7 @@ func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { t.Fatalf("expected persisted omnibus match, got %+v", stored) } author, err := idx.Author("j-r-r-tolkien", "") - if err != nil || author.GoodReadsAuthorURL != results[3].AuthorURL { + if err != nil || author.DataSourceLinks["goodreads"] != results[3].AuthorURL { t.Fatalf("expected author URL on author record, got %+v, %v", author, err) } if want := []string{title + " J. R. R. Tolkien", title}; !slices.Equal(source.Queries(), want) { diff --git a/internal/index/goodreads_fields_test.go b/internal/index/goodreads_fields_test.go index 240fb2f7..c0ade3f8 100644 --- a/internal/index/goodreads_fields_test.go +++ b/internal/index/goodreads_fields_test.go @@ -1,6 +1,7 @@ package index_test import ( + "maps" "slices" "testing" @@ -252,30 +253,35 @@ func legacyBookIDQuery(id int) *query.NumericRangeQuery { return q } -func TestAuthorGoodReadsURLIsStoredButNotSearchable(t *testing.T) { +func TestAuthorDataSourceLinksAreStoredButNotSearchable(t *testing.T) { authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) if err != nil { t.Fatal(err) } defer authors.Close() + idx := index.NewBleve(nil, authors, nil, "", nil, index.Config{}) const authorURL = "/author/show/151155.Zyxwv_Author" - if err := authors.Index("alice", index.Author{Slug: "alice", Name: "Alice Borchardt", GoodReadsAuthorURL: authorURL}); err != nil { + links := map[string]string{ + "goodreads": authorURL, + "other": "https://example.org/Zyxwv_Author", + } + if err := idx.IndexAuthor(index.Author{Slug: "alice", Name: "Alice Borchardt", DataSourceLinks: links}); err != nil { t.Fatal(err) } - req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{"alice"})) - req.Fields = []string{"GoodReadsAuthorURL"} - result, err := authors.Search(req) + author, err := idx.Author("alice", "") if err != nil { t.Fatal(err) } - if len(result.Hits) != 1 || result.Hits[0].Fields["GoodReadsAuthorURL"] != authorURL { - t.Fatalf("expected stored author URL, got %+v", result.Hits) + if !maps.Equal(author.DataSourceLinks, links) { + t.Fatalf("expected stored author links, got %+v", author.DataSourceLinks) } for _, q := range []query.Query{ - fieldTermQuery("GoodReadsAuthorURL", authorURL), + fieldTermQuery("DataSourceLinks", authorURL), + fieldTermQuery("DataSourceLinks.goodreads", authorURL), + fieldTermQuery("DataSourceLinks.other", links["other"]), bleve.NewTermQuery(authorURL), bleve.NewMatchQuery("Zyxwv_Author"), - bleve.NewQueryStringQuery(`GoodReadsAuthorURL:"` + authorURL + `"`), + bleve.NewQueryStringQuery(`DataSourceLinks:"` + authorURL + `"`), } { result, err := authors.Search(bleve.NewSearchRequest(q)) if err != nil { diff --git a/internal/index/rebuild_authors_test.go b/internal/index/rebuild_authors_test.go index 27390e6b..82dbec9d 100644 --- a/internal/index/rebuild_authors_test.go +++ b/internal/index/rebuild_authors_test.go @@ -63,7 +63,10 @@ func TestRebuildAuthorsFromDocuments(t *testing.T) { if author.DocumentCount != 1 { t.Fatalf("expected DocumentCount=1 for George Orwell, got %d", author.DocumentCount) } - author.GoodReadsAuthorURL = "/author/show/3706.George_Orwell" + author.DataSourceLinks = map[string]string{ + "goodreads": "/author/show/3706.George_Orwell", + "other": "https://example.org/authors/orwell", + } if err := idx.IndexAuthor(author); err != nil { t.Fatal(err) } @@ -71,7 +74,8 @@ func TestRebuildAuthorsFromDocuments(t *testing.T) { t.Fatal(err) } author, err = idx.Author("george-orwell", "en") - if err != nil || author.GoodReadsAuthorURL != "/author/show/3706.George_Orwell" { + if err != nil || author.DataSourceLinks["goodreads"] != "/author/show/3706.George_Orwell" || + author.DataSourceLinks["other"] != "https://example.org/authors/orwell" { t.Fatalf("expected existing author URL preserved during rebuild, got %+v, %v", author, err) } diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index 2322cfd1..a0536a17 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -47,11 +47,12 @@ func (b *BiblioReads) Name() string { return "Goodreads" } -// DocumentLink returns the indexed book link without making a network request. +// AuthorLink returns the indexed author link without making a network request. func (b *BiblioReads) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { - return goodReadsDataSourceLink(author.GoodReadsAuthorURL, "/author/show/") + return goodReadsDataSourceLink(author.DataSourceLinks["goodreads"], "/author/show/") } +// DocumentLink returns the indexed book link without making a network request. func (b *BiblioReads) DocumentLink(document index.Document) datasourcemodel.DataSourceLink { return goodReadsDataSourceLink(document.DataSourceLink, "/book/show/") } diff --git a/internal/reviews/biblioreads_test.go b/internal/reviews/biblioreads_test.go index 0a0acb08..afecb007 100644 --- a/internal/reviews/biblioreads_test.go +++ b/internal/reviews/biblioreads_test.go @@ -10,6 +10,35 @@ import ( "github.com/svera/coreander/v5/internal/reviews" ) +func TestBiblioReadsDataSourceLinks(t *testing.T) { + provider := reviews.NewBiblioReads(nil, nil) + author := provider.AuthorLink(index.Author{ + DataSourceLinks: map[string]string{"goodreads": "/author/show/123.Jane_Doe"}, + }, "en") + if author.URL != "https://www.goodreads.com/author/show/123.Jane_Doe" || + author.Source != "Goodreads" || author.Icon != "/images/goodreads.svg" || !author.IconInvertible { + t.Fatalf("unexpected Goodreads author link: %+v", author) + } + + document := provider.DocumentLink(index.Document{ + DataSourceLink: "/book/show/456.A_Book", + }) + if document.URL != "https://www.goodreads.com/book/show/456.A_Book" || + document.Source != "Goodreads" || document.Icon != "/images/goodreads.svg" || !document.IconInvertible { + t.Fatalf("unexpected Goodreads document link: %+v", document) + } + for _, links := range []map[string]string{ + nil, + {}, + {"other": "/author/show/123.Jane_Doe"}, + {"goodreads": "/book/show/456.A_Book"}, + } { + if link := provider.AuthorLink(index.Author{DataSourceLinks: links}, "en"); link.URL != "" { + t.Errorf("unexpected Goodreads link for %v: %+v", links, link) + } + } +} + type reviewLinkIndex struct { link string err error diff --git a/internal/reviews/data_source_link_test.go b/internal/reviews/data_source_link_test.go deleted file mode 100644 index 609381f0..00000000 --- a/internal/reviews/data_source_link_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package reviews_test - -import ( - "testing" - - "github.com/svera/coreander/v5/internal/index" - "github.com/svera/coreander/v5/internal/reviews" -) - -func TestBiblioReadsDataSourceLinks(t *testing.T) { - provider := reviews.NewBiblioReads(nil, nil) - author := provider.AuthorLink(index.Author{ - GoodReadsAuthorURL: "/author/show/123.Jane_Doe", - }, "en") - if author.URL != "https://www.goodreads.com/author/show/123.Jane_Doe" || - author.Source != "Goodreads" || author.Icon != "/images/goodreads.svg" || !author.IconInvertible { - t.Fatalf("unexpected Goodreads author link: %+v", author) - } - - document := provider.DocumentLink(index.Document{ - DataSourceLink: "/book/show/456.A_Book", - }) - if document.URL != "https://www.goodreads.com/book/show/456.A_Book" || - document.Source != "Goodreads" || document.Icon != "/images/goodreads.svg" || !document.IconInvertible { - t.Fatalf("unexpected Goodreads document link: %+v", document) - } -} diff --git a/internal/webserver/author_summary_test.go b/internal/webserver/author_summary_test.go index f5ec6b29..12ef3639 100644 --- a/internal/webserver/author_summary_test.go +++ b/internal/webserver/author_summary_test.go @@ -164,11 +164,11 @@ func TestAuthorSummaryDataSourceLinkTranslations(t *testing.T) { author := index.Author{ Slug: tc.name, Name: "Alice Borchardt", DataSourceID: tc.dataSourceID, RetrievedOn: time.Now().UTC(), - WikipediaLink: make(map[string]string), + DataSourceLinks: map[string]string{"website": "https://author.example/" + tc.name}, } if tc.wikipedia { for _, locale := range locales { - author.WikipediaLink[locale.lang] = strings.Replace(wikipediaURL, "en.wikipedia.org", locale.lang+".wikipedia.org", 1) + author.DataSourceLinks["wikipedia:"+locale.lang] = strings.Replace(wikipediaURL, "en.wikipedia.org", locale.lang+".wikipedia.org", 1) } } if err := idx.IndexAuthor(author); err != nil { @@ -196,6 +196,10 @@ func TestAuthorSummaryDataSourceLinkTranslations(t *testing.T) { t.Fatal(err) } wikipediaLink := page.Find(`a[href="https://` + locale.lang + `.wikipedia.org/wiki/Alice_Borchardt"]`) + websiteLink := page.Find(`a[href="https://author.example/` + tc.name + `"]`) + if websiteLink.Length() != 1 || websiteLink.PrevFiltered("i.bi-globe-europe-africa").Length() != 1 { + t.Fatal("expected the stored website link and globe icon") + } if tc.wikipedia && (wikipediaLink.Length() != 1 || strings.TrimSpace(wikipediaLink.Text()) != locale.wikipediaLabel) { t.Fatalf("expected localized Wikipedia data-source link %q, got %d links with text %q", locale.wikipediaLabel, wikipediaLink.Length(), wikipediaLink.Text()) } diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index e10d187b..c16cdf0d 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -93,33 +93,33 @@ type Config struct { } type Controller struct { - hlRepository highlightsRepository - usersRepository usersRepository - readingRepository readingRepository - annotationsRepository annotationsRepository - idx IdxReaderWriter - sender Sender - config Config - appFs afero.Fs - translator i18n.Translator - reviewProvider ReviewProvider - dataSourceLinkProviders []authorcontroller.DataSourceLinkProvider + hlRepository highlightsRepository + usersRepository usersRepository + readingRepository readingRepository + annotationsRepository annotationsRepository + idx IdxReaderWriter + sender Sender + config Config + appFs afero.Fs + translator i18n.Translator + reviewProvider ReviewProvider + linkProviders []authorcontroller.DataSourceLinkProvider } // NewController creates a document controller. A nil reviewProvider disables scores and reviews. -func NewController(hlRepository highlightsRepository, usersRepository usersRepository, readingRepository readingRepository, annotationsRepository annotationsRepository, sender Sender, idx IdxReaderWriter, appFs afero.Fs, cfg Config, translator i18n.Translator, reviewProvider ReviewProvider, dataSourceLinkProviders ...authorcontroller.DataSourceLinkProvider) *Controller { +func NewController(hlRepository highlightsRepository, usersRepository usersRepository, readingRepository readingRepository, annotationsRepository annotationsRepository, sender Sender, idx IdxReaderWriter, appFs afero.Fs, cfg Config, translator i18n.Translator, reviewProvider ReviewProvider, linkProviders ...authorcontroller.DataSourceLinkProvider) *Controller { return &Controller{ - hlRepository: hlRepository, - usersRepository: usersRepository, - readingRepository: readingRepository, - annotationsRepository: annotationsRepository, - idx: idx, - sender: sender, - config: cfg, - appFs: appFs, - translator: translator, - reviewProvider: reviewProvider, - dataSourceLinkProviders: dataSourceLinkProviders, + hlRepository: hlRepository, + usersRepository: usersRepository, + readingRepository: readingRepository, + annotationsRepository: annotationsRepository, + idx: idx, + sender: sender, + config: cfg, + appFs: appFs, + translator: translator, + reviewProvider: reviewProvider, + linkProviders: linkProviders, } } diff --git a/internal/webserver/controller/document/detail.go b/internal/webserver/controller/document/detail.go index f6f747a3..86f4998f 100644 --- a/internal/webserver/controller/document/detail.go +++ b/internal/webserver/controller/document/detail.go @@ -75,11 +75,11 @@ func (d *Controller) Detail(c fiber.Ctx) error { "IllustratorSummaries": illustratorSummaries, "Debug": c.RequestCtx().QueryArgs().Has("debug"), "ReviewsEnabled": d.reviewProvider != nil, - "DocumentLink": d.documentLink(document), + "DocumentLink": d.dataSourceDocumentLink(document), }, "layout") } -func (d *Controller) documentLink(document index.Document) datasourcemodel.DataSourceLink { +func (d *Controller) dataSourceDocumentLink(document index.Document) datasourcemodel.DataSourceLink { if provider, ok := d.reviewProvider.(interface { DocumentLink(index.Document) datasourcemodel.DataSourceLink }); ok { @@ -131,7 +131,7 @@ func (d *Controller) authorSummaryFor(authorSlug, lang string) authorSummary { log.Println(err) } - links := authorcontroller.DataSourceLinks(author, lang, d.dataSourceLinkProviders...) + links := authorcontroller.DataSourceLinks(author, lang, d.linkProviders...) return authorSummary{Author: author, ImageVersion: fsutil.AuthorImageVersion(d.appFs, d.config.CacheDir, author.Slug), DataSourceLinks: links} } diff --git a/internal/webserver/controller/document/reviews.go b/internal/webserver/controller/document/reviews.go index 00073f78..14c05f73 100644 --- a/internal/webserver/controller/document/reviews.go +++ b/internal/webserver/controller/document/reviews.go @@ -51,7 +51,7 @@ func (d *Controller) Reviews(c fiber.Ctx) error { return fiber.ErrInternalServerError } if !page.Available { - link := d.documentLink(document) + link := d.dataSourceDocumentLink(document) if link.URL == "" { return c.SendString("") } diff --git a/internal/webserver/controller/document/score.go b/internal/webserver/controller/document/score.go index c1dee23b..893ae4ae 100644 --- a/internal/webserver/controller/document/score.go +++ b/internal/webserver/controller/document/score.go @@ -35,7 +35,7 @@ func (d *Controller) Score(c fiber.Ctx) error { return c.SendString("") } score.Value, score.Count = "", "" - score.URL = d.documentLink(document).URL + score.URL = d.dataSourceDocumentLink(document).URL } return c.Render("partials/document-score", fiber.Map{ "Source": score.Source, diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 0aa0a214..ca327140 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -668,7 +668,7 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { t.Errorf("expected the found link to be stored, got %+v", stored) } author, err := idx.Author("jane-doe", "") - if err != nil || author.GoodReadsAuthorURL != "/author/show/1.Jane_Doe" || author.DocumentCount != 1 { + if err != nil || author.DataSourceLinks["goodreads"] != "/author/show/1.Jane_Doe" || author.DocumentCount != 1 { t.Fatalf("expected missing author entry created with URL and document count, got %+v, %v", author, err) } if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { diff --git a/internal/webserver/embedded/views/partials/author-summary-doc-detail.html b/internal/webserver/embedded/views/partials/author-summary-doc-detail.html index 946679e6..cf8d944a 100644 --- a/internal/webserver/embedded/views/partials/author-summary-doc-detail.html +++ b/internal/webserver/embedded/views/partials/author-summary-doc-detail.html @@ -71,12 +71,12 @@

    {{.Author.Name}}

    {{end}} {{end}} - {{if or .Author.Website .DataSourceLinks}} + {{if or (index .Author.DataSourceLinks "website") .DataSourceLinks}}
      - {{if .Author.Website}} + {{with index .Author.DataSourceLinks "website"}}
    • - {{t .Lang "Website"}} + {{t $.Lang "Website"}}
      • {{end}} diff --git a/internal/webserver/embedded/views/partials/author-summary.html b/internal/webserver/embedded/views/partials/author-summary.html index c13c0fe8..42ca8889 100644 --- a/internal/webserver/embedded/views/partials/author-summary.html +++ b/internal/webserver/embedded/views/partials/author-summary.html @@ -71,12 +71,12 @@

      {{.Author.Name}}

      {{end}} {{end}} - {{if or .Author.Website .DataSourceLinks}} + {{if or (index .Author.DataSourceLinks "website") .DataSourceLinks}}
        - {{if .Author.Website}} + {{with index .Author.DataSourceLinks "website"}}
      • - {{t .Lang "Website"}} + {{t $.Lang "Website"}}
        • {{end}} From 8d65382cc4ef2fcd962ddd3336ba823b0b249736 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 16:44:44 +0100 Subject: [PATCH 21/50] WIP --- index_migration_test.go | 32 ++- internal/datasource/biblioreads/document.go | 12 ++ internal/index/author_search_test.go | 2 +- internal/index/bleve.go | 22 +- internal/index/bleve_author_migrate.go | 22 +- internal/index/bleve_document_migrate.go | 115 +++++++++++ internal/index/bleve_document_migrate_test.go | 185 +++++++++++++++++ internal/index/bleve_document_read.go | 51 +++-- internal/index/bleve_document_write.go | 12 +- internal/index/bleve_file_hash_test.go | 2 +- internal/index/bleve_goodreads_write.go | 128 +++++++----- internal/index/bleve_goodreads_write_test.go | 188 ++++++++++-------- internal/index/bleve_textrank_write.go | 21 +- internal/index/bleve_textrank_write_test.go | 8 +- internal/index/document.go | 77 +++++-- internal/index/document_data_test.go | 141 +++++++++++++ internal/index/goodreads_concurrency_test.go | 2 +- internal/index/goodreads_fields_test.go | 65 +++--- internal/index/rebuild_authors_test.go | 2 +- internal/index/search_filter_scoring_test.go | 2 +- internal/index/textrank_related_test.go | 2 +- internal/reviews/biblioreads.go | 46 ++++- internal/reviews/biblioreads_test.go | 12 +- internal/webserver/document_detail_test.go | 4 +- internal/webserver/document_goodreads_test.go | 73 ++++--- .../views/partials/document-metadata.html | 7 +- .../infrastructure/template_engine.go | 2 + internal/webserver/search_test.go | 48 +++-- internal/webserver/webserver_test.go | 6 +- main.go | 14 +- 30 files changed, 986 insertions(+), 317 deletions(-) create mode 100644 internal/datasource/biblioreads/document.go create mode 100644 internal/index/bleve_document_migrate.go create mode 100644 internal/index/bleve_document_migrate_test.go create mode 100644 internal/index/document_data_test.go diff --git a/index_migration_test.go b/index_migration_test.go index d7454f7c..3d883b1a 100644 --- a/index_migration_test.go +++ b/index_migration_test.go @@ -8,10 +8,19 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/mapping" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" ) +func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + t.Fatal(err) + } + return document +} + func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { previousHome, previousInput := homeDir, input t.Cleanup(func() { homeDir, input = previousHome, previousInput }) @@ -23,7 +32,7 @@ func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { } documents := index.CreateDocumentsIndex(homeDir + documentsIndexPath) for key, value := range map[string]string{ - "version": index.DocumentVersion, + "version": "v16", "illustrated_min_size": "0.25", "min_occurrence_ratio": "0.1", "max_textrank_words": "10000", @@ -32,12 +41,12 @@ func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { t.Fatal(err) } } - doc := index.Document{ + doc := withGoodReadsData(t, index.Document{ ID: "book", Slug: "book", AuthorsSlugs: []string{"alice-borchardt"}, - Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, - GoodReadsEnriched: true, - } - if err := documents.Index(doc.ID, doc); err != nil { + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + }, biblioreads.DocumentData{Enriched: true}) + + if err := index.IndexDocument(documents, doc.ID, doc); err != nil { t.Fatal(err) } if err := documents.Close(); err != nil { @@ -71,8 +80,19 @@ func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { if count, err := documents.DocCount(); err != nil || count != 1 { t.Fatalf("document index was not preserved: count=%d, err=%v", count, err) } + stored, err := idx.Document("book") + if err != nil { + t.Fatal(err) + } + data, err := index.DataSourceData[biblioreads.DocumentData](stored, biblioreads.Source) + if err != nil || !data.Enriched { + t.Fatalf("document migration lost enrichment state: %+v, %v", data, err) + } author, err := idx.Author("alice-borchardt", "") if err != nil || author.DataSourceID != "Q1" || !author.RetrievedOn.Equal(retrieved) { t.Fatalf("expected existing author metadata preserved: %+v, %v", author, err) } + if pending, err := idx.AuthorsWithoutInfo(); err != nil || len(pending) != 0 { + t.Fatalf("index migrations queued enriched authors again: %+v, %v", pending, err) + } } diff --git a/internal/datasource/biblioreads/document.go b/internal/datasource/biblioreads/document.go new file mode 100644 index 00000000..1ff3b016 --- /dev/null +++ b/internal/datasource/biblioreads/document.go @@ -0,0 +1,12 @@ +package biblioreads + +const Source = "goodreads" + +// DocumentData is the persisted Goodreads state used by BiblioReads. +type DocumentData struct { + URL string `json:"url,omitempty"` + RatingCount string `json:"ratingCount,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + LegacyBookID int `json:"legacyBookId,omitempty"` + Enriched bool `json:"enriched"` +} diff --git a/internal/index/author_search_test.go b/internal/index/author_search_test.go index 6a9879ad..5304cd74 100644 --- a/internal/index/author_search_test.go +++ b/internal/index/author_search_test.go @@ -258,7 +258,7 @@ func TestSearchAuthors(t *testing.T) { }, } for _, doc := range docs { - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } } diff --git a/internal/index/bleve.go b/internal/index/bleve.go index b02f0df8..a9c7aaf8 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -30,7 +30,7 @@ import ( // DocumentVersion identifies the mapping used for indexing documents. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. -const DocumentVersion = "v16" +const DocumentVersion = "v17" // AuthorVersion identifies the mapping used for indexing authors. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. @@ -401,10 +401,8 @@ func CreateDocumentsMapping() mapping.IndexMapping { storedOnlyFieldMapping.IncludeInAll = false storedOnlyFieldMapping.DocValues = false - storedOnlyNumericFieldMapping := bleve.NewNumericFieldMapping() - storedOnlyNumericFieldMapping.Index = false - storedOnlyNumericFieldMapping.IncludeInAll = false - storedOnlyNumericFieldMapping.DocValues = false + enrichedSourcesFieldMapping := bleve.NewKeywordFieldMapping() + enrichedSourcesFieldMapping.IncludeInAll = false simpleTextFieldMapping := bleve.NewTextFieldMapping() simpleTextFieldMapping.Analyzer = defaultAnalyzer @@ -464,12 +462,9 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.TypeMapping[lang].AddFieldMappingsAt("Illustrations", numericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSourceLink", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSources", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("EnrichedDataSources", enrichedSourcesFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSourceScoreValue", numericFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSourceScoreCount", storedOnlyFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) - indexMapping.TypeMapping[lang].AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) } indexMapping.DefaultMapping.DefaultAnalyzer = defaultAnalyzer @@ -494,12 +489,9 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("Illustrations", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceLink", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("DataSources", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("EnrichedDataSources", enrichedSourcesFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceScoreValue", numericFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceScoreCount", storedOnlyFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsResourceID", storedOnlyFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsLegacyBookID", storedOnlyNumericFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsEnriched", booleanFieldMapping) return indexMapping } diff --git a/internal/index/bleve_author_migrate.go b/internal/index/bleve_author_migrate.go index cef2ce2d..87bd206f 100644 --- a/internal/index/bleve_author_migrate.go +++ b/internal/index/bleve_author_migrate.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" ) // OpenAuthorsIndex upgrades version 2 without discarding stored author metadata. @@ -40,6 +41,7 @@ func OpenAuthorsIndex(path string, batchSize int) (bleve.Index, error) { return nil, errors.Join(err, authors.Close()) } authors = upgraded + log.Printf("Migrated authors index to version %s", AuthorVersion) } else if string(version) != AuthorVersion { return authors, nil } @@ -50,36 +52,39 @@ func OpenAuthorsIndex(path string, batchSize int) (bleve.Index, error) { } func migrateAuthorsIndex(authors bleve.Index, path string, batchSize int) (bleve.Index, error) { + return migrateStoredIndex(authors, path, path+".v2-backup", batchSize, CreateAuthorsMapping(), copyAuthors) +} + +func migrateStoredIndex(source bleve.Index, path, backup string, batchSize int, targetMapping mapping.IndexMapping, copyDocuments func(bleve.Index, bleve.Index, int) error) (bleve.Index, error) { if batchSize <= 0 { - return nil, fmt.Errorf("authors migration batch size must be positive") + return nil, fmt.Errorf("index migration batch size must be positive") } - backup := path + ".v2-backup" if _, err := os.Stat(backup); err == nil { - return nil, fmt.Errorf("authors migration backup already exists: %s", backup) + return nil, fmt.Errorf("index migration backup already exists: %s", backup) } else if !os.IsNotExist(err) { return nil, err } - stagingDir, err := os.MkdirTemp(filepath.Dir(path), ".authors-migration-") + stagingDir, err := os.MkdirTemp(filepath.Dir(path), ".index-migration-") if err != nil { return nil, err } defer func() { if err := os.RemoveAll(stagingDir); err != nil { - log.Printf("Error removing authors migration staging directory: %s", err) + log.Printf("Error removing index migration staging directory: %s", err) } }() stagingPath := filepath.Join(stagingDir, "index") - staged, err := bleve.New(stagingPath, CreateAuthorsMapping()) + staged, err := bleve.New(stagingPath, targetMapping) if err != nil { return nil, err } - if err := copyAuthors(authors, staged, batchSize); err != nil { + if err := copyDocuments(source, staged, batchSize); err != nil { return nil, errors.Join(err, staged.Close()) } if err := staged.Close(); err != nil { return nil, err } - if err := authors.Close(); err != nil { + if err := source.Close(); err != nil { return nil, err } if err := os.Rename(path, backup); err != nil { @@ -88,7 +93,6 @@ func migrateAuthorsIndex(authors bleve.Index, path string, batchSize int) (bleve if err := os.Rename(stagingPath, path); err != nil { return nil, errors.Join(err, os.Rename(backup, path)) } - log.Printf("Migrated authors index to version %s", AuthorVersion) return bleve.Open(path) } diff --git a/internal/index/bleve_document_migrate.go b/internal/index/bleve_document_migrate.go new file mode 100644 index 00000000..b78e2f47 --- /dev/null +++ b/internal/index/bleve_document_migrate.go @@ -0,0 +1,115 @@ +package index + +import ( + "errors" + "fmt" + "log" + "os" + + "github.com/blevesearch/bleve/v2" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +var previousDocumentVersions = []string{"v13", "v14", "v15", "v16"} + +// OpenDocumentsIndex preserves document metadata while upgrading provider storage. +func OpenDocumentsIndex(path string, batchSize int) (bleve.Index, error) { + documents, err := bleve.Open(path) + for _, version := range previousDocumentVersions { + if !errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { + break + } + backup := path + "." + version + "-backup" + if _, statErr := os.Stat(backup); statErr == nil { + if err := os.Rename(backup, path); err != nil { + return nil, fmt.Errorf("restoring documents index: %w", err) + } + documents, err = bleve.Open(path) + } else if !os.IsNotExist(statErr) { + return nil, fmt.Errorf("checking documents index backup: %w", statErr) + } + } + if err != nil { + return nil, err + } + version, err := documents.GetInternal(internalVersion) + if err != nil { + return nil, errors.Join(err, documents.Close()) + } + for _, previous := range previousDocumentVersions { + if string(version) == previous { + upgraded, err := migrateStoredIndex(documents, path, path+"."+previous+"-backup", batchSize, CreateDocumentsMapping(), copyDocuments) + if err != nil { + return nil, errors.Join(err, documents.Close()) + } + documents = upgraded + version = []byte(DocumentVersion) + log.Printf("Migrated documents index to version %s", DocumentVersion) + break + } + } + if string(version) != DocumentVersion { + return nil, errors.Join(fmt.Errorf("unsupported documents index version %q; index was not modified", version), documents.Close()) + } + for _, previous := range previousDocumentVersions { + if err := os.RemoveAll(path + "." + previous + "-backup"); err != nil { + return nil, errors.Join(fmt.Errorf("removing previous documents index: %w", err), documents.Close()) + } + } + return documents, nil +} + +func copyDocuments(source, destination bleve.Index, batchSize int) error { + for from := 0; ; from += batchSize { + req := bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), batchSize, from, false) + req.Fields = []string{"*"} + req.SortBy([]string{"_id"}) + result, err := source.Search(req) + if err != nil { + return err + } + if len(result.Hits) == 0 { + break + } + batch := destination.NewBatch() + for _, hit := range result.Hits { + document := hydrateDocument(hit) + document.DataSources, err = decodeDataSources(hit.Fields, hit.ID) + if err != nil { + return err + } + link, _ := hit.Fields["DataSourceLink"].(string) + count, _ := hit.Fields["DataSourceScoreCount"].(string) + resource, _ := hit.Fields["GoodReadsResourceID"].(string) + legacyID, _ := hit.Fields["GoodReadsLegacyBookID"].(float64) + enriched, _ := hit.Fields["GoodReadsEnriched"].(bool) + if link != "" || count != "" || resource != "" || legacyID != 0 || enriched { + if _, exists := document.DataSources[biblioreads.Source]; !exists { + if err := SetDataSourceData(&document, biblioreads.Source, biblioreads.DocumentData{ + URL: link, RatingCount: count, ResourceID: resource, LegacyBookID: int(legacyID), Enriched: enriched, + }); err != nil { + return err + } + } + } + if err := IndexDocument(batch, hit.ID, document); err != nil { + return err + } + } + if err := destination.Batch(batch); err != nil { + return err + } + } + for _, key := range [][]byte{internalLanguages, internalIllustratedMinSize, internalMinOccurrenceRatio, internalMaxTextRankWords, internalDocCountAtLastPrune} { + value, err := source.GetInternal(key) + if err != nil { + return err + } + if value != nil { + if err := destination.SetInternal(key, value); err != nil { + return err + } + } + } + return destination.SetInternal(internalVersion, []byte(DocumentVersion)) +} diff --git a/internal/index/bleve_document_migrate_test.go b/internal/index/bleve_document_migrate_test.go new file mode 100644 index 00000000..4170a334 --- /dev/null +++ b/internal/index/bleve_document_migrate_test.go @@ -0,0 +1,185 @@ +package index + +import ( + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/metadata" +) + +func legacyDocumentsIndex(t *testing.T, path, version string) bleve.Index { + t.Helper() + m := CreateDocumentsMapping().(*mapping.IndexMappingImpl) + mappings := []*mapping.DocumentMapping{m.DefaultMapping} + for _, documentMapping := range m.TypeMapping { + mappings = append(mappings, documentMapping) + } + for _, documentMapping := range mappings { + delete(documentMapping.Properties, "DataSources") + delete(documentMapping.Properties, "EnrichedDataSources") + for _, field := range []string{"DataSourceLink", "DataSourceScoreCount", "GoodReadsResourceID"} { + stored := bleve.NewKeywordFieldMapping() + stored.Index, stored.IncludeInAll, stored.DocValues = false, false, false + documentMapping.AddFieldMappingsAt(field, stored) + } + storedNumber := bleve.NewNumericFieldMapping() + storedNumber.Index, storedNumber.IncludeInAll, storedNumber.DocValues = false, false, false + documentMapping.AddFieldMappingsAt("GoodReadsLegacyBookID", storedNumber) + documentMapping.AddFieldMappingsAt("GoodReadsEnriched", bleve.NewBooleanFieldMapping()) + } + idx, err := bleve.New(path, m) + if err != nil { + t.Fatal(err) + } + if err := idx.SetInternal(internalVersion, []byte(version)); err != nil { + t.Fatal(err) + } + return idx +} + +func TestOpenDocumentsIndexMigratesStoredMetadata(t *testing.T) { + for _, version := range previousDocumentVersions { + t.Run(version, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "documents") + legacy := legacyDocumentsIndex(t, path, version) + want := make(map[string]Document) + for _, data := range []biblioreads.DocumentData{ + {URL: "/book/show/123", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, Enriched: true}, + {Enriched: true}, + {}, + } { + id := "linked" + if data.URL == "" { + id = "pending" + if data.Enriched { + id = "unmatched" + } + } + document := Document{ + ID: id, Slug: id, + Metadata: metadata.Metadata{Title: "Book " + id, Authors: []string{"Writer"}, Language: "es", Format: "EPUB"}, + AuthorsSlugs: []string{"writer"}, AddedOn: time.Date(2026, 11, 3, 10, 0, 0, 0, time.UTC), + TextRankWords: []string{"atomic", "history"}, TextRankPhrases: []string{"atomic history"}, TextRankEnriched: true, + DataSourceScoreValue: 4.25, + } + if err := legacy.Index(id, struct { + Document + DataSourceLink string + DataSourceScoreCount string + GoodReadsResourceID string + GoodReadsLegacyBookID int + GoodReadsEnriched bool + }{document, data.URL, data.RatingCount, data.ResourceID, data.LegacyBookID, data.Enriched}); err != nil { + t.Fatal(err) + } + reader := NewBleve(legacy, nil, nil, "", nil, Config{}) + stored, err := reader.documentByIndexID(id) + if err != nil { + t.Fatal(err) + } + if data != (biblioreads.DocumentData{}) { + stored = withGoodReadsData(t, stored, data) + } + want[id] = stored + } + internal := map[string]string{ + string(internalLanguages): "es,en", string(internalIllustratedMinSize): "0.25", + string(internalMinOccurrenceRatio): "0.1", string(internalMaxTextRankWords): "10000", + string(internalDocCountAtLastPrune): "3", + } + for key, value := range internal { + if err := legacy.SetInternal([]byte(key), []byte(value)); err != nil { + t.Fatal(err) + } + } + if err := legacy.Close(); err != nil { + t.Fatal(err) + } + upgraded, err := OpenDocumentsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + if err := upgraded.Close(); err != nil { + t.Fatal(err) + } + reopened, err := OpenDocumentsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + idx := NewBleve(reopened, nil, nil, "", nil, Config{}) + for id, expected := range want { + got, err := idx.documentByIndexID(id) + if err != nil || !reflect.DeepEqual(got, expected) { + t.Fatalf("migration changed document:\ngot: %#v\nwant: %#v\nerror: %v", got, expected, err) + } + } + if _, total, err := idx.documentsNeedingGoodReads(10); err != nil || total != 1 { + t.Fatalf("migration changed enrichment queue: %d pending, %v", total, err) + } + for key, expected := range internal { + got, err := reopened.GetInternal([]byte(key)) + if err != nil || string(got) != expected { + t.Fatalf("migration lost internal metadata %s: %q, %v", key, got, err) + } + } + if _, err := os.Stat(path + "." + version + "-backup"); !os.IsNotExist(err) { + t.Fatalf("migration backup was not removed: %v", err) + } + }) + } +} + +func TestOpenDocumentsIndexPreservesOriginalOnFailure(t *testing.T) { + for _, version := range []string{"v16", "unsupported"} { + t.Run(version, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "documents") + legacy := legacyDocumentsIndex(t, path, version) + if err := legacy.Index("saved", Document{ID: "saved"}); err != nil { + t.Fatal(err) + } + if err := legacy.Close(); err != nil { + t.Fatal(err) + } + if _, err := OpenDocumentsIndex(path, 0); err == nil { + t.Fatal("expected invalid batch size or unsupported version error") + } + original, err := bleve.Open(path) + if err != nil { + t.Fatal(err) + } + defer original.Close() + if count, err := original.DocCount(); err != nil || count != 1 { + t.Fatalf("original index lost: %d, %v", count, err) + } + if got, err := original.GetInternal(internalVersion); err != nil || string(got) != version { + t.Fatalf("original version changed: %q, %v", got, err) + } + }) + } +} + +func TestOpenDocumentsIndexRecoversInterruptedSwap(t *testing.T) { + path := filepath.Join(t.TempDir(), "documents") + legacy := legacyDocumentsIndex(t, path+".v16-backup", "v16") + if err := legacy.Index("saved", Document{ID: "saved", Slug: "saved", Metadata: metadata.Metadata{Title: "Saved"}}); err != nil { + t.Fatal(err) + } + if err := legacy.Close(); err != nil { + t.Fatal(err) + } + upgraded, err := OpenDocumentsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + defer upgraded.Close() + if count, err := upgraded.DocCount(); err != nil || count != 1 { + t.Fatalf("interrupted migration lost documents: %d, %v", count, err) + } +} diff --git a/internal/index/bleve_document_read.go b/internal/index/bleve_document_read.go index c96f404f..45d40bd2 100644 --- a/internal/index/bleve_document_read.go +++ b/internal/index/bleve_document_read.go @@ -3,10 +3,12 @@ package index import ( "cmp" "crypto/sha256" + "encoding/json" "errors" "fmt" "html/template" "image" + "log" "math" "net/url" "os" @@ -1082,12 +1084,7 @@ func hydrateDocument(match *search.DocumentMatch) Document { textRankEnriched = match.Fields["TextRankEnriched"].(bool) } - reviewLink, _ := match.Fields["DataSourceLink"].(string) reviewScoreValue, _ := match.Fields["DataSourceScoreValue"].(float64) - reviewScoreCount, _ := match.Fields["DataSourceScoreCount"].(string) - goodReadsResourceID, _ := match.Fields["GoodReadsResourceID"].(string) - goodReadsLegacyBookID, _ := match.Fields["GoodReadsLegacyBookID"].(float64) - goodReadsEnriched, _ := match.Fields["GoodReadsEnriched"].(bool) doc := Document{ ID: match.ID, @@ -1106,26 +1103,40 @@ func hydrateDocument(match *search.DocumentMatch) Document { Illustrations: illustrations, Format: match.Fields["Format"].(string), }, - Slug: match.Fields["Slug"].(string), - AuthorsSlugs: slicer(match.Fields["AuthorsSlugs"]), - IllustratorsSlugs: illustratorsSlugs, - SeriesSlug: match.Fields["SeriesSlug"].(string), - SubjectsSlugs: slicer(match.Fields["SubjectsSlugs"]), - AddedOn: addedOn, - TextRankPhrases: textRankPhrases, - TextRankWords: textRankWords, - TextRankEnriched: textRankEnriched, - DataSourceLink: reviewLink, - DataSourceScoreValue: reviewScoreValue, - DataSourceScoreCount: reviewScoreCount, - GoodReadsResourceID: goodReadsResourceID, - GoodReadsLegacyBookID: int(goodReadsLegacyBookID), - GoodReadsEnriched: goodReadsEnriched, + Slug: match.Fields["Slug"].(string), + AuthorsSlugs: slicer(match.Fields["AuthorsSlugs"]), + IllustratorsSlugs: illustratorsSlugs, + SeriesSlug: match.Fields["SeriesSlug"].(string), + SubjectsSlugs: slicer(match.Fields["SubjectsSlugs"]), + AddedOn: addedOn, + TextRankPhrases: textRankPhrases, + TextRankWords: textRankWords, + TextRankEnriched: textRankEnriched, + DataSources: hydrateDataSources(match.Fields, match.ID), + DataSourceScoreValue: reviewScoreValue, } return doc } +func hydrateDataSources(fields map[string]any, documentID string) map[string]json.RawMessage { + sources, err := decodeDataSources(fields, documentID) + if err != nil { + log.Println(err) + } + return sources +} + +func decodeDataSources(fields map[string]any, documentID string) (map[string]json.RawMessage, error) { + var sources map[string]json.RawMessage + if raw, ok := fields["DataSources"].(string); ok { + if err := json.Unmarshal([]byte(raw), &sources); err != nil { + return nil, fmt.Errorf("reading data sources for document %s: %w", documentID, err) + } + } + return sources, nil +} + func slicer(val any) []string { var ( terms []any diff --git a/internal/index/bleve_document_write.go b/internal/index/bleve_document_write.go index 225959d4..abbdee31 100644 --- a/internal/index/bleve_document_write.go +++ b/internal/index/bleve_document_write.go @@ -147,9 +147,11 @@ func (b *BleveIndexer) indexFileLocked(file string) (string, error) { document.TextRankEnriched = !b.supportsTextRank(file) documents := []Document{document} b.documentsMu.Lock() - b.preserveGoodReadsLocked(documents) - document = documents[0] - err = b.documentsIdx.Index(id, document) + err = b.preserveDataSourcesLocked(documents) + if err == nil { + document = documents[0] + err = IndexDocument(b.documentsIdx, id, document) + } b.documentsMu.Unlock() if err != nil { return "", fmt.Errorf("error updating indexed file %s: %w", file, err) @@ -167,7 +169,7 @@ func (b *BleveIndexer) indexFileLocked(file string) (string, error) { document.TextRankEnriched = !b.supportsTextRank(file) b.documentsMu.Lock() - err = b.documentsIdx.Index(document.ID, document) + err = IndexDocument(b.documentsIdx, document.ID, document) b.documentsMu.Unlock() if err != nil { return "", fmt.Errorf("error indexing file %s: %s", file, err) @@ -332,7 +334,7 @@ func (b *BleveIndexer) AddLibrary(batchSize int, forceIndexing bool, metadataWor // consider them. document.TextRankEnriched = !b.supportsTextRank(job.path) - if err = batch.Index(document.ID, document); err != nil { + if err = IndexDocument(batch, document.ID, document); err != nil { log.Printf("Error indexing file %s: %s\n", job.path, err) continue } diff --git a/internal/index/bleve_file_hash_test.go b/internal/index/bleve_file_hash_test.go index b812776a..32441fab 100644 --- a/internal/index/bleve_file_hash_test.go +++ b/internal/index/bleve_file_hash_test.go @@ -146,7 +146,7 @@ func TestIndexFileUpdatesHashWithoutMetadataChanges(t *testing.T) { t.Fatal(err) } doc.TextRankWords = []string{"enriched"} - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } for _, content := range []string{"original content", "modified content", "modified content"} { diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go index 5ef4ff42..72a61c6b 100644 --- a/internal/index/bleve_goodreads_write.go +++ b/internal/index/bleve_goodreads_write.go @@ -2,9 +2,11 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "log" + "maps" "math" "regexp" "strconv" @@ -29,7 +31,7 @@ type GoodReadsSource interface { SearchBooks(ctx context.Context, query string) (biblioreads.SearchBooksResponse, error) } -var goodReadsFields = []string{"DataSourceLink", "DataSourceScoreValue", "DataSourceScoreCount", "GoodReadsResourceID", "GoodReadsLegacyBookID", "GoodReadsEnriched"} +var dataSourceFields = []string{"DataSources", "DataSourceScoreValue"} // Strip named series annotations and omnibus ranges, not standalone volume numbers. var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) @@ -257,8 +259,11 @@ func goodReadsQuery(document Document) string { } func documentsNeedingGoodReadsQuery() query.Query { - notEnriched := bleve.NewBoolFieldQuery(false) - notEnriched.SetField("GoodReadsEnriched") + enriched := bleve.NewTermQuery(biblioreads.Source) + enriched.SetField("EnrichedDataSources") + notEnriched := bleve.NewBooleanQuery() + notEnriched.AddMust(bleve.NewMatchAllQuery()) + notEnriched.AddMustNot(enriched) return notEnriched } @@ -279,7 +284,7 @@ func (b *BleveIndexer) documentsNeedingGoodReads(size int) ([]Document, uint64, } // EnrichGoodReads looks up on Goodreads every document not processed yet (see -// Document.GoodReadsEnriched), storing the book URL on the document and the +// biblioreads.DocumentData.Enriched), storing the book URL on the document and the // author URL on the matching author. The source handles per-instance throttling. // It stops at the first error that means Goodreads can't be reached right // now, leaving the remaining documents for the next run. @@ -356,8 +361,12 @@ func (b *BleveIndexer) enrichGoodReadsBatch(documents []Document) error { // GoodReadsBook returns the stored book URL and rating, or searches a document // not looked up yet and persists its match. Returns an empty result if there's no match. func (b *BleveIndexer) GoodReadsBook(ctx context.Context, document Document) (biblioreads.SearchResult, error) { - if document.DataSourceLink != "" || document.GoodReadsEnriched || b.goodReadsSource == nil { - return biblioreads.SearchResult{BookURL: document.DataSourceLink, Rating: fmt.Sprintf("%.2f", document.DataSourceScoreValue), RatingCount: document.DataSourceScoreCount}, nil + data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + return biblioreads.SearchResult{}, err + } + if data.URL != "" || data.Enriched || b.goodReadsSource == nil { + return biblioreads.SearchResult{BookURL: data.URL, Rating: fmt.Sprintf("%.2f", document.DataSourceScoreValue), RatingCount: data.RatingCount}, nil } return b.enrichGoodReads(ctx, document) } @@ -373,16 +382,18 @@ func (b *BleveIndexer) UpdateDataSourceLink(documentID, reviewLink string) error if !found { return ErrDocumentNotFound } - if current.DataSourceLink == reviewLink { + data, err := DataSourceData[biblioreads.DocumentData](current, biblioreads.Source) + if err != nil { + return err + } + if data.URL == reviewLink { return nil } - current.DataSourceLink = reviewLink current.DataSourceScoreValue = 0 - current.DataSourceScoreCount = "" - current.GoodReadsResourceID = "" - current.GoodReadsLegacyBookID = 0 - current.GoodReadsEnriched = true - if err := b.documentsIdx.Index(current.ID, current); err != nil { + if err := SetDataSourceData(¤t, biblioreads.Source, biblioreads.DocumentData{URL: reviewLink, Enriched: true}); err != nil { + return err + } + if err := IndexDocument(b.documentsIdx, current.ID, current); err != nil { return fmt.Errorf("saving review-link correction for document %s: %w", documentID, err) } return nil @@ -407,12 +418,16 @@ func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, if !found { return ErrDocumentNotFound } - if goodReadsQuery(current) != goodReadsQuery(document) || current.DataSourceLink != bookURL { + data, err := DataSourceData[biblioreads.DocumentData](current, biblioreads.Source) + if err != nil { + return err + } + if goodReadsQuery(current) != goodReadsQuery(document) || data.URL != bookURL { return fmt.Errorf("GoodReads match changed for document %s", document.ID) } changed := false - if book.RatingCount != "" && current.DataSourceScoreCount != book.RatingCount { - current.DataSourceScoreCount = book.RatingCount + if book.RatingCount != "" && data.RatingCount != book.RatingCount { + data.RatingCount = book.RatingCount changed = true } if book.Rating != "" { @@ -422,15 +437,18 @@ func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, changed = true } } - if hasIDs && (current.GoodReadsResourceID != book.ResourceID || current.GoodReadsLegacyBookID != book.LegacyBookID) { - current.GoodReadsResourceID = book.ResourceID - current.GoodReadsLegacyBookID = book.LegacyBookID + if hasIDs && (data.ResourceID != book.ResourceID || data.LegacyBookID != book.LegacyBookID) { + data.ResourceID = book.ResourceID + data.LegacyBookID = book.LegacyBookID changed = true } if !changed { return nil } - if err := b.documentsIdx.Index(current.ID, current); err != nil { + if err := SetDataSourceData(¤t, biblioreads.Source, data); err != nil { + return err + } + if err := IndexDocument(b.documentsIdx, current.ID, current); err != nil { return fmt.Errorf("error storing GoodReads details for document %s: %w", document.ID, err) } return nil @@ -440,6 +458,10 @@ func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, // book URL, and stores matching URLs on the document and author respectively. // Unavailability or persistence errors leave the document unenriched for retry. func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) (biblioreads.SearchResult, error) { + data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + return biblioreads.SearchResult{}, err + } select { case b.goodReadsSlots <- struct{}{}: defer func() { <-b.goodReadsSlots }() @@ -447,7 +469,7 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) ( return biblioreads.SearchResult{}, ctx.Err() } q := goodReadsQuery(document) - result := biblioreads.SearchResult{BookURL: document.DataSourceLink} + result := biblioreads.SearchResult{BookURL: data.URL} if result.BookURL == "" && document.Title != "" { ctx, cancel := context.WithTimeout(ctx, goodReadsRequestTimeout) defer cancel() @@ -487,26 +509,33 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) ( if !found || goodReadsQuery(current) != q { return result, nil } + data, err = DataSourceData[biblioreads.DocumentData](current, biblioreads.Source) + if err != nil { + return result, err + } // Looked up concurrently (e.g. by the background enrichment and a detail // page request at the same time): keep the stored result. - if current.GoodReadsEnriched { - result = biblioreads.SearchResult{BookURL: current.DataSourceLink, Rating: fmt.Sprintf("%.2f", current.DataSourceScoreValue), RatingCount: current.DataSourceScoreCount} + if data.Enriched { + result = biblioreads.SearchResult{BookURL: data.URL, Rating: fmt.Sprintf("%.2f", current.DataSourceScoreValue), RatingCount: data.RatingCount} return result, nil } - if current.DataSourceLink != "" && current.DataSourceLink != result.BookURL { - result = biblioreads.SearchResult{BookURL: current.DataSourceLink, Rating: fmt.Sprintf("%.2f", current.DataSourceScoreValue), RatingCount: current.DataSourceScoreCount} + if data.URL != "" && data.URL != result.BookURL { + result = biblioreads.SearchResult{BookURL: data.URL, Rating: fmt.Sprintf("%.2f", current.DataSourceScoreValue), RatingCount: data.RatingCount} } - current.DataSourceLink = result.BookURL + data.URL = result.BookURL if result.Rating == "" && current.DataSourceScoreValue > 0 { result.Rating = fmt.Sprintf("%.2f", current.DataSourceScoreValue) } current.DataSourceScoreValue = parseGoodReadsRating(result.Rating, current.ID) - current.DataSourceScoreCount = result.RatingCount - current.GoodReadsEnriched = true + data.RatingCount = result.RatingCount + data.Enriched = true + if err := SetDataSourceData(¤t, biblioreads.Source, data); err != nil { + return result, err + } if err := b.storeGoodReadsAuthorURL(current, result.AuthorURL, result.Author); err != nil { return result, fmt.Errorf("error storing GoodReads author URL for document %s: %w", current.ID, err) } - if err := b.documentsIdx.Index(current.ID, current); err != nil { + if err := IndexDocument(b.documentsIdx, current.ID, current); err != nil { return result, fmt.Errorf("error indexing GoodReads-enriched document %s: %w", current.ID, err) } return result, nil @@ -533,7 +562,12 @@ func (b *BleveIndexer) runGoodReadsEnrichWorker() { log.Printf("Error reading document %s for GoodReads enrichment: %s\n", id, err) continue } - if document.ID == "" || document.GoodReadsEnriched { + data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + log.Println(err) + continue + } + if document.ID == "" || data.Enriched { continue } if _, err := b.enrichGoodReads(context.Background(), document); err != nil { @@ -564,43 +598,43 @@ func (b *BleveIndexer) documentByIndexIDLocked(id string) (Document, bool, error return hydrateDocument(searchResult.Hits[0]), true, nil } -// preserveGoodReadsLocked copies the currently stored GoodReads fields into +// preserveDataSourcesLocked copies currently stored provider data into // documents, so writers that rebuilt them from an earlier read don't discard // a lookup persisted in the meantime. Must be called with documentsMu held // for writing, right before writing documents back. -func (b *BleveIndexer) preserveGoodReadsLocked(documents []Document) { +func (b *BleveIndexer) preserveDataSourcesLocked(documents []Document) error { if len(documents) == 0 { - return + return nil } ids := make([]string, len(documents)) for i, document := range documents { ids[i] = document.ID } searchReq := bleve.NewSearchRequestOptions(bleve.NewDocIDQuery(ids), len(ids), 0, false) - searchReq.Fields = goodReadsFields + searchReq.Fields = dataSourceFields searchResult, err := b.documentsIdx.Search(searchReq) if err != nil { - log.Printf("Error reading stored GoodReads fields: %s\n", err) - return + return fmt.Errorf("reading stored data source fields: %w", err) } stored := make(map[string]Document, len(searchResult.Hits)) for _, hit := range searchResult.Hits { - bookURL, _ := hit.Fields["DataSourceLink"].(string) ratingValue, _ := hit.Fields["DataSourceScoreValue"].(float64) - ratingCount, _ := hit.Fields["DataSourceScoreCount"].(string) - resourceID, _ := hit.Fields["GoodReadsResourceID"].(string) - legacyBookID, _ := hit.Fields["GoodReadsLegacyBookID"].(float64) - enriched, _ := hit.Fields["GoodReadsEnriched"].(bool) - stored[hit.ID] = Document{DataSourceLink: bookURL, DataSourceScoreValue: ratingValue, DataSourceScoreCount: ratingCount, GoodReadsResourceID: resourceID, GoodReadsLegacyBookID: int(legacyBookID), GoodReadsEnriched: enriched} + sources, err := decodeDataSources(hit.Fields, hit.ID) + if err != nil { + return err + } + stored[hit.ID] = Document{DataSources: sources, DataSourceScoreValue: ratingValue} } for i := range documents { if s, ok := stored[documents[i].ID]; ok { - documents[i].DataSourceLink = s.DataSourceLink + merged := maps.Clone(documents[i].DataSources) + if merged == nil && len(s.DataSources) > 0 { + merged = make(map[string]json.RawMessage) + } + maps.Copy(merged, s.DataSources) + documents[i].DataSources = merged documents[i].DataSourceScoreValue = s.DataSourceScoreValue - documents[i].DataSourceScoreCount = s.DataSourceScoreCount - documents[i].GoodReadsResourceID = s.GoodReadsResourceID - documents[i].GoodReadsLegacyBookID = s.GoodReadsLegacyBookID - documents[i].GoodReadsEnriched = s.GoodReadsEnriched } } + return nil } diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go index a7518c3c..32f46275 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/index/bleve_goodreads_write_test.go @@ -15,6 +15,30 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) +func mustGoodReadsData(t *testing.T, document Document) biblioreads.DocumentData { + t.Helper() + data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + t.Fatal(err) + } + return data +} + +func withGoodReadsData(t *testing.T, document Document, data biblioreads.DocumentData) Document { + t.Helper() + if err := SetDataSourceData(&document, biblioreads.Source, data); err != nil { + t.Fatal(err) + } + return document +} + +func updateGoodReadsData(t *testing.T, document *Document, update func(*biblioreads.DocumentData)) { + t.Helper() + data := mustGoodReadsData(t, *document) + update(&data) + *document = withGoodReadsData(t, *document, data) +} + func assertNoDocumentAuthorURL(t *testing.T, documents bleve.Index) { t.Helper() req := bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), 100, 0, false) @@ -104,17 +128,17 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { if err != nil { t.Fatal(err) } - if doc.DataSourceLink != wantDataSourceLink { - t.Errorf("unexpected book URL %q", doc.DataSourceLink) + if mustGoodReadsData(t, doc).URL != wantDataSourceLink { + t.Errorf("unexpected book URL %q", mustGoodReadsData(t, doc).URL) } if doc.DataSourceScoreValue != 3.82 { t.Errorf("unexpected Goodreads rating %v", doc.DataSourceScoreValue) } - if doc.DataSourceScoreCount != "2,285" { - t.Errorf("unexpected Goodreads rating count %q", doc.DataSourceScoreCount) + if mustGoodReadsData(t, doc).RatingCount != "2,285" { + t.Errorf("unexpected Goodreads rating count %q", mustGoodReadsData(t, doc).RatingCount) } assertNoDocumentAuthorURL(t, idx.documentsIdx) - if !doc.GoodReadsEnriched { + if !mustGoodReadsData(t, doc).Enriched { t.Error("expected document to be marked as GoodReads-enriched") } author, err := idx.Author("alice-borchardt", "") @@ -157,7 +181,7 @@ func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.DataSourceLink != "" { + if !mustGoodReadsData(t, doc).Enriched || mustGoodReadsData(t, doc).URL != "" { t.Errorf("expected document enriched with no URLs, got %+v", doc) } if want := []string{"La Reina Dragon Alice Borchardt", "La Reina Dragon"}; !slices.Equal(source.Queries(), want) { @@ -234,7 +258,7 @@ func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { if err != nil { t.Fatal(err) } - if doc.GoodReadsEnriched { + if mustGoodReadsData(t, doc).Enriched { t.Error("expected document not to be marked as GoodReads-enriched") } @@ -245,8 +269,8 @@ func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { if err := idx.EnrichGoodReads(10); err != nil { t.Fatal(err) } - if doc, _ = idx.documentByIndexID(goodReadsTestID); doc.DataSourceLink != wantDataSourceLink { - t.Errorf("expected document to be enriched on retry, got %q", doc.DataSourceLink) + if doc, _ = idx.documentByIndexID(goodReadsTestID); mustGoodReadsData(t, doc).URL != wantDataSourceLink { + t.Errorf("expected document to be enriched on retry, got %q", mustGoodReadsData(t, doc).URL) } } @@ -264,9 +288,9 @@ func TestNewFileSchedulesGoodReadsEnrichment(t *testing.T) { if err != nil { t.Fatal(err) } - if doc.GoodReadsEnriched { - if doc.DataSourceLink != wantDataSourceLink { - t.Errorf("unexpected book URL %q", doc.DataSourceLink) + if mustGoodReadsData(t, doc).Enriched { + if mustGoodReadsData(t, doc).URL != wantDataSourceLink { + t.Errorf("unexpected book URL %q", mustGoodReadsData(t, doc).URL) } assertNoDocumentAuthorURL(t, idx.documentsIdx) return @@ -306,10 +330,10 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.DataSourceLink != wantDataSourceLink || doc.DataSourceScoreValue != 3.82 || doc.DataSourceScoreCount != "2,285" { + if !mustGoodReadsData(t, doc).Enriched || mustGoodReadsData(t, doc).URL != wantDataSourceLink || doc.DataSourceScoreValue != 3.82 || mustGoodReadsData(t, doc).RatingCount != "2,285" { t.Errorf("expected GoodReads fields to survive a TextRank write, got %+v", doc) } - if doc.GoodReadsResourceID != "work-id" || doc.GoodReadsLegacyBookID != 12207444 { + if mustGoodReadsData(t, doc).ResourceID != "work-id" || mustGoodReadsData(t, doc).LegacyBookID != 12207444 { t.Errorf("expected GoodReads identifiers to survive a TextRank write, got %+v", doc) } if !doc.TextRankEnriched { @@ -318,10 +342,12 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { // A GoodReads write must keep TextRank fields written after the document was read. stale, _ = idx.documentByIndexID(goodReadsTestID) - stale.GoodReadsEnriched = false + updateGoodReadsData(t, &stale, func(data *biblioreads.DocumentData) { + data.Enriched = false + }) doc.TextRankWords = []string{"guinevere"} idx.documentsMu.Lock() - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } idx.documentsMu.Unlock() @@ -348,7 +374,7 @@ func TestEnrichGoodReadsWithoutSourceIsNoop(t *testing.T) { if err != nil { t.Fatal(err) } - if doc.GoodReadsEnriched { + if mustGoodReadsData(t, doc).Enriched { t.Error("expected document not to be GoodReads-enriched when the source is disabled") } } @@ -363,8 +389,10 @@ func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if err != nil { t.Fatal(err) } - doc.DataSourceLink = "/book/show/1-existing" - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + updateGoodReadsData(t, &doc, func(data *biblioreads.DocumentData) { + data.URL = "/book/show/1-existing" + }) + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } @@ -379,7 +407,7 @@ func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if err != nil { t.Fatal(err) } - if !doc.GoodReadsEnriched || doc.DataSourceLink != "/book/show/1-existing" { + if !mustGoodReadsData(t, doc).Enriched || mustGoodReadsData(t, doc).URL != "/book/show/1-existing" { t.Errorf("expected existing link kept and document marked enriched, got %+v", doc) } } @@ -406,7 +434,7 @@ func TestGoodReadsBook(t *testing.T) { if err != nil { t.Fatal(err) } - if !stored.GoodReadsEnriched || stored.DataSourceLink != wantDataSourceLink { + if !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != wantDataSourceLink { t.Errorf("expected the link to be persisted, got %+v", stored) } @@ -460,7 +488,7 @@ func TestGoodReadsBookReturnsErrorWhenUnavailable(t *testing.T) { if _, err := idx.GoodReadsBook(context.Background(), doc); !errors.Is(err, biblioreads.ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } - if doc, _ = idx.documentByIndexID(goodReadsTestID); doc.GoodReadsEnriched { + if doc, _ = idx.documentByIndexID(goodReadsTestID); mustGoodReadsData(t, doc).Enriched { t.Error("expected document to stay unenriched for a later retry") } } @@ -580,7 +608,7 @@ func TestGoodReadsBookMatchesKennedyMiddleName(t *testing.T) { ID: goodReadsTestID, AuthorsSlugs: []string{"paul-kennedy", "another-writer"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"Paul Kennedy", "Another Writer"}}, } - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } got, err := idx.GoodReadsBook(context.Background(), doc) @@ -588,7 +616,7 @@ func TestGoodReadsBookMatchesKennedyMiddleName(t *testing.T) { t.Fatalf("expected Kennedy's book and rating, got %+v, %v", got, err) } stored, err := idx.documentByIndexID(doc.ID) - if err != nil || !stored.GoodReadsEnriched || stored.DataSourceLink != bookURL || stored.DataSourceScoreValue != 4.14 { + if err != nil || !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != bookURL || stored.DataSourceScoreValue != 4.14 { t.Fatalf("expected persisted book match and rating, got %+v, %v", stored, err) } author, err := idx.Author("paul-kennedy", "") @@ -775,7 +803,7 @@ func TestGoodReadsBookMatchesDolnickSubtitle(t *testing.T) { ID: goodReadsTestID, AuthorsSlugs: []string{"edward-dolnick"}, Metadata: metadata.Metadata{Title: title, Authors: []string{author}}, } - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } got, err := idx.GoodReadsBook(context.Background(), doc) @@ -783,7 +811,7 @@ func TestGoodReadsBookMatchesDolnickSubtitle(t *testing.T) { t.Fatalf("expected Dolnick's book and score, got %+v, %v", got, err) } stored, err := idx.documentByIndexID(doc.ID) - if err != nil || !stored.GoodReadsEnriched || stored.DataSourceLink != bookURL { + if err != nil || !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != bookURL { t.Fatalf("expected subtitle match persisted, got %+v, %v", stored, err) } if want := []string{title + " " + author}; !slices.Equal(source.Queries(), want) { @@ -807,7 +835,7 @@ func TestGoodReadsTitleFallback(t *testing.T) { }} idx := newGoodReadsTestIndexer(t, source) doc := Document{ID: goodReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } if fallbackUnavailable { @@ -819,7 +847,7 @@ func TestGoodReadsTitleFallback(t *testing.T) { t.Fatalf("expected unavailable error, got %+v, %v", got, err) } stored, readErr := idx.documentByIndexID(doc.ID) - if readErr != nil || stored.GoodReadsEnriched { + if readErr != nil || mustGoodReadsData(t, stored).Enriched { t.Fatalf("expected document left for retry, got %+v, %v", stored, readErr) } source.queryErrors = nil @@ -853,7 +881,7 @@ func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { source := &fakeGoodReadsSource{byQuery: map[string][]biblioreads.SearchResult{title: results}} idx := newGoodReadsTestIndexer(t, source) doc := Document{ID: goodReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } got, err := idx.GoodReadsBook(context.Background(), doc) @@ -864,7 +892,7 @@ func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { if err != nil { t.Fatal(err) } - if !stored.GoodReadsEnriched || stored.DataSourceLink != wantURL { + if !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != wantURL { t.Fatalf("expected persisted omnibus match, got %+v", stored) } author, err := idx.Author("j-r-r-tolkien", "") @@ -886,11 +914,13 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { if err != nil { t.Fatal(err) } - doc.DataSourceLink = "/book/show/1-wrong" - doc.GoodReadsResourceID = "wrong-work" - doc.GoodReadsLegacyBookID = 1 - doc.GoodReadsEnriched = true - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + updateGoodReadsData(t, &doc, func(data *biblioreads.DocumentData) { + data.URL = "/book/show/1-wrong" + data.ResourceID = "wrong-work" + data.LegacyBookID = 1 + data.Enriched = true + }) + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } if err := idx.AddLibrary(10, true, 1); err != nil { @@ -903,11 +933,11 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { if err != nil { t.Fatal(err) } - if doc.DataSourceLink != wantDataSourceLink { - t.Fatalf("expected refreshed book link, got %q", doc.DataSourceLink) + if mustGoodReadsData(t, doc).URL != wantDataSourceLink { + t.Fatalf("expected refreshed book link, got %q", mustGoodReadsData(t, doc).URL) } - if doc.GoodReadsResourceID != "" || doc.GoodReadsLegacyBookID != 0 { - t.Fatalf("expected old identifiers to be cleared, got %q and %d", doc.GoodReadsResourceID, doc.GoodReadsLegacyBookID) + if mustGoodReadsData(t, doc).ResourceID != "" || mustGoodReadsData(t, doc).LegacyBookID != 0 { + t.Fatalf("expected old identifiers to be cleared, got %q and %d", mustGoodReadsData(t, doc).ResourceID, mustGoodReadsData(t, doc).LegacyBookID) } } @@ -915,28 +945,32 @@ func TestStoreGoodReadsDetails(t *testing.T) { for _, scenario := range []string{"linked", "newly linked", "changed metadata", "changed match", "deleted", "partial identifiers"} { t.Run(scenario, func(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) - doc := Document{ - ID: goodReadsTestID, - Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, - DataSourceLink: wantDataSourceLink, - GoodReadsEnriched: true, - TextRankWords: []string{"dragon"}, - } + doc := withGoodReadsData(t, Document{ + ID: goodReadsTestID, + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + + TextRankWords: []string{"dragon"}, + }, biblioreads.DocumentData{URL: wantDataSourceLink, Enriched: true}) + stale := doc resourceID := "work-id" switch scenario { case "newly linked": - stale.DataSourceLink = "" - stale.GoodReadsEnriched = false + updateGoodReadsData(t, &stale, func(data *biblioreads.DocumentData) { + data.URL = "" + data.Enriched = false + }) case "changed metadata": doc.Title = "Another title" case "changed match": - doc.DataSourceLink = "/book/show/2-other" + updateGoodReadsData(t, &doc, func(data *biblioreads.DocumentData) { + data.URL = "/book/show/2-other" + }) case "partial identifiers": resourceID = "" } if scenario != "deleted" { - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } } @@ -963,7 +997,7 @@ func TestStoreGoodReadsDetails(t *testing.T) { if writes.count != 0 { t.Fatal("stale details must not cause any index writes") } - if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || stored.DataSourceScoreValue != 0 { + if mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 || stored.DataSourceScoreValue != 0 { t.Fatalf("unexpected details persisted: %+v", stored) } return @@ -972,13 +1006,13 @@ func TestStoreGoodReadsDetails(t *testing.T) { if scenario == "partial identifiers" { wantLegacyBookID = 0 } - if stored.GoodReadsResourceID != resourceID || stored.GoodReadsLegacyBookID != wantLegacyBookID { + if mustGoodReadsData(t, stored).ResourceID != resourceID || mustGoodReadsData(t, stored).LegacyBookID != wantLegacyBookID { t.Fatalf("identifiers not persisted: %+v", stored) } if stored.DataSourceScoreValue != 4.12 { t.Fatalf("rating not persisted with identifiers: %+v", stored) } - if !stored.GoodReadsEnriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { + if !mustGoodReadsData(t, stored).Enriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("existing enrichment was lost: %+v", stored) } book.Rating = "4.120" @@ -1025,32 +1059,30 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) - doc := Document{ - ID: goodReadsTestID, DataSourceLink: wantDataSourceLink, + doc := withGoodReadsData(t, Document{ + ID: goodReadsTestID, DataSourceScoreValue: 3.82, - DataSourceScoreCount: "2,285", - GoodReadsResourceID: "old-work", GoodReadsLegacyBookID: 1, - } - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + }, biblioreads.DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 1}) + + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - if err := idx.StoreGoodReadsDetails(doc, doc.DataSourceLink, tt.book); err != nil { + if err := idx.StoreGoodReadsDetails(doc, mustGoodReadsData(t, doc).URL, tt.book); err != nil { t.Fatal(err) } stored, err := idx.documentByIndexID(doc.ID) if err != nil { t.Fatal(err) } - if stored.DataSourceScoreValue != parseGoodReadsRating(tt.wantRating, doc.ID) || - stored.GoodReadsResourceID != tt.wantWork || stored.GoodReadsLegacyBookID != tt.wantID { + if stored.DataSourceScoreValue != parseGoodReadsRating(tt.wantRating, doc.ID) || mustGoodReadsData(t, stored).ResourceID != tt.wantWork || mustGoodReadsData(t, stored).LegacyBookID != tt.wantID { t.Fatalf("partial details changed fields that were not supplied: %+v", stored) } wantCount := tt.wantCount if wantCount == "" { - wantCount = doc.DataSourceScoreCount + wantCount = mustGoodReadsData(t, doc).RatingCount } - if stored.DataSourceScoreCount != wantCount { - t.Fatalf("expected rating count %q, got %q", wantCount, stored.DataSourceScoreCount) + if mustGoodReadsData(t, stored).RatingCount != wantCount { + t.Fatalf("expected rating count %q, got %q", wantCount, mustGoodReadsData(t, stored).RatingCount) } }) } @@ -1058,18 +1090,15 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { func TestUpdateDataSourceLink(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) - doc := Document{ + doc := withGoodReadsData(t, Document{ ID: goodReadsTestID, Slug: "la-reina-dragon", - Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, - TextRankWords: []string{"dragon"}, - DataSourceLink: wantDataSourceLink, - DataSourceScoreValue: 3.82, - DataSourceScoreCount: "2,285", - GoodReadsResourceID: "old-work", - GoodReadsLegacyBookID: 12207444, - GoodReadsEnriched: true, - } - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + TextRankWords: []string{"dragon"}, + + DataSourceScoreValue: 3.82, + }, biblioreads.DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 12207444, Enriched: true}) + + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } const newLink = "provider-specific-document-reference" @@ -1080,15 +1109,14 @@ func TestUpdateDataSourceLink(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.DataSourceLink != newLink || !stored.GoodReadsEnriched || - stored.DataSourceScoreValue != 0 || stored.DataSourceScoreCount != "" || - stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { + if mustGoodReadsData(t, stored).URL != newLink || !mustGoodReadsData(t, stored).Enriched || + stored.DataSourceScoreValue != 0 || mustGoodReadsData(t, stored).RatingCount != "" || mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 { t.Fatalf("updating review link must clear cached score and provider lookup data: %+v", stored) } if stored.Title != doc.Title || stored.Slug != doc.Slug || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("review-link update changed unrelated document data: %+v", stored) } - if err := idx.StoreGoodReadsDetails(doc, doc.DataSourceLink, biblioreads.BookResponse{Rating: "4.5"}); err == nil { + if err := idx.StoreGoodReadsDetails(doc, mustGoodReadsData(t, doc).URL, biblioreads.BookResponse{Rating: "4.5"}); err == nil { t.Fatal("stale score write must not overwrite the updated link") } } diff --git a/internal/index/bleve_textrank_write.go b/internal/index/bleve_textrank_write.go index f0f38104..8600c504 100644 --- a/internal/index/bleve_textrank_write.go +++ b/internal/index/bleve_textrank_write.go @@ -87,8 +87,10 @@ func (b *BleveIndexer) enrichTextRankAndReindex(document Document) { enriched := b.rankDocument(document) documents := []Document{enriched} b.documentsMu.Lock() - b.preserveGoodReadsLocked(documents) - err := b.documentsIdx.Index(enriched.ID, documents[0]) + err := b.preserveDataSourcesLocked(documents) + if err == nil { + err = IndexDocument(b.documentsIdx, enriched.ID, documents[0]) + } b.documentsMu.Unlock() if err != nil { log.Printf("Error indexing TextRank-enriched document %s: %s\n", enriched.ID, err) @@ -266,10 +268,14 @@ func (b *BleveIndexer) EnrichTextRankKeywords(batchSize, workers int) error { enriched := b.rankDocuments(chunk, workers) b.documentsMu.Lock() - b.preserveGoodReadsLocked(enriched) + if err := b.preserveDataSourcesLocked(enriched); err != nil { + b.documentsMu.Unlock() + b.endTextRankEnrichment() + return err + } batch := b.documentsIdx.NewBatch() for _, document := range enriched { - if err := batch.Index(document.ID, document); err != nil { + if err := IndexDocument(batch, document.ID, document); err != nil { log.Printf("Error indexing enriched document %s: %s\n", document.ID, err) } } @@ -545,10 +551,13 @@ func (b *BleveIndexer) rewriteCommonTextRankEntries(docCount uint64, batchSize i } b.documentsMu.Lock() - b.preserveGoodReadsLocked(documents) + if err := b.preserveDataSourcesLocked(documents); err != nil { + b.documentsMu.Unlock() + return pruned, err + } batch := b.documentsIdx.NewBatch() for _, document := range documents { - if err := batch.Index(document.ID, document); err != nil { + if err := IndexDocument(batch, document.ID, document); err != nil { log.Printf("Error indexing document %s while pruning common TextRank entries: %s\n", document.ID, err) continue } diff --git a/internal/index/bleve_textrank_write_test.go b/internal/index/bleve_textrank_write_test.go index 36eea4cf..d869c730 100644 --- a/internal/index/bleve_textrank_write_test.go +++ b/internal/index/bleve_textrank_write_test.go @@ -36,7 +36,7 @@ func TestCommonTextRankEntriesNeedPrune(t *testing.T) { }) for i := 0; i < 4; i++ { - if err := documentsIndexMem.Index(strconv.Itoa(i), Document{ID: strconv.Itoa(i)}); err != nil { + if err := IndexDocument(documentsIndexMem, strconv.Itoa(i), Document{ID: strconv.Itoa(i)}); err != nil { t.Fatal(err) } } @@ -60,7 +60,7 @@ func TestCommonTextRankEntriesNeedPrune(t *testing.T) { // PruneChangeTriggerRatio is 0.5 of the 4 documents recorded above (i.e. // 2), so 3 more documents is enough to cross it. for i := 0; i < 3; i++ { - if err := documentsIndexMem.Index("extra"+strconv.Itoa(i), Document{ID: "extra" + strconv.Itoa(i)}); err != nil { + if err := IndexDocument(documentsIndexMem, "extra"+strconv.Itoa(i), Document{ID: "extra" + strconv.Itoa(i)}); err != nil { t.Fatal(err) } } @@ -83,7 +83,7 @@ func TestPruningReportsProgress(t *testing.T) { TextRankPhrases: []string{"phrase " + id}, TextRankWords: []string{"word" + id}, } - if err := documentsIndexMem.Index(id, document); err != nil { + if err := IndexDocument(documentsIndexMem, id, document); err != nil { t.Fatal(err) } } @@ -140,7 +140,7 @@ func TestPruneCommonTextRankEntriesRewritesCommonEntries(t *testing.T) { TextRankPhrases: []string{"common phrase", "commonword distinctive" + id, "unique phrase " + id}, TextRankWords: []string{"commonword", "unique" + id}, } - if err := documentsIndexMem.Index(id, document); err != nil { + if err := IndexDocument(documentsIndexMem, id, document); err != nil { t.Fatal(err) } } diff --git a/internal/index/document.go b/internal/index/document.go index 3117c786..e9459cd3 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -1,6 +1,9 @@ package index import ( + "encoding/json" + "fmt" + "maps" "time" "github.com/rickb777/date/v2" @@ -82,20 +85,70 @@ type Document struct { // in the background. See documentsNeedingTextRank, which drives that pass // off this field so it survives (and resumes correctly across) restarts. TextRankEnriched bool - // DataSourceLink is the configured data source's document link. - // It is stored but not indexed. - DataSourceLink string + // DataSources holds opaque provider-owned JSON objects, stored but not indexed. + DataSources map[string]json.RawMessage `json:"-"` // DataSourceScoreValue is the configured data source's score, stored and indexed for filtering and sorting. DataSourceScoreValue float64 - DataSourceScoreCount string - // GoodReadsResourceID identifies the work; GoodReadsLegacyBookID identifies - // the edition. Both are stored only, populated when book details are fetched, - // and used together by BiblioReads to retrieve reviews. - GoodReadsResourceID string - GoodReadsLegacyBookID int - // GoodReadsEnriched is false until the GoodReads enrichment has looked - // this document up, whether or not a match was found. - GoodReadsEnriched bool +} + +// DataSourceData decodes a provider's payload. Missing providers have zero-valued data. +func DataSourceData[T any](document Document, source string) (T, error) { + var data T + if raw, ok := document.DataSources[source]; ok { + if err := json.Unmarshal(raw, &data); err != nil { + return data, fmt.Errorf("decoding %s data for document %s: %w", source, document.ID, err) + } + } + return data, nil +} + +// SetDataSourceData replaces one provider's payload without mutating shared document maps. +func SetDataSourceData(document *Document, source string, data any) error { + raw, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("encoding %s data for document %s: %w", source, document.ID, err) + } + var object map[string]json.RawMessage + if err := json.Unmarshal(raw, &object); err != nil || object == nil { + return fmt.Errorf("data for provider %s must be a JSON object", source) + } + document.DataSources = maps.Clone(document.DataSources) + if document.DataSources == nil { + document.DataSources = make(map[string]json.RawMessage) + } + document.DataSources[source] = raw + return nil +} + +// IndexDocument stores provider JSON without exposing its contents to Bleve's dynamic mapping. +func IndexDocument(destination interface{ Index(string, any) error }, id string, document Document) error { + for source, raw := range document.DataSources { + var object map[string]json.RawMessage + if err := json.Unmarshal(raw, &object); err != nil || object == nil { + return fmt.Errorf("invalid JSON object for provider %s in document %s", source, id) + } + } + raw, err := json.Marshal(document.DataSources) + if err != nil { + return fmt.Errorf("encoding data sources for document %s: %w", id, err) + } + var enriched []string + for source, payload := range document.DataSources { + var state struct { + Enriched bool `json:"enriched"` + } + if err := json.Unmarshal(payload, &state); err != nil { + return fmt.Errorf("reading enrichment state for provider %s in document %s: %w", source, id, err) + } + if state.Enriched { + enriched = append(enriched, source) + } + } + return destination.Index(id, struct { + Document + DataSources string + EnrichedDataSources []string + }{document, string(raw), enriched}) } func (d Document) DataSourceScore() float64 { diff --git a/internal/index/document_data_test.go b/internal/index/document_data_test.go new file mode 100644 index 00000000..0801d413 --- /dev/null +++ b/internal/index/document_data_test.go @@ -0,0 +1,141 @@ +package index + +import ( + "context" + "encoding/json" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/metadata" +) + +func TestDocumentDataSourcesRoundTrip(t *testing.T) { + idx := newGoodReadsTestIndexer(t, nil) + for _, language := range []string{"es", ""} { + document := withGoodReadsData(t, Document{ + ID: "book-" + language, Slug: "book-" + language, + Metadata: metadata.Metadata{Title: "A book", Language: language, Format: "EPUB"}, + DataSourceScoreValue: 4.25, + }, biblioreads.DocumentData{URL: "/book/show/123", Enriched: true}) + other := map[string]any{ + "id": "opaqueprovideridentifier", "nested": map[string]any{"items": []any{"privatecatalogtoken", float64(42)}}, + } + if err := SetDataSourceData(&document, "other", other); err != nil { + t.Fatal(err) + } + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + got, err := idx.documentByIndexID(document.ID) + if err != nil { + t.Fatal(err) + } + decoded, err := DataSourceData[map[string]any](got, "other") + if err != nil || !reflect.DeepEqual(decoded, other) { + t.Fatalf("provider payload changed: %+v, %v", decoded, err) + } + if mustGoodReadsData(t, got) != mustGoodReadsData(t, document) || got.DataSourceScoreValue != 4.25 { + t.Fatalf("Goodreads payload or score changed: %+v", got) + } + for _, term := range []string{"opaqueprovideridentifier", "privatecatalogtoken", "goodreads", "other"} { + request := bleve.NewSearchRequest(bleve.NewMatchQuery(term)) + result, err := idx.documentsIdx.Search(request) + if err != nil || result.Total != 0 { + t.Fatalf("opaque data affected keyword search for %q: %+v, %v", term, result, err) + } + } + } + if _, total, err := idx.documentsNeedingGoodReads(10); err != nil || total != 0 { + t.Fatalf("enriched payloads were queued again: %d, %v", total, err) + } + pending := Document{ID: "other-only", Slug: "other-only", Metadata: metadata.Metadata{Title: "Pending book"}} + if err := SetDataSourceData(&pending, "other", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + if err := IndexDocument(idx.documentsIdx, pending.ID, pending); err != nil { + t.Fatal(err) + } + if documents, total, err := idx.documentsNeedingGoodReads(10); err != nil || total != 1 || documents[0].ID != pending.ID { + t.Fatalf("another provider marked Goodreads as enriched: %+v, %d, %v", documents, total, err) + } +} + +func TestDocumentDataSourcesRejectInvalidData(t *testing.T) { + idx := newGoodReadsTestIndexer(t, nil) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} + for _, invalid := range []any{nil, "not an object", []int{1}, make(chan int)} { + if err := SetDataSourceData(&document, "invalid", invalid); err == nil { + t.Fatalf("accepted non-object provider payload: %#v", invalid) + } + if document.DataSources != nil { + t.Fatal("failed update mutated document data") + } + } + for _, raw := range []string{`{"url":`, `null`, `[]`} { + document.DataSources = map[string]json.RawMessage{biblioreads.Source: json.RawMessage(raw)} + if err := IndexDocument(idx.documentsIdx, document.ID, document); err == nil { + t.Fatalf("indexed invalid payload: %s", raw) + } + } + document.DataSources = map[string]json.RawMessage{biblioreads.Source: json.RawMessage(`{"url":42}`)} + if _, err := idx.GoodReadsBook(context.Background(), document); err == nil { + t.Fatal("Goodreads accepted an incorrectly typed URL") + } +} + +func TestDocumentDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { + idx := newGoodReadsTestIndexer(t, &fakeGoodReadsSource{results: goodReadsTestResults}) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + stale, err := idx.documentByIndexID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + current := stale + other := map[string]any{"id": "external-id", "nested": map[string]any{"count": float64(3)}} + if err := SetDataSourceData(¤t, "other", other); err != nil { + t.Fatal(err) + } + if _, exists := stale.DataSources["other"]; exists { + t.Fatal("updating a provider mutated another document's map") + } + if err := IndexDocument(idx.documentsIdx, current.ID, current); err != nil { + t.Fatal(err) + } + if _, err := idx.enrichGoodReads(context.Background(), stale); err != nil { + t.Fatal(err) + } + if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, biblioreads.BookResponse{ + Rating: "4.5", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, + }); err != nil { + t.Fatal(err) + } + if err := idx.UpdateDataSourceLink(current.ID, "/book/show/456"); err != nil { + t.Fatal(err) + } + stale.TextRankWords = []string{"updated"} + documents := []Document{stale} + idx.documentsMu.Lock() + err = idx.preserveDataSourcesLocked(documents) + if err == nil { + err = IndexDocument(idx.documentsIdx, stale.ID, documents[0]) + } + idx.documentsMu.Unlock() + if err != nil { + t.Fatal(err) + } + got, err := idx.documentByIndexID(stale.ID) + if err != nil { + t.Fatal(err) + } + decoded, err := DataSourceData[map[string]any](got, "other") + if err != nil || !reflect.DeepEqual(decoded, other) { + t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) + } + if data := mustGoodReadsData(t, got); data != (biblioreads.DocumentData{URL: "/book/show/456", Enriched: true}) { + t.Fatalf("stale write restored old Goodreads state: %+v", data) + } +} diff --git a/internal/index/goodreads_concurrency_test.go b/internal/index/goodreads_concurrency_test.go index b4443492..b0eb6349 100644 --- a/internal/index/goodreads_concurrency_test.go +++ b/internal/index/goodreads_concurrency_test.go @@ -37,7 +37,7 @@ func TestGoodReadsEnrichmentUsesBoundedConcurrency(t *testing.T) { ID: fmt.Sprintf("book-%d.pdf", i), Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, } - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } documents = append(documents, doc) diff --git a/internal/index/goodreads_fields_test.go b/internal/index/goodreads_fields_test.go index c0ade3f8..692f5935 100644 --- a/internal/index/goodreads_fields_test.go +++ b/internal/index/goodreads_fields_test.go @@ -8,10 +8,28 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search/query" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" ) +func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { + t.Helper() + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + t.Fatal(err) + } + return data +} + +func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + t.Fatal(err) + } + return document +} + func TestReviewScoreSorting(t *testing.T) { docs := []index.Document{ {ID: "high", Slug: "high", DataSourceScoreValue: 4.75}, @@ -104,7 +122,7 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) legacyBookID = 12207444 ) for _, lang := range []string{"es", ""} { - doc := index.Document{ + doc := withGoodReadsData(t, index.Document{ ID: "doc-" + lang, Slug: "doc-" + lang, Metadata: metadata.Metadata{ @@ -113,14 +131,12 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) Language: lang, Format: "EPUB", }, - AuthorsSlugs: []string{"alice-borchardt"}, - DataSourceLink: bookURL, - DataSourceScoreValue: 4.25, - DataSourceScoreCount: "2285", - GoodReadsResourceID: resourceID, - GoodReadsLegacyBookID: legacyBookID, - } - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + AuthorsSlugs: []string{"alice-borchardt"}, + + DataSourceScoreValue: 4.25, + }, biblioreads.DocumentData{URL: bookURL, RatingCount: "2285", ResourceID: resourceID, LegacyBookID: legacyBookID}) + + if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } @@ -128,20 +144,20 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) if err != nil { t.Fatal(err) } - if got.DataSourceLink != bookURL { - t.Errorf("language %q: expected stored book URL, got %q", lang, got.DataSourceLink) + if mustGoodReadsData(t, got).URL != bookURL { + t.Errorf("language %q: expected stored book URL, got %q", lang, mustGoodReadsData(t, got).URL) } if got.DataSourceScoreValue != 4.25 { t.Errorf("language %q: expected stored rating, got %v", lang, got.DataSourceScoreValue) } - if got.DataSourceScoreCount != "2285" { - t.Errorf("language %q: expected stored rating count, got %q", lang, got.DataSourceScoreCount) + if mustGoodReadsData(t, got).RatingCount != "2285" { + t.Errorf("language %q: expected stored rating count, got %q", lang, mustGoodReadsData(t, got).RatingCount) } - if got.GoodReadsResourceID != resourceID || got.GoodReadsLegacyBookID != legacyBookID { - t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, got.GoodReadsResourceID, got.GoodReadsLegacyBookID) + if mustGoodReadsData(t, got).ResourceID != resourceID || mustGoodReadsData(t, got).LegacyBookID != legacyBookID { + t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, mustGoodReadsData(t, got).ResourceID, mustGoodReadsData(t, got).LegacyBookID) } } - if err := documentsIndexMem.Index("unscored", index.Document{ + if err := index.IndexDocument(documentsIndexMem, "unscored", index.Document{ ID: "unscored", Slug: "unscored", Metadata: metadata.Metadata{ @@ -175,13 +191,14 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) } queries := map[string]bleve.SearchRequest{ - "book URL field term": *bleve.NewSearchRequest(fieldTermQuery("DataSourceLink", bookURL)), - "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsAuthorURL", authorURL)), - "resource ID field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsResourceID", resourceID)), - "legacy ID numeric field": *bleve.NewSearchRequest(legacyBookIDQuery(legacyBookID)), - "_all match": *bleve.NewSearchRequest(bleve.NewMatchQuery("zyxwvbook")), - "_all term": *bleve.NewSearchRequest(bleve.NewTermQuery(bookURL)), - "query string field": *bleve.NewSearchRequest(bleve.NewQueryStringQuery(`DataSourceLink:"` + bookURL + `"`)), + "provider data field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources", bookURL)), + "provider URL field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources.goodreads.url", bookURL)), + "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsAuthorURL", authorURL)), + "resource ID field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources.goodreads.resourceId", resourceID)), + "legacy ID numeric field": *bleve.NewSearchRequest(legacyBookIDQuery(legacyBookID)), + "_all match": *bleve.NewSearchRequest(bleve.NewMatchQuery("zyxwvbook")), + "_all term": *bleve.NewSearchRequest(bleve.NewTermQuery(bookURL)), + "query string field": *bleve.NewSearchRequest(bleve.NewQueryStringQuery(`DataSources:"` + bookURL + `"`)), } for name, req := range queries { res, err := documentsIndexMem.Search(&req) @@ -249,7 +266,7 @@ func TestReviewScoreSortingMissingFieldsAndTies(t *testing.T) { func legacyBookIDQuery(id int) *query.NumericRangeQuery { value := float64(id) q := bleve.NewNumericRangeQuery(&value, &value) - q.SetField("GoodReadsLegacyBookID") + q.SetField("DataSources.goodreads.legacyBookId") return q } diff --git a/internal/index/rebuild_authors_test.go b/internal/index/rebuild_authors_test.go index 82dbec9d..b9cf9971 100644 --- a/internal/index/rebuild_authors_test.go +++ b/internal/index/rebuild_authors_test.go @@ -35,7 +35,7 @@ func TestRebuildAuthorsFromDocuments(t *testing.T) { }, } for _, doc := range docs { - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } } diff --git a/internal/index/search_filter_scoring_test.go b/internal/index/search_filter_scoring_test.go index 816d0989..0873f3a9 100644 --- a/internal/index/search_filter_scoring_test.go +++ b/internal/index/search_filter_scoring_test.go @@ -68,7 +68,7 @@ func TestAddFiltersDoesNotAffectScoring(t *testing.T) { } idx := NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, Config{}) for _, doc := range docs { - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + if err := IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } } diff --git a/internal/index/textrank_related_test.go b/internal/index/textrank_related_test.go index 4751abdd..2fd7ca1d 100644 --- a/internal/index/textrank_related_test.go +++ b/internal/index/textrank_related_test.go @@ -35,7 +35,7 @@ func newTextRankTestIndexWithConfig(t *testing.T, docs []index.Document, cfg ind idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, cfg) for _, doc := range docs { - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } } diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index a0536a17..ace23c84 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -54,7 +54,12 @@ func (b *BiblioReads) AuthorLink(author index.Author, _ string) datasourcemodel. // DocumentLink returns the indexed book link without making a network request. func (b *BiblioReads) DocumentLink(document index.Document) datasourcemodel.DataSourceLink { - return goodReadsDataSourceLink(document.DataSourceLink, "/book/show/") + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + log.Println(err) + return datasourcemodel.DataSourceLink{} + } + return goodReadsDataSourceLink(data.URL, "/book/show/") } func goodReadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink { @@ -65,6 +70,18 @@ func goodReadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink return link } +// StoredScore builds a score from indexed data without making a network request. +func (b *BiblioReads) StoredScore(document index.Document) (Score, error) { + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + return Score{}, err + } + return Score{ + Source: b.Name(), Value: fmt.Sprintf("%.2f", document.DataSourceScoreValue), + Count: data.RatingCount, URL: goodReadsDataSourceLink(data.URL, goodReadsBookPrefix).URL, + }, nil +} + // CanEditDocumentLink reports that Goodreads book links can be corrected. func (b *BiblioReads) CanEditDocumentLink() bool { return true @@ -105,7 +122,11 @@ func bookID(bookURL string) (string, error) { } func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblioreads.BookResponse, error) { - id, err := bookID(document.DataSourceLink) + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + return biblioreads.BookResponse{}, err + } + id, err := bookID(data.URL) if err != nil { return biblioreads.BookResponse{}, err } @@ -118,7 +139,7 @@ func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblio book.Rating = fmt.Sprintf("%.2f", document.DataSourceScoreValue) details.Rating = "" } - if err := b.index.StoreGoodReadsDetails(document, document.DataSourceLink, details); err != nil { + if err := b.index.StoreGoodReadsDetails(document, data.URL, details); err != nil { log.Printf("Error saving Goodreads details for %s: %s", document.ID, err) } return book, nil @@ -131,7 +152,11 @@ func isZeroRating(rating string) bool { func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score, error) { score := Score{Source: "Goodreads"} - if document.DataSourceLink == "" { + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + return score, err + } + if data.URL == "" { result, err := b.index.GoodReadsBook(ctx, document) if err != nil { return score, err @@ -142,7 +167,10 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score if _, err := bookID(result.BookURL); err != nil { return score, err } - document.DataSourceLink = result.BookURL + data.URL = result.BookURL + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + return score, err + } score.Value, score.Count, score.URL = result.Rating, result.RatingCount, b.DocumentLink(document).URL return score, nil } @@ -155,7 +183,11 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score } func (b *BiblioReads) Reviews(ctx context.Context, document index.Document, language string) (Page, error) { - if document.DataSourceLink == "" { + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + return Page{}, err + } + if data.URL == "" { return Page{}, nil } book, err := b.book(ctx, document) @@ -165,7 +197,7 @@ func (b *BiblioReads) Reviews(ctx context.Context, document index.Document, lang if book.Rating == "" { return Page{}, nil } - resourceID, legacyBookID := document.GoodReadsResourceID, document.GoodReadsLegacyBookID + resourceID, legacyBookID := data.ResourceID, data.LegacyBookID if book.ResourceID != "" && book.LegacyBookID > 0 { resourceID, legacyBookID = book.ResourceID, book.LegacyBookID } diff --git a/internal/reviews/biblioreads_test.go b/internal/reviews/biblioreads_test.go index afecb007..588597c2 100644 --- a/internal/reviews/biblioreads_test.go +++ b/internal/reviews/biblioreads_test.go @@ -10,6 +10,14 @@ import ( "github.com/svera/coreander/v5/internal/reviews" ) +func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + t.Fatal(err) + } + return document +} + func TestBiblioReadsDataSourceLinks(t *testing.T) { provider := reviews.NewBiblioReads(nil, nil) author := provider.AuthorLink(index.Author{ @@ -20,9 +28,7 @@ func TestBiblioReadsDataSourceLinks(t *testing.T) { t.Fatalf("unexpected Goodreads author link: %+v", author) } - document := provider.DocumentLink(index.Document{ - DataSourceLink: "/book/show/456.A_Book", - }) + document := provider.DocumentLink(withGoodReadsData(t, index.Document{}, biblioreads.DocumentData{URL: "/book/show/456.A_Book"})) if document.URL != "https://www.goodreads.com/book/show/456.A_Book" || document.Source != "Goodreads" || document.Icon != "/images/goodreads.svg" || !document.IconInvertible { t.Fatalf("unexpected Goodreads document link: %+v", document) diff --git a/internal/webserver/document_detail_test.go b/internal/webserver/document_detail_test.go index 3f17cea5..bb02c6b4 100644 --- a/internal/webserver/document_detail_test.go +++ b/internal/webserver/document_detail_test.go @@ -183,7 +183,7 @@ func TestRelatedDocumentHeadersShowFullTotals(t *testing.T) { Metadata: metadata.Metadata{Title: "Source", Series: "Collection", Format: "EPUB"}, TextRankPhrases: []string{"atomic bomb", "los alamos"}, } - if err := documents.Index(source.ID, source); err != nil { + if err := index.IndexDocument(documents, source.ID, source); err != nil { t.Fatal(err) } for i := 0; i < 6; i++ { @@ -198,7 +198,7 @@ func TestRelatedDocumentHeadersShowFullTotals(t *testing.T) { } else { doc.TextRankPhrases = source.TextRankPhrases } - if err := documents.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documents, doc.ID, doc); err != nil { t.Fatal(err) } } diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index ca327140..6a553299 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -41,6 +41,23 @@ type fakeBookSource struct { reviewCalls []reviewCall } +func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { + t.Helper() + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + t.Fatal(err) + } + return data +} + +func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + t.Fatal(err) + } + return document +} + func TestDocumentReviewsCanSelectSupportedLanguage(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "4", ResourceID: "work-id", LegacyBookID: 1, @@ -326,26 +343,21 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide goodReadsLinks := reviews.NewBiblioReads(nil, idx) t.Cleanup(func() { _ = idx.Close() }) - for _, doc := range []index.Document{ - { - ID: "linked.epub", Slug: "linked", - Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, - DataSourceLink: "/book/show/12207444-la-reina-dragon", - DataSourceScoreValue: 3.82, - GoodReadsEnriched: true, - }, - { - ID: "unlinked.epub", Slug: "unlinked", - Metadata: metadata.Metadata{Title: "Unlinked", Authors: []string{"Jane Doe", "John Doe"}, Format: "EPUB"}, - AuthorsSlugs: []string{"jane-doe", "john-doe"}, - }, - { - ID: "unmatched.epub", Slug: "unmatched", - Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, - GoodReadsEnriched: true, - }, + for _, doc := range []index.Document{withGoodReadsData(t, index.Document{ + ID: "linked.epub", Slug: "linked", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, + + DataSourceScoreValue: 3.82, + }, biblioreads.DocumentData{URL: "/book/show/12207444-la-reina-dragon", Enriched: true}), { + ID: "unlinked.epub", Slug: "unlinked", + Metadata: metadata.Metadata{Title: "Unlinked", Authors: []string{"Jane Doe", "John Doe"}, Format: "EPUB"}, + AuthorsSlugs: []string{"jane-doe", "john-doe"}, + }, withGoodReadsData(t, index.Document{ + ID: "unmatched.epub", Slug: "unmatched", + Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, + }, biblioreads.DocumentData{Enriched: true}), } { - if err := documents.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documents, doc.ID, doc); err != nil { t.Fatal(err) } } @@ -428,7 +440,7 @@ func TestDocumentDetailGoodReadsRating(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.GoodReadsResourceID != source.book.ResourceID || stored.GoodReadsLegacyBookID != source.book.LegacyBookID { + if mustGoodReadsData(t, stored).ResourceID != source.book.ResourceID || mustGoodReadsData(t, stored).LegacyBookID != source.book.LegacyBookID { t.Fatalf("expected book identifiers to be stored, got %+v", stored) } } @@ -524,10 +536,9 @@ func TestDocumentUpdateGoodReads(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.DataSourceLink != "/book/show/1368314.El_Cercano_Oriente" || - stored.DataSourceScoreValue != 0 || - stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 || - !stored.GoodReadsEnriched { + if mustGoodReadsData(t, stored).URL != "/book/show/1368314.El_Cercano_Oriente" || + stored.DataSourceScoreValue != 0 || mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 || + !mustGoodReadsData(t, stored).Enriched { t.Fatalf("expected correction persisted without stale rating/review identifiers: %+v", stored) } source.book = biblioreads.BookResponse{Rating: "4.2", ResourceID: "new-work", LegacyBookID: 1368314} @@ -568,7 +579,7 @@ func TestDocumentUpdateGoodReadsValidation(t *testing.T) { t.Fatalf("invalid input should be retained for correction, got %q", got) } stored, err := idx.Document("linked") - if err != nil || stored.DataSourceLink != "/book/show/12207444-la-reina-dragon" || stored.DataSourceScoreValue != 3.82 { + if err != nil || mustGoodReadsData(t, stored).URL != "/book/show/12207444-la-reina-dragon" || stored.DataSourceScoreValue != 3.82 { t.Fatalf("invalid input must not change the stored match: %+v, %v", stored, err) } } @@ -664,14 +675,14 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.DataSourceLink != "/book/show/1-unlinked" || !stored.GoodReadsEnriched { + if mustGoodReadsData(t, stored).URL != "/book/show/1-unlinked" || !mustGoodReadsData(t, stored).Enriched { t.Errorf("expected the found link to be stored, got %+v", stored) } author, err := idx.Author("jane-doe", "") if err != nil || author.DataSourceLinks["goodreads"] != "/author/show/1.Jane_Doe" || author.DocumentCount != 1 { t.Fatalf("expected missing author entry created with URL and document count, got %+v, %v", author, err) } - if stored.GoodReadsResourceID != "" || stored.GoodReadsLegacyBookID != 0 { + if mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 { t.Fatalf("search-only lookup must not populate scraper identifiers, got %+v", stored) } @@ -743,7 +754,7 @@ func TestDocumentDetailGoodReadsIdentifiersWithoutRating(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.GoodReadsResourceID != "work-id" || stored.GoodReadsLegacyBookID != 12207444 { + if mustGoodReadsData(t, stored).ResourceID != "work-id" || mustGoodReadsData(t, stored).LegacyBookID != 12207444 { t.Fatalf("expected identifiers even without a rating, got %+v", stored) } @@ -758,7 +769,7 @@ func TestDocumentDetailGoodReadsIdentifiersWithoutRating(t *testing.T) { if err != nil { t.Fatal(err) } - if stored.GoodReadsResourceID != "work-id" || stored.GoodReadsLegacyBookID != 12207444 { + if mustGoodReadsData(t, stored).ResourceID != "work-id" || mustGoodReadsData(t, stored).LegacyBookID != 12207444 { t.Fatalf("incomplete response overwrote stored identifiers: %+v", stored) } } @@ -830,7 +841,7 @@ func TestDocumentDetailGoodReadsSearchDoesNotScrape(t *testing.T) { t.Errorf("expected %d searches and no book-scraper calls, got queries %q and ids %q", tt.queries, source.queries, source.ids) } stored, err := idx.Document("unlinked") - if err != nil || stored.GoodReadsEnriched != tt.enriched { + if err != nil || mustGoodReadsData(t, stored).Enriched != tt.enriched { t.Fatalf("unexpected stored lookup status: %+v, %v", stored, err) } }) @@ -1016,7 +1027,7 @@ func TestDocumentDetailGoodReadsReviews(t *testing.T) { t.Fatal("expected review dates in ISO format for client-side localization") } stored, err := idx.Document("linked") - if err != nil || stored.GoodReadsResourceID != "work-id" || stored.GoodReadsLegacyBookID != 12207444 { + if err != nil || mustGoodReadsData(t, stored).ResourceID != "work-id" || mustGoodReadsData(t, stored).LegacyBookID != 12207444 { t.Fatalf("expected review identifiers to be stored: %+v, %v", stored, err) } } diff --git a/internal/webserver/embedded/views/partials/document-metadata.html b/internal/webserver/embedded/views/partials/document-metadata.html index 1ff5532a..42d31b6b 100644 --- a/internal/webserver/embedded/views/partials/document-metadata.html +++ b/internal/webserver/embedded/views/partials/document-metadata.html @@ -15,8 +15,11 @@ {{ if .ReviewsEnabled }}
        • {{ end }} - {{if and .ShowScore .Document.DataSourceLink}} - {{template "partials/document-score" dict "Lang" .Lang "Source" "Goodreads" "Rating" (printf "%.2f" .Document.DataSourceScoreValue) "RatingCount" .Document.DataSourceScoreCount "Scored" (gt .Document.DataSourceScoreValue 0.0) "URL" (printf "https://www.goodreads.com%s" .Document.DataSourceLink)}} + {{if .ShowScore}} + {{$score := storedDocumentScore .Document.Document}} + {{if $score.URL}} + {{template "partials/document-score" dict "Lang" .Lang "Source" $score.Source "Rating" $score.Value "RatingCount" $score.Count "Scored" (gt .Document.DataSourceScoreValue 0.0) "URL" $score.URL}} + {{end}} {{end}} {{ if and .IllustratedMinAmount (ge .Document.Illustrations .IllustratedMinAmount) }}
      • {{t .Lang "Illustrated"}}
        • diff --git a/internal/webserver/infrastructure/template_engine.go b/internal/webserver/infrastructure/template_engine.go index afc18664..410b52df 100644 --- a/internal/webserver/infrastructure/template_engine.go +++ b/internal/webserver/infrastructure/template_engine.go @@ -12,6 +12,7 @@ import ( "github.com/gofiber/template/html/v3" "github.com/gosimple/slug" "github.com/svera/coreander/v5/internal/i18n" + "github.com/svera/coreander/v5/internal/reviews" ) // TemplateEngine builds the html/template engine. assetVersion is a cache-busting @@ -56,6 +57,7 @@ func TemplateEngine(viewsFS fs.FS, translator i18n.Translator, assetVersion stri }) engine.AddFunc("notLast", notLast[string]) + engine.AddFunc("storedDocumentScore", reviews.NewBiblioReads(nil, nil).StoredScore) engine.AddFunc("basename", func(path string) string { return filepath.Base(path) diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index cfc5ed1c..ea7c5d6d 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -9,6 +9,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v3" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/webserver" @@ -280,13 +281,13 @@ func TestSearchResultsShowStoredDataSourceScore(t *testing.T) { {score: 5, want: "5.00"}, } { t.Run(tt.want, func(t *testing.T) { - app, _ := newSearchResultsTestApp(t, []index.Document{{ + app, _ := newSearchResultsTestApp(t, []index.Document{withGoodReadsData(t, index.Document{ ID: "linked.epub", Slug: "linked", - Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, - DataSourceLink: "/provider/items/linked", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, + DataSourceScoreValue: tt.score, - DataSourceScoreCount: "2,285", - }}) + }, biblioreads.DocumentData{URL: "/book/show/linked", RatingCount: "2,285"}), + }) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) if err != nil { t.Fatal(err) @@ -303,22 +304,19 @@ func TestSearchResultsShowStoredDataSourceScore(t *testing.T) { } func TestSearchResultsSortByDataSourceScore(t *testing.T) { - app, _ := newSearchResultsTestApp(t, []index.Document{ - { - ID: "linked.epub", Slug: "linked", - Metadata: metadata.Metadata{Title: "Linked", Format: "EPUB"}, - DataSourceLink: "/provider/items/linked", DataSourceScoreValue: 3.2, - }, - { - ID: "unlinked.epub", Slug: "unlinked", - Metadata: metadata.Metadata{Title: "Unlinked", Format: "EPUB"}, - DataSourceLink: "/provider/items/unlinked", DataSourceScoreValue: 4.1, - }, - { - ID: "unmatched.epub", Slug: "unmatched", - Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, - DataSourceLink: "/provider/items/unmatched", DataSourceScoreValue: 4.8, - }, + app, _ := newSearchResultsTestApp(t, []index.Document{withGoodReadsData(t, index.Document{ + ID: "linked.epub", Slug: "linked", + Metadata: metadata.Metadata{Title: "Linked", Format: "EPUB"}, + DataSourceScoreValue: 3.2, + }, biblioreads.DocumentData{URL: "/book/show/linked"}), withGoodReadsData(t, index.Document{ + ID: "unlinked.epub", Slug: "unlinked", + Metadata: metadata.Metadata{Title: "Unlinked", Format: "EPUB"}, + DataSourceScoreValue: 4.1, + }, biblioreads.DocumentData{URL: "/book/show/unlinked"}), withGoodReadsData(t, index.Document{ + ID: "unmatched.epub", Slug: "unmatched", + Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, + DataSourceScoreValue: 4.8, + }, biblioreads.DocumentData{URL: "/book/show/unmatched"}), }) for _, tt := range []struct { direction string @@ -346,11 +344,11 @@ func TestSearchResultsSortByDataSourceScore(t *testing.T) { } func TestSearchResultsShowStoredZeroRatingAsNotScored(t *testing.T) { - app, _ := newSearchResultsTestApp(t, []index.Document{{ + app, _ := newSearchResultsTestApp(t, []index.Document{withGoodReadsData(t, index.Document{ ID: "unscored.epub", Slug: "unscored", - Metadata: metadata.Metadata{Title: "Unscored", Format: "EPUB"}, - DataSourceLink: "/provider/items/unscored", - }}) + Metadata: metadata.Metadata{Title: "Unscored", Format: "EPUB"}, + }, biblioreads.DocumentData{URL: "/book/show/unscored"}), + }) body := getBody(t, app, "/search?type=documents&search=Unscored") page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) if err != nil { diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index 498b3cfa..79a70c40 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -123,7 +123,7 @@ func newSearchResultsTestApp(t *testing.T, documents []index.Document) (*fiber.A idx := index.NewBleve(documentIndex, authorsIndex, appFS, testLibraryDir, nil, index.Config{}) t.Cleanup(func() { _ = idx.Close() }) for _, doc := range documents { - if err := documentIndex.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documentIndex, doc.ID, doc); err != nil { t.Fatal(err) } } @@ -139,7 +139,7 @@ func newSearchResultsTestApp(t *testing.T, documents []index.Document) (*fiber.A // setReviewScore seeds a document's review score through the current provider-backed index storage. func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Document, score, count string) { t.Helper() - bookURL := doc.DataSourceLink + bookURL := mustGoodReadsData(t, doc).URL if bookURL == "" { bookURL = "https://www.goodreads.com/book/show/1-" + doc.Slug if err := idx.UpdateDataSourceLink(doc.ID, bookURL); err != nil { @@ -150,7 +150,7 @@ func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Documen if err != nil { t.Fatalf("reload %s: %v", doc.Slug, err) } - if err := idx.StoreGoodReadsDetails(updated, updated.DataSourceLink, biblioreads.BookResponse{ + if err := idx.StoreGoodReadsDetails(updated, mustGoodReadsData(t, updated).URL, biblioreads.BookResponse{ Rating: score, RatingCount: count, }); err != nil { t.Fatalf("set review score for %s: %v", doc.Slug, err) diff --git a/main.go b/main.go index a064c9eb..1a08bb72 100644 --- a/main.go +++ b/main.go @@ -257,11 +257,13 @@ func getIndexes(fs afero.Fs, illustratedMinSize, minOccurrenceRatio float64, max needsReindex := false // Open or create documents index - documentsIndex, err := bleve.Open(homeDir + documentsIndexPath) + documentsIndex, err := index.OpenDocumentsIndex(homeDir+documentsIndexPath, input.BatchSize) if err == bleve.ErrorIndexPathDoesNotExist { log.Println("No documents index found, creating a new one.") documentsIndex = index.CreateDocumentsIndex(homeDir + documentsIndexPath) needsReindex = true + } else if err != nil { + log.Fatal(err) } // Open or create authors index @@ -280,15 +282,7 @@ func getIndexes(fs afero.Fs, illustratedMinSize, minOccurrenceRatio float64, max log.Fatal(err) } if string(version) == "" || string(version) != index.DocumentVersion { - log.Println("Old version documents index found, recreating with new mapping.") - if err = documentsIndex.Close(); err != nil { - log.Fatal(err) - } - if err = fs.RemoveAll(homeDir + documentsIndexPath); err != nil { - log.Fatal(err) - } - documentsIndex = index.CreateDocumentsIndex(homeDir + documentsIndexPath) - needsReindex = true + log.Fatalf("Unsupported documents index version %q; preserving the index instead of recreating it.", version) } // Rebuild index if TextRank or illustration configuration changed (stored in index metadata). From d28c7220d03c2e27c13a04c7470caa3e1e372501 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 16:46:31 +0100 Subject: [PATCH 22/50] WIP --- internal/index/bleve_document_migrate.go | 15 ------- internal/index/bleve_document_migrate_test.go | 41 +++---------------- 2 files changed, 5 insertions(+), 51 deletions(-) diff --git a/internal/index/bleve_document_migrate.go b/internal/index/bleve_document_migrate.go index b78e2f47..7bdb0665 100644 --- a/internal/index/bleve_document_migrate.go +++ b/internal/index/bleve_document_migrate.go @@ -7,7 +7,6 @@ import ( "os" "github.com/blevesearch/bleve/v2" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" ) var previousDocumentVersions = []string{"v13", "v14", "v15", "v16"} @@ -78,20 +77,6 @@ func copyDocuments(source, destination bleve.Index, batchSize int) error { if err != nil { return err } - link, _ := hit.Fields["DataSourceLink"].(string) - count, _ := hit.Fields["DataSourceScoreCount"].(string) - resource, _ := hit.Fields["GoodReadsResourceID"].(string) - legacyID, _ := hit.Fields["GoodReadsLegacyBookID"].(float64) - enriched, _ := hit.Fields["GoodReadsEnriched"].(bool) - if link != "" || count != "" || resource != "" || legacyID != 0 || enriched { - if _, exists := document.DataSources[biblioreads.Source]; !exists { - if err := SetDataSourceData(&document, biblioreads.Source, biblioreads.DocumentData{ - URL: link, RatingCount: count, ResourceID: resource, LegacyBookID: int(legacyID), Enriched: enriched, - }); err != nil { - return err - } - } - } if err := IndexDocument(batch, hit.ID, document); err != nil { return err } diff --git a/internal/index/bleve_document_migrate_test.go b/internal/index/bleve_document_migrate_test.go index 4170a334..2927906a 100644 --- a/internal/index/bleve_document_migrate_test.go +++ b/internal/index/bleve_document_migrate_test.go @@ -9,7 +9,6 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/mapping" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/metadata" ) @@ -23,15 +22,7 @@ func legacyDocumentsIndex(t *testing.T, path, version string) bleve.Index { for _, documentMapping := range mappings { delete(documentMapping.Properties, "DataSources") delete(documentMapping.Properties, "EnrichedDataSources") - for _, field := range []string{"DataSourceLink", "DataSourceScoreCount", "GoodReadsResourceID"} { - stored := bleve.NewKeywordFieldMapping() - stored.Index, stored.IncludeInAll, stored.DocValues = false, false, false - documentMapping.AddFieldMappingsAt(field, stored) - } - storedNumber := bleve.NewNumericFieldMapping() - storedNumber.Index, storedNumber.IncludeInAll, storedNumber.DocValues = false, false, false - documentMapping.AddFieldMappingsAt("GoodReadsLegacyBookID", storedNumber) - documentMapping.AddFieldMappingsAt("GoodReadsEnriched", bleve.NewBooleanFieldMapping()) + delete(documentMapping.Properties, "DataSourceScoreValue") } idx, err := bleve.New(path, m) if err != nil { @@ -49,33 +40,14 @@ func TestOpenDocumentsIndexMigratesStoredMetadata(t *testing.T) { path := filepath.Join(t.TempDir(), "documents") legacy := legacyDocumentsIndex(t, path, version) want := make(map[string]Document) - for _, data := range []biblioreads.DocumentData{ - {URL: "/book/show/123", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, Enriched: true}, - {Enriched: true}, - {}, - } { - id := "linked" - if data.URL == "" { - id = "pending" - if data.Enriched { - id = "unmatched" - } - } + for _, id := range []string{"first", "second", "third"} { document := Document{ ID: id, Slug: id, Metadata: metadata.Metadata{Title: "Book " + id, Authors: []string{"Writer"}, Language: "es", Format: "EPUB"}, AuthorsSlugs: []string{"writer"}, AddedOn: time.Date(2026, 11, 3, 10, 0, 0, 0, time.UTC), TextRankWords: []string{"atomic", "history"}, TextRankPhrases: []string{"atomic history"}, TextRankEnriched: true, - DataSourceScoreValue: 4.25, } - if err := legacy.Index(id, struct { - Document - DataSourceLink string - DataSourceScoreCount string - GoodReadsResourceID string - GoodReadsLegacyBookID int - GoodReadsEnriched bool - }{document, data.URL, data.RatingCount, data.ResourceID, data.LegacyBookID, data.Enriched}); err != nil { + if err := legacy.Index(id, document); err != nil { t.Fatal(err) } reader := NewBleve(legacy, nil, nil, "", nil, Config{}) @@ -83,9 +55,6 @@ func TestOpenDocumentsIndexMigratesStoredMetadata(t *testing.T) { if err != nil { t.Fatal(err) } - if data != (biblioreads.DocumentData{}) { - stored = withGoodReadsData(t, stored, data) - } want[id] = stored } internal := map[string]string{ @@ -120,8 +89,8 @@ func TestOpenDocumentsIndexMigratesStoredMetadata(t *testing.T) { t.Fatalf("migration changed document:\ngot: %#v\nwant: %#v\nerror: %v", got, expected, err) } } - if _, total, err := idx.documentsNeedingGoodReads(10); err != nil || total != 1 { - t.Fatalf("migration changed enrichment queue: %d pending, %v", total, err) + if _, total, err := idx.documentsNeedingGoodReads(10); err != nil || total != 3 { + t.Fatalf("expected documents without provider data to need enrichment: %d pending, %v", total, err) } for key, expected := range internal { got, err := reopened.GetInternal([]byte(key)) From a2268a7e3c1ec8b8ee8c2285012c4c8b356a6a20 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 16:59:39 +0100 Subject: [PATCH 23/50] WIP --- index_migration_test.go | 4 +- internal/index/bleve_author_migrate.go | 20 +-- internal/index/bleve_document_migrate.go | 100 ------------ internal/index/bleve_document_migrate_test.go | 154 ------------------ main.go | 2 +- 5 files changed, 11 insertions(+), 269 deletions(-) delete mode 100644 internal/index/bleve_document_migrate.go delete mode 100644 internal/index/bleve_document_migrate_test.go diff --git a/index_migration_test.go b/index_migration_test.go index 3d883b1a..4c5c53e7 100644 --- a/index_migration_test.go +++ b/index_migration_test.go @@ -32,7 +32,7 @@ func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { } documents := index.CreateDocumentsIndex(homeDir + documentsIndexPath) for key, value := range map[string]string{ - "version": "v16", + "version": index.DocumentVersion, "illustrated_min_size": "0.25", "min_occurrence_ratio": "0.1", "max_textrank_words": "10000", @@ -86,7 +86,7 @@ func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { } data, err := index.DataSourceData[biblioreads.DocumentData](stored, biblioreads.Source) if err != nil || !data.Enriched { - t.Fatalf("document migration lost enrichment state: %+v, %v", data, err) + t.Fatalf("author migration changed document enrichment state: %+v, %v", data, err) } author, err := idx.Author("alice-borchardt", "") if err != nil || author.DataSourceID != "Q1" || !author.RetrievedOn.Equal(retrieved) { diff --git a/internal/index/bleve_author_migrate.go b/internal/index/bleve_author_migrate.go index 87bd206f..4fdc6900 100644 --- a/internal/index/bleve_author_migrate.go +++ b/internal/index/bleve_author_migrate.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/mapping" ) // OpenAuthorsIndex upgrades version 2 without discarding stored author metadata. @@ -52,39 +51,36 @@ func OpenAuthorsIndex(path string, batchSize int) (bleve.Index, error) { } func migrateAuthorsIndex(authors bleve.Index, path string, batchSize int) (bleve.Index, error) { - return migrateStoredIndex(authors, path, path+".v2-backup", batchSize, CreateAuthorsMapping(), copyAuthors) -} - -func migrateStoredIndex(source bleve.Index, path, backup string, batchSize int, targetMapping mapping.IndexMapping, copyDocuments func(bleve.Index, bleve.Index, int) error) (bleve.Index, error) { if batchSize <= 0 { - return nil, fmt.Errorf("index migration batch size must be positive") + return nil, fmt.Errorf("authors migration batch size must be positive") } + backup := path + ".v2-backup" if _, err := os.Stat(backup); err == nil { - return nil, fmt.Errorf("index migration backup already exists: %s", backup) + return nil, fmt.Errorf("authors migration backup already exists: %s", backup) } else if !os.IsNotExist(err) { return nil, err } - stagingDir, err := os.MkdirTemp(filepath.Dir(path), ".index-migration-") + stagingDir, err := os.MkdirTemp(filepath.Dir(path), ".authors-migration-") if err != nil { return nil, err } defer func() { if err := os.RemoveAll(stagingDir); err != nil { - log.Printf("Error removing index migration staging directory: %s", err) + log.Printf("Error removing authors migration staging directory: %s", err) } }() stagingPath := filepath.Join(stagingDir, "index") - staged, err := bleve.New(stagingPath, targetMapping) + staged, err := bleve.New(stagingPath, CreateAuthorsMapping()) if err != nil { return nil, err } - if err := copyDocuments(source, staged, batchSize); err != nil { + if err := copyAuthors(authors, staged, batchSize); err != nil { return nil, errors.Join(err, staged.Close()) } if err := staged.Close(); err != nil { return nil, err } - if err := source.Close(); err != nil { + if err := authors.Close(); err != nil { return nil, err } if err := os.Rename(path, backup); err != nil { diff --git a/internal/index/bleve_document_migrate.go b/internal/index/bleve_document_migrate.go deleted file mode 100644 index 7bdb0665..00000000 --- a/internal/index/bleve_document_migrate.go +++ /dev/null @@ -1,100 +0,0 @@ -package index - -import ( - "errors" - "fmt" - "log" - "os" - - "github.com/blevesearch/bleve/v2" -) - -var previousDocumentVersions = []string{"v13", "v14", "v15", "v16"} - -// OpenDocumentsIndex preserves document metadata while upgrading provider storage. -func OpenDocumentsIndex(path string, batchSize int) (bleve.Index, error) { - documents, err := bleve.Open(path) - for _, version := range previousDocumentVersions { - if !errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { - break - } - backup := path + "." + version + "-backup" - if _, statErr := os.Stat(backup); statErr == nil { - if err := os.Rename(backup, path); err != nil { - return nil, fmt.Errorf("restoring documents index: %w", err) - } - documents, err = bleve.Open(path) - } else if !os.IsNotExist(statErr) { - return nil, fmt.Errorf("checking documents index backup: %w", statErr) - } - } - if err != nil { - return nil, err - } - version, err := documents.GetInternal(internalVersion) - if err != nil { - return nil, errors.Join(err, documents.Close()) - } - for _, previous := range previousDocumentVersions { - if string(version) == previous { - upgraded, err := migrateStoredIndex(documents, path, path+"."+previous+"-backup", batchSize, CreateDocumentsMapping(), copyDocuments) - if err != nil { - return nil, errors.Join(err, documents.Close()) - } - documents = upgraded - version = []byte(DocumentVersion) - log.Printf("Migrated documents index to version %s", DocumentVersion) - break - } - } - if string(version) != DocumentVersion { - return nil, errors.Join(fmt.Errorf("unsupported documents index version %q; index was not modified", version), documents.Close()) - } - for _, previous := range previousDocumentVersions { - if err := os.RemoveAll(path + "." + previous + "-backup"); err != nil { - return nil, errors.Join(fmt.Errorf("removing previous documents index: %w", err), documents.Close()) - } - } - return documents, nil -} - -func copyDocuments(source, destination bleve.Index, batchSize int) error { - for from := 0; ; from += batchSize { - req := bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), batchSize, from, false) - req.Fields = []string{"*"} - req.SortBy([]string{"_id"}) - result, err := source.Search(req) - if err != nil { - return err - } - if len(result.Hits) == 0 { - break - } - batch := destination.NewBatch() - for _, hit := range result.Hits { - document := hydrateDocument(hit) - document.DataSources, err = decodeDataSources(hit.Fields, hit.ID) - if err != nil { - return err - } - if err := IndexDocument(batch, hit.ID, document); err != nil { - return err - } - } - if err := destination.Batch(batch); err != nil { - return err - } - } - for _, key := range [][]byte{internalLanguages, internalIllustratedMinSize, internalMinOccurrenceRatio, internalMaxTextRankWords, internalDocCountAtLastPrune} { - value, err := source.GetInternal(key) - if err != nil { - return err - } - if value != nil { - if err := destination.SetInternal(key, value); err != nil { - return err - } - } - } - return destination.SetInternal(internalVersion, []byte(DocumentVersion)) -} diff --git a/internal/index/bleve_document_migrate_test.go b/internal/index/bleve_document_migrate_test.go deleted file mode 100644 index 2927906a..00000000 --- a/internal/index/bleve_document_migrate_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package index - -import ( - "os" - "path/filepath" - "reflect" - "testing" - "time" - - "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/mapping" - "github.com/svera/coreander/v5/internal/metadata" -) - -func legacyDocumentsIndex(t *testing.T, path, version string) bleve.Index { - t.Helper() - m := CreateDocumentsMapping().(*mapping.IndexMappingImpl) - mappings := []*mapping.DocumentMapping{m.DefaultMapping} - for _, documentMapping := range m.TypeMapping { - mappings = append(mappings, documentMapping) - } - for _, documentMapping := range mappings { - delete(documentMapping.Properties, "DataSources") - delete(documentMapping.Properties, "EnrichedDataSources") - delete(documentMapping.Properties, "DataSourceScoreValue") - } - idx, err := bleve.New(path, m) - if err != nil { - t.Fatal(err) - } - if err := idx.SetInternal(internalVersion, []byte(version)); err != nil { - t.Fatal(err) - } - return idx -} - -func TestOpenDocumentsIndexMigratesStoredMetadata(t *testing.T) { - for _, version := range previousDocumentVersions { - t.Run(version, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "documents") - legacy := legacyDocumentsIndex(t, path, version) - want := make(map[string]Document) - for _, id := range []string{"first", "second", "third"} { - document := Document{ - ID: id, Slug: id, - Metadata: metadata.Metadata{Title: "Book " + id, Authors: []string{"Writer"}, Language: "es", Format: "EPUB"}, - AuthorsSlugs: []string{"writer"}, AddedOn: time.Date(2026, 11, 3, 10, 0, 0, 0, time.UTC), - TextRankWords: []string{"atomic", "history"}, TextRankPhrases: []string{"atomic history"}, TextRankEnriched: true, - } - if err := legacy.Index(id, document); err != nil { - t.Fatal(err) - } - reader := NewBleve(legacy, nil, nil, "", nil, Config{}) - stored, err := reader.documentByIndexID(id) - if err != nil { - t.Fatal(err) - } - want[id] = stored - } - internal := map[string]string{ - string(internalLanguages): "es,en", string(internalIllustratedMinSize): "0.25", - string(internalMinOccurrenceRatio): "0.1", string(internalMaxTextRankWords): "10000", - string(internalDocCountAtLastPrune): "3", - } - for key, value := range internal { - if err := legacy.SetInternal([]byte(key), []byte(value)); err != nil { - t.Fatal(err) - } - } - if err := legacy.Close(); err != nil { - t.Fatal(err) - } - upgraded, err := OpenDocumentsIndex(path, 1) - if err != nil { - t.Fatal(err) - } - if err := upgraded.Close(); err != nil { - t.Fatal(err) - } - reopened, err := OpenDocumentsIndex(path, 1) - if err != nil { - t.Fatal(err) - } - defer reopened.Close() - idx := NewBleve(reopened, nil, nil, "", nil, Config{}) - for id, expected := range want { - got, err := idx.documentByIndexID(id) - if err != nil || !reflect.DeepEqual(got, expected) { - t.Fatalf("migration changed document:\ngot: %#v\nwant: %#v\nerror: %v", got, expected, err) - } - } - if _, total, err := idx.documentsNeedingGoodReads(10); err != nil || total != 3 { - t.Fatalf("expected documents without provider data to need enrichment: %d pending, %v", total, err) - } - for key, expected := range internal { - got, err := reopened.GetInternal([]byte(key)) - if err != nil || string(got) != expected { - t.Fatalf("migration lost internal metadata %s: %q, %v", key, got, err) - } - } - if _, err := os.Stat(path + "." + version + "-backup"); !os.IsNotExist(err) { - t.Fatalf("migration backup was not removed: %v", err) - } - }) - } -} - -func TestOpenDocumentsIndexPreservesOriginalOnFailure(t *testing.T) { - for _, version := range []string{"v16", "unsupported"} { - t.Run(version, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "documents") - legacy := legacyDocumentsIndex(t, path, version) - if err := legacy.Index("saved", Document{ID: "saved"}); err != nil { - t.Fatal(err) - } - if err := legacy.Close(); err != nil { - t.Fatal(err) - } - if _, err := OpenDocumentsIndex(path, 0); err == nil { - t.Fatal("expected invalid batch size or unsupported version error") - } - original, err := bleve.Open(path) - if err != nil { - t.Fatal(err) - } - defer original.Close() - if count, err := original.DocCount(); err != nil || count != 1 { - t.Fatalf("original index lost: %d, %v", count, err) - } - if got, err := original.GetInternal(internalVersion); err != nil || string(got) != version { - t.Fatalf("original version changed: %q, %v", got, err) - } - }) - } -} - -func TestOpenDocumentsIndexRecoversInterruptedSwap(t *testing.T) { - path := filepath.Join(t.TempDir(), "documents") - legacy := legacyDocumentsIndex(t, path+".v16-backup", "v16") - if err := legacy.Index("saved", Document{ID: "saved", Slug: "saved", Metadata: metadata.Metadata{Title: "Saved"}}); err != nil { - t.Fatal(err) - } - if err := legacy.Close(); err != nil { - t.Fatal(err) - } - upgraded, err := OpenDocumentsIndex(path, 1) - if err != nil { - t.Fatal(err) - } - defer upgraded.Close() - if count, err := upgraded.DocCount(); err != nil || count != 1 { - t.Fatalf("interrupted migration lost documents: %d, %v", count, err) - } -} diff --git a/main.go b/main.go index 1a08bb72..091a1d34 100644 --- a/main.go +++ b/main.go @@ -257,7 +257,7 @@ func getIndexes(fs afero.Fs, illustratedMinSize, minOccurrenceRatio float64, max needsReindex := false // Open or create documents index - documentsIndex, err := index.OpenDocumentsIndex(homeDir+documentsIndexPath, input.BatchSize) + documentsIndex, err := bleve.Open(homeDir + documentsIndexPath) if err == bleve.ErrorIndexPathDoesNotExist { log.Println("No documents index found, creating a new one.") documentsIndex = index.CreateDocumentsIndex(homeDir + documentsIndexPath) From 80a5df1e42032943c2df9629cd21b3e4695fa23e Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 17:37:36 +0100 Subject: [PATCH 24/50] WIP --- internal/datasource/model/author.go | 6 +- internal/datasource/wikidata/author.go | 22 ++++++-- internal/datasource/wikidata/service_test.go | 38 +++++++++++++ internal/index/author.go | 2 + internal/index/authors_enrich_test.go | 59 +++++++++++++------- internal/index/bleve_author_write.go | 15 +++-- 6 files changed, 107 insertions(+), 35 deletions(-) diff --git a/internal/datasource/model/author.go b/internal/datasource/model/author.go index 599f00ce..82356915 100644 --- a/internal/datasource/model/author.go +++ b/internal/datasource/model/author.go @@ -14,8 +14,10 @@ type Author interface { DateOfBirth() precisiondate.PrecisionDate DateOfDeath() precisiondate.PrecisionDate Image() string - Website() string - WikipediaLink(language string) string + // Links returns the current links owned by this datasource, including language-specific keys. + Links() map[string]string + // OwnsLink identifies entries replaced by a metadata refresh, even if absent from Links. + OwnsLink(key string) bool SourceID() string RetrievedOn() time.Time Pseudonyms() []string diff --git a/internal/datasource/wikidata/author.go b/internal/datasource/wikidata/author.go index 21f3b73a..70780204 100644 --- a/internal/datasource/wikidata/author.go +++ b/internal/datasource/wikidata/author.go @@ -1,6 +1,7 @@ package wikidata import ( + "strings" "time" "github.com/svera/coreander/v5/internal/precisiondate" @@ -90,8 +91,21 @@ func (a Author) DateOfDeath() precisiondate.PrecisionDate { return a.dateOfDeath } -func (a Author) Website() string { - return a.website +func (a Author) Links() map[string]string { + links := make(map[string]string, len(a.wikipediaLink)+1) + if a.website != "" { + links["website"] = a.website + } + for language, url := range a.wikipediaLink { + if url != "" { + links["wikipedia:"+language] = url + } + } + return links +} + +func (a Author) OwnsLink(key string) bool { + return key == "website" || strings.HasPrefix(key, "wikipedia:") } func (a Author) Image() string { @@ -102,10 +116,6 @@ func (a Author) InstanceOf() float64 { return a.instanceOf } -func (a Author) WikipediaLink(language string) string { - return a.wikipediaLink[language] -} - func (a Author) Gender() float64 { return a.gender } diff --git a/internal/datasource/wikidata/service_test.go b/internal/datasource/wikidata/service_test.go index 2924399e..aa00113e 100644 --- a/internal/datasource/wikidata/service_test.go +++ b/internal/datasource/wikidata/service_test.go @@ -10,6 +10,44 @@ import ( "github.com/svera/coreander/v5/internal/precisiondate" ) +func TestAuthorLinks(t *testing.T) { + author := Author{ + website: "https://author.example", + wikipediaLink: map[string]string{ + "en": "https://en.wikipedia.org/wiki/Example", + "es": "https://es.wikipedia.org/wiki/Ejemplo", + "fr": "", + }, + } + want := map[string]string{ + "website": "https://author.example", + "wikipedia:en": "https://en.wikipedia.org/wiki/Example", + "wikipedia:es": "https://es.wikipedia.org/wiki/Ejemplo", + } + if got := author.Links(); !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected author links: %+v", got) + } + links := author.Links() + links["website"] = "https://changed.example" + links["wikipedia:en"] = "https://changed.example/en" + if got := author.Links(); !reflect.DeepEqual(got, want) { + t.Fatalf("caller mutated author links: %+v", got) + } + for _, key := range []string{"website", "wikipedia:en", "wikipedia:de"} { + if !author.OwnsLink(key) { + t.Errorf("Wikidata should own %q, including missing language links", key) + } + } + for _, key := range []string{"goodreads", "catalog", "wikipedia", "website:other"} { + if author.OwnsLink(key) { + t.Errorf("Wikidata should not own %q", key) + } + } + if links := (Author{}).Links(); len(links) != 0 { + t.Fatalf("expected no links for an empty author: %+v", links) + } +} + func TestWikipediaDataSourceLink(t *testing.T) { source := NewWikidataSource(Gowikidata{}) link := source.AuthorLink(index.Author{ diff --git a/internal/index/author.go b/internal/index/author.go index 14ccf7be..d247ebee 100644 --- a/internal/index/author.go +++ b/internal/index/author.go @@ -24,6 +24,8 @@ type Author struct { Gender float64 Pseudonyms []string DocumentCount uint64 + // ownsRefreshedLink is set only while applying an external metadata refresh. + ownsRefreshedLink func(string) bool } func (a Author) YearOfBirthAbs() int { diff --git a/internal/index/authors_enrich_test.go b/internal/index/authors_enrich_test.go index be3e975a..375e41ae 100644 --- a/internal/index/authors_enrich_test.go +++ b/internal/index/authors_enrich_test.go @@ -2,6 +2,8 @@ package index_test import ( "errors" + "maps" + "strings" "sync" "testing" "time" @@ -76,9 +78,9 @@ func (m *mockAuthorDataSource) RetrieveAuthors(candidates map[string][]string, _ } type stubAuthor struct { - sourceID string - website string - wikipedia map[string]string + sourceID string + links map[string]string + linkPrefix string } func (a stubAuthor) BirthName() string { return "" } @@ -88,11 +90,13 @@ func (a stubAuthor) Gender() float64 { return 0 } func (a stubAuthor) DateOfBirth() precisiondate.PrecisionDate { return precisiondate.PrecisionDate{} } func (a stubAuthor) DateOfDeath() precisiondate.PrecisionDate { return precisiondate.PrecisionDate{} } func (a stubAuthor) Image() string { return "" } -func (a stubAuthor) Website() string { return a.website } -func (a stubAuthor) WikipediaLink(lang string) string { return a.wikipedia[lang] } -func (a stubAuthor) SourceID() string { return a.sourceID } -func (a stubAuthor) RetrievedOn() time.Time { return time.Now().UTC() } -func (a stubAuthor) Pseudonyms() []string { return nil } +func (a stubAuthor) Links() map[string]string { return maps.Clone(a.links) } +func (a stubAuthor) OwnsLink(key string) bool { + return a.linkPrefix != "" && strings.HasPrefix(key, a.linkPrefix) +} +func (a stubAuthor) SourceID() string { return a.sourceID } +func (a stubAuthor) RetrievedOn() time.Time { return time.Now().UTC() } +func (a stubAuthor) Pseudonyms() []string { return nil } func TestAuthorMetadataRefreshUpdatesLinks(t *testing.T) { authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) @@ -104,21 +108,31 @@ func TestAuthorMetadataRefreshUpdatesLinks(t *testing.T) { author := index.Author{ Slug: "writer", Name: "Writer", DataSourceLinks: map[string]string{ - "website": "https://old.example", - "wikipedia:en": "https://en.wikipedia.org/wiki/Old", - "wikipedia:es": "https://es.wikipedia.org/wiki/Old", - "goodreads": "/author/show/1.Writer", - "catalog": "https://catalog.example/writer", + "example:website": "https://old.example", + "example:en": "https://en.example/Old", + "example:es": "https://es.example/Old", + "goodreads": "/author/show/1.Writer", + "catalog": "https://catalog.example/writer", }, } if err := idx.IndexAuthor(author); err != nil { t.Fatal(err) } for _, source := range []stubAuthor{ - {sourceID: "Q1", website: "https://new.example", wikipedia: map[string]string{"en": "https://en.wikipedia.org/wiki/New"}}, - {sourceID: "Q1"}, + {sourceID: "Q1", linkPrefix: "example:", links: map[string]string{ + "example:website": "https://new.example", "example:en": "https://en.example/New", + }}, + {sourceID: "Q1", linkPrefix: "example:"}, } { index.CombineWithDataSource(&author, source, []string{"en", "es"}) + current, err := idx.Author("writer", "en") + if err != nil { + t.Fatal(err) + } + current.DataSourceLinks["another-provider"] = "https://another.example/writer" + if err := idx.IndexAuthor(current); err != nil { + t.Fatal(err) + } if err := idx.IndexAuthor(author); err != nil { t.Fatal(err) } @@ -126,15 +140,22 @@ func TestAuthorMetadataRefreshUpdatesLinks(t *testing.T) { if err != nil { t.Fatal(err) } - if got.DataSourceLinks["website"] != source.website || - got.DataSourceLinks["wikipedia:en"] != source.wikipedia["en"] || - got.DataSourceLinks["wikipedia:es"] != "" { + if got.DataSourceLinks["example:website"] != source.links["example:website"] || + got.DataSourceLinks["example:en"] != source.links["example:en"] || + got.DataSourceLinks["example:es"] != "" { t.Fatalf("metadata links were not refreshed: %+v", got.DataSourceLinks) } if got.DataSourceLinks["goodreads"] != "/author/show/1.Writer" || - got.DataSourceLinks["catalog"] != "https://catalog.example/writer" { + got.DataSourceLinks["catalog"] != "https://catalog.example/writer" || + got.DataSourceLinks["another-provider"] != "https://another.example/writer" { t.Fatalf("metadata refresh lost independent provider links: %+v", got.DataSourceLinks) } + if source.links != nil { + author.DataSourceLinks["example:website"] = "https://caller.example" + if source.links["example:website"] != "https://new.example" { + t.Fatal("combining metadata retained a mutable datasource map") + } + } } } diff --git a/internal/index/bleve_author_write.go b/internal/index/bleve_author_write.go index 45e94790..4f25ae9c 100644 --- a/internal/index/bleve_author_write.go +++ b/internal/index/bleve_author_write.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "maps" - "strings" "time" "github.com/blevesearch/bleve/v2" @@ -180,7 +179,7 @@ func (b *BleveIndexer) IndexAuthor(author Author) error { links = make(map[string]string) } for source, link := range current.DataSourceLinks { - if source != "website" && !strings.HasPrefix(source, "wikipedia:") { + if author.ownsRefreshedLink == nil || !author.ownsRefreshedLink(source) { links[source] = link } } @@ -233,11 +232,12 @@ func CombineWithDataSource(author *Author, authorDataSource datasourcemodel.Auth author.BirthName = authorDataSource.BirthName() author.RetrievedOn = authorDataSource.RetrievedOn() author.DataSourceLinks = maps.Clone(author.DataSourceLinks) + author.ownsRefreshedLink = authorDataSource.OwnsLink if author.DataSourceLinks == nil { author.DataSourceLinks = make(map[string]string) } for source := range author.DataSourceLinks { - if source == "website" || strings.HasPrefix(source, "wikipedia:") { + if authorDataSource.OwnsLink(source) { delete(author.DataSourceLinks, source) } } @@ -245,8 +245,10 @@ func CombineWithDataSource(author *Author, authorDataSource datasourcemodel.Auth author.Description = make(map[string]string) author.DateOfBirth = authorDataSource.DateOfBirth() author.DateOfDeath = authorDataSource.DateOfDeath() - if website := authorDataSource.Website(); website != "" { - author.DataSourceLinks["website"] = website + for key, link := range authorDataSource.Links() { + if link != "" { + author.DataSourceLinks[key] = link + } } author.DataSourceImage = authorDataSource.Image() author.Gender = authorDataSource.Gender() @@ -259,9 +261,6 @@ func CombineWithDataSource(author *Author, authorDataSource datasourcemodel.Auth } for _, lang := range supportedLanguages { - if link := authorDataSource.WikipediaLink(lang); link != "" { - author.DataSourceLinks["wikipedia:"+lang] = link - } author.Description[lang] = authorDataSource.Description(lang) } } From 0857c0e975bb0236cee0cd1c0d386d3c0579d907 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 18:39:41 +0100 Subject: [PATCH 25/50] WIP --- internal/index/bleve.go | 25 +-- internal/index/bleve_document_enrich.go | 161 +++++++++++++++ internal/index/bleve_document_enrich_test.go | 121 ++++++++++++ internal/index/bleve_document_write.go | 2 +- internal/index/bleve_goodreads_write.go | 183 +++++------------- internal/index/bleve_goodreads_write_test.go | 28 +-- internal/index/goodreads_concurrency_test.go | 2 +- internal/webserver/document_goodreads_test.go | 4 +- main.go | 10 +- 9 files changed, 362 insertions(+), 174 deletions(-) create mode 100644 internal/index/bleve_document_enrich.go create mode 100644 internal/index/bleve_document_enrich_test.go diff --git a/internal/index/bleve.go b/internal/index/bleve.go index a9c7aaf8..bccc04d6 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -4,6 +4,7 @@ import ( "errors" "log" "path/filepath" + "slices" "strconv" "strings" "sync" @@ -30,7 +31,7 @@ import ( // DocumentVersion identifies the mapping used for indexing documents. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. -const DocumentVersion = "v17" +const DocumentVersion = "v14" // AuthorVersion identifies the mapping used for indexing authors. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. @@ -182,9 +183,7 @@ type Config struct { // to already be resolved (e.g. via ResolveMetadataWorkers); <= 0 falls // back to 1. TextRankEnrichWorkers int - // GoodReadsSource, if set, is used to fill each document's GoodReads URLs - // in the background (see EnrichGoodReads). Nil disables it. - GoodReadsSource GoodReadsSource + DocumentEnrichers []DocumentEnricher } type BleveIndexer struct { @@ -218,11 +217,8 @@ type BleveIndexer struct { // textRankEnrichJobs feeds documents queued by indexFile to a fixed-size // pool of long-lived worker goroutines; see runTextRankEnrichWorker. textRankEnrichJobs chan Document - // goodReadsEnrichJobs feeds IDs of documents queued by indexFile to - // runGoodReadsEnrichWorker; only used when goodReadsSource is set. - goodReadsEnrichJobs chan string - goodReadsSource GoodReadsSource - goodReadsSlots chan struct{} + documentEnrichJobs chan string + documentEnrichers []DocumentEnricher // fileLocks holds a *sync.Mutex per document ID, serializing concurrent // indexFile calls for the same file (upload vs. file watcher reacting to // the same write). @@ -327,18 +323,17 @@ func NewBleve(documentsIndex bleve.Index, authorsIndex bleve.Index, fs afero.Fs, minCommonTextRankAbsoluteCount: minCommonTextRankAbsoluteCount, pruneChangeTriggerRatio: pruneChangeTriggerRatio, textRankEnrichJobs: make(chan Document, textRankEnrichWorkers), - goodReadsSource: cfg.GoodReadsSource, - goodReadsSlots: make(chan struct{}, goodReadsEnrichWorkers), + documentEnrichers: slices.Clone(cfg.DocumentEnrichers), } for range textRankEnrichWorkers { go b.runTextRankEnrichWorker() } - if b.goodReadsSource != nil { - b.goodReadsEnrichJobs = make(chan string, goodReadsEnrichWorkers) - for range goodReadsEnrichWorkers { - go b.runGoodReadsEnrichWorker() + if len(b.documentEnrichers) > 0 { + b.documentEnrichJobs = make(chan string, documentEnrichWorkers) + for range documentEnrichWorkers { + go b.runDocumentEnrichWorker() } } diff --git a/internal/index/bleve_document_enrich.go b/internal/index/bleve_document_enrich.go new file mode 100644 index 00000000..4661415f --- /dev/null +++ b/internal/index/bleve_document_enrich.go @@ -0,0 +1,161 @@ +package index + +import ( + "context" + "errors" + "fmt" + "log" + "sync" + + "github.com/blevesearch/bleve/v2" +) + +const documentEnrichWorkers = 4 + +// DocumentEnricher owns a provider's lookup policy and persisted payload. +type DocumentEnricher interface { + Source() string + NeedsEnrichment(Document) (bool, error) + // Enrich persists an enriched=true payload for completed lookups, including no matches. + // Transient failures must leave the document pending. + Enrich(context.Context, *BleveIndexer, Document) error +} + +func (b *BleveIndexer) documentsNeedingProvider(source string, size int) ([]Document, uint64, error) { + enriched := bleve.NewTermQuery(source) + enriched.SetField("EnrichedDataSources") + q := bleve.NewBooleanQuery() + q.AddMust(bleve.NewMatchAllQuery()) + q.AddMustNot(enriched) + req := bleve.NewSearchRequestOptions(q, size, 0, false) + req.Fields = []string{"*"} + b.documentsMu.RLock() + result, err := b.documentsIdx.Search(req) + b.documentsMu.RUnlock() + if err != nil { + return nil, 0, err + } + documents := make([]Document, 0, len(result.Hits)) + for _, hit := range result.Hits { + documents = append(documents, hydrateDocument(hit)) + } + return documents, result.Total, nil +} + +// EnrichDocuments resumes pending lookups for every configured provider. +// A provider failure does not prevent independent providers from running. +func (b *BleveIndexer) EnrichDocuments(batchSize int) error { + if batchSize <= 0 { + return fmt.Errorf("document enrichment batch size must be positive") + } + var failures []error + for _, provider := range b.documentEnrichers { + if err := b.enrichProvider(provider, batchSize); err != nil { + failures = append(failures, fmt.Errorf("enriching documents with %s: %w", provider.Source(), err)) + } + } + return errors.Join(failures...) +} + +func (b *BleveIndexer) enrichProvider(provider DocumentEnricher, batchSize int) error { + _, total, err := b.documentsNeedingProvider(provider.Source(), 0) + if err != nil || total == 0 { + return err + } + log.Printf("Enriching %d documents with %s data", total, provider.Source()) + for iteration := 0; iteration < int(total)+10; iteration++ { + documents, _, err := b.documentsNeedingProvider(provider.Source(), batchSize) + if err != nil { + return err + } + if len(documents) == 0 { + return nil + } + if err := b.enrichProviderBatch(provider, documents); err != nil { + return err + } + } + return fmt.Errorf("document enrichment did not finish; some documents remain pending") +} + +func (b *BleveIndexer) enrichProviderBatch(provider DocumentEnricher, documents []Document) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + jobs := make(chan Document, len(documents)) + for _, document := range documents { + jobs <- document + } + close(jobs) + var workers sync.WaitGroup + failures := make(chan error, 1) + for range min(documentEnrichWorkers, len(documents)) { + workers.Go(func() { + for document := range jobs { + needed, err := provider.NeedsEnrichment(document) + if err == nil && needed { + err = provider.Enrich(ctx, b, document) + } + if err != nil { + select { + case failures <- err: + cancel() + default: + } + return + } + } + }) + } + workers.Wait() + select { + case err := <-failures: + return err + default: + return nil + } +} + +func (b *BleveIndexer) runDocumentEnrichWorker() { + for id := range b.documentEnrichJobs { + for _, provider := range b.documentEnrichers { + document, err := b.documentByIndexID(id) + if err != nil { + log.Printf("Error reading document %s for %s enrichment: %s", id, provider.Source(), err) + continue + } + if document.ID == "" { + break + } + needed, err := provider.NeedsEnrichment(document) + if err == nil && needed { + err = provider.Enrich(context.Background(), b, document) + } + if err != nil { + log.Printf("Error enriching document %s with %s: %s", id, provider.Source(), err) + } + } + } +} + +func (b *BleveIndexer) scheduleDocumentEnrichment(id string) { + if len(b.documentEnrichers) > 0 { + go func() { b.documentEnrichJobs <- id }() + } +} + +// StoreDocumentData replaces only one provider's payload under the document write lock. +func (b *BleveIndexer) StoreDocumentData(documentID, source string, data any) error { + b.documentsMu.Lock() + defer b.documentsMu.Unlock() + document, found, err := b.documentByIndexIDLocked(documentID) + if err != nil { + return err + } + if !found { + return ErrDocumentNotFound + } + if err := SetDataSourceData(&document, source, data); err != nil { + return err + } + return IndexDocument(b.documentsIdx, documentID, document) +} diff --git a/internal/index/bleve_document_enrich_test.go b/internal/index/bleve_document_enrich_test.go new file mode 100644 index 00000000..4d9aefed --- /dev/null +++ b/internal/index/bleve_document_enrich_test.go @@ -0,0 +1,121 @@ +package index + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/metadata" +) + +type testDocumentEnricher struct { + source string + calls atomic.Int32 + failure error +} + +func (p *testDocumentEnricher) Source() string { return p.source } + +func (p *testDocumentEnricher) NeedsEnrichment(document Document) (bool, error) { + data, err := DataSourceData[struct{ Enriched bool }](document, p.source) + return !data.Enriched, err +} + +func (p *testDocumentEnricher) Enrich(_ context.Context, idx *BleveIndexer, document Document) error { + p.calls.Add(1) + if p.failure != nil { + return p.failure + } + return idx.StoreDocumentData(document.ID, p.source, map[string]any{ + "enriched": true, "reference": p.source + "-reference", + }) +} + +func TestDocumentEnrichmentSupportsIndependentProviders(t *testing.T) { + first := &testDocumentEnricher{source: "first"} + second := &testDocumentEnricher{source: "second"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{first, second}}) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + for range 2 { + if err := idx.EnrichDocuments(1); err != nil { + t.Fatal(err) + } + } + stored, err := idx.documentByIndexID(document.ID) + if err != nil { + t.Fatal(err) + } + for _, provider := range []*testDocumentEnricher{first, second} { + if provider.calls.Load() != 1 { + t.Fatalf("%s enriched an already processed document", provider.source) + } + data, err := DataSourceData[map[string]any](stored, provider.source) + if err != nil || data["reference"] != provider.source+"-reference" { + t.Fatalf("provider payload lost: %+v, %v", data, err) + } + } +} + +func TestDocumentEnrichmentContinuesAfterProviderFailure(t *testing.T) { + unavailable := errors.New("provider unavailable") + first := &testDocumentEnricher{source: "first", failure: unavailable} + second := &testDocumentEnricher{source: "second"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{first, second}}) + if err := IndexDocument(idx.documentsIdx, "book", Document{ID: "book", Metadata: metadata.Metadata{Title: "Book"}}); err != nil { + t.Fatal(err) + } + if err := idx.EnrichDocuments(1); !errors.Is(err, unavailable) { + t.Fatalf("expected provider error, got %v", err) + } + if second.calls.Load() != 1 { + t.Fatal("one provider failure prevented another provider from running") + } + first.failure = nil + if err := idx.EnrichDocuments(1); err != nil { + t.Fatal(err) + } + if first.calls.Load() != 2 || second.calls.Load() != 1 { + t.Fatal("retry did not resume only the failed provider") + } +} + +func TestDocumentEnrichmentQueueUsesConfiguredProviders(t *testing.T) { + first := &testDocumentEnricher{source: "first"} + second := &testDocumentEnricher{source: "second"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{first, second}}) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + idx.scheduleDocumentEnrichment(document.ID) + deadline := time.Now().Add(2 * time.Second) + for { + stored, err := idx.documentByIndexID(document.ID) + if err != nil { + t.Fatal(err) + } + a, err := first.NeedsEnrichment(stored) + if err != nil { + t.Fatal(err) + } + b, err := second.NeedsEnrichment(stored) + if err != nil { + t.Fatal(err) + } + if !a && !b { + break + } + if time.Now().After(deadline) { + t.Fatal("queued document was not enriched by both providers") + } + time.Sleep(time.Millisecond) + } + if err := idx.EnrichDocuments(0); err == nil { + t.Fatal("accepted invalid batch size") + } +} diff --git a/internal/index/bleve_document_write.go b/internal/index/bleve_document_write.go index abbdee31..0a2d57e7 100644 --- a/internal/index/bleve_document_write.go +++ b/internal/index/bleve_document_write.go @@ -186,7 +186,7 @@ func (b *BleveIndexer) indexFileLocked(file string) (string, error) { if !document.TextRankEnriched { b.scheduleTextRankEnrichment(document) } - b.scheduleGoodReadsEnrichment(document.ID) + b.scheduleDocumentEnrichment(document.ID) b.maybePruneForLibraryChange() diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go index 72a61c6b..66e47d48 100644 --- a/internal/index/bleve_goodreads_write.go +++ b/internal/index/bleve_goodreads_write.go @@ -11,18 +11,16 @@ import ( "regexp" "strconv" "strings" - "sync" "time" "unicode" "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/search/query" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "golang.org/x/text/unicode/norm" ) const ( - goodReadsEnrichWorkers = 4 + goodReadsEnrichWorkers = documentEnrichWorkers goodReadsRequestTimeout = 2 * time.Minute ) @@ -31,6 +29,37 @@ type GoodReadsSource interface { SearchBooks(ctx context.Context, query string) (biblioreads.SearchBooksResponse, error) } +// GoodReadsEnricher adapts Goodreads matching to the document enrichment pipeline. +type GoodReadsEnricher struct { + source GoodReadsSource + slots chan struct{} +} + +func NewGoodReadsEnricher(source GoodReadsSource) *GoodReadsEnricher { + return &GoodReadsEnricher{source: source, slots: make(chan struct{}, goodReadsEnrichWorkers)} +} + +func (*GoodReadsEnricher) Source() string { return biblioreads.Source } + +func (*GoodReadsEnricher) NeedsEnrichment(document Document) (bool, error) { + data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + return !data.Enriched, err +} + +func (p *GoodReadsEnricher) Enrich(ctx context.Context, b *BleveIndexer, document Document) error { + _, err := p.enrich(ctx, b, document) + return err +} + +func (b *BleveIndexer) goodReadsProvider() *GoodReadsEnricher { + for _, provider := range b.documentEnrichers { + if p, ok := provider.(*GoodReadsEnricher); ok { + return p + } + } + return nil +} + var dataSourceFields = []string{"DataSources", "DataSourceScoreValue"} // Strip named series annotations and omnibus ranges, not standalone volume numbers. @@ -258,104 +287,8 @@ func goodReadsQuery(document Document) string { return q } -func documentsNeedingGoodReadsQuery() query.Query { - enriched := bleve.NewTermQuery(biblioreads.Source) - enriched.SetField("EnrichedDataSources") - notEnriched := bleve.NewBooleanQuery() - notEnriched.AddMust(bleve.NewMatchAllQuery()) - notEnriched.AddMustNot(enriched) - return notEnriched -} - func (b *BleveIndexer) documentsNeedingGoodReads(size int) ([]Document, uint64, error) { - searchReq := bleve.NewSearchRequestOptions(documentsNeedingGoodReadsQuery(), size, 0, false) - searchReq.Fields = []string{"*"} - b.documentsMu.RLock() - searchResult, err := b.documentsIdx.Search(searchReq) - b.documentsMu.RUnlock() - if err != nil { - return nil, 0, err - } - documents := make([]Document, 0, len(searchResult.Hits)) - for _, hit := range searchResult.Hits { - documents = append(documents, hydrateDocument(hit)) - } - return documents, searchResult.Total, nil -} - -// EnrichGoodReads looks up on Goodreads every document not processed yet (see -// biblioreads.DocumentData.Enriched), storing the book URL on the document and the -// author URL on the matching author. The source handles per-instance throttling. -// It stops at the first error that means Goodreads can't be reached right -// now, leaving the remaining documents for the next run. -func (b *BleveIndexer) EnrichGoodReads(batchSize int) error { - if b.goodReadsSource == nil { - return nil - } - - _, total, err := b.documentsNeedingGoodReads(0) - if err != nil || total == 0 { - return err - } - log.Printf("Enriching %d documents with GoodReads data", total) - - // Bounds the loop if a document's write keeps failing, which would leave - // it in documentsNeedingGoodReads' results forever. - maxIterations := int(total) + 10 - for iterations := 0; ; iterations++ { - if iterations >= maxIterations { - return fmt.Errorf("GoodReads enrichment did not finish after %d iterations, some documents may be stuck unenriched", iterations) - } - // Enriched documents drop out of the query, so it always starts from 0. - chunk, _, err := b.documentsNeedingGoodReads(batchSize) - if err != nil { - return err - } - if len(chunk) == 0 { - break - } - if err := b.enrichGoodReadsBatch(chunk); err != nil { - return err - } - } - - log.Printf("GoodReads enrichment finished") - return nil -} - -func (b *BleveIndexer) enrichGoodReadsBatch(documents []Document) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - jobs := make(chan Document, len(documents)) - for _, document := range documents { - jobs <- document - } - close(jobs) - var wg sync.WaitGroup - errs := make(chan error, 1) - for range min(goodReadsEnrichWorkers, len(documents)) { - wg.Add(1) - go func() { - defer wg.Done() - for document := range jobs { - if _, err := b.enrichGoodReads(ctx, document); err != nil { - select { - case errs <- err: - cancel() - default: - } - return - } - } - }() - } - wg.Wait() - select { - case err := <-errs: - return err - default: - return nil - } + return b.documentsNeedingProvider(biblioreads.Source, size) } // GoodReadsBook returns the stored book URL and rating, or searches a document @@ -365,7 +298,7 @@ func (b *BleveIndexer) GoodReadsBook(ctx context.Context, document Document) (bi if err != nil { return biblioreads.SearchResult{}, err } - if data.URL != "" || data.Enriched || b.goodReadsSource == nil { + if data.URL != "" || data.Enriched || b.goodReadsProvider() == nil { return biblioreads.SearchResult{BookURL: data.URL, Rating: fmt.Sprintf("%.2f", document.DataSourceScoreValue), RatingCount: data.RatingCount}, nil } return b.enrichGoodReads(ctx, document) @@ -458,13 +391,21 @@ func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, // book URL, and stores matching URLs on the document and author respectively. // Unavailability or persistence errors leave the document unenriched for retry. func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) (biblioreads.SearchResult, error) { + provider := b.goodReadsProvider() + if provider == nil { + return biblioreads.SearchResult{}, fmt.Errorf("Goodreads enrichment is not configured") + } + return provider.enrich(ctx, b, document) +} + +func (p *GoodReadsEnricher) enrich(ctx context.Context, b *BleveIndexer, document Document) (biblioreads.SearchResult, error) { data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) if err != nil { return biblioreads.SearchResult{}, err } select { - case b.goodReadsSlots <- struct{}{}: - defer func() { <-b.goodReadsSlots }() + case p.slots <- struct{}{}: + defer func() { <-p.slots }() case <-ctx.Done(): return biblioreads.SearchResult{}, ctx.Err() } @@ -478,7 +419,7 @@ func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) ( queries = append(queries, title) } for _, query := range queries { - res, err := b.goodReadsSource.SearchBooks(ctx, query) + res, err := p.source.SearchBooks(ctx, query) switch { case errors.Is(err, biblioreads.ErrUnavailable), errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): return biblioreads.SearchResult{}, fmt.Errorf("error searching %q on GoodReads: %w", query, err) @@ -553,40 +494,6 @@ func parseGoodReadsRating(rating, documentID string) float64 { return value } -// runGoodReadsEnrichWorker looks up documents queued by indexFile for the -// indexer's lifetime. -func (b *BleveIndexer) runGoodReadsEnrichWorker() { - for id := range b.goodReadsEnrichJobs { - document, err := b.documentByIndexID(id) - if err != nil { - log.Printf("Error reading document %s for GoodReads enrichment: %s\n", id, err) - continue - } - data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) - if err != nil { - log.Println(err) - continue - } - if document.ID == "" || data.Enriched { - continue - } - if _, err := b.enrichGoodReads(context.Background(), document); err != nil { - log.Println(err) - } - } -} - -// scheduleGoodReadsEnrichment queues the document with the given ID for -// lookup from a throwaway goroutine, so indexFile never blocks on it. -func (b *BleveIndexer) scheduleGoodReadsEnrichment(id string) { - if b.goodReadsSource == nil { - return - } - go func() { - b.goodReadsEnrichJobs <- id - }() -} - // documentByIndexIDLocked is documentByIndexID for callers already holding documentsMu. func (b *BleveIndexer) documentByIndexIDLocked(id string) (Document, bool, error) { searchReq := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/index/bleve_goodreads_write_test.go index 32f46275..5bcceb3a 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/index/bleve_goodreads_write_test.go @@ -102,7 +102,11 @@ const ( func newGoodReadsTestIndexer(t *testing.T, source GoodReadsSource) *BleveIndexer { t.Helper() - idx, _ := newTestTextRankIndexer(t, Config{GoodReadsSource: source}) + var providers []DocumentEnricher + if source != nil { + providers = append(providers, NewGoodReadsEnricher(source)) + } + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: providers}) appFS := afero.NewMemMapFs() if err := afero.WriteFile(appFS, "lib/book.pdf", []byte(""), 0644); err != nil { t.Fatal(err) @@ -120,7 +124,7 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { t.Fatal(err) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } @@ -153,7 +157,7 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { t.Errorf("expected queries %q, got %q", want, source.Queries()) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } if len(source.Queries()) != 1 { @@ -174,7 +178,7 @@ func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { t.Fatal(err) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } doc, err := idx.documentByIndexID(goodReadsTestID) @@ -208,7 +212,7 @@ func TestEnrichGoodReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) if err := idx.IndexAuthor(stale); err != nil { t.Fatal(err) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } author, err := idx.Author("second-author", "") @@ -251,7 +255,7 @@ func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { t.Fatal(err) } - if err := idx.EnrichGoodReads(10); !errors.Is(err, biblioreads.ErrUnavailable) { + if err := idx.EnrichDocuments(10); !errors.Is(err, biblioreads.ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } doc, err := idx.documentByIndexID(goodReadsTestID) @@ -266,7 +270,7 @@ func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { source.err = nil source.results = goodReadsTestResults source.mu.Unlock() - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } if doc, _ = idx.documentByIndexID(goodReadsTestID); mustGoodReadsData(t, doc).URL != wantDataSourceLink { @@ -316,7 +320,7 @@ func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { t.Fatal(err) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } stale.TextRankEnriched = false @@ -367,7 +371,7 @@ func TestEnrichGoodReadsWithoutSourceIsNoop(t *testing.T) { if _, err := idx.NewFile("new.pdf", []byte("")); err != nil { t.Fatal(err) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } doc, err := idx.documentByIndexID(goodReadsTestID) @@ -396,7 +400,7 @@ func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { t.Fatal(err) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } @@ -452,7 +456,7 @@ func TestGoodReadsBook(t *testing.T) { t.Errorf("rating must not be stored in document field %s", field) } } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } if len(source.Queries()) != 1 { @@ -926,7 +930,7 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } doc, err = idx.documentByIndexID(goodReadsTestID) diff --git a/internal/index/goodreads_concurrency_test.go b/internal/index/goodreads_concurrency_test.go index b0eb6349..2962b82a 100644 --- a/internal/index/goodreads_concurrency_test.go +++ b/internal/index/goodreads_concurrency_test.go @@ -43,7 +43,7 @@ func TestGoodReadsEnrichmentUsesBoundedConcurrency(t *testing.T) { documents = append(documents, doc) } done := make(chan error, 1) - go func() { done <- idx.enrichGoodReadsBatch(documents) }() + go func() { done <- idx.enrichProviderBatch(idx.goodReadsProvider(), documents) }() defer func() { close(source.release) if err := <-done; err != nil { diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 6a553299..3d38778a 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -334,7 +334,7 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide appFS := afero.NewMemMapFs() idxCfg := index.Config{} if source != nil { - idxCfg.GoodReadsSource = source + idxCfg.DocumentEnrichers = []index.DocumentEnricher{index.NewGoodReadsEnricher(source)} } idx := index.NewBleve(documents, authors, appFS, testLibraryDir, nil, idxCfg) if source != nil { @@ -686,7 +686,7 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { t.Fatalf("search-only lookup must not populate scraper identifiers, got %+v", stored) } - if err := idx.EnrichGoodReads(10); err != nil { + if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } page, err = goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unlinked/score"))) diff --git a/main.go b/main.go index 091a1d34..68a5d347 100644 --- a/main.go +++ b/main.go @@ -102,13 +102,13 @@ func main() { var documentsIndex, authorsIndex bleve.Index var needsReindex bool documentsIndex, authorsIndex, needsReindex = getIndexes(appFs, input.IllustratedMinSize, input.MinOccurrenceRatio, maxTextRankWords) - var goodReadsSource index.GoodReadsSource + var documentEnrichers []index.DocumentEnricher if !input.DisableBiblioreads { biblioreadsClient = biblioreads.NewClient( biblioreads.WithInstances(input.BiblioreadsInstances), biblioreads.WithCacheTTL(time.Duration(input.BiblioreadsCacheTTL)*time.Second), ) - goodReadsSource = biblioreadsClient + documentEnrichers = append(documentEnrichers, index.NewGoodReadsEnricher(biblioreadsClient)) } idx = index.NewBleve(documentsIndex, authorsIndex, appFs, input.LibPath, metadataReaders, index.Config{ IllustratedMinAmount: input.IllustratedMinAmount, @@ -122,7 +122,7 @@ func main() { MinCommonTextRankAbsoluteCount: input.MinCommonTextRankAbsoluteCount, PruneChangeTriggerRatio: input.PruneChangeTriggerRatio, TextRankEnrichWorkers: resolvedIndexWorkers, - GoodReadsSource: goodReadsSource, + DocumentEnrichers: documentEnrichers, }) // If index was newly created or recreated, force reindexing if needsReindex { @@ -236,8 +236,8 @@ func startIndex(idx *index.BleveIndexer, batchSize int, libPath string, indexWor // Network-bound, so it runs alongside the CPU-bound TextRank enrichment. go func() { - if err := idx.EnrichGoodReads(batchSize); err != nil { - log.Printf("Error enriching documents with GoodReads data: %s", err) + if err := idx.EnrichDocuments(batchSize); err != nil { + log.Printf("Error enriching documents with provider data: %s", err) } }() From 15c1d7ee341c307b89eb4d4ca441a90aedcd0600 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Tue, 3 Nov 2026 20:09:02 +0100 Subject: [PATCH 26/50] WIP --- internal/index/bleve_document_enrich_test.go | 51 +++++++++++++++ internal/index/goodreads_concurrency_test.go | 65 -------------------- 2 files changed, 51 insertions(+), 65 deletions(-) delete mode 100644 internal/index/goodreads_concurrency_test.go diff --git a/internal/index/bleve_document_enrich_test.go b/internal/index/bleve_document_enrich_test.go index 4d9aefed..409c6717 100644 --- a/internal/index/bleve_document_enrich_test.go +++ b/internal/index/bleve_document_enrich_test.go @@ -16,6 +16,57 @@ type testDocumentEnricher struct { failure error } +type blockingDocumentEnricher struct { + started chan struct{} + release chan struct{} +} + +func (*blockingDocumentEnricher) Source() string { return "blocking" } + +func (*blockingDocumentEnricher) NeedsEnrichment(Document) (bool, error) { + return true, nil +} + +func (p *blockingDocumentEnricher) Enrich(ctx context.Context, _ *BleveIndexer, _ Document) error { + p.started <- struct{}{} + select { + case <-p.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func TestDocumentEnrichmentUsesBoundedConcurrency(t *testing.T) { + const documentCount = 2 * documentEnrichWorkers + provider := &blockingDocumentEnricher{ + started: make(chan struct{}, documentCount), + release: make(chan struct{}), + } + idx, _ := newTestTextRankIndexer(t, Config{}) + documents := make([]Document, documentCount) + done := make(chan error, 1) + go func() { done <- idx.enrichProviderBatch(provider, documents) }() + defer func() { + close(provider.release) + if err := <-done; err != nil { + t.Errorf("enrichment failed: %v", err) + } + }() + for range documentEnrichWorkers { + select { + case <-provider.started: + case <-time.After(time.Second): + t.Fatal("expected concurrent provider lookups") + } + } + select { + case <-provider.started: + t.Fatal("enrichment exceeded its concurrency bound") + case <-time.After(30 * time.Millisecond): + } +} + func (p *testDocumentEnricher) Source() string { return p.source } func (p *testDocumentEnricher) NeedsEnrichment(document Document) (bool, error) { diff --git a/internal/index/goodreads_concurrency_test.go b/internal/index/goodreads_concurrency_test.go deleted file mode 100644 index 2962b82a..00000000 --- a/internal/index/goodreads_concurrency_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package index - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/svera/coreander/v5/internal/datasource/biblioreads" - "github.com/svera/coreander/v5/internal/metadata" -) - -type blockingGoodReadsSource struct { - started chan struct{} - release chan struct{} -} - -func (s *blockingGoodReadsSource) SearchBooks(ctx context.Context, _ string) (biblioreads.SearchBooksResponse, error) { - s.started <- struct{}{} - select { - case <-s.release: - return biblioreads.SearchBooksResponse{Result: goodReadsTestResults}, nil - case <-ctx.Done(): - return biblioreads.SearchBooksResponse{}, ctx.Err() - } -} - -func TestGoodReadsEnrichmentUsesBoundedConcurrency(t *testing.T) { - source := &blockingGoodReadsSource{ - started: make(chan struct{}, 8), - release: make(chan struct{}), - } - idx := newGoodReadsTestIndexer(t, source) - var documents []Document - for i := range 8 { - doc := Document{ - ID: fmt.Sprintf("book-%d.pdf", i), - Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, - } - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { - t.Fatal(err) - } - documents = append(documents, doc) - } - done := make(chan error, 1) - go func() { done <- idx.enrichProviderBatch(idx.goodReadsProvider(), documents) }() - defer func() { - close(source.release) - if err := <-done; err != nil { - t.Errorf("enrichment failed: %v", err) - } - }() - for range goodReadsEnrichWorkers { - select { - case <-source.started: - case <-time.After(time.Second): - t.Fatal("expected concurrent Goodreads lookups") - } - } - select { - case <-source.started: - t.Fatal("enrichment exceeded its concurrency bound") - case <-time.After(30 * time.Millisecond): - } -} From 637510950f95f49388cf2f292e7ca428ae586949 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 10:26:45 +0100 Subject: [PATCH 27/50] WIP --- internal/enrichment/goodreads/adapter.go | 202 +++++++ .../goodreads/adapter_test.go} | 271 +++++---- internal/enrichment/goodreads/matching.go | 214 +++++++ internal/index/bleve_author_migrate_test.go | 4 +- internal/index/bleve_document_data.go | 137 +++++ internal/index/bleve_document_data_test.go | 178 ++++++ internal/index/bleve_goodreads_write.go | 547 ------------------ .../index/bleve_provider_preservation_test.go | 71 +++ internal/index/document_data_test.go | 75 ++- internal/webserver/document_goodreads_test.go | 14 +- internal/webserver/webserver_test.go | 11 +- main.go | 8 +- 12 files changed, 1025 insertions(+), 707 deletions(-) create mode 100644 internal/enrichment/goodreads/adapter.go rename internal/{index/bleve_goodreads_write_test.go => enrichment/goodreads/adapter_test.go} (86%) create mode 100644 internal/enrichment/goodreads/matching.go create mode 100644 internal/index/bleve_document_data.go create mode 100644 internal/index/bleve_document_data_test.go delete mode 100644 internal/index/bleve_goodreads_write.go create mode 100644 internal/index/bleve_provider_preservation_test.go diff --git a/internal/enrichment/goodreads/adapter.go b/internal/enrichment/goodreads/adapter.go new file mode 100644 index 00000000..029be873 --- /dev/null +++ b/internal/enrichment/goodreads/adapter.go @@ -0,0 +1,202 @@ +// Package goodreads adapts Goodreads matching and review storage to the generic index. +package goodreads + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" +) + +const ( + goodReadsEnrichWorkers = 4 + goodReadsRequestTimeout = 2 * time.Minute +) + +type GoodReadsSource interface { + SearchBooks(context.Context, string) (biblioreads.SearchBooksResponse, error) +} + +type Enricher struct { + source GoodReadsSource + idx *index.BleveIndexer + slots chan struct{} +} + +func New(source GoodReadsSource) *Enricher { + return &Enricher{source: source, slots: make(chan struct{}, goodReadsEnrichWorkers)} +} + +// SetIndex binds the adapter before starting background or on-demand enrichment. +func (p *Enricher) SetIndex(idx *index.BleveIndexer) { + p.idx = idx +} + +func (*Enricher) Source() string { return biblioreads.Source } + +func (*Enricher) NeedsEnrichment(document index.Document) (bool, error) { + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + return !data.Enriched, err +} + +func (p *Enricher) Enrich(ctx context.Context, idx *index.BleveIndexer, document index.Document) error { + _, err := p.enrich(ctx, idx, document) + return err +} + +// GoodReadsBook returns a stored match or performs an on-demand lookup. +func (p *Enricher) GoodReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) { + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + return biblioreads.SearchResult{}, err + } + if data.URL != "" || data.Enriched || p.source == nil { + return storedResult(document, data), nil + } + return p.enrich(ctx, p.idx, document) +} + +func storedResult(document index.Document, data biblioreads.DocumentData) biblioreads.SearchResult { + return biblioreads.SearchResult{ + BookURL: data.URL, Rating: fmt.Sprintf("%.2f", document.DataSourceScoreValue), RatingCount: data.RatingCount, + } +} + +// UpdateDataSourceLink clears cached Goodreads data when an administrator corrects a match. +func (p *Enricher) UpdateDataSourceLink(documentID, reviewLink string) error { + if p.idx == nil { + return fmt.Errorf("Goodreads adapter has no index") + } + return p.idx.UpdateDocument(documentID, func(current *index.DocumentUpdate) (bool, error) { + data, err := index.DataSourceData[biblioreads.DocumentData](current.Document, biblioreads.Source) + if err != nil || data.URL == reviewLink { + return false, err + } + current.DataSourceScoreValue = 0 + err = index.SetDataSourceData(¤t.Document, biblioreads.Source, biblioreads.DocumentData{URL: reviewLink, Enriched: true}) + return true, err + }) +} + +// StoreGoodReadsDetails rejects responses for a match or metadata changed during the fetch. +func (p *Enricher) StoreGoodReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error { + if p.idx == nil { + return fmt.Errorf("Goodreads adapter has no index") + } + if bookURL == "" { + return fmt.Errorf("invalid GoodReads book URL for document %s", document.ID) + } + hasIDs := book.ResourceID != "" && book.LegacyBookID > 0 + if book.Rating == "" && book.RatingCount == "" && !hasIDs { + return nil + } + return p.idx.UpdateDocument(document.ID, func(current *index.DocumentUpdate) (bool, error) { + data, err := index.DataSourceData[biblioreads.DocumentData](current.Document, biblioreads.Source) + if err != nil { + return false, err + } + if goodReadsQuery(current.Document) != goodReadsQuery(document) || data.URL != bookURL { + return false, fmt.Errorf("GoodReads match changed for document %s", document.ID) + } + changed := false + if book.RatingCount != "" && data.RatingCount != book.RatingCount { + data.RatingCount = book.RatingCount + changed = true + } + if book.Rating != "" { + value := parseGoodReadsRating(book.Rating, document.ID) + if current.DataSourceScoreValue != value { + current.DataSourceScoreValue = value + changed = true + } + } + if hasIDs && (data.ResourceID != book.ResourceID || data.LegacyBookID != book.LegacyBookID) { + data.ResourceID, data.LegacyBookID = book.ResourceID, book.LegacyBookID + changed = true + } + if !changed { + return false, nil + } + return true, index.SetDataSourceData(¤t.Document, biblioreads.Source, data) + }) +} + +func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document index.Document) (biblioreads.SearchResult, error) { + if idx == nil || p.source == nil { + return biblioreads.SearchResult{}, fmt.Errorf("Goodreads enrichment requires a source and index") + } + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + return biblioreads.SearchResult{}, err + } + select { + case p.slots <- struct{}{}: + defer func() { <-p.slots }() + case <-ctx.Done(): + return biblioreads.SearchResult{}, ctx.Err() + } + q := goodReadsQuery(document) + result := biblioreads.SearchResult{BookURL: data.URL} + if result.BookURL == "" && document.Title != "" { + ctx, cancel := context.WithTimeout(ctx, goodReadsRequestTimeout) + defer cancel() + queries := []string{q} + if title := strings.TrimSpace(document.Title); title != q { + queries = append(queries, title) + } + for _, query := range queries { + response, err := p.source.SearchBooks(ctx, query) + switch { + case errors.Is(err, biblioreads.ErrUnavailable), errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): + return biblioreads.SearchResult{}, fmt.Errorf("error searching %q on GoodReads: %w", query, err) + case err != nil: + log.Printf("Error searching %q on GoodReads: %s", query, err) + default: + if match, ok := matchingGoodReadsBook(document, response.Result); ok { + result = match + } + } + if result.BookURL != "" { + break + } + } + } + err = idx.UpdateDocument(document.ID, func(current *index.DocumentUpdate) (bool, error) { + if goodReadsQuery(current.Document) != q { + return false, nil + } + data, err := index.DataSourceData[biblioreads.DocumentData](current.Document, biblioreads.Source) + if err != nil { + return false, err + } + if data.Enriched { + result = storedResult(current.Document, data) + return false, nil + } + if data.URL != "" && data.URL != result.BookURL { + result = storedResult(current.Document, data) + } + data.URL, data.RatingCount, data.Enriched = result.BookURL, result.RatingCount, true + if result.Rating == "" && current.DataSourceScoreValue > 0 { + result.Rating = fmt.Sprintf("%.2f", current.DataSourceScoreValue) + } + current.DataSourceScoreValue = parseGoodReadsRating(result.Rating, current.ID) + if result.AuthorURL != "" { + if slug, name := goodReadsAuthor(current.Document, result.Author); slug != "" { + current.AuthorLinks = append(current.AuthorLinks, index.AuthorLink{ + Slug: slug, Name: name, Source: biblioreads.Source, URL: result.AuthorURL, + }) + } + } + return true, index.SetDataSourceData(¤t.Document, biblioreads.Source, data) + }) + if errors.Is(err, index.ErrDocumentNotFound) { + return result, nil + } + return result, err +} diff --git a/internal/index/bleve_goodreads_write_test.go b/internal/enrichment/goodreads/adapter_test.go similarity index 86% rename from internal/index/bleve_goodreads_write_test.go rename to internal/enrichment/goodreads/adapter_test.go index 5bcceb3a..dfa9314f 100644 --- a/internal/index/bleve_goodreads_write_test.go +++ b/internal/enrichment/goodreads/adapter_test.go @@ -1,38 +1,42 @@ -package index +package goodreads import ( "context" + "encoding/json" "errors" "image" + "reflect" "slices" "sync" + "sync/atomic" "testing" "time" "github.com/blevesearch/bleve/v2" "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" ) -func mustGoodReadsData(t *testing.T, document Document) biblioreads.DocumentData { +func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { t.Helper() - data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) if err != nil { t.Fatal(err) } return data } -func withGoodReadsData(t *testing.T, document Document, data biblioreads.DocumentData) Document { +func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { t.Helper() - if err := SetDataSourceData(&document, biblioreads.Source, data); err != nil { + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { t.Fatal(err) } return document } -func updateGoodReadsData(t *testing.T, document *Document, update func(*biblioreads.DocumentData)) { +func updateGoodReadsData(t *testing.T, document *index.Document, update func(*biblioreads.DocumentData)) { t.Helper() data := mustGoodReadsData(t, *document) update(&data) @@ -100,21 +104,38 @@ const ( goodReadsTestID = "book.pdf" ) -func newGoodReadsTestIndexer(t *testing.T, source GoodReadsSource) *BleveIndexer { +type goodReadsTestIndexer struct { + *index.BleveIndexer + *Enricher + documentsIdx *goodReadsWriteCountingIndex + authorsIdx bleve.Index + fs afero.Fs +} + +func newGoodReadsTestIndexer(t *testing.T, source GoodReadsSource) *goodReadsTestIndexer { t.Helper() - var providers []DocumentEnricher + documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + writes := &goodReadsWriteCountingIndex{countedGoodReadsIndex: documents} + adapter := New(source) + var providers []index.DocumentEnricher if source != nil { - providers = append(providers, NewGoodReadsEnricher(source)) + providers = append(providers, adapter) } - idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: providers}) appFS := afero.NewMemMapFs() if err := afero.WriteFile(appFS, "lib/book.pdf", []byte(""), 0644); err != nil { t.Fatal(err) } - idx.fs = appFS - idx.libraryPath = "lib" - idx.reader = map[string]metadata.Reader{".pdf": goodReadsTestReader{}} - return idx + idx := index.NewBleve(writes, authors, appFS, "lib", map[string]metadata.Reader{".pdf": goodReadsTestReader{}}, index.Config{DocumentEnrichers: providers}) + adapter.SetIndex(idx) + t.Cleanup(func() { _ = idx.Close() }) + return &goodReadsTestIndexer{BleveIndexer: idx, Enricher: adapter, documentsIdx: writes, authorsIdx: authors, fs: appFS} } func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { @@ -128,7 +149,7 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { t.Fatal(err) } - doc, err := idx.documentByIndexID(goodReadsTestID) + doc, err := idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -181,7 +202,7 @@ func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err := idx.documentByIndexID(goodReadsTestID) + doc, err := idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -239,7 +260,7 @@ func TestEnrichGoodReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) author.DataSourceID != "Q2" || author.DocumentCount != 1 { t.Fatalf("stale author edit lost URL or metadata: %+v, %v", author, err) } - if err := idx.incrementAuthorCount(author.Name, author.Slug); err != nil { + if _, err := idx.NewFile("second.pdf", []byte("")); err != nil { t.Fatal(err) } author, err = idx.Author("second-author", "") @@ -258,7 +279,7 @@ func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { if err := idx.EnrichDocuments(10); !errors.Is(err, biblioreads.ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } - doc, err := idx.documentByIndexID(goodReadsTestID) + doc, err := idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -273,7 +294,7 @@ func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - if doc, _ = idx.documentByIndexID(goodReadsTestID); mustGoodReadsData(t, doc).URL != wantDataSourceLink { + if doc, _ = idx.DocumentByID(goodReadsTestID); mustGoodReadsData(t, doc).URL != wantDataSourceLink { t.Errorf("expected document to be enriched on retry, got %q", mustGoodReadsData(t, doc).URL) } } @@ -288,7 +309,7 @@ func TestNewFileSchedulesGoodReadsEnrichment(t *testing.T) { deadline := time.Now().Add(2 * time.Second) for { - doc, err := idx.documentByIndexID("new.pdf") + doc, err := idx.DocumentByID("new.pdf") if err != nil { t.Fatal(err) } @@ -306,63 +327,6 @@ func TestNewFileSchedulesGoodReadsEnrichment(t *testing.T) { } } -// TestTextRankWritesPreserveGoodReadsFields guards against TextRank -// enrichment, which writes back documents read before ranking them, -// discarding GoodReads URLs persisted in the meantime, and vice versa. -func TestTextRankWritesPreserveGoodReadsFields(t *testing.T) { - source := &fakeGoodReadsSource{results: goodReadsTestResults} - idx := newGoodReadsTestIndexer(t, source) - if err := idx.AddLibrary(10, true, 1); err != nil { - t.Fatal(err) - } - stale, err := idx.documentByIndexID(goodReadsTestID) - if err != nil { - t.Fatal(err) - } - - if err := idx.EnrichDocuments(10); err != nil { - t.Fatal(err) - } - stale.TextRankEnriched = false - if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}); err != nil { - t.Fatal(err) - } - stale.TextRankWords = []string{"dragon"} - idx.enrichTextRankAndReindex(stale) - - doc, err := idx.documentByIndexID(goodReadsTestID) - if err != nil { - t.Fatal(err) - } - if !mustGoodReadsData(t, doc).Enriched || mustGoodReadsData(t, doc).URL != wantDataSourceLink || doc.DataSourceScoreValue != 3.82 || mustGoodReadsData(t, doc).RatingCount != "2,285" { - t.Errorf("expected GoodReads fields to survive a TextRank write, got %+v", doc) - } - if mustGoodReadsData(t, doc).ResourceID != "work-id" || mustGoodReadsData(t, doc).LegacyBookID != 12207444 { - t.Errorf("expected GoodReads identifiers to survive a TextRank write, got %+v", doc) - } - if !doc.TextRankEnriched { - t.Error("expected TextRank write to be persisted") - } - - // A GoodReads write must keep TextRank fields written after the document was read. - stale, _ = idx.documentByIndexID(goodReadsTestID) - updateGoodReadsData(t, &stale, func(data *biblioreads.DocumentData) { - data.Enriched = false - }) - doc.TextRankWords = []string{"guinevere"} - idx.documentsMu.Lock() - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { - t.Fatal(err) - } - idx.documentsMu.Unlock() - if _, err := idx.enrichGoodReads(context.Background(), stale); err != nil { - t.Fatal(err) - } - if doc, _ = idx.documentByIndexID(goodReadsTestID); !slices.Equal(doc.TextRankWords, []string{"guinevere"}) { - t.Errorf("expected TextRank words to survive a GoodReads write, got %q", doc.TextRankWords) - } -} - func TestEnrichGoodReadsWithoutSourceIsNoop(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) if err := idx.AddLibrary(10, true, 1); err != nil { @@ -374,7 +338,7 @@ func TestEnrichGoodReadsWithoutSourceIsNoop(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err := idx.documentByIndexID(goodReadsTestID) + doc, err := idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -389,14 +353,14 @@ func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.documentByIndexID(goodReadsTestID) + doc, err := idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } updateGoodReadsData(t, &doc, func(data *biblioreads.DocumentData) { data.URL = "/book/show/1-existing" }) - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } @@ -407,7 +371,7 @@ func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if len(source.Queries()) != 0 { t.Errorf("expected no search for a document with a link, got %q", source.Queries()) } - doc, err = idx.documentByIndexID(goodReadsTestID) + doc, err = idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -422,7 +386,7 @@ func TestGoodReadsBook(t *testing.T) { if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.documentByIndexID(goodReadsTestID) + doc, err := idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -434,7 +398,7 @@ func TestGoodReadsBook(t *testing.T) { if got.BookURL != wantDataSourceLink || got.Rating != "3.82" { t.Errorf("expected fresh book URL and search rating, got %+v", got) } - stored, err := idx.documentByIndexID(goodReadsTestID) + stored, err := idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -488,11 +452,11 @@ func TestGoodReadsBookReturnsErrorWhenUnavailable(t *testing.T) { if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, _ := idx.documentByIndexID(goodReadsTestID) + doc, _ := idx.DocumentByID(goodReadsTestID) if _, err := idx.GoodReadsBook(context.Background(), doc); !errors.Is(err, biblioreads.ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } - if doc, _ = idx.documentByIndexID(goodReadsTestID); mustGoodReadsData(t, doc).Enriched { + if doc, _ = idx.DocumentByID(goodReadsTestID); mustGoodReadsData(t, doc).Enriched { t.Error("expected document to stay unenriched for a later retry") } } @@ -587,7 +551,7 @@ func TestGoodReadsAuthorNameMatches(t *testing.T) { } func TestGoodReadsAuthorPrefersExactAndRejectsAmbiguousPartialMatches(t *testing.T) { - doc := Document{ + doc := index.Document{ Metadata: metadata.Metadata{Authors: []string{"Paul Kennedy", "Paul M. Kennedy"}}, AuthorsSlugs: []string{"paul-kennedy", "paul-m-kennedy"}, } @@ -608,18 +572,18 @@ func TestGoodReadsBookMatchesKennedyMiddleName(t *testing.T) { Title: title, Author: "Paul M. Kennedy", BookURL: bookURL, AuthorURL: authorURL, Rating: "4.14", }}} idx := newGoodReadsTestIndexer(t, source) - doc := Document{ + doc := index.Document{ ID: goodReadsTestID, AuthorsSlugs: []string{"paul-kennedy", "another-writer"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"Paul Kennedy", "Another Writer"}}, } - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } got, err := idx.GoodReadsBook(context.Background(), doc) if err != nil || got.BookURL != bookURL || got.Rating != "4.14" { t.Fatalf("expected Kennedy's book and rating, got %+v, %v", got, err) } - stored, err := idx.documentByIndexID(doc.ID) + stored, err := idx.DocumentByID(doc.ID) if err != nil || !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != bookURL || stored.DataSourceScoreValue != 4.14 { t.Fatalf("expected persisted book match and rating, got %+v, %v", stored, err) } @@ -699,7 +663,7 @@ func TestMatchingGoodReadsBook(t *testing.T) { if authors == nil { authors = []string{"J. R. R. Tolkien"} } - match, ok := matchingGoodReadsBook(Document{Metadata: metadata.Metadata{Title: title, Authors: authors}}, tt.results) + match, ok := matchingGoodReadsBook(index.Document{Metadata: metadata.Metadata{Title: title, Authors: authors}}, tt.results) if match.BookURL != tt.want || ok != (tt.want != "") { t.Fatalf("got %q, %v; want %q", match.BookURL, ok, tt.want) } @@ -708,7 +672,7 @@ func TestMatchingGoodReadsBook(t *testing.T) { } func TestMatchingGoodReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { - document := Document{Metadata: metadata.Metadata{ + document := index.Document{Metadata: metadata.Metadata{ Title: "Los asesinatos de Manhattan", Authors: []string{"Douglas Preston", "Lincoln Child"}, }} popular := biblioreads.SearchResult{ @@ -751,7 +715,7 @@ func TestDataSourceScoreCount(t *testing.T) { func TestMatchingGoodReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { const olderBookURL = "/book/show/1368314.El_Cercano_Oriente" const newerBookURL = "/book/show/243175949-el-cercano-oriente" - document := Document{Metadata: metadata.Metadata{ + document := index.Document{Metadata: metadata.Metadata{ Title: "El cercano Oriente", Authors: []string{"Isaac Asimov"}, }} older := biblioreads.SearchResult{ @@ -772,7 +736,7 @@ func TestMatchingGoodReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { } func TestMatchingGoodReadsBookPrefersRatedParentheticalTitle(t *testing.T) { - doc := Document{Metadata: metadata.Metadata{Title: "La promesa", Authors: []string{"Pearl S. Buck"}}} + doc := index.Document{Metadata: metadata.Metadata{Title: "La promesa", Authors: []string{"Pearl S. Buck"}}} rated := biblioreads.SearchResult{ Title: "La promesa (The Promise)", Author: "Pearl S. Buck", BookURL: "/book/show/7377104-la-promesa", Rating: "4.04", @@ -803,18 +767,18 @@ func TestGoodReadsBookMatchesDolnickSubtitle(t *testing.T) { BookURL: bookURL, Rating: "4.19", }}} idx := newGoodReadsTestIndexer(t, source) - doc := Document{ + doc := index.Document{ ID: goodReadsTestID, AuthorsSlugs: []string{"edward-dolnick"}, Metadata: metadata.Metadata{Title: title, Authors: []string{author}}, } - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } got, err := idx.GoodReadsBook(context.Background(), doc) if err != nil || got.BookURL != bookURL || got.Rating != "4.19" { t.Fatalf("expected Dolnick's book and score, got %+v, %v", got, err) } - stored, err := idx.documentByIndexID(doc.ID) + stored, err := idx.DocumentByID(doc.ID) if err != nil || !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != bookURL { t.Fatalf("expected subtitle match persisted, got %+v, %v", stored, err) } @@ -838,8 +802,8 @@ func TestGoodReadsTitleFallback(t *testing.T) { title: {parody, correct}, }} idx := newGoodReadsTestIndexer(t, source) - doc := Document{ID: goodReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + doc := index.Document{ID: goodReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } if fallbackUnavailable { @@ -850,7 +814,7 @@ func TestGoodReadsTitleFallback(t *testing.T) { if !errors.Is(err, biblioreads.ErrUnavailable) || got.BookURL != "" { t.Fatalf("expected unavailable error, got %+v, %v", got, err) } - stored, readErr := idx.documentByIndexID(doc.ID) + stored, readErr := idx.DocumentByID(doc.ID) if readErr != nil || mustGoodReadsData(t, stored).Enriched { t.Fatalf("expected document left for retry, got %+v, %v", stored, readErr) } @@ -884,15 +848,15 @@ func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { } source := &fakeGoodReadsSource{byQuery: map[string][]biblioreads.SearchResult{title: results}} idx := newGoodReadsTestIndexer(t, source) - doc := Document{ID: goodReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + doc := index.Document{ID: goodReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } got, err := idx.GoodReadsBook(context.Background(), doc) if err != nil || got.BookURL != wantURL { t.Fatalf("got %+v, %v; want %q", got, err, wantURL) } - stored, err := idx.documentByIndexID(doc.ID) + stored, err := idx.DocumentByID(doc.ID) if err != nil { t.Fatal(err) } @@ -914,7 +878,7 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.documentByIndexID(goodReadsTestID) + doc, err := idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -924,7 +888,7 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { data.LegacyBookID = 1 data.Enriched = true }) - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } if err := idx.AddLibrary(10, true, 1); err != nil { @@ -933,7 +897,7 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err = idx.documentByIndexID(goodReadsTestID) + doc, err = idx.DocumentByID(goodReadsTestID) if err != nil { t.Fatal(err) } @@ -949,7 +913,7 @@ func TestStoreGoodReadsDetails(t *testing.T) { for _, scenario := range []string{"linked", "newly linked", "changed metadata", "changed match", "deleted", "partial identifiers"} { t.Run(scenario, func(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) - doc := withGoodReadsData(t, Document{ + doc := withGoodReadsData(t, index.Document{ ID: goodReadsTestID, Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, @@ -974,31 +938,31 @@ func TestStoreGoodReadsDetails(t *testing.T) { resourceID = "" } if scenario != "deleted" { - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } } book := biblioreads.BookResponse{Rating: "4.12", ResourceID: resourceID, LegacyBookID: 12207444} - writes := &goodReadsWriteCountingIndex{countedGoodReadsIndex: idx.documentsIdx} - idx.documentsIdx = writes + writes := idx.documentsIdx + writes.count.Store(0) err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, book) wantError := scenario != "linked" && scenario != "newly linked" && scenario != "partial identifiers" if (err != nil) != wantError { t.Fatalf("StoreGoodReadsDetails error = %v, want error %v", err, wantError) } if scenario == "deleted" { - if !errors.Is(err, ErrDocumentNotFound) { - t.Fatalf("expected ErrDocumentNotFound, got %v", err) + if !errors.Is(err, index.ErrDocumentNotFound) { + t.Fatalf("expected index.ErrDocumentNotFound, got %v", err) } return } - stored, err := idx.documentByIndexID(doc.ID) + stored, err := idx.DocumentByID(doc.ID) if err != nil { t.Fatal(err) } if wantError { - if writes.count != 0 { + if writes.count.Load() != 0 { t.Fatal("stale details must not cause any index writes") } if mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 || stored.DataSourceScoreValue != 0 { @@ -1023,8 +987,8 @@ func TestStoreGoodReadsDetails(t *testing.T) { if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, book); err != nil { t.Fatalf("repeated store failed: %v", err) } - if writes.count != 1 { - t.Fatalf("expected one combined write and no write for unchanged details, got %d", writes.count) + if writes.count.Load() != 1 { + t.Fatalf("expected one combined write and no write for unchanged details, got %d", writes.count.Load()) } }) } @@ -1036,11 +1000,11 @@ type countedGoodReadsIndex interface { type goodReadsWriteCountingIndex struct { countedGoodReadsIndex - count int + count atomic.Int64 } func (i *goodReadsWriteCountingIndex) Index(id string, data any) error { - i.count++ + i.count.Add(1) return i.countedGoodReadsIndex.Index(id, data) } @@ -1063,18 +1027,18 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) - doc := withGoodReadsData(t, Document{ + doc := withGoodReadsData(t, index.Document{ ID: goodReadsTestID, DataSourceScoreValue: 3.82, }, biblioreads.DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 1}) - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } if err := idx.StoreGoodReadsDetails(doc, mustGoodReadsData(t, doc).URL, tt.book); err != nil { t.Fatal(err) } - stored, err := idx.documentByIndexID(doc.ID) + stored, err := idx.DocumentByID(doc.ID) if err != nil { t.Fatal(err) } @@ -1094,7 +1058,7 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { func TestUpdateDataSourceLink(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) - doc := withGoodReadsData(t, Document{ + doc := withGoodReadsData(t, index.Document{ ID: goodReadsTestID, Slug: "la-reina-dragon", Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, TextRankWords: []string{"dragon"}, @@ -1102,14 +1066,14 @@ func TestUpdateDataSourceLink(t *testing.T) { DataSourceScoreValue: 3.82, }, biblioreads.DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 12207444, Enriched: true}) - if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } const newLink = "provider-specific-document-reference" if err := idx.UpdateDataSourceLink(doc.ID, newLink); err != nil { t.Fatal(err) } - stored, err := idx.documentByIndexID(doc.ID) + stored, err := idx.DocumentByID(doc.ID) if err != nil { t.Fatal(err) } @@ -1127,7 +1091,66 @@ func TestUpdateDataSourceLink(t *testing.T) { func TestUpdateDataSourceLinkMissingDocument(t *testing.T) { idx := newGoodReadsTestIndexer(t, nil) - if err := idx.UpdateDataSourceLink("missing", "provider-specific-document-reference"); !errors.Is(err, ErrDocumentNotFound) { + if err := idx.UpdateDataSourceLink("missing", "provider-specific-document-reference"); !errors.Is(err, index.ErrDocumentNotFound) { t.Fatalf("expected missing-document error, got %v", err) } } + +func TestGoodReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { + idx := newGoodReadsTestIndexer(t, &fakeGoodReadsSource{results: goodReadsTestResults}) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + stale, err := idx.DocumentByID(goodReadsTestID) + if err != nil { + t.Fatal(err) + } + current := stale + other := map[string]any{"id": "external-id", "nested": map[string]any{"count": float64(3)}} + if err := index.SetDataSourceData(¤t, "other", other); err != nil { + t.Fatal(err) + } + if _, exists := stale.DataSources["other"]; exists { + t.Fatal("updating a provider mutated another document's map") + } + if err := index.IndexDocument(idx.documentsIdx, current.ID, current); err != nil { + t.Fatal(err) + } + if err := idx.Enricher.Enrich(context.Background(), idx.BleveIndexer, stale); err != nil { + t.Fatal(err) + } + if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, biblioreads.BookResponse{ + Rating: "4.5", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, + }); err != nil { + t.Fatal(err) + } + if err := idx.UpdateDataSourceLink(current.ID, "/book/show/456"); err != nil { + t.Fatal(err) + } + stale.TextRankWords = []string{"updated"} + if err := idx.UpdateDocument(stale.ID, func(update *index.DocumentUpdate) (bool, error) { + update.TextRankWords = stale.TextRankWords + return true, nil + }); err != nil { + t.Fatal(err) + } + got, err := idx.DocumentByID(stale.ID) + if err != nil { + t.Fatal(err) + } + decoded, err := index.DataSourceData[map[string]any](got, "other") + if err != nil || !reflect.DeepEqual(decoded, other) { + t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) + } + if data := mustGoodReadsData(t, got); data != (biblioreads.DocumentData{URL: "/book/show/456", Enriched: true}) { + t.Fatalf("stale write restored old Goodreads state: %+v", data) + } +} + +func TestGoodReadsBookRejectsInvalidProviderData(t *testing.T) { + idx := newGoodReadsTestIndexer(t, nil) + document := index.Document{DataSources: map[string]json.RawMessage{biblioreads.Source: json.RawMessage(`{"url":42}`)}} + if _, err := idx.GoodReadsBook(context.Background(), document); err == nil { + t.Fatal("Goodreads accepted an incorrectly typed URL") + } +} diff --git a/internal/enrichment/goodreads/matching.go b/internal/enrichment/goodreads/matching.go new file mode 100644 index 00000000..d8f44942 --- /dev/null +++ b/internal/enrichment/goodreads/matching.go @@ -0,0 +1,214 @@ +package goodreads + +import ( + "log" + "math" + "regexp" + "strconv" + "strings" + "unicode" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" + "golang.org/x/text/unicode/norm" +) + +// Strip named series annotations and omnibus ranges, not standalone volume numbers. +var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) + +var goodReadsSubtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+`) + +var goodReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) + +var goodReadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) + +func normalizeGoodReadsName(value string) string { + var result strings.Builder + for _, r := range norm.NFD.String(strings.ToLower(value)) { + switch { + case unicode.Is(unicode.Mn, r): + continue + case unicode.IsLetter(r), unicode.IsNumber(r): + result.WriteRune(r) + default: + result.WriteByte(' ') + } + } + return strings.Join(strings.Fields(result.String()), " ") +} + +func goodReadsAuthorNameMatches(name, candidate string) bool { + name = normalizeGoodReadsName(name) + candidate = normalizeGoodReadsName(candidate) + if name == "" || candidate == "" { + return false + } + if strings.ReplaceAll(name, " ", "") == strings.ReplaceAll(candidate, " ", "") { + return true + } + short, long := strings.Fields(name), strings.Fields(candidate) + if len(short) > len(long) { + short, long = long, short + } + if len(short) < 2 || short[0] != long[0] || short[len(short)-1] != long[len(long)-1] { + return false + } + // Allow omitted middle names, but not conflicting ones. + next := 1 + for _, part := range long[1 : len(long)-1] { + if next < len(short)-1 && part == short[next] { + next++ + } + } + return next == len(short)-1 +} + +func goodReadsAuthor(document index.Document, matchedName string) (string, string) { + nameKey := strings.ReplaceAll(normalizeGoodReadsName(matchedName), " ", "") + if nameKey == "" { + return "", "" + } + for i, name := range document.Authors { + if i < len(document.AuthorsSlugs) && strings.ReplaceAll(normalizeGoodReadsName(name), " ", "") == nameKey { + return document.AuthorsSlugs[i], name + } + } + var slug, authorName string + for i, name := range document.Authors { + if i < len(document.AuthorsSlugs) && goodReadsAuthorNameMatches(name, matchedName) { + if slug != "" { + return "", "" + } + slug, authorName = document.AuthorsSlugs[i], name + } + } + return slug, authorName +} + +func goodReadsTitle(value string) string { + return normalizeGoodReadsName(goodReadsSeriesSuffix.ReplaceAllString(value, "")) +} + +func goodReadsMainTitle(value string) string { + value = goodReadsSeriesSuffix.ReplaceAllString(value, "") + if separator := goodReadsSubtitleSeparator.FindStringIndex(value); separator != nil { + value = value[:separator[0]] + } + return normalizeGoodReadsName(value) +} + +func goodReadsTitleMatchScore(candidate, title, mainTitle string) int { + candidateTitle := goodReadsTitle(candidate) + if candidateTitle == "" { + return 0 + } + if candidateTitle == title { + return 3 + } + if goodReadsMainTitle(candidate) == title || mainTitle == candidateTitle || + goodReadsTitle(goodReadsParentheticalTitle.ReplaceAllString(candidate, "")) == title { + return 1 + } + return 0 +} + +func preferGoodReadsBook(candidate, best biblioreads.SearchResult, score, bestScore int) bool { + if bestScore == 0 { + return true + } + if score == bestScore { + count, bestCount := goodReadsRatingCount(candidate.RatingCount), goodReadsRatingCount(best.RatingCount) + if count != bestCount { + return count > bestCount + } + id, bestID := goodReadsBookID(candidate.BookURL), goodReadsBookID(best.BookURL) + if id > 0 && bestID > 0 && id != bestID { + return id < bestID + } + } + hasRating, bestHasRating := hasGoodReadsRating(candidate.Rating), hasGoodReadsRating(best.Rating) + if hasRating != bestHasRating { + return hasRating + } + return score > bestScore +} + +func goodReadsRatingCount(value string) uint64 { + if strings.TrimSpace(value) == "" { + return 0 + } + count, err := strconv.ParseUint(strings.ReplaceAll(strings.TrimSpace(value), ",", ""), 10, 64) + if err != nil { + log.Printf("Invalid GoodReads rating count: %q", value) + return 0 + } + return count +} + +func goodReadsBookID(bookURL string) uint64 { + match := goodReadsBookPath.FindStringSubmatch(bookURL) + if match == nil { + return 0 + } + id, err := strconv.ParseUint(match[1], 10, 64) + if err != nil { + log.Printf("Invalid GoodReads book ID in URL: %q", bookURL) + return 0 + } + return id +} + +func matchingGoodReadsBook(document index.Document, results []biblioreads.SearchResult) (biblioreads.SearchResult, bool) { + title := goodReadsTitle(document.Title) + if title == "" { + return biblioreads.SearchResult{}, false + } + mainTitle := goodReadsMainTitle(document.Title) + var best biblioreads.SearchResult + bestScore := 0 + for _, result := range results { + if result.BookURL == "" { + continue + } + score := goodReadsTitleMatchScore(result.Title, title, mainTitle) + if score == 0 { + continue + } + for _, author := range document.Authors { + if goodReadsAuthorNameMatches(author, result.Author) { + if preferGoodReadsBook(result, best, score, bestScore) { + best, bestScore = result, score + } + break + } + } + } + return best, bestScore > 0 +} + +func hasGoodReadsRating(rating string) bool { + value, err := strconv.ParseFloat(rating, 64) + return err == nil && value > 0 && value <= 5 && !math.IsNaN(value) && !math.IsInf(value, 0) +} + +// goodReadsQuery returns the Goodreads search string for document: its title +// followed by its first author, if any. +func goodReadsQuery(document index.Document) string { + q := strings.TrimSpace(document.Title) + if len(document.Authors) > 0 { + q = strings.TrimSpace(q + " " + document.Authors[0]) + } + return q +} + +func parseGoodReadsRating(rating, documentID string) float64 { + if rating == "" { + return 0 + } + value, err := strconv.ParseFloat(rating, 64) + if err != nil || value < 0 || value > 5 || math.IsNaN(value) || math.IsInf(value, 0) { + log.Printf("Invalid GoodReads rating for document %s: %q", documentID, rating) + return 0 + } + return value +} diff --git a/internal/index/bleve_author_migrate_test.go b/internal/index/bleve_author_migrate_test.go index 89e16fb7..4d03ef04 100644 --- a/internal/index/bleve_author_migrate_test.go +++ b/internal/index/bleve_author_migrate_test.go @@ -90,7 +90,7 @@ func TestOpenAuthorsIndexMigratesStoredMetadata(t *testing.T) { t.Fatalf("expected original index removed after successful migration, got %v", err) } - got.DataSourceLinks["goodreads"] = wantGoodReadsAuthorURL + got.DataSourceLinks["goodreads"] = "https://example.org/authors/writer" if err := idx.IndexAuthor(got); err != nil { t.Fatal(err) } @@ -108,7 +108,7 @@ func TestOpenAuthorsIndexMigratesStoredMetadata(t *testing.T) { if err != nil || len(result.Hits) != 1 { t.Fatalf("reading reopened author: %v", err) } - want.DataSourceLinks["goodreads"] = wantGoodReadsAuthorURL + want.DataSourceLinks["goodreads"] = "https://example.org/authors/writer" if got := hydrateAuthor(result.Hits[0]); !reflect.DeepEqual(got, want) { t.Fatalf("migrated mapping or URL did not survive reopening: %#v", got) } diff --git a/internal/index/bleve_document_data.go b/internal/index/bleve_document_data.go new file mode 100644 index 00000000..2317e0da --- /dev/null +++ b/internal/index/bleve_document_data.go @@ -0,0 +1,137 @@ +package index + +import ( + "encoding/json" + "fmt" + "maps" + + "github.com/blevesearch/bleve/v2" +) + +// AuthorLink is a provider-owned link discovered while enriching a document. +type AuthorLink struct { + Slug string + Name string + Source string + URL string +} + +// DocumentUpdate groups document changes and discovered author links. +type DocumentUpdate struct { + Document + AuthorLinks []AuthorLink +} + +// DocumentByID retrieves a document using its index ID rather than its public slug. +func (b *BleveIndexer) DocumentByID(id string) (Document, error) { + return b.documentByIndexID(id) +} + +// UpdateDocument applies changes to the latest stored document under the write lock. +// The callback returns false when no write is needed. It must not call other index methods. +func (b *BleveIndexer) UpdateDocument(id string, update func(*DocumentUpdate) (bool, error)) error { + b.documentsMu.Lock() + defer b.documentsMu.Unlock() + document, found, err := b.documentByIndexIDLocked(id) + if err != nil { + return fmt.Errorf("reading document %s for update: %w", id, err) + } + if !found { + return ErrDocumentNotFound + } + pending := DocumentUpdate{Document: document} + changed, err := update(&pending) + if err != nil || !changed { + return err + } + for _, link := range pending.AuthorLinks { + if err := b.storeAuthorLinkLocked(link); err != nil { + return err + } + } + return IndexDocument(b.documentsIdx, id, pending.Document) +} + +// storeAuthorLinkLocked requires documentsMu to be held. +func (b *BleveIndexer) storeAuthorLinkLocked(link AuthorLink) error { + if link.Slug == "" || link.Source == "" || link.URL == "" { + return fmt.Errorf("author link must have a slug, source and URL") + } + b.authorsMu.Lock() + defer b.authorsMu.Unlock() + author, err := b.authorBySlugLocked(link.Slug) + if err != nil { + return err + } + if author.Slug == "" { + author = Author{Slug: link.Slug, Name: link.Name} + for _, field := range []string{"AuthorsSlugs", "IllustratorsSlugs"} { + q := bleve.NewTermQuery(link.Slug) + q.SetField(field) + result, err := b.documentsIdx.Search(bleve.NewSearchRequestOptions(q, 0, 0, false)) + if err != nil { + return fmt.Errorf("counting documents for author %s: %w", link.Slug, err) + } + author.DocumentCount += result.Total + } + } + if author.DataSourceLinks[link.Source] != "" { + return nil + } + if author.DataSourceLinks == nil { + author.DataSourceLinks = make(map[string]string) + } + author.DataSourceLinks[link.Source] = link.URL + return indexAuthor(b.authorsIdx, link.Slug, author) +} + +func (b *BleveIndexer) documentByIndexIDLocked(id string) (Document, bool, error) { + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) + req.Fields = []string{"*"} + result, err := b.documentsIdx.Search(req) + if err != nil || len(result.Hits) == 0 { + return Document{}, false, err + } + document := hydrateDocument(result.Hits[0]) + document.DataSources, err = decodeDataSources(result.Hits[0].Fields, id) + return document, true, err +} + +// preserveDataSourcesLocked protects provider updates from stale metadata writers. +// It requires documentsMu to be held for writing. +func (b *BleveIndexer) preserveDataSourcesLocked(documents []Document) error { + if len(documents) == 0 { + return nil + } + ids := make([]string, len(documents)) + for i, document := range documents { + ids[i] = document.ID + } + req := bleve.NewSearchRequestOptions(bleve.NewDocIDQuery(ids), len(ids), 0, false) + req.Fields = []string{"DataSources", "DataSourceScoreValue"} + result, err := b.documentsIdx.Search(req) + if err != nil { + return fmt.Errorf("reading stored data source fields: %w", err) + } + stored := make(map[string]Document, len(result.Hits)) + for _, hit := range result.Hits { + ratingValue, _ := hit.Fields["DataSourceScoreValue"].(float64) + sources, err := decodeDataSources(hit.Fields, hit.ID) + if err != nil { + return err + } + stored[hit.ID] = Document{DataSources: sources, DataSourceScoreValue: ratingValue} + } + for i := range documents { + if current, ok := stored[documents[i].ID]; ok { + merged := maps.Clone(documents[i].DataSources) + if merged == nil && len(current.DataSources) > 0 { + merged = make(map[string]json.RawMessage) + } + maps.Copy(merged, current.DataSources) + documents[i].DataSources = merged + documents[i].DataSourceScoreValue = current.DataSourceScoreValue + } + } + return nil +} diff --git a/internal/index/bleve_document_data_test.go b/internal/index/bleve_document_data_test.go new file mode 100644 index 00000000..039b6e31 --- /dev/null +++ b/internal/index/bleve_document_data_test.go @@ -0,0 +1,178 @@ +package index + +import ( + "encoding/json" + "errors" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/svera/coreander/v5/internal/metadata" +) + +type documentUpdateBaseIndex interface { + bleve.Index +} + +type documentUpdateCountingIndex struct { + documentUpdateBaseIndex + writes int +} + +func (i *documentUpdateCountingIndex) Index(id string, data any) error { + i.writes++ + return i.documentUpdateBaseIndex.Index(id, data) +} + +func newDocumentUpdateTestIndexer(t *testing.T) (*BleveIndexer, *documentUpdateCountingIndex, *documentUpdateCountingIndex) { + t.Helper() + documents, err := bleve.NewMemOnly(CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authors, err := bleve.NewMemOnly(CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + documentWrites := &documentUpdateCountingIndex{documentUpdateBaseIndex: documents} + authorWrites := &documentUpdateCountingIndex{documentUpdateBaseIndex: authors} + idx := NewBleve(documentWrites, authorWrites, nil, "", nil, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + return idx, documentWrites, authorWrites +} + +func TestUpdateDocumentSkipsUnchangedAndFailedUpdates(t *testing.T) { + failure := errors.New("provider failed") + for _, scenario := range []struct { + name string + changed bool + err error + }{ + {name: "unchanged"}, + {name: "failed", changed: true, err: failure}, + } { + t.Run(scenario.name, func(t *testing.T) { + idx, documents, authors := newDocumentUpdateTestIndexer(t) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Original"}} + if err := IndexDocument(documents, document.ID, document); err != nil { + t.Fatal(err) + } + before, err := idx.DocumentByID(document.ID) + if err != nil { + t.Fatal(err) + } + documents.writes = 0 + called := false + err = idx.UpdateDocument(document.ID, func(update *DocumentUpdate) (bool, error) { + called = true + update.Title = "Changed" + update.AuthorLinks = []AuthorLink{{Slug: "writer", Name: "Writer", Source: "dummy", URL: "reference"}} + if err := SetDataSourceData(&update.Document, "dummy", testProviderData{Enriched: true}); err != nil { + t.Fatal(err) + } + return scenario.changed, scenario.err + }) + if !called || !errors.Is(err, scenario.err) { + t.Fatalf("callback called = %v, error = %v, want %v", called, err, scenario.err) + } + if documents.writes != 0 || authors.writes != 0 { + t.Fatalf("skipped update wrote documents/authors: %d/%d", documents.writes, authors.writes) + } + after, err := idx.DocumentByID(document.ID) + if err != nil || !reflect.DeepEqual(after, before) { + t.Fatalf("skipped update changed document: %+v, %v", after, err) + } + }) + } +} + +func TestUpdateDocumentMissingDocumentDoesNotCallCallback(t *testing.T) { + idx, documents, authors := newDocumentUpdateTestIndexer(t) + called := false + err := idx.UpdateDocument("missing", func(*DocumentUpdate) (bool, error) { + called = true + return true, nil + }) + if !errors.Is(err, ErrDocumentNotFound) || called || documents.writes != 0 || authors.writes != 0 { + t.Fatalf("missing-document update: error %v, callback %v, writes %d/%d", err, called, documents.writes, authors.writes) + } +} + +func TestUpdateDocumentRejectsMalformedProviderData(t *testing.T) { + for _, raw := range []string{`{"url":`, `null`, `[]`} { + t.Run(raw, func(t *testing.T) { + idx, documents, _ := newDocumentUpdateTestIndexer(t) + if err := IndexDocument(documents, "book", Document{ID: "book", Metadata: metadata.Metadata{Title: "Original"}}); err != nil { + t.Fatal(err) + } + documents.writes = 0 + err := idx.UpdateDocument("book", func(update *DocumentUpdate) (bool, error) { + update.Title = "Invalid update" + update.DataSources = map[string]json.RawMessage{"dummy": json.RawMessage(raw)} + return true, nil + }) + if err == nil || documents.writes != 0 { + t.Fatalf("malformed data accepted: error %v, writes %d", err, documents.writes) + } + stored, err := idx.DocumentByID("book") + if err != nil || stored.Title != "Original" || len(stored.DataSources) != 0 { + t.Fatalf("malformed update changed document: %+v, %v", stored, err) + } + }) + } +} + +func TestUpdateDocumentStoresAuthorLinksFirstWins(t *testing.T) { + for _, existing := range []bool{false, true} { + name := "new author" + if existing { + name = "existing author" + } + t.Run(name, func(t *testing.T) { + idx, documents, _ := newDocumentUpdateTestIndexer(t) + for _, document := range []Document{ + {ID: "book", AuthorsSlugs: []string{"writer"}}, + {ID: "second", AuthorsSlugs: []string{"writer"}}, + {ID: "illustrated", IllustratorsSlugs: []string{"writer"}}, + {ID: "unrelated", AuthorsSlugs: []string{"other"}}, + } { + if err := IndexDocument(documents, document.ID, document); err != nil { + t.Fatal(err) + } + } + if existing { + if err := idx.IndexAuthor(Author{ + Name: "Existing Writer", Slug: "writer", DocumentCount: 42, DataSourceID: "external-id", + DataSourceLinks: map[string]string{"other": "other-reference"}, + }); err != nil { + t.Fatal(err) + } + } + for _, url := range []string{"first-reference", "second-reference"} { + if err := idx.UpdateDocument("book", func(update *DocumentUpdate) (bool, error) { + update.Title = url + update.AuthorLinks = []AuthorLink{{Slug: "writer", Name: "Writer", Source: "dummy", URL: url}} + return true, nil + }); err != nil { + t.Fatal(err) + } + } + author, err := idx.Author("writer", "") + if err != nil || author.DataSourceLinks["dummy"] != "first-reference" { + t.Fatalf("first author link lost: %+v, %v", author, err) + } + if existing { + if author.Name != "Existing Writer" || author.DocumentCount != 42 || + author.DataSourceID != "external-id" || author.DataSourceLinks["other"] != "other-reference" { + t.Fatalf("existing author metadata changed: %+v", author) + } + } else if author.Name != "Writer" || author.DocumentCount != 3 { + t.Fatalf("new author name or author/illustrator count incorrect: %+v", author) + } + document, err := idx.DocumentByID("book") + if err != nil || document.Title != "second-reference" { + t.Fatalf("author link prevented document update: %+v, %v", document, err) + } + }) + } +} diff --git a/internal/index/bleve_goodreads_write.go b/internal/index/bleve_goodreads_write.go deleted file mode 100644 index 66e47d48..00000000 --- a/internal/index/bleve_goodreads_write.go +++ /dev/null @@ -1,547 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log" - "maps" - "math" - "regexp" - "strconv" - "strings" - "time" - "unicode" - - "github.com/blevesearch/bleve/v2" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" - "golang.org/x/text/unicode/norm" -) - -const ( - goodReadsEnrichWorkers = documentEnrichWorkers - goodReadsRequestTimeout = 2 * time.Minute -) - -// GoodReadsSource searches books on Goodreads; implemented by *biblioreads.Client. -type GoodReadsSource interface { - SearchBooks(ctx context.Context, query string) (biblioreads.SearchBooksResponse, error) -} - -// GoodReadsEnricher adapts Goodreads matching to the document enrichment pipeline. -type GoodReadsEnricher struct { - source GoodReadsSource - slots chan struct{} -} - -func NewGoodReadsEnricher(source GoodReadsSource) *GoodReadsEnricher { - return &GoodReadsEnricher{source: source, slots: make(chan struct{}, goodReadsEnrichWorkers)} -} - -func (*GoodReadsEnricher) Source() string { return biblioreads.Source } - -func (*GoodReadsEnricher) NeedsEnrichment(document Document) (bool, error) { - data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) - return !data.Enriched, err -} - -func (p *GoodReadsEnricher) Enrich(ctx context.Context, b *BleveIndexer, document Document) error { - _, err := p.enrich(ctx, b, document) - return err -} - -func (b *BleveIndexer) goodReadsProvider() *GoodReadsEnricher { - for _, provider := range b.documentEnrichers { - if p, ok := provider.(*GoodReadsEnricher); ok { - return p - } - } - return nil -} - -var dataSourceFields = []string{"DataSources", "DataSourceScoreValue"} - -// Strip named series annotations and omnibus ranges, not standalone volume numbers. -var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) - -var goodReadsSubtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+`) - -var goodReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) - -var goodReadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) - -func normalizeGoodReadsName(value string) string { - var result strings.Builder - for _, r := range norm.NFD.String(strings.ToLower(value)) { - switch { - case unicode.Is(unicode.Mn, r): - continue - case unicode.IsLetter(r), unicode.IsNumber(r): - result.WriteRune(r) - default: - result.WriteByte(' ') - } - } - return strings.Join(strings.Fields(result.String()), " ") -} - -func goodReadsAuthorNameMatches(name, candidate string) bool { - name = normalizeGoodReadsName(name) - candidate = normalizeGoodReadsName(candidate) - if name == "" || candidate == "" { - return false - } - if strings.ReplaceAll(name, " ", "") == strings.ReplaceAll(candidate, " ", "") { - return true - } - short, long := strings.Fields(name), strings.Fields(candidate) - if len(short) > len(long) { - short, long = long, short - } - if len(short) < 2 || short[0] != long[0] || short[len(short)-1] != long[len(long)-1] { - return false - } - // Allow omitted middle names, but not conflicting ones. - next := 1 - for _, part := range long[1 : len(long)-1] { - if next < len(short)-1 && part == short[next] { - next++ - } - } - return next == len(short)-1 -} - -func goodReadsAuthor(document Document, matchedName string) (string, string) { - nameKey := strings.ReplaceAll(normalizeGoodReadsName(matchedName), " ", "") - if nameKey == "" { - return "", "" - } - for i, name := range document.Authors { - if i < len(document.AuthorsSlugs) && strings.ReplaceAll(normalizeGoodReadsName(name), " ", "") == nameKey { - return document.AuthorsSlugs[i], name - } - } - var slug, authorName string - for i, name := range document.Authors { - if i < len(document.AuthorsSlugs) && goodReadsAuthorNameMatches(name, matchedName) { - if slug != "" { - return "", "" - } - slug, authorName = document.AuthorsSlugs[i], name - } - } - return slug, authorName -} - -// storeGoodReadsAuthorURL requires documentsMu to be held. -func (b *BleveIndexer) storeGoodReadsAuthorURL(document Document, authorURL, matchedName string) error { - if authorURL == "" { - return nil - } - authorSlug, name := goodReadsAuthor(document, matchedName) - if authorSlug == "" { - return nil - } - b.authorsMu.Lock() - defer b.authorsMu.Unlock() - author, err := b.authorBySlugLocked(authorSlug) - if err != nil { - return err - } - if author.Slug == "" { - author = Author{Slug: authorSlug, Name: name} - for _, field := range []string{"AuthorsSlugs", "IllustratorsSlugs"} { - q := bleve.NewTermQuery(authorSlug) - q.SetField(field) - result, err := b.documentsIdx.Search(bleve.NewSearchRequestOptions(q, 0, 0, false)) - if err != nil { - return fmt.Errorf("counting documents for author %s: %w", authorSlug, err) - } - author.DocumentCount += result.Total - } - } - if author.DataSourceLinks["goodreads"] != "" { - return nil - } - if author.DataSourceLinks == nil { - author.DataSourceLinks = make(map[string]string) - } - author.DataSourceLinks["goodreads"] = authorURL - return indexAuthor(b.authorsIdx, authorSlug, author) -} - -func goodReadsTitle(value string) string { - return normalizeGoodReadsName(goodReadsSeriesSuffix.ReplaceAllString(value, "")) -} - -func goodReadsMainTitle(value string) string { - value = goodReadsSeriesSuffix.ReplaceAllString(value, "") - if separator := goodReadsSubtitleSeparator.FindStringIndex(value); separator != nil { - value = value[:separator[0]] - } - return normalizeGoodReadsName(value) -} - -func goodReadsTitleMatchScore(candidate, title, mainTitle string) int { - candidateTitle := goodReadsTitle(candidate) - if candidateTitle == "" { - return 0 - } - if candidateTitle == title { - return 3 - } - if goodReadsMainTitle(candidate) == title || mainTitle == candidateTitle || - goodReadsTitle(goodReadsParentheticalTitle.ReplaceAllString(candidate, "")) == title { - return 1 - } - return 0 -} - -func preferGoodReadsBook(candidate, best biblioreads.SearchResult, score, bestScore int) bool { - if bestScore == 0 { - return true - } - if score == bestScore { - count, bestCount := goodReadsRatingCount(candidate.RatingCount), goodReadsRatingCount(best.RatingCount) - if count != bestCount { - return count > bestCount - } - id, bestID := goodReadsBookID(candidate.BookURL), goodReadsBookID(best.BookURL) - if id > 0 && bestID > 0 && id != bestID { - return id < bestID - } - } - hasRating, bestHasRating := hasGoodReadsRating(candidate.Rating), hasGoodReadsRating(best.Rating) - if hasRating != bestHasRating { - return hasRating - } - return score > bestScore -} - -func goodReadsRatingCount(value string) uint64 { - if strings.TrimSpace(value) == "" { - return 0 - } - count, err := strconv.ParseUint(strings.ReplaceAll(strings.TrimSpace(value), ",", ""), 10, 64) - if err != nil { - log.Printf("Invalid GoodReads rating count: %q", value) - return 0 - } - return count -} - -func goodReadsBookID(bookURL string) uint64 { - match := goodReadsBookPath.FindStringSubmatch(bookURL) - if match == nil { - return 0 - } - id, err := strconv.ParseUint(match[1], 10, 64) - if err != nil { - log.Printf("Invalid GoodReads book ID in URL: %q", bookURL) - return 0 - } - return id -} - -func matchingGoodReadsBook(document Document, results []biblioreads.SearchResult) (biblioreads.SearchResult, bool) { - title := goodReadsTitle(document.Title) - if title == "" { - return biblioreads.SearchResult{}, false - } - mainTitle := goodReadsMainTitle(document.Title) - var best biblioreads.SearchResult - bestScore := 0 - for _, result := range results { - if result.BookURL == "" { - continue - } - score := goodReadsTitleMatchScore(result.Title, title, mainTitle) - if score == 0 { - continue - } - for _, author := range document.Authors { - if goodReadsAuthorNameMatches(author, result.Author) { - if preferGoodReadsBook(result, best, score, bestScore) { - best, bestScore = result, score - } - break - } - } - } - return best, bestScore > 0 -} - -func hasGoodReadsRating(rating string) bool { - value, err := strconv.ParseFloat(rating, 64) - return err == nil && value > 0 && value <= 5 && !math.IsNaN(value) && !math.IsInf(value, 0) -} - -// goodReadsQuery returns the Goodreads search string for document: its title -// followed by its first author, if any. -func goodReadsQuery(document Document) string { - q := strings.TrimSpace(document.Title) - if len(document.Authors) > 0 { - q = strings.TrimSpace(q + " " + document.Authors[0]) - } - return q -} - -func (b *BleveIndexer) documentsNeedingGoodReads(size int) ([]Document, uint64, error) { - return b.documentsNeedingProvider(biblioreads.Source, size) -} - -// GoodReadsBook returns the stored book URL and rating, or searches a document -// not looked up yet and persists its match. Returns an empty result if there's no match. -func (b *BleveIndexer) GoodReadsBook(ctx context.Context, document Document) (biblioreads.SearchResult, error) { - data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) - if err != nil { - return biblioreads.SearchResult{}, err - } - if data.URL != "" || data.Enriched || b.goodReadsProvider() == nil { - return biblioreads.SearchResult{BookURL: data.URL, Rating: fmt.Sprintf("%.2f", document.DataSourceScoreValue), RatingCount: data.RatingCount}, nil - } - return b.enrichGoodReads(ctx, document) -} - -// UpdateDataSourceLink saves a provider-normalized link and clears data cached for the previous match. -func (b *BleveIndexer) UpdateDataSourceLink(documentID, reviewLink string) error { - b.documentsMu.Lock() - defer b.documentsMu.Unlock() - current, found, err := b.documentByIndexIDLocked(documentID) - if err != nil { - return fmt.Errorf("reading document %s for review-link correction: %w", documentID, err) - } - if !found { - return ErrDocumentNotFound - } - data, err := DataSourceData[biblioreads.DocumentData](current, biblioreads.Source) - if err != nil { - return err - } - if data.URL == reviewLink { - return nil - } - current.DataSourceScoreValue = 0 - if err := SetDataSourceData(¤t, biblioreads.Source, biblioreads.DocumentData{URL: reviewLink, Enriched: true}); err != nil { - return err - } - if err := IndexDocument(b.documentsIdx, current.ID, current); err != nil { - return fmt.Errorf("saving review-link correction for document %s: %w", documentID, err) - } - return nil -} - -// StoreGoodReadsDetails persists a nonempty rating, rating count, and complete review identifiers, -// provided the document still has the metadata and match read before the fetch. -func (b *BleveIndexer) StoreGoodReadsDetails(document Document, bookURL string, book biblioreads.BookResponse) error { - if bookURL == "" { - return fmt.Errorf("invalid GoodReads book URL for document %s", document.ID) - } - hasIDs := book.ResourceID != "" && book.LegacyBookID > 0 - if book.Rating == "" && book.RatingCount == "" && !hasIDs { - return nil - } - b.documentsMu.Lock() - defer b.documentsMu.Unlock() - current, found, err := b.documentByIndexIDLocked(document.ID) - if err != nil { - return fmt.Errorf("error reading document %s for GoodReads details: %w", document.ID, err) - } - if !found { - return ErrDocumentNotFound - } - data, err := DataSourceData[biblioreads.DocumentData](current, biblioreads.Source) - if err != nil { - return err - } - if goodReadsQuery(current) != goodReadsQuery(document) || data.URL != bookURL { - return fmt.Errorf("GoodReads match changed for document %s", document.ID) - } - changed := false - if book.RatingCount != "" && data.RatingCount != book.RatingCount { - data.RatingCount = book.RatingCount - changed = true - } - if book.Rating != "" { - ratingValue := parseGoodReadsRating(book.Rating, document.ID) - if current.DataSourceScoreValue != ratingValue { - current.DataSourceScoreValue = ratingValue - changed = true - } - } - if hasIDs && (data.ResourceID != book.ResourceID || data.LegacyBookID != book.LegacyBookID) { - data.ResourceID = book.ResourceID - data.LegacyBookID = book.LegacyBookID - changed = true - } - if !changed { - return nil - } - if err := SetDataSourceData(¤t, biblioreads.Source, data); err != nil { - return err - } - if err := IndexDocument(b.documentsIdx, current.ID, current); err != nil { - return fmt.Errorf("error storing GoodReads details for document %s: %w", document.ID, err) - } - return nil -} - -// enrichGoodReads searches document on Goodreads, unless it already has a -// book URL, and stores matching URLs on the document and author respectively. -// Unavailability or persistence errors leave the document unenriched for retry. -func (b *BleveIndexer) enrichGoodReads(ctx context.Context, document Document) (biblioreads.SearchResult, error) { - provider := b.goodReadsProvider() - if provider == nil { - return biblioreads.SearchResult{}, fmt.Errorf("Goodreads enrichment is not configured") - } - return provider.enrich(ctx, b, document) -} - -func (p *GoodReadsEnricher) enrich(ctx context.Context, b *BleveIndexer, document Document) (biblioreads.SearchResult, error) { - data, err := DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) - if err != nil { - return biblioreads.SearchResult{}, err - } - select { - case p.slots <- struct{}{}: - defer func() { <-p.slots }() - case <-ctx.Done(): - return biblioreads.SearchResult{}, ctx.Err() - } - q := goodReadsQuery(document) - result := biblioreads.SearchResult{BookURL: data.URL} - if result.BookURL == "" && document.Title != "" { - ctx, cancel := context.WithTimeout(ctx, goodReadsRequestTimeout) - defer cancel() - queries := []string{q} - if title := strings.TrimSpace(document.Title); title != q { - queries = append(queries, title) - } - for _, query := range queries { - res, err := p.source.SearchBooks(ctx, query) - switch { - case errors.Is(err, biblioreads.ErrUnavailable), errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): - return biblioreads.SearchResult{}, fmt.Errorf("error searching %q on GoodReads: %w", query, err) - case err != nil: - log.Printf("Error searching %q on GoodReads: %s\n", query, err) - default: - if match, ok := matchingGoodReadsBook(document, res.Result); ok { - result = match - } - } - if result.BookURL != "" { - break - } - } - } - - // Re-read the document under the write lock, so concurrent writes of other - // fields (e.g. TextRank enrichment) aren't overwritten with stale values. - b.documentsMu.Lock() - defer b.documentsMu.Unlock() - current, found, err := b.documentByIndexIDLocked(document.ID) - if err != nil { - log.Printf("Error reading document %s for GoodReads enrichment: %s\n", document.ID, err) - return result, nil - } - // Removed, or reindexed with a different title/author in the meantime - // (which schedules its own lookup). - if !found || goodReadsQuery(current) != q { - return result, nil - } - data, err = DataSourceData[biblioreads.DocumentData](current, biblioreads.Source) - if err != nil { - return result, err - } - // Looked up concurrently (e.g. by the background enrichment and a detail - // page request at the same time): keep the stored result. - if data.Enriched { - result = biblioreads.SearchResult{BookURL: data.URL, Rating: fmt.Sprintf("%.2f", current.DataSourceScoreValue), RatingCount: data.RatingCount} - return result, nil - } - if data.URL != "" && data.URL != result.BookURL { - result = biblioreads.SearchResult{BookURL: data.URL, Rating: fmt.Sprintf("%.2f", current.DataSourceScoreValue), RatingCount: data.RatingCount} - } - data.URL = result.BookURL - if result.Rating == "" && current.DataSourceScoreValue > 0 { - result.Rating = fmt.Sprintf("%.2f", current.DataSourceScoreValue) - } - current.DataSourceScoreValue = parseGoodReadsRating(result.Rating, current.ID) - data.RatingCount = result.RatingCount - data.Enriched = true - if err := SetDataSourceData(¤t, biblioreads.Source, data); err != nil { - return result, err - } - if err := b.storeGoodReadsAuthorURL(current, result.AuthorURL, result.Author); err != nil { - return result, fmt.Errorf("error storing GoodReads author URL for document %s: %w", current.ID, err) - } - if err := IndexDocument(b.documentsIdx, current.ID, current); err != nil { - return result, fmt.Errorf("error indexing GoodReads-enriched document %s: %w", current.ID, err) - } - return result, nil -} - -func parseGoodReadsRating(rating, documentID string) float64 { - if rating == "" { - return 0 - } - value, err := strconv.ParseFloat(rating, 64) - if err != nil || value < 0 || value > 5 || math.IsNaN(value) || math.IsInf(value, 0) { - log.Printf("Invalid GoodReads rating for document %s: %q", documentID, rating) - return 0 - } - return value -} - -// documentByIndexIDLocked is documentByIndexID for callers already holding documentsMu. -func (b *BleveIndexer) documentByIndexIDLocked(id string) (Document, bool, error) { - searchReq := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) - searchReq.Fields = []string{"*"} - searchResult, err := b.documentsIdx.Search(searchReq) - if err != nil || len(searchResult.Hits) == 0 { - return Document{}, false, err - } - return hydrateDocument(searchResult.Hits[0]), true, nil -} - -// preserveDataSourcesLocked copies currently stored provider data into -// documents, so writers that rebuilt them from an earlier read don't discard -// a lookup persisted in the meantime. Must be called with documentsMu held -// for writing, right before writing documents back. -func (b *BleveIndexer) preserveDataSourcesLocked(documents []Document) error { - if len(documents) == 0 { - return nil - } - ids := make([]string, len(documents)) - for i, document := range documents { - ids[i] = document.ID - } - searchReq := bleve.NewSearchRequestOptions(bleve.NewDocIDQuery(ids), len(ids), 0, false) - searchReq.Fields = dataSourceFields - searchResult, err := b.documentsIdx.Search(searchReq) - if err != nil { - return fmt.Errorf("reading stored data source fields: %w", err) - } - stored := make(map[string]Document, len(searchResult.Hits)) - for _, hit := range searchResult.Hits { - ratingValue, _ := hit.Fields["DataSourceScoreValue"].(float64) - sources, err := decodeDataSources(hit.Fields, hit.ID) - if err != nil { - return err - } - stored[hit.ID] = Document{DataSources: sources, DataSourceScoreValue: ratingValue} - } - for i := range documents { - if s, ok := stored[documents[i].ID]; ok { - merged := maps.Clone(documents[i].DataSources) - if merged == nil && len(s.DataSources) > 0 { - merged = make(map[string]json.RawMessage) - } - maps.Copy(merged, s.DataSources) - documents[i].DataSources = merged - documents[i].DataSourceScoreValue = s.DataSourceScoreValue - } - } - return nil -} diff --git a/internal/index/bleve_provider_preservation_test.go b/internal/index/bleve_provider_preservation_test.go new file mode 100644 index 00000000..bd1a6e40 --- /dev/null +++ b/internal/index/bleve_provider_preservation_test.go @@ -0,0 +1,71 @@ +package index + +import ( + "context" + "slices" + "testing" + + "github.com/svera/coreander/v5/internal/metadata" +) + +// TestTextRankWritesPreserveProviderFields guards against TextRank +// enrichment, which writes back documents read before ranking them, +// discarding provider data persisted in the meantime, and vice versa. +func TestTextRankWritesPreserveProviderFields(t *testing.T) { + source := &testDocumentEnricher{source: "dummy"} + idx, documents := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{source}}) + t.Cleanup(func() { _ = idx.Close() }) + if err := IndexDocument(documents, "book", Document{ID: "book", Metadata: metadata.Metadata{Title: "Book"}}); err != nil { + t.Fatal(err) + } + stale, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + stale.TextRankEnriched = false + want := testProviderData{ + URL: "provider-reference", Enriched: true, RatingCount: "2,285", + ResourceID: "work-id", LegacyBookID: 12207444, + } + if err := idx.UpdateDocument(stale.ID, func(update *DocumentUpdate) (bool, error) { + update.DataSourceScoreValue = 3.82 + return true, SetDataSourceData(&update.Document, source.Source(), want) + }); err != nil { + t.Fatal(err) + } + stale.TextRankWords = []string{"dragon"} + idx.enrichTextRankAndReindex(stale) + + doc, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + if data := mustProviderData(t, doc); data != want || doc.DataSourceScoreValue != 3.82 { + t.Errorf("expected provider fields and identifiers to survive a TextRank write, got %+v", doc) + } + if !doc.TextRankEnriched { + t.Error("expected TextRank write to be persisted") + } + + // A Provider write must keep TextRank fields written after the document was read. + stale, _ = idx.documentByIndexID("book") + if err := SetDataSourceData(&stale, source.Source(), testProviderData{Enriched: false}); err != nil { + t.Fatal(err) + } + doc.TextRankWords = []string{"guinevere"} + idx.documentsMu.Lock() + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + idx.documentsMu.Unlock() + if err := source.Enrich(context.Background(), idx, stale); err != nil { + t.Fatal(err) + } + if doc, _ = idx.documentByIndexID("book"); !slices.Equal(doc.TextRankWords, []string{"guinevere"}) { + t.Errorf("expected TextRank words to survive a Provider write, got %q", doc.TextRankWords) + } +} diff --git a/internal/index/document_data_test.go b/internal/index/document_data_test.go index 0801d413..0ebac9a8 100644 --- a/internal/index/document_data_test.go +++ b/internal/index/document_data_test.go @@ -1,24 +1,23 @@ package index import ( - "context" "encoding/json" "reflect" "testing" "github.com/blevesearch/bleve/v2" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/metadata" ) func TestDocumentDataSourcesRoundTrip(t *testing.T) { - idx := newGoodReadsTestIndexer(t, nil) + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) for _, language := range []string{"es", ""} { - document := withGoodReadsData(t, Document{ + document := withProviderData(t, Document{ ID: "book-" + language, Slug: "book-" + language, Metadata: metadata.Metadata{Title: "A book", Language: language, Format: "EPUB"}, DataSourceScoreValue: 4.25, - }, biblioreads.DocumentData{URL: "/book/show/123", Enriched: true}) + }, testProviderData{URL: "/book/show/123", Enriched: true}) other := map[string]any{ "id": "opaqueprovideridentifier", "nested": map[string]any{"items": []any{"privatecatalogtoken", float64(42)}}, } @@ -36,10 +35,10 @@ func TestDocumentDataSourcesRoundTrip(t *testing.T) { if err != nil || !reflect.DeepEqual(decoded, other) { t.Fatalf("provider payload changed: %+v, %v", decoded, err) } - if mustGoodReadsData(t, got) != mustGoodReadsData(t, document) || got.DataSourceScoreValue != 4.25 { - t.Fatalf("Goodreads payload or score changed: %+v", got) + if mustProviderData(t, got) != mustProviderData(t, document) || got.DataSourceScoreValue != 4.25 { + t.Fatalf("provider payload or score changed: %+v", got) } - for _, term := range []string{"opaqueprovideridentifier", "privatecatalogtoken", "goodreads", "other"} { + for _, term := range []string{"opaqueprovideridentifier", "privatecatalogtoken", "dummy", "other"} { request := bleve.NewSearchRequest(bleve.NewMatchQuery(term)) result, err := idx.documentsIdx.Search(request) if err != nil || result.Total != 0 { @@ -47,7 +46,7 @@ func TestDocumentDataSourcesRoundTrip(t *testing.T) { } } } - if _, total, err := idx.documentsNeedingGoodReads(10); err != nil || total != 0 { + if _, total, err := idx.documentsNeedingProvider("dummy", 10); err != nil || total != 0 { t.Fatalf("enriched payloads were queued again: %d, %v", total, err) } pending := Document{ID: "other-only", Slug: "other-only", Metadata: metadata.Metadata{Title: "Pending book"}} @@ -57,13 +56,14 @@ func TestDocumentDataSourcesRoundTrip(t *testing.T) { if err := IndexDocument(idx.documentsIdx, pending.ID, pending); err != nil { t.Fatal(err) } - if documents, total, err := idx.documentsNeedingGoodReads(10); err != nil || total != 1 || documents[0].ID != pending.ID { - t.Fatalf("another provider marked Goodreads as enriched: %+v, %d, %v", documents, total, err) + if documents, total, err := idx.documentsNeedingProvider("dummy", 10); err != nil || total != 1 || documents[0].ID != pending.ID { + t.Fatalf("another provider marked provider as enriched: %+v, %d, %v", documents, total, err) } } func TestDocumentDataSourcesRejectInvalidData(t *testing.T) { - idx := newGoodReadsTestIndexer(t, nil) + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} for _, invalid := range []any{nil, "not an object", []int{1}, make(chan int)} { if err := SetDataSourceData(&document, "invalid", invalid); err == nil { @@ -74,23 +74,20 @@ func TestDocumentDataSourcesRejectInvalidData(t *testing.T) { } } for _, raw := range []string{`{"url":`, `null`, `[]`} { - document.DataSources = map[string]json.RawMessage{biblioreads.Source: json.RawMessage(raw)} + document.DataSources = map[string]json.RawMessage{"dummy": json.RawMessage(raw)} if err := IndexDocument(idx.documentsIdx, document.ID, document); err == nil { t.Fatalf("indexed invalid payload: %s", raw) } } - document.DataSources = map[string]json.RawMessage{biblioreads.Source: json.RawMessage(`{"url":42}`)} - if _, err := idx.GoodReadsBook(context.Background(), document); err == nil { - t.Fatal("Goodreads accepted an incorrectly typed URL") - } } func TestDocumentDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { - idx := newGoodReadsTestIndexer(t, &fakeGoodReadsSource{results: goodReadsTestResults}) - if err := idx.AddLibrary(10, true, 1); err != nil { + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + if err := IndexDocument(idx.documentsIdx, "book", Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}}); err != nil { t.Fatal(err) } - stale, err := idx.documentByIndexID(goodReadsTestID) + stale, err := idx.documentByIndexID("book") if err != nil { t.Fatal(err) } @@ -105,15 +102,16 @@ func TestDocumentDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { if err := IndexDocument(idx.documentsIdx, current.ID, current); err != nil { t.Fatal(err) } - if _, err := idx.enrichGoodReads(context.Background(), stale); err != nil { + if err := idx.StoreDocumentData(stale.ID, "dummy", testProviderData{URL: "/book/show/123", Enriched: true}); err != nil { t.Fatal(err) } - if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, biblioreads.BookResponse{ - Rating: "4.5", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, + if err := idx.UpdateDocument(stale.ID, func(update *DocumentUpdate) (bool, error) { + update.DataSourceScoreValue = 4.5 + return true, SetDataSourceData(&update.Document, "dummy", testProviderData{URL: "/book/show/123", Enriched: true, RatingCount: "100", ResourceID: "work", LegacyBookID: 123}) }); err != nil { t.Fatal(err) } - if err := idx.UpdateDataSourceLink(current.ID, "/book/show/456"); err != nil { + if err := idx.StoreDocumentData(stale.ID, "dummy", testProviderData{URL: "/book/show/456", Enriched: true}); err != nil { t.Fatal(err) } stale.TextRankWords = []string{"updated"} @@ -135,7 +133,32 @@ func TestDocumentDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { if err != nil || !reflect.DeepEqual(decoded, other) { t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) } - if data := mustGoodReadsData(t, got); data != (biblioreads.DocumentData{URL: "/book/show/456", Enriched: true}) { - t.Fatalf("stale write restored old Goodreads state: %+v", data) + if data := mustProviderData(t, got); data != (testProviderData{URL: "/book/show/456", Enriched: true}) { + t.Fatalf("stale write restored old provider state: %+v", data) + } +} + +type testProviderData struct { + URL string `json:"url"` + Enriched bool `json:"enriched"` + RatingCount string `json:"ratingCount,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + LegacyBookID int `json:"legacyBookId,omitempty"` +} + +func mustProviderData(t *testing.T, document Document) testProviderData { + t.Helper() + data, err := DataSourceData[testProviderData](document, "dummy") + if err != nil { + t.Fatal(err) + } + return data +} + +func withProviderData(t *testing.T, document Document, data testProviderData) Document { + t.Helper() + if err := SetDataSourceData(&document, "dummy", data); err != nil { + t.Fatal(err) } + return document } diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_goodreads_test.go index 3d38778a..894d5214 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_goodreads_test.go @@ -18,6 +18,7 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" + "github.com/svera/coreander/v5/internal/enrichment/goodreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -333,14 +334,21 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide } appFS := afero.NewMemMapFs() idxCfg := index.Config{} + var adapter *goodreads.Enricher if source != nil { - idxCfg.DocumentEnrichers = []index.DocumentEnricher{index.NewGoodReadsEnricher(source)} + adapter = goodreads.New(source) + } else { + adapter = goodreads.New(nil) + } + if source != nil { + idxCfg.DocumentEnrichers = []index.DocumentEnricher{adapter} } idx := index.NewBleve(documents, authors, appFS, testLibraryDir, nil, idxCfg) + adapter.SetIndex(idx) if source != nil { - provider = reviews.NewBiblioReads(source, idx) + provider = reviews.NewBiblioReads(source, adapter) } - goodReadsLinks := reviews.NewBiblioReads(nil, idx) + goodReadsLinks := reviews.NewBiblioReads(nil, adapter) t.Cleanup(func() { _ = idx.Close() }) for _, doc := range []index.Document{withGoodReadsData(t, index.Document{ diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index 79a70c40..a6c9c57d 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -18,6 +18,7 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" + "github.com/svera/coreander/v5/internal/enrichment/goodreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -96,7 +97,9 @@ func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.F if err != nil { log.Fatal(err) } - controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, dataSource, reviews.NewBiblioReads(nil, idx)) + adapter := goodreads.New(nil) + adapter.SetIndex(idx) + controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, dataSource, reviews.NewBiblioReads(nil, adapter)) usersRepository := &model.UserRepository{DB: db} return webserver.New(webserverConfig, controllers, sender, idx, usersRepository), idx } @@ -139,10 +142,12 @@ func newSearchResultsTestApp(t *testing.T, documents []index.Document) (*fiber.A // setReviewScore seeds a document's review score through the current provider-backed index storage. func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Document, score, count string) { t.Helper() + adapter := goodreads.New(nil) + adapter.SetIndex(idx) bookURL := mustGoodReadsData(t, doc).URL if bookURL == "" { bookURL = "https://www.goodreads.com/book/show/1-" + doc.Slug - if err := idx.UpdateDataSourceLink(doc.ID, bookURL); err != nil { + if err := adapter.UpdateDataSourceLink(doc.ID, bookURL); err != nil { t.Fatalf("set review link for %s: %v", doc.Slug, err) } } @@ -150,7 +155,7 @@ func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Documen if err != nil { t.Fatalf("reload %s: %v", doc.Slug, err) } - if err := idx.StoreGoodReadsDetails(updated, mustGoodReadsData(t, updated).URL, biblioreads.BookResponse{ + if err := adapter.StoreGoodReadsDetails(updated, mustGoodReadsData(t, updated).URL, biblioreads.BookResponse{ Rating: score, RatingCount: count, }); err != nil { t.Fatalf("set review score for %s: %v", doc.Slug, err) diff --git a/main.go b/main.go index 68a5d347..3c6bb1c8 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" + "github.com/svera/coreander/v5/internal/enrichment/goodreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -103,12 +104,14 @@ func main() { var needsReindex bool documentsIndex, authorsIndex, needsReindex = getIndexes(appFs, input.IllustratedMinSize, input.MinOccurrenceRatio, maxTextRankWords) var documentEnrichers []index.DocumentEnricher + goodReadsAdapter := goodreads.New(nil) if !input.DisableBiblioreads { biblioreadsClient = biblioreads.NewClient( biblioreads.WithInstances(input.BiblioreadsInstances), biblioreads.WithCacheTTL(time.Duration(input.BiblioreadsCacheTTL)*time.Second), ) - documentEnrichers = append(documentEnrichers, index.NewGoodReadsEnricher(biblioreadsClient)) + goodReadsAdapter = goodreads.New(biblioreadsClient) + documentEnrichers = append(documentEnrichers, goodReadsAdapter) } idx = index.NewBleve(documentsIndex, authorsIndex, appFs, input.LibPath, metadataReaders, index.Config{ IllustratedMinAmount: input.IllustratedMinAmount, @@ -124,6 +127,7 @@ func main() { TextRankEnrichWorkers: resolvedIndexWorkers, DocumentEnrichers: documentEnrichers, }) + goodReadsAdapter.SetIndex(idx) // If index was newly created or recreated, force reindexing if needsReindex { input.ForceIndexing = true @@ -205,7 +209,7 @@ func main() { dataSource := wikidata.NewWikidataSource(wikidata.Gowikidata{}) var reviewProvider document.ReviewProvider - biblioReads := reviews.NewBiblioReads(biblioreadsClient, idx) + biblioReads := reviews.NewBiblioReads(biblioreadsClient, goodReadsAdapter) if biblioreadsClient != nil { reviewProvider = biblioReads } From 7887f244e1993a1290341af4438d5608e8aa2d1f Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 10:35:46 +0100 Subject: [PATCH 28/50] Rename to biblioreads --- index_migration_test.go | 4 +- internal/datasource/biblioreads/document.go | 1 + .../{goodreads => biblioreads}/adapter.go | 56 +-- .../adapter_test.go | 366 +++++++++--------- .../{goodreads => biblioreads}/matching.go | 86 ++-- ...lds_test.go => biblioreads_fields_test.go} | 18 +- internal/reviews/biblioreads.go | 40 +- internal/reviews/biblioreads_test.go | 8 +- ...s_test.go => document_biblioreads_test.go} | 142 +++---- internal/webserver/search_test.go | 10 +- internal/webserver/webserver_test.go | 10 +- main.go | 12 +- 12 files changed, 377 insertions(+), 376 deletions(-) rename internal/enrichment/{goodreads => biblioreads}/adapter.go (73%) rename internal/enrichment/{goodreads => biblioreads}/adapter_test.go (74%) rename internal/enrichment/{goodreads => biblioreads}/matching.go (53%) rename internal/index/{goodreads_fields_test.go => biblioreads_fields_test.go} (94%) rename internal/webserver/{document_goodreads_test.go => document_biblioreads_test.go} (89%) diff --git a/index_migration_test.go b/index_migration_test.go index 4c5c53e7..d5766738 100644 --- a/index_migration_test.go +++ b/index_migration_test.go @@ -13,7 +13,7 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) -func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { +func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { t.Helper() if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { t.Fatal(err) @@ -41,7 +41,7 @@ func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { t.Fatal(err) } } - doc := withGoodReadsData(t, index.Document{ + doc := withBiblioReadsData(t, index.Document{ ID: "book", Slug: "book", AuthorsSlugs: []string{"alice-borchardt"}, Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, }, biblioreads.DocumentData{Enriched: true}) diff --git a/internal/datasource/biblioreads/document.go b/internal/datasource/biblioreads/document.go index 1ff3b016..23fce34f 100644 --- a/internal/datasource/biblioreads/document.go +++ b/internal/datasource/biblioreads/document.go @@ -1,5 +1,6 @@ package biblioreads +// Source retains the persisted key for Goodreads data fetched through BiblioReads. const Source = "goodreads" // DocumentData is the persisted Goodreads state used by BiblioReads. diff --git a/internal/enrichment/goodreads/adapter.go b/internal/enrichment/biblioreads/adapter.go similarity index 73% rename from internal/enrichment/goodreads/adapter.go rename to internal/enrichment/biblioreads/adapter.go index 029be873..6c207df9 100644 --- a/internal/enrichment/goodreads/adapter.go +++ b/internal/enrichment/biblioreads/adapter.go @@ -1,5 +1,5 @@ -// Package goodreads adapts Goodreads matching and review storage to the generic index. -package goodreads +// Package biblioreads adapts BiblioReads matching and review storage to the generic index. +package biblioreads import ( "context" @@ -14,22 +14,22 @@ import ( ) const ( - goodReadsEnrichWorkers = 4 - goodReadsRequestTimeout = 2 * time.Minute + biblioReadsEnrichWorkers = 4 + biblioReadsRequestTimeout = 2 * time.Minute ) -type GoodReadsSource interface { +type BiblioReadsSource interface { SearchBooks(context.Context, string) (biblioreads.SearchBooksResponse, error) } type Enricher struct { - source GoodReadsSource + source BiblioReadsSource idx *index.BleveIndexer slots chan struct{} } -func New(source GoodReadsSource) *Enricher { - return &Enricher{source: source, slots: make(chan struct{}, goodReadsEnrichWorkers)} +func New(source BiblioReadsSource) *Enricher { + return &Enricher{source: source, slots: make(chan struct{}, biblioReadsEnrichWorkers)} } // SetIndex binds the adapter before starting background or on-demand enrichment. @@ -49,8 +49,8 @@ func (p *Enricher) Enrich(ctx context.Context, idx *index.BleveIndexer, document return err } -// GoodReadsBook returns a stored match or performs an on-demand lookup. -func (p *Enricher) GoodReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) { +// BiblioReadsBook returns a stored match or performs an on-demand lookup. +func (p *Enricher) BiblioReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) { data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) if err != nil { return biblioreads.SearchResult{}, err @@ -67,10 +67,10 @@ func storedResult(document index.Document, data biblioreads.DocumentData) biblio } } -// UpdateDataSourceLink clears cached Goodreads data when an administrator corrects a match. +// UpdateDataSourceLink clears cached BiblioReads data when an administrator corrects a match. func (p *Enricher) UpdateDataSourceLink(documentID, reviewLink string) error { if p.idx == nil { - return fmt.Errorf("Goodreads adapter has no index") + return fmt.Errorf("BiblioReads adapter has no index") } return p.idx.UpdateDocument(documentID, func(current *index.DocumentUpdate) (bool, error) { data, err := index.DataSourceData[biblioreads.DocumentData](current.Document, biblioreads.Source) @@ -83,13 +83,13 @@ func (p *Enricher) UpdateDataSourceLink(documentID, reviewLink string) error { }) } -// StoreGoodReadsDetails rejects responses for a match or metadata changed during the fetch. -func (p *Enricher) StoreGoodReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error { +// StoreBiblioReadsDetails rejects responses for a match or metadata changed during the fetch. +func (p *Enricher) StoreBiblioReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error { if p.idx == nil { - return fmt.Errorf("Goodreads adapter has no index") + return fmt.Errorf("BiblioReads adapter has no index") } if bookURL == "" { - return fmt.Errorf("invalid GoodReads book URL for document %s", document.ID) + return fmt.Errorf("invalid Goodreads book URL for document %s", document.ID) } hasIDs := book.ResourceID != "" && book.LegacyBookID > 0 if book.Rating == "" && book.RatingCount == "" && !hasIDs { @@ -100,8 +100,8 @@ func (p *Enricher) StoreGoodReadsDetails(document index.Document, bookURL string if err != nil { return false, err } - if goodReadsQuery(current.Document) != goodReadsQuery(document) || data.URL != bookURL { - return false, fmt.Errorf("GoodReads match changed for document %s", document.ID) + if biblioReadsQuery(current.Document) != biblioReadsQuery(document) || data.URL != bookURL { + return false, fmt.Errorf("BiblioReads match changed for document %s", document.ID) } changed := false if book.RatingCount != "" && data.RatingCount != book.RatingCount { @@ -109,7 +109,7 @@ func (p *Enricher) StoreGoodReadsDetails(document index.Document, bookURL string changed = true } if book.Rating != "" { - value := parseGoodReadsRating(book.Rating, document.ID) + value := parseBiblioReadsRating(book.Rating, document.ID) if current.DataSourceScoreValue != value { current.DataSourceScoreValue = value changed = true @@ -128,7 +128,7 @@ func (p *Enricher) StoreGoodReadsDetails(document index.Document, bookURL string func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document index.Document) (biblioreads.SearchResult, error) { if idx == nil || p.source == nil { - return biblioreads.SearchResult{}, fmt.Errorf("Goodreads enrichment requires a source and index") + return biblioreads.SearchResult{}, fmt.Errorf("BiblioReads enrichment requires a source and index") } data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) if err != nil { @@ -140,10 +140,10 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document case <-ctx.Done(): return biblioreads.SearchResult{}, ctx.Err() } - q := goodReadsQuery(document) + q := biblioReadsQuery(document) result := biblioreads.SearchResult{BookURL: data.URL} if result.BookURL == "" && document.Title != "" { - ctx, cancel := context.WithTimeout(ctx, goodReadsRequestTimeout) + ctx, cancel := context.WithTimeout(ctx, biblioReadsRequestTimeout) defer cancel() queries := []string{q} if title := strings.TrimSpace(document.Title); title != q { @@ -153,11 +153,11 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document response, err := p.source.SearchBooks(ctx, query) switch { case errors.Is(err, biblioreads.ErrUnavailable), errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): - return biblioreads.SearchResult{}, fmt.Errorf("error searching %q on GoodReads: %w", query, err) + return biblioreads.SearchResult{}, fmt.Errorf("error searching %q on BiblioReads: %w", query, err) case err != nil: - log.Printf("Error searching %q on GoodReads: %s", query, err) + log.Printf("Error searching %q on BiblioReads: %s", query, err) default: - if match, ok := matchingGoodReadsBook(document, response.Result); ok { + if match, ok := matchingBiblioReadsBook(document, response.Result); ok { result = match } } @@ -167,7 +167,7 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document } } err = idx.UpdateDocument(document.ID, func(current *index.DocumentUpdate) (bool, error) { - if goodReadsQuery(current.Document) != q { + if biblioReadsQuery(current.Document) != q { return false, nil } data, err := index.DataSourceData[biblioreads.DocumentData](current.Document, biblioreads.Source) @@ -185,9 +185,9 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document if result.Rating == "" && current.DataSourceScoreValue > 0 { result.Rating = fmt.Sprintf("%.2f", current.DataSourceScoreValue) } - current.DataSourceScoreValue = parseGoodReadsRating(result.Rating, current.ID) + current.DataSourceScoreValue = parseBiblioReadsRating(result.Rating, current.ID) if result.AuthorURL != "" { - if slug, name := goodReadsAuthor(current.Document, result.Author); slug != "" { + if slug, name := biblioReadsAuthor(current.Document, result.Author); slug != "" { current.AuthorLinks = append(current.AuthorLinks, index.AuthorLink{ Slug: slug, Name: name, Source: biblioreads.Source, URL: result.AuthorURL, }) diff --git a/internal/enrichment/goodreads/adapter_test.go b/internal/enrichment/biblioreads/adapter_test.go similarity index 74% rename from internal/enrichment/goodreads/adapter_test.go rename to internal/enrichment/biblioreads/adapter_test.go index dfa9314f..8c004b52 100644 --- a/internal/enrichment/goodreads/adapter_test.go +++ b/internal/enrichment/biblioreads/adapter_test.go @@ -1,4 +1,4 @@ -package goodreads +package biblioreads import ( "context" @@ -19,7 +19,7 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) -func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { +func mustBiblioReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { t.Helper() data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) if err != nil { @@ -28,7 +28,7 @@ func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.Docume return data } -func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { +func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { t.Helper() if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { t.Fatal(err) @@ -36,11 +36,11 @@ func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.D return document } -func updateGoodReadsData(t *testing.T, document *index.Document, update func(*biblioreads.DocumentData)) { +func updateBiblioReadsData(t *testing.T, document *index.Document, update func(*biblioreads.DocumentData)) { t.Helper() - data := mustGoodReadsData(t, *document) + data := mustBiblioReadsData(t, *document) update(&data) - *document = withGoodReadsData(t, *document, data) + *document = withBiblioReadsData(t, *document, data) } func assertNoDocumentAuthorURL(t *testing.T, documents bleve.Index) { @@ -58,7 +58,7 @@ func assertNoDocumentAuthorURL(t *testing.T, documents bleve.Index) { } } -type fakeGoodReadsSource struct { +type fakeBiblioReadsSource struct { mu sync.Mutex queries []string results []biblioreads.SearchResult @@ -67,7 +67,7 @@ type fakeGoodReadsSource struct { err error } -func (f *fakeGoodReadsSource) SearchBooks(_ context.Context, query string) (biblioreads.SearchBooksResponse, error) { +func (f *fakeBiblioReadsSource) SearchBooks(_ context.Context, query string) (biblioreads.SearchBooksResponse, error) { f.mu.Lock() defer f.mu.Unlock() f.queries = append(f.queries, query) @@ -77,42 +77,42 @@ func (f *fakeGoodReadsSource) SearchBooks(_ context.Context, query string) (bibl return biblioreads.SearchBooksResponse{Result: f.results}, f.err } -func (f *fakeGoodReadsSource) Queries() []string { +func (f *fakeBiblioReadsSource) Queries() []string { f.mu.Lock() defer f.mu.Unlock() return slices.Clone(f.queries) } -type goodReadsTestReader struct{} +type biblioReadsTestReader struct{} -func (goodReadsTestReader) Metadata(string) (metadata.Metadata, error) { +func (biblioReadsTestReader) Metadata(string) (metadata.Metadata, error) { return metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt", "Second Author"}, Format: "PDF"}, nil } -func (goodReadsTestReader) Cover(string, int) (image.Image, error) { +func (biblioReadsTestReader) Cover(string, int) (image.Image, error) { return nil, nil } -var goodReadsTestResults = []biblioreads.SearchResult{ +var biblioReadsTestResults = []biblioreads.SearchResult{ {Title: "La Reina Dragon", Author: "Alice Borchardt", BookURL: "/book/show/12207444-la-reina-dragon", AuthorURL: "/author/show/151155.Alice_Borchardt", Rating: "3.82", RatingCount: "2,285"}, {BookURL: "/book/show/2-other", AuthorURL: "/author/show/2.Other"}, } const ( - wantDataSourceLink = "/book/show/12207444-la-reina-dragon" - wantGoodReadsAuthorURL = "/author/show/151155.Alice_Borchardt" - goodReadsTestID = "book.pdf" + wantDataSourceLink = "/book/show/12207444-la-reina-dragon" + wantBiblioReadsAuthorURL = "/author/show/151155.Alice_Borchardt" + biblioReadsTestID = "book.pdf" ) -type goodReadsTestIndexer struct { +type biblioReadsTestIndexer struct { *index.BleveIndexer *Enricher - documentsIdx *goodReadsWriteCountingIndex + documentsIdx *biblioReadsWriteCountingIndex authorsIdx bleve.Index fs afero.Fs } -func newGoodReadsTestIndexer(t *testing.T, source GoodReadsSource) *goodReadsTestIndexer { +func newBiblioReadsTestIndexer(t *testing.T, source BiblioReadsSource) *biblioReadsTestIndexer { t.Helper() documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) if err != nil { @@ -122,7 +122,7 @@ func newGoodReadsTestIndexer(t *testing.T, source GoodReadsSource) *goodReadsTes if err != nil { t.Fatal(err) } - writes := &goodReadsWriteCountingIndex{countedGoodReadsIndex: documents} + writes := &biblioReadsWriteCountingIndex{countedBiblioReadsIndex: documents} adapter := New(source) var providers []index.DocumentEnricher if source != nil { @@ -132,15 +132,15 @@ func newGoodReadsTestIndexer(t *testing.T, source GoodReadsSource) *goodReadsTes if err := afero.WriteFile(appFS, "lib/book.pdf", []byte(""), 0644); err != nil { t.Fatal(err) } - idx := index.NewBleve(writes, authors, appFS, "lib", map[string]metadata.Reader{".pdf": goodReadsTestReader{}}, index.Config{DocumentEnrichers: providers}) + idx := index.NewBleve(writes, authors, appFS, "lib", map[string]metadata.Reader{".pdf": biblioReadsTestReader{}}, index.Config{DocumentEnrichers: providers}) adapter.SetIndex(idx) t.Cleanup(func() { _ = idx.Close() }) - return &goodReadsTestIndexer{BleveIndexer: idx, Enricher: adapter, documentsIdx: writes, authorsIdx: authors, fs: appFS} + return &biblioReadsTestIndexer{BleveIndexer: idx, Enricher: adapter, documentsIdx: writes, authorsIdx: authors, fs: appFS} } -func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { - source := &fakeGoodReadsSource{results: goodReadsTestResults} - idx := newGoodReadsTestIndexer(t, source) +func TestEnrichBiblioReadsStoresMatchingResult(t *testing.T) { + source := &fakeBiblioReadsSource{results: biblioReadsTestResults} + idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -149,25 +149,25 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { t.Fatal(err) } - doc, err := idx.DocumentByID(goodReadsTestID) + doc, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, doc).URL != wantDataSourceLink { - t.Errorf("unexpected book URL %q", mustGoodReadsData(t, doc).URL) + if mustBiblioReadsData(t, doc).URL != wantDataSourceLink { + t.Errorf("unexpected book URL %q", mustBiblioReadsData(t, doc).URL) } if doc.DataSourceScoreValue != 3.82 { t.Errorf("unexpected Goodreads rating %v", doc.DataSourceScoreValue) } - if mustGoodReadsData(t, doc).RatingCount != "2,285" { - t.Errorf("unexpected Goodreads rating count %q", mustGoodReadsData(t, doc).RatingCount) + if mustBiblioReadsData(t, doc).RatingCount != "2,285" { + t.Errorf("unexpected Goodreads rating count %q", mustBiblioReadsData(t, doc).RatingCount) } assertNoDocumentAuthorURL(t, idx.documentsIdx) - if !mustGoodReadsData(t, doc).Enriched { - t.Error("expected document to be marked as GoodReads-enriched") + if !mustBiblioReadsData(t, doc).Enriched { + t.Error("expected document to be marked as BiblioReads-enriched") } author, err := idx.Author("alice-borchardt", "") - if err != nil || author.DataSourceLinks["goodreads"] != wantGoodReadsAuthorURL { + if err != nil || author.DataSourceLinks["goodreads"] != wantBiblioReadsAuthorURL { t.Fatalf("expected matching author URL stored, got %+v, %v", author, err) } other, err := idx.Author("second-author", "") @@ -186,15 +186,15 @@ func TestEnrichGoodReadsStoresMatchingResult(t *testing.T) { } } -func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { - for name, source := range map[string]*fakeGoodReadsSource{ +func TestEnrichBiblioReadsMarksDocumentsWithoutMatch(t *testing.T) { + for name, source := range map[string]*fakeBiblioReadsSource{ "no results": {}, "non-transient err": {err: errors.New("biblioreads: request failed with status code 504")}, "wrong title": {results: []biblioreads.SearchResult{{Title: "Another book", Author: "Alice Borchardt", BookURL: "/book/show/1"}}}, "wrong author": {results: []biblioreads.SearchResult{{Title: "La Reina Dragon", Author: "Another writer", BookURL: "/book/show/1"}}}, } { t.Run(name, func(t *testing.T) { - idx := newGoodReadsTestIndexer(t, source) + idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -202,11 +202,11 @@ func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(goodReadsTestID) + doc, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - if !mustGoodReadsData(t, doc).Enriched || mustGoodReadsData(t, doc).URL != "" { + if !mustBiblioReadsData(t, doc).Enriched || mustBiblioReadsData(t, doc).URL != "" { t.Errorf("expected document enriched with no URLs, got %+v", doc) } if want := []string{"La Reina Dragon Alice Borchardt", "La Reina Dragon"}; !slices.Equal(source.Queries(), want) { @@ -216,12 +216,12 @@ func TestEnrichGoodReadsMarksDocumentsWithoutMatch(t *testing.T) { } } -func TestEnrichGoodReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) { +func TestEnrichBiblioReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) { const authorURL = "/author/show/2" - source := &fakeGoodReadsSource{results: []biblioreads.SearchResult{ + source := &fakeBiblioReadsSource{results: []biblioreads.SearchResult{ {Title: "La Reina Dragon", Author: "Second Author", BookURL: wantDataSourceLink, AuthorURL: authorURL}, }} - idx := newGoodReadsTestIndexer(t, source) + idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -269,9 +269,9 @@ func TestEnrichGoodReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) } } -func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { - source := &fakeGoodReadsSource{err: biblioreads.ErrUnavailable} - idx := newGoodReadsTestIndexer(t, source) +func TestEnrichBiblioReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { + source := &fakeBiblioReadsSource{err: biblioreads.ErrUnavailable} + idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -279,29 +279,29 @@ func TestEnrichGoodReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { if err := idx.EnrichDocuments(10); !errors.Is(err, biblioreads.ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } - doc, err := idx.DocumentByID(goodReadsTestID) + doc, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, doc).Enriched { - t.Error("expected document not to be marked as GoodReads-enriched") + if mustBiblioReadsData(t, doc).Enriched { + t.Error("expected document not to be marked as BiblioReads-enriched") } source.mu.Lock() source.err = nil - source.results = goodReadsTestResults + source.results = biblioReadsTestResults source.mu.Unlock() if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - if doc, _ = idx.DocumentByID(goodReadsTestID); mustGoodReadsData(t, doc).URL != wantDataSourceLink { - t.Errorf("expected document to be enriched on retry, got %q", mustGoodReadsData(t, doc).URL) + if doc, _ = idx.DocumentByID(biblioReadsTestID); mustBiblioReadsData(t, doc).URL != wantDataSourceLink { + t.Errorf("expected document to be enriched on retry, got %q", mustBiblioReadsData(t, doc).URL) } } -func TestNewFileSchedulesGoodReadsEnrichment(t *testing.T) { - source := &fakeGoodReadsSource{results: goodReadsTestResults} - idx := newGoodReadsTestIndexer(t, source) +func TestNewFileSchedulesBiblioReadsEnrichment(t *testing.T) { + source := &fakeBiblioReadsSource{results: biblioReadsTestResults} + idx := newBiblioReadsTestIndexer(t, source) if _, err := idx.NewFile("new.pdf", []byte("")); err != nil { t.Fatal(err) @@ -313,22 +313,22 @@ func TestNewFileSchedulesGoodReadsEnrichment(t *testing.T) { if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, doc).Enriched { - if mustGoodReadsData(t, doc).URL != wantDataSourceLink { - t.Errorf("unexpected book URL %q", mustGoodReadsData(t, doc).URL) + if mustBiblioReadsData(t, doc).Enriched { + if mustBiblioReadsData(t, doc).URL != wantDataSourceLink { + t.Errorf("unexpected book URL %q", mustBiblioReadsData(t, doc).URL) } assertNoDocumentAuthorURL(t, idx.documentsIdx) return } if time.Now().After(deadline) { - t.Fatal("timed out waiting for background GoodReads enrichment") + t.Fatal("timed out waiting for background BiblioReads enrichment") } time.Sleep(10 * time.Millisecond) } } -func TestEnrichGoodReadsWithoutSourceIsNoop(t *testing.T) { - idx := newGoodReadsTestIndexer(t, nil) +func TestEnrichBiblioReadsWithoutSourceIsNoop(t *testing.T) { + idx := newBiblioReadsTestIndexer(t, nil) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -338,26 +338,26 @@ func TestEnrichGoodReadsWithoutSourceIsNoop(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(goodReadsTestID) + doc, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, doc).Enriched { - t.Error("expected document not to be GoodReads-enriched when the source is disabled") + if mustBiblioReadsData(t, doc).Enriched { + t.Error("expected document not to be BiblioReads-enriched when the source is disabled") } } -func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { - source := &fakeGoodReadsSource{results: goodReadsTestResults} - idx := newGoodReadsTestIndexer(t, source) +func TestEnrichBiblioReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { + source := &fakeBiblioReadsSource{results: biblioReadsTestResults} + idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(goodReadsTestID) + doc, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - updateGoodReadsData(t, &doc, func(data *biblioreads.DocumentData) { + updateBiblioReadsData(t, &doc, func(data *biblioreads.DocumentData) { data.URL = "/book/show/1-existing" }) if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { @@ -371,42 +371,42 @@ func TestEnrichGoodReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if len(source.Queries()) != 0 { t.Errorf("expected no search for a document with a link, got %q", source.Queries()) } - doc, err = idx.DocumentByID(goodReadsTestID) + doc, err = idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - if !mustGoodReadsData(t, doc).Enriched || mustGoodReadsData(t, doc).URL != "/book/show/1-existing" { + if !mustBiblioReadsData(t, doc).Enriched || mustBiblioReadsData(t, doc).URL != "/book/show/1-existing" { t.Errorf("expected existing link kept and document marked enriched, got %+v", doc) } } -func TestGoodReadsBook(t *testing.T) { - source := &fakeGoodReadsSource{results: goodReadsTestResults} - idx := newGoodReadsTestIndexer(t, source) +func TestBiblioReadsBook(t *testing.T) { + source := &fakeBiblioReadsSource{results: biblioReadsTestResults} + idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(goodReadsTestID) + doc, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - got, err := idx.GoodReadsBook(context.Background(), doc) + got, err := idx.BiblioReadsBook(context.Background(), doc) if err != nil { t.Fatal(err) } if got.BookURL != wantDataSourceLink || got.Rating != "3.82" { t.Errorf("expected fresh book URL and search rating, got %+v", got) } - stored, err := idx.DocumentByID(goodReadsTestID) + stored, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - if !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != wantDataSourceLink { + if !mustBiblioReadsData(t, stored).Enriched || mustBiblioReadsData(t, stored).URL != wantDataSourceLink { t.Errorf("expected the link to be persisted, got %+v", stored) } - if got, _ := idx.GoodReadsBook(context.Background(), stored); got.BookURL != wantDataSourceLink || got.Rating != "3.82" { + if got, _ := idx.BiblioReadsBook(context.Background(), stored); got.BookURL != wantDataSourceLink || got.Rating != "3.82" { t.Errorf("expected stored URL and rating, got %+v", got) } req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{stored.ID})) @@ -428,7 +428,7 @@ func TestGoodReadsBook(t *testing.T) { } } -func TestParseGoodReadsRating(t *testing.T) { +func TestParseBiblioReadsRating(t *testing.T) { for _, tt := range []struct { rating string want float64 @@ -439,29 +439,29 @@ func TestParseGoodReadsRating(t *testing.T) { {rating: "unavailable", want: 0}, } { t.Run(tt.rating, func(t *testing.T) { - if got := parseGoodReadsRating(tt.rating, goodReadsTestID); got != tt.want { - t.Errorf("parseGoodReadsRating(%q) = %v, want %v", tt.rating, got, tt.want) + if got := parseBiblioReadsRating(tt.rating, biblioReadsTestID); got != tt.want { + t.Errorf("parseBiblioReadsRating(%q) = %v, want %v", tt.rating, got, tt.want) } }) } } -func TestGoodReadsBookReturnsErrorWhenUnavailable(t *testing.T) { - source := &fakeGoodReadsSource{err: biblioreads.ErrUnavailable} - idx := newGoodReadsTestIndexer(t, source) +func TestBiblioReadsBookReturnsErrorWhenUnavailable(t *testing.T) { + source := &fakeBiblioReadsSource{err: biblioreads.ErrUnavailable} + idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, _ := idx.DocumentByID(goodReadsTestID) - if _, err := idx.GoodReadsBook(context.Background(), doc); !errors.Is(err, biblioreads.ErrUnavailable) { + doc, _ := idx.DocumentByID(biblioReadsTestID) + if _, err := idx.BiblioReadsBook(context.Background(), doc); !errors.Is(err, biblioreads.ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } - if doc, _ = idx.DocumentByID(goodReadsTestID); mustGoodReadsData(t, doc).Enriched { + if doc, _ = idx.DocumentByID(biblioReadsTestID); mustBiblioReadsData(t, doc).Enriched { t.Error("expected document to stay unenriched for a later retry") } } -func TestGoodReadsTitleMatchScore(t *testing.T) { +func TestBiblioReadsTitleMatchScore(t *testing.T) { for _, tt := range []struct { name string title string @@ -480,7 +480,7 @@ func TestGoodReadsTitleMatchScore(t *testing.T) { {name: "empty candidate", title: "La promesa"}, } { t.Run(tt.name, func(t *testing.T) { - got := goodReadsTitleMatchScore(tt.candidate, goodReadsTitle(tt.title), goodReadsMainTitle(tt.title)) + got := biblioReadsTitleMatchScore(tt.candidate, biblioReadsTitle(tt.title), biblioReadsMainTitle(tt.title)) if got != tt.want { t.Fatalf("title match score = %d, want %d", got, tt.want) } @@ -488,7 +488,7 @@ func TestGoodReadsTitleMatchScore(t *testing.T) { } } -func TestPreferGoodReadsBook(t *testing.T) { +func TestPreferBiblioReadsBook(t *testing.T) { for _, tt := range []struct { name string candidate, best biblioreads.SearchResult @@ -510,14 +510,14 @@ func TestPreferGoodReadsBook(t *testing.T) { {name: "complete tie retains first match", candidate: biblioreads.SearchResult{ID: 1, Rating: "4"}, best: biblioreads.SearchResult{ID: 1, Rating: "3"}, score: 3, bestScore: 3}, } { t.Run(tt.name, func(t *testing.T) { - if got := preferGoodReadsBook(tt.candidate, tt.best, tt.score, tt.bestScore); got != tt.want { + if got := preferBiblioReadsBook(tt.candidate, tt.best, tt.score, tt.bestScore); got != tt.want { t.Fatalf("candidate preference = %v, want %v", got, tt.want) } }) } } -func TestGoodReadsAuthorNameMatches(t *testing.T) { +func TestBiblioReadsAuthorNameMatches(t *testing.T) { tests := []struct { name, candidate string want bool @@ -540,51 +540,51 @@ func TestGoodReadsAuthorNameMatches(t *testing.T) { } for _, tt := range tests { t.Run(tt.name+"/"+tt.candidate, func(t *testing.T) { - if got := goodReadsAuthorNameMatches(tt.name, tt.candidate); got != tt.want { + if got := biblioReadsAuthorNameMatches(tt.name, tt.candidate); got != tt.want { t.Errorf("match = %v, want %v", got, tt.want) } - if got := goodReadsAuthorNameMatches(tt.candidate, tt.name); got != tt.want { + if got := biblioReadsAuthorNameMatches(tt.candidate, tt.name); got != tt.want { t.Errorf("reverse match = %v, want %v", got, tt.want) } }) } } -func TestGoodReadsAuthorPrefersExactAndRejectsAmbiguousPartialMatches(t *testing.T) { +func TestBiblioReadsAuthorPrefersExactAndRejectsAmbiguousPartialMatches(t *testing.T) { doc := index.Document{ Metadata: metadata.Metadata{Authors: []string{"Paul Kennedy", "Paul M. Kennedy"}}, AuthorsSlugs: []string{"paul-kennedy", "paul-m-kennedy"}, } - if slug, name := goodReadsAuthor(doc, "Paul M. Kennedy"); slug != "paul-m-kennedy" || name != "Paul M. Kennedy" { + if slug, name := biblioReadsAuthor(doc, "Paul M. Kennedy"); slug != "paul-m-kennedy" || name != "Paul M. Kennedy" { t.Fatalf("expected exact author, got %q, %q", slug, name) } doc.Authors = []string{"Paul M. Kennedy", "Paul A. Kennedy"} - if slug, name := goodReadsAuthor(doc, "Paul Kennedy"); slug != "" || name != "" { + if slug, name := biblioReadsAuthor(doc, "Paul Kennedy"); slug != "" || name != "" { t.Fatalf("ambiguous partial match must not select an author, got %q, %q", slug, name) } } -func TestGoodReadsBookMatchesKennedyMiddleName(t *testing.T) { +func TestBiblioReadsBookMatchesKennedyMiddleName(t *testing.T) { const title = "Auge y caída de las grandes potencias" const bookURL = "/book/show/27214" const authorURL = "/author/show/8390124.Paul_M_Kennedy" - source := &fakeGoodReadsSource{results: []biblioreads.SearchResult{{ + source := &fakeBiblioReadsSource{results: []biblioreads.SearchResult{{ Title: title, Author: "Paul M. Kennedy", BookURL: bookURL, AuthorURL: authorURL, Rating: "4.14", }}} - idx := newGoodReadsTestIndexer(t, source) + idx := newBiblioReadsTestIndexer(t, source) doc := index.Document{ - ID: goodReadsTestID, AuthorsSlugs: []string{"paul-kennedy", "another-writer"}, + ID: biblioReadsTestID, AuthorsSlugs: []string{"paul-kennedy", "another-writer"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"Paul Kennedy", "Another Writer"}}, } if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - got, err := idx.GoodReadsBook(context.Background(), doc) + got, err := idx.BiblioReadsBook(context.Background(), doc) if err != nil || got.BookURL != bookURL || got.Rating != "4.14" { t.Fatalf("expected Kennedy's book and rating, got %+v, %v", got, err) } stored, err := idx.DocumentByID(doc.ID) - if err != nil || !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != bookURL || stored.DataSourceScoreValue != 4.14 { + if err != nil || !mustBiblioReadsData(t, stored).Enriched || mustBiblioReadsData(t, stored).URL != bookURL || stored.DataSourceScoreValue != 4.14 { t.Fatalf("expected persisted book match and rating, got %+v, %v", stored, err) } author, err := idx.Author("paul-kennedy", "") @@ -600,7 +600,7 @@ func TestGoodReadsBookMatchesKennedyMiddleName(t *testing.T) { } } -func TestMatchingGoodReadsBook(t *testing.T) { +func TestMatchingBiblioReadsBook(t *testing.T) { correct := biblioreads.SearchResult{ Title: "El señor de los anillos", Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810", AuthorURL: "/author/show/656983.J_R_R_Tolkien", @@ -663,7 +663,7 @@ func TestMatchingGoodReadsBook(t *testing.T) { if authors == nil { authors = []string{"J. R. R. Tolkien"} } - match, ok := matchingGoodReadsBook(index.Document{Metadata: metadata.Metadata{Title: title, Authors: authors}}, tt.results) + match, ok := matchingBiblioReadsBook(index.Document{Metadata: metadata.Metadata{Title: title, Authors: authors}}, tt.results) if match.BookURL != tt.want || ok != (tt.want != "") { t.Fatalf("got %q, %v; want %q", match.BookURL, ok, tt.want) } @@ -671,7 +671,7 @@ func TestMatchingGoodReadsBook(t *testing.T) { } } -func TestMatchingGoodReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { +func TestMatchingBiblioReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { document := index.Document{Metadata: metadata.Metadata{ Title: "Los asesinatos de Manhattan", Authors: []string{"Douglas Preston", "Lincoln Child"}, }} @@ -684,13 +684,13 @@ func TestMatchingGoodReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { BookURL: "/book/show/152308668-los-asesinatos-de-manhattan", Rating: "3.67", RatingCount: "3", } for _, results := range [][]biblioreads.SearchResult{{popular, obscure}, {obscure, popular}} { - match, ok := matchingGoodReadsBook(document, results) + match, ok := matchingBiblioReadsBook(document, results) if !ok || match.BookURL != popular.BookURL { t.Fatalf("expected popular series entry, got %+v, %v", match, ok) } } popular.Author = "Another author" - match, ok := matchingGoodReadsBook(document, []biblioreads.SearchResult{popular, obscure}) + match, ok := matchingBiblioReadsBook(document, []biblioreads.SearchResult{popular, obscure}) if !ok || match.BookURL != obscure.BookURL { t.Fatalf("rating count must not bypass author matching, got %+v, %v", match, ok) } @@ -705,14 +705,14 @@ func TestDataSourceScoreCount(t *testing.T) { {"unknown", 0}, {"-1", 0}, {"3.5", 0}, {"18446744073709551616", 0}, } { t.Run(tt.value, func(t *testing.T) { - if got := goodReadsRatingCount(tt.value); got != tt.want { + if got := biblioReadsRatingCount(tt.value); got != tt.want { t.Fatalf("rating count = %d, want %d", got, tt.want) } }) } } -func TestMatchingGoodReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { +func TestMatchingBiblioReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { const olderBookURL = "/book/show/1368314.El_Cercano_Oriente" const newerBookURL = "/book/show/243175949-el-cercano-oriente" document := index.Document{Metadata: metadata.Metadata{ @@ -728,14 +728,14 @@ func TestMatchingGoodReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { } for _, results := range [][]biblioreads.SearchResult{{newer, older}, {older, newer}} { - match, ok := matchingGoodReadsBook(document, results) + match, ok := matchingBiblioReadsBook(document, results) if !ok || match.BookURL != olderBookURL { t.Fatalf("expected lower Goodreads ID %q, got %+v, %v", olderBookURL, match, ok) } } } -func TestMatchingGoodReadsBookPrefersRatedParentheticalTitle(t *testing.T) { +func TestMatchingBiblioReadsBookPrefersRatedParentheticalTitle(t *testing.T) { doc := index.Document{Metadata: metadata.Metadata{Title: "La promesa", Authors: []string{"Pearl S. Buck"}}} rated := biblioreads.SearchResult{ Title: "La promesa (The Promise)", Author: "Pearl S. Buck", @@ -746,40 +746,40 @@ func TestMatchingGoodReadsBookPrefersRatedParentheticalTitle(t *testing.T) { BookURL: "/book/show/210898619-la-promesa", Rating: "0.00", } for _, results := range [][]biblioreads.SearchResult{{rated, unrated}, {unrated, rated}} { - match, ok := matchingGoodReadsBook(doc, results) + match, ok := matchingBiblioReadsBook(doc, results) if !ok || match.BookURL != rated.BookURL || match.Rating != rated.Rating { t.Fatalf("expected rated translated-title entry, got %+v, %v", match, ok) } } rated.Author = "Another author" - match, ok := matchingGoodReadsBook(doc, []biblioreads.SearchResult{rated, unrated}) + match, ok := matchingBiblioReadsBook(doc, []biblioreads.SearchResult{rated, unrated}) if !ok || match.BookURL != unrated.BookURL { t.Fatalf("rated entry by another author must not win, got %+v, %v", match, ok) } } -func TestGoodReadsBookMatchesDolnickSubtitle(t *testing.T) { +func TestBiblioReadsBookMatchesDolnickSubtitle(t *testing.T) { const title = "La escritura de los dioses" const author = "Edward Dolnick" const bookURL = "/book/show/208277727-la-escritura-de-los-dioses-descifrando-la-piedra-de-rosetta" - source := &fakeGoodReadsSource{results: []biblioreads.SearchResult{{ + source := &fakeBiblioReadsSource{results: []biblioreads.SearchResult{{ Title: title + ". Descifrando la piedra de Rosetta", Author: author, BookURL: bookURL, Rating: "4.19", }}} - idx := newGoodReadsTestIndexer(t, source) + idx := newBiblioReadsTestIndexer(t, source) doc := index.Document{ - ID: goodReadsTestID, AuthorsSlugs: []string{"edward-dolnick"}, + ID: biblioReadsTestID, AuthorsSlugs: []string{"edward-dolnick"}, Metadata: metadata.Metadata{Title: title, Authors: []string{author}}, } if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - got, err := idx.GoodReadsBook(context.Background(), doc) + got, err := idx.BiblioReadsBook(context.Background(), doc) if err != nil || got.BookURL != bookURL || got.Rating != "4.19" { t.Fatalf("expected Dolnick's book and score, got %+v, %v", got, err) } stored, err := idx.DocumentByID(doc.ID) - if err != nil || !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != bookURL { + if err != nil || !mustBiblioReadsData(t, stored).Enriched || mustBiblioReadsData(t, stored).URL != bookURL { t.Fatalf("expected subtitle match persisted, got %+v, %v", stored, err) } if want := []string{title + " " + author}; !slices.Equal(source.Queries(), want) { @@ -787,7 +787,7 @@ func TestGoodReadsBookMatchesDolnickSubtitle(t *testing.T) { } } -func TestGoodReadsTitleFallback(t *testing.T) { +func TestBiblioReadsTitleFallback(t *testing.T) { const title = "El señor de los anillos" correct := biblioreads.SearchResult{Title: title, Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810"} parody := biblioreads.SearchResult{Title: "El sopor de los anillos", Author: "The Harvard Lampoon", BookURL: "/book/show/7297403"} @@ -797,29 +797,29 @@ func TestGoodReadsTitleFallback(t *testing.T) { name = "unavailable" } t.Run(name, func(t *testing.T) { - source := &fakeGoodReadsSource{byQuery: map[string][]biblioreads.SearchResult{ + source := &fakeBiblioReadsSource{byQuery: map[string][]biblioreads.SearchResult{ title + " J. R. R. Tolkien": {parody}, title: {parody, correct}, }} - idx := newGoodReadsTestIndexer(t, source) - doc := index.Document{ID: goodReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + idx := newBiblioReadsTestIndexer(t, source) + doc := index.Document{ID: biblioReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } if fallbackUnavailable { source.queryErrors = map[string]error{title: biblioreads.ErrUnavailable} } - got, err := idx.GoodReadsBook(context.Background(), doc) + got, err := idx.BiblioReadsBook(context.Background(), doc) if fallbackUnavailable { if !errors.Is(err, biblioreads.ErrUnavailable) || got.BookURL != "" { t.Fatalf("expected unavailable error, got %+v, %v", got, err) } stored, readErr := idx.DocumentByID(doc.ID) - if readErr != nil || mustGoodReadsData(t, stored).Enriched { + if readErr != nil || mustBiblioReadsData(t, stored).Enriched { t.Fatalf("expected document left for retry, got %+v, %v", stored, readErr) } source.queryErrors = nil - got, err = idx.GoodReadsBook(context.Background(), stored) + got, err = idx.BiblioReadsBook(context.Background(), stored) } if err != nil || got.BookURL != correct.BookURL { t.Fatalf("got %+v, %v; want %q", got, err, correct.BookURL) @@ -835,7 +835,7 @@ func TestGoodReadsTitleFallback(t *testing.T) { } } -func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { +func TestBiblioReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { const title = "El señor de los anillos" const wantURL = "/book/show/37910810-el-se-or-de-los-anillos" // Titles and ordering from BiblioReads' actual title-only search response. @@ -846,13 +846,13 @@ func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { {Title: "El señor de los anillos (#1-#3)", Author: "J.R.R. Tolkien", BookURL: wantURL, AuthorURL: "/author/show/656983.J_R_R_Tolkien"}, {Title: "Apéndices (El Señor de los Anillos, #4)", Author: "J.R.R. Tolkien", BookURL: "/book/show/54694.Ap_ndices"}, } - source := &fakeGoodReadsSource{byQuery: map[string][]biblioreads.SearchResult{title: results}} - idx := newGoodReadsTestIndexer(t, source) - doc := index.Document{ID: goodReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + source := &fakeBiblioReadsSource{byQuery: map[string][]biblioreads.SearchResult{title: results}} + idx := newBiblioReadsTestIndexer(t, source) + doc := index.Document{ID: biblioReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - got, err := idx.GoodReadsBook(context.Background(), doc) + got, err := idx.BiblioReadsBook(context.Background(), doc) if err != nil || got.BookURL != wantURL { t.Fatalf("got %+v, %v; want %q", got, err, wantURL) } @@ -860,7 +860,7 @@ func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { if err != nil { t.Fatal(err) } - if !mustGoodReadsData(t, stored).Enriched || mustGoodReadsData(t, stored).URL != wantURL { + if !mustBiblioReadsData(t, stored).Enriched || mustBiblioReadsData(t, stored).URL != wantURL { t.Fatalf("expected persisted omnibus match, got %+v", stored) } author, err := idx.Author("j-r-r-tolkien", "") @@ -872,17 +872,17 @@ func TestGoodReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { } } -func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { - source := &fakeGoodReadsSource{results: goodReadsTestResults} - idx := newGoodReadsTestIndexer(t, source) +func TestForceIndexingRefreshesBiblioReadsMatch(t *testing.T) { + source := &fakeBiblioReadsSource{results: biblioReadsTestResults} + idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(goodReadsTestID) + doc, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - updateGoodReadsData(t, &doc, func(data *biblioreads.DocumentData) { + updateBiblioReadsData(t, &doc, func(data *biblioreads.DocumentData) { data.URL = "/book/show/1-wrong" data.ResourceID = "wrong-work" data.LegacyBookID = 1 @@ -897,24 +897,24 @@ func TestForceIndexingRefreshesGoodReadsMatch(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err = idx.DocumentByID(goodReadsTestID) + doc, err = idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, doc).URL != wantDataSourceLink { - t.Fatalf("expected refreshed book link, got %q", mustGoodReadsData(t, doc).URL) + if mustBiblioReadsData(t, doc).URL != wantDataSourceLink { + t.Fatalf("expected refreshed book link, got %q", mustBiblioReadsData(t, doc).URL) } - if mustGoodReadsData(t, doc).ResourceID != "" || mustGoodReadsData(t, doc).LegacyBookID != 0 { - t.Fatalf("expected old identifiers to be cleared, got %q and %d", mustGoodReadsData(t, doc).ResourceID, mustGoodReadsData(t, doc).LegacyBookID) + if mustBiblioReadsData(t, doc).ResourceID != "" || mustBiblioReadsData(t, doc).LegacyBookID != 0 { + t.Fatalf("expected old identifiers to be cleared, got %q and %d", mustBiblioReadsData(t, doc).ResourceID, mustBiblioReadsData(t, doc).LegacyBookID) } } -func TestStoreGoodReadsDetails(t *testing.T) { +func TestStoreBiblioReadsDetails(t *testing.T) { for _, scenario := range []string{"linked", "newly linked", "changed metadata", "changed match", "deleted", "partial identifiers"} { t.Run(scenario, func(t *testing.T) { - idx := newGoodReadsTestIndexer(t, nil) - doc := withGoodReadsData(t, index.Document{ - ID: goodReadsTestID, + idx := newBiblioReadsTestIndexer(t, nil) + doc := withBiblioReadsData(t, index.Document{ + ID: biblioReadsTestID, Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, TextRankWords: []string{"dragon"}, @@ -924,14 +924,14 @@ func TestStoreGoodReadsDetails(t *testing.T) { resourceID := "work-id" switch scenario { case "newly linked": - updateGoodReadsData(t, &stale, func(data *biblioreads.DocumentData) { + updateBiblioReadsData(t, &stale, func(data *biblioreads.DocumentData) { data.URL = "" data.Enriched = false }) case "changed metadata": doc.Title = "Another title" case "changed match": - updateGoodReadsData(t, &doc, func(data *biblioreads.DocumentData) { + updateBiblioReadsData(t, &doc, func(data *biblioreads.DocumentData) { data.URL = "/book/show/2-other" }) case "partial identifiers": @@ -946,10 +946,10 @@ func TestStoreGoodReadsDetails(t *testing.T) { book := biblioreads.BookResponse{Rating: "4.12", ResourceID: resourceID, LegacyBookID: 12207444} writes := idx.documentsIdx writes.count.Store(0) - err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, book) + err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, book) wantError := scenario != "linked" && scenario != "newly linked" && scenario != "partial identifiers" if (err != nil) != wantError { - t.Fatalf("StoreGoodReadsDetails error = %v, want error %v", err, wantError) + t.Fatalf("StoreBiblioReadsDetails error = %v, want error %v", err, wantError) } if scenario == "deleted" { if !errors.Is(err, index.ErrDocumentNotFound) { @@ -965,7 +965,7 @@ func TestStoreGoodReadsDetails(t *testing.T) { if writes.count.Load() != 0 { t.Fatal("stale details must not cause any index writes") } - if mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 || stored.DataSourceScoreValue != 0 { + if mustBiblioReadsData(t, stored).ResourceID != "" || mustBiblioReadsData(t, stored).LegacyBookID != 0 || stored.DataSourceScoreValue != 0 { t.Fatalf("unexpected details persisted: %+v", stored) } return @@ -974,17 +974,17 @@ func TestStoreGoodReadsDetails(t *testing.T) { if scenario == "partial identifiers" { wantLegacyBookID = 0 } - if mustGoodReadsData(t, stored).ResourceID != resourceID || mustGoodReadsData(t, stored).LegacyBookID != wantLegacyBookID { + if mustBiblioReadsData(t, stored).ResourceID != resourceID || mustBiblioReadsData(t, stored).LegacyBookID != wantLegacyBookID { t.Fatalf("identifiers not persisted: %+v", stored) } if stored.DataSourceScoreValue != 4.12 { t.Fatalf("rating not persisted with identifiers: %+v", stored) } - if !mustGoodReadsData(t, stored).Enriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { + if !mustBiblioReadsData(t, stored).Enriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("existing enrichment was lost: %+v", stored) } book.Rating = "4.120" - if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, book); err != nil { + if err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, book); err != nil { t.Fatalf("repeated store failed: %v", err) } if writes.count.Load() != 1 { @@ -994,21 +994,21 @@ func TestStoreGoodReadsDetails(t *testing.T) { } } -type countedGoodReadsIndex interface { +type countedBiblioReadsIndex interface { bleve.Index } -type goodReadsWriteCountingIndex struct { - countedGoodReadsIndex +type biblioReadsWriteCountingIndex struct { + countedBiblioReadsIndex count atomic.Int64 } -func (i *goodReadsWriteCountingIndex) Index(id string, data any) error { +func (i *biblioReadsWriteCountingIndex) Index(id string, data any) error { i.count.Add(1) - return i.countedGoodReadsIndex.Index(id, data) + return i.countedBiblioReadsIndex.Index(id, data) } -func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { +func TestStoreBiblioReadsDetailsPreservesMissingFields(t *testing.T) { for _, tt := range []struct { name string book biblioreads.BookResponse @@ -1026,40 +1026,40 @@ func TestStoreGoodReadsDetailsPreservesMissingFields(t *testing.T) { {name: "zero count", book: biblioreads.BookResponse{RatingCount: "0"}, wantRating: "3.82", wantCount: "0", wantWork: "old-work", wantID: 1}, } { t.Run(tt.name, func(t *testing.T) { - idx := newGoodReadsTestIndexer(t, nil) - doc := withGoodReadsData(t, index.Document{ - ID: goodReadsTestID, + idx := newBiblioReadsTestIndexer(t, nil) + doc := withBiblioReadsData(t, index.Document{ + ID: biblioReadsTestID, DataSourceScoreValue: 3.82, }, biblioreads.DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 1}) if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - if err := idx.StoreGoodReadsDetails(doc, mustGoodReadsData(t, doc).URL, tt.book); err != nil { + if err := idx.StoreBiblioReadsDetails(doc, mustBiblioReadsData(t, doc).URL, tt.book); err != nil { t.Fatal(err) } stored, err := idx.DocumentByID(doc.ID) if err != nil { t.Fatal(err) } - if stored.DataSourceScoreValue != parseGoodReadsRating(tt.wantRating, doc.ID) || mustGoodReadsData(t, stored).ResourceID != tt.wantWork || mustGoodReadsData(t, stored).LegacyBookID != tt.wantID { + if stored.DataSourceScoreValue != parseBiblioReadsRating(tt.wantRating, doc.ID) || mustBiblioReadsData(t, stored).ResourceID != tt.wantWork || mustBiblioReadsData(t, stored).LegacyBookID != tt.wantID { t.Fatalf("partial details changed fields that were not supplied: %+v", stored) } wantCount := tt.wantCount if wantCount == "" { - wantCount = mustGoodReadsData(t, doc).RatingCount + wantCount = mustBiblioReadsData(t, doc).RatingCount } - if mustGoodReadsData(t, stored).RatingCount != wantCount { - t.Fatalf("expected rating count %q, got %q", wantCount, mustGoodReadsData(t, stored).RatingCount) + if mustBiblioReadsData(t, stored).RatingCount != wantCount { + t.Fatalf("expected rating count %q, got %q", wantCount, mustBiblioReadsData(t, stored).RatingCount) } }) } } func TestUpdateDataSourceLink(t *testing.T) { - idx := newGoodReadsTestIndexer(t, nil) - doc := withGoodReadsData(t, index.Document{ - ID: goodReadsTestID, Slug: "la-reina-dragon", + idx := newBiblioReadsTestIndexer(t, nil) + doc := withBiblioReadsData(t, index.Document{ + ID: biblioReadsTestID, Slug: "la-reina-dragon", Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, TextRankWords: []string{"dragon"}, @@ -1077,31 +1077,31 @@ func TestUpdateDataSourceLink(t *testing.T) { if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, stored).URL != newLink || !mustGoodReadsData(t, stored).Enriched || - stored.DataSourceScoreValue != 0 || mustGoodReadsData(t, stored).RatingCount != "" || mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 { + if mustBiblioReadsData(t, stored).URL != newLink || !mustBiblioReadsData(t, stored).Enriched || + stored.DataSourceScoreValue != 0 || mustBiblioReadsData(t, stored).RatingCount != "" || mustBiblioReadsData(t, stored).ResourceID != "" || mustBiblioReadsData(t, stored).LegacyBookID != 0 { t.Fatalf("updating review link must clear cached score and provider lookup data: %+v", stored) } if stored.Title != doc.Title || stored.Slug != doc.Slug || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("review-link update changed unrelated document data: %+v", stored) } - if err := idx.StoreGoodReadsDetails(doc, mustGoodReadsData(t, doc).URL, biblioreads.BookResponse{Rating: "4.5"}); err == nil { + if err := idx.StoreBiblioReadsDetails(doc, mustBiblioReadsData(t, doc).URL, biblioreads.BookResponse{Rating: "4.5"}); err == nil { t.Fatal("stale score write must not overwrite the updated link") } } func TestUpdateDataSourceLinkMissingDocument(t *testing.T) { - idx := newGoodReadsTestIndexer(t, nil) + idx := newBiblioReadsTestIndexer(t, nil) if err := idx.UpdateDataSourceLink("missing", "provider-specific-document-reference"); !errors.Is(err, index.ErrDocumentNotFound) { t.Fatalf("expected missing-document error, got %v", err) } } -func TestGoodReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { - idx := newGoodReadsTestIndexer(t, &fakeGoodReadsSource{results: goodReadsTestResults}) +func TestBiblioReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { + idx := newBiblioReadsTestIndexer(t, &fakeBiblioReadsSource{results: biblioReadsTestResults}) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - stale, err := idx.DocumentByID(goodReadsTestID) + stale, err := idx.DocumentByID(biblioReadsTestID) if err != nil { t.Fatal(err) } @@ -1119,7 +1119,7 @@ func TestGoodReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { if err := idx.Enricher.Enrich(context.Background(), idx.BleveIndexer, stale); err != nil { t.Fatal(err) } - if err := idx.StoreGoodReadsDetails(stale, wantDataSourceLink, biblioreads.BookResponse{ + if err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, biblioreads.BookResponse{ Rating: "4.5", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, }); err != nil { t.Fatal(err) @@ -1142,15 +1142,15 @@ func TestGoodReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { if err != nil || !reflect.DeepEqual(decoded, other) { t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) } - if data := mustGoodReadsData(t, got); data != (biblioreads.DocumentData{URL: "/book/show/456", Enriched: true}) { + if data := mustBiblioReadsData(t, got); data != (biblioreads.DocumentData{URL: "/book/show/456", Enriched: true}) { t.Fatalf("stale write restored old Goodreads state: %+v", data) } } -func TestGoodReadsBookRejectsInvalidProviderData(t *testing.T) { - idx := newGoodReadsTestIndexer(t, nil) +func TestBiblioReadsBookRejectsInvalidProviderData(t *testing.T) { + idx := newBiblioReadsTestIndexer(t, nil) document := index.Document{DataSources: map[string]json.RawMessage{biblioreads.Source: json.RawMessage(`{"url":42}`)}} - if _, err := idx.GoodReadsBook(context.Background(), document); err == nil { + if _, err := idx.BiblioReadsBook(context.Background(), document); err == nil { t.Fatal("Goodreads accepted an incorrectly typed URL") } } diff --git a/internal/enrichment/goodreads/matching.go b/internal/enrichment/biblioreads/matching.go similarity index 53% rename from internal/enrichment/goodreads/matching.go rename to internal/enrichment/biblioreads/matching.go index d8f44942..2686e9db 100644 --- a/internal/enrichment/goodreads/matching.go +++ b/internal/enrichment/biblioreads/matching.go @@ -1,4 +1,4 @@ -package goodreads +package biblioreads import ( "log" @@ -14,15 +14,15 @@ import ( ) // Strip named series annotations and omnibus ranges, not standalone volume numbers. -var goodReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) +var biblioReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) -var goodReadsSubtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+`) +var biblioReadsSubtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+`) -var goodReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) +var biblioReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) -var goodReadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) +var goodreadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) -func normalizeGoodReadsName(value string) string { +func normalizeBiblioReadsName(value string) string { var result strings.Builder for _, r := range norm.NFD.String(strings.ToLower(value)) { switch { @@ -37,9 +37,9 @@ func normalizeGoodReadsName(value string) string { return strings.Join(strings.Fields(result.String()), " ") } -func goodReadsAuthorNameMatches(name, candidate string) bool { - name = normalizeGoodReadsName(name) - candidate = normalizeGoodReadsName(candidate) +func biblioReadsAuthorNameMatches(name, candidate string) bool { + name = normalizeBiblioReadsName(name) + candidate = normalizeBiblioReadsName(candidate) if name == "" || candidate == "" { return false } @@ -63,19 +63,19 @@ func goodReadsAuthorNameMatches(name, candidate string) bool { return next == len(short)-1 } -func goodReadsAuthor(document index.Document, matchedName string) (string, string) { - nameKey := strings.ReplaceAll(normalizeGoodReadsName(matchedName), " ", "") +func biblioReadsAuthor(document index.Document, matchedName string) (string, string) { + nameKey := strings.ReplaceAll(normalizeBiblioReadsName(matchedName), " ", "") if nameKey == "" { return "", "" } for i, name := range document.Authors { - if i < len(document.AuthorsSlugs) && strings.ReplaceAll(normalizeGoodReadsName(name), " ", "") == nameKey { + if i < len(document.AuthorsSlugs) && strings.ReplaceAll(normalizeBiblioReadsName(name), " ", "") == nameKey { return document.AuthorsSlugs[i], name } } var slug, authorName string for i, name := range document.Authors { - if i < len(document.AuthorsSlugs) && goodReadsAuthorNameMatches(name, matchedName) { + if i < len(document.AuthorsSlugs) && biblioReadsAuthorNameMatches(name, matchedName) { if slug != "" { return "", "" } @@ -85,98 +85,98 @@ func goodReadsAuthor(document index.Document, matchedName string) (string, strin return slug, authorName } -func goodReadsTitle(value string) string { - return normalizeGoodReadsName(goodReadsSeriesSuffix.ReplaceAllString(value, "")) +func biblioReadsTitle(value string) string { + return normalizeBiblioReadsName(biblioReadsSeriesSuffix.ReplaceAllString(value, "")) } -func goodReadsMainTitle(value string) string { - value = goodReadsSeriesSuffix.ReplaceAllString(value, "") - if separator := goodReadsSubtitleSeparator.FindStringIndex(value); separator != nil { +func biblioReadsMainTitle(value string) string { + value = biblioReadsSeriesSuffix.ReplaceAllString(value, "") + if separator := biblioReadsSubtitleSeparator.FindStringIndex(value); separator != nil { value = value[:separator[0]] } - return normalizeGoodReadsName(value) + return normalizeBiblioReadsName(value) } -func goodReadsTitleMatchScore(candidate, title, mainTitle string) int { - candidateTitle := goodReadsTitle(candidate) +func biblioReadsTitleMatchScore(candidate, title, mainTitle string) int { + candidateTitle := biblioReadsTitle(candidate) if candidateTitle == "" { return 0 } if candidateTitle == title { return 3 } - if goodReadsMainTitle(candidate) == title || mainTitle == candidateTitle || - goodReadsTitle(goodReadsParentheticalTitle.ReplaceAllString(candidate, "")) == title { + if biblioReadsMainTitle(candidate) == title || mainTitle == candidateTitle || + biblioReadsTitle(biblioReadsParentheticalTitle.ReplaceAllString(candidate, "")) == title { return 1 } return 0 } -func preferGoodReadsBook(candidate, best biblioreads.SearchResult, score, bestScore int) bool { +func preferBiblioReadsBook(candidate, best biblioreads.SearchResult, score, bestScore int) bool { if bestScore == 0 { return true } if score == bestScore { - count, bestCount := goodReadsRatingCount(candidate.RatingCount), goodReadsRatingCount(best.RatingCount) + count, bestCount := biblioReadsRatingCount(candidate.RatingCount), biblioReadsRatingCount(best.RatingCount) if count != bestCount { return count > bestCount } - id, bestID := goodReadsBookID(candidate.BookURL), goodReadsBookID(best.BookURL) + id, bestID := goodreadsBookID(candidate.BookURL), goodreadsBookID(best.BookURL) if id > 0 && bestID > 0 && id != bestID { return id < bestID } } - hasRating, bestHasRating := hasGoodReadsRating(candidate.Rating), hasGoodReadsRating(best.Rating) + hasRating, bestHasRating := hasBiblioReadsRating(candidate.Rating), hasBiblioReadsRating(best.Rating) if hasRating != bestHasRating { return hasRating } return score > bestScore } -func goodReadsRatingCount(value string) uint64 { +func biblioReadsRatingCount(value string) uint64 { if strings.TrimSpace(value) == "" { return 0 } count, err := strconv.ParseUint(strings.ReplaceAll(strings.TrimSpace(value), ",", ""), 10, 64) if err != nil { - log.Printf("Invalid GoodReads rating count: %q", value) + log.Printf("Invalid BiblioReads rating count: %q", value) return 0 } return count } -func goodReadsBookID(bookURL string) uint64 { - match := goodReadsBookPath.FindStringSubmatch(bookURL) +func goodreadsBookID(bookURL string) uint64 { + match := goodreadsBookPath.FindStringSubmatch(bookURL) if match == nil { return 0 } id, err := strconv.ParseUint(match[1], 10, 64) if err != nil { - log.Printf("Invalid GoodReads book ID in URL: %q", bookURL) + log.Printf("Invalid Goodreads book ID in URL: %q", bookURL) return 0 } return id } -func matchingGoodReadsBook(document index.Document, results []biblioreads.SearchResult) (biblioreads.SearchResult, bool) { - title := goodReadsTitle(document.Title) +func matchingBiblioReadsBook(document index.Document, results []biblioreads.SearchResult) (biblioreads.SearchResult, bool) { + title := biblioReadsTitle(document.Title) if title == "" { return biblioreads.SearchResult{}, false } - mainTitle := goodReadsMainTitle(document.Title) + mainTitle := biblioReadsMainTitle(document.Title) var best biblioreads.SearchResult bestScore := 0 for _, result := range results { if result.BookURL == "" { continue } - score := goodReadsTitleMatchScore(result.Title, title, mainTitle) + score := biblioReadsTitleMatchScore(result.Title, title, mainTitle) if score == 0 { continue } for _, author := range document.Authors { - if goodReadsAuthorNameMatches(author, result.Author) { - if preferGoodReadsBook(result, best, score, bestScore) { + if biblioReadsAuthorNameMatches(author, result.Author) { + if preferBiblioReadsBook(result, best, score, bestScore) { best, bestScore = result, score } break @@ -186,14 +186,14 @@ func matchingGoodReadsBook(document index.Document, results []biblioreads.Search return best, bestScore > 0 } -func hasGoodReadsRating(rating string) bool { +func hasBiblioReadsRating(rating string) bool { value, err := strconv.ParseFloat(rating, 64) return err == nil && value > 0 && value <= 5 && !math.IsNaN(value) && !math.IsInf(value, 0) } -// goodReadsQuery returns the Goodreads search string for document: its title +// biblioReadsQuery returns the BiblioReads search string for document: its title // followed by its first author, if any. -func goodReadsQuery(document index.Document) string { +func biblioReadsQuery(document index.Document) string { q := strings.TrimSpace(document.Title) if len(document.Authors) > 0 { q = strings.TrimSpace(q + " " + document.Authors[0]) @@ -201,13 +201,13 @@ func goodReadsQuery(document index.Document) string { return q } -func parseGoodReadsRating(rating, documentID string) float64 { +func parseBiblioReadsRating(rating, documentID string) float64 { if rating == "" { return 0 } value, err := strconv.ParseFloat(rating, 64) if err != nil || value < 0 || value > 5 || math.IsNaN(value) || math.IsInf(value, 0) { - log.Printf("Invalid GoodReads rating for document %s: %q", documentID, rating) + log.Printf("Invalid BiblioReads rating for document %s: %q", documentID, rating) return 0 } return value diff --git a/internal/index/goodreads_fields_test.go b/internal/index/biblioreads_fields_test.go similarity index 94% rename from internal/index/goodreads_fields_test.go rename to internal/index/biblioreads_fields_test.go index 692f5935..97419b69 100644 --- a/internal/index/goodreads_fields_test.go +++ b/internal/index/biblioreads_fields_test.go @@ -13,7 +13,7 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) -func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { +func mustBiblioReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { t.Helper() data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) if err != nil { @@ -22,7 +22,7 @@ func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.Docume return data } -func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { +func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { t.Helper() if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { t.Fatal(err) @@ -122,7 +122,7 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) legacyBookID = 12207444 ) for _, lang := range []string{"es", ""} { - doc := withGoodReadsData(t, index.Document{ + doc := withBiblioReadsData(t, index.Document{ ID: "doc-" + lang, Slug: "doc-" + lang, Metadata: metadata.Metadata{ @@ -144,17 +144,17 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, got).URL != bookURL { - t.Errorf("language %q: expected stored book URL, got %q", lang, mustGoodReadsData(t, got).URL) + if mustBiblioReadsData(t, got).URL != bookURL { + t.Errorf("language %q: expected stored book URL, got %q", lang, mustBiblioReadsData(t, got).URL) } if got.DataSourceScoreValue != 4.25 { t.Errorf("language %q: expected stored rating, got %v", lang, got.DataSourceScoreValue) } - if mustGoodReadsData(t, got).RatingCount != "2285" { - t.Errorf("language %q: expected stored rating count, got %q", lang, mustGoodReadsData(t, got).RatingCount) + if mustBiblioReadsData(t, got).RatingCount != "2285" { + t.Errorf("language %q: expected stored rating count, got %q", lang, mustBiblioReadsData(t, got).RatingCount) } - if mustGoodReadsData(t, got).ResourceID != resourceID || mustGoodReadsData(t, got).LegacyBookID != legacyBookID { - t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, mustGoodReadsData(t, got).ResourceID, mustGoodReadsData(t, got).LegacyBookID) + if mustBiblioReadsData(t, got).ResourceID != resourceID || mustBiblioReadsData(t, got).LegacyBookID != legacyBookID { + t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, mustBiblioReadsData(t, got).ResourceID, mustBiblioReadsData(t, got).LegacyBookID) } } if err := index.IndexDocument(documentsIndexMem, "unscored", index.Document{ diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index ace23c84..c0495a6d 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -17,29 +17,29 @@ import ( ) const ( - goodReadsBaseURL = "https://www.goodreads.com" - goodReadsBookPrefix = "/book/show/" + goodreadsBaseURL = "https://www.goodreads.com" + goodreadsBookPrefix = "/book/show/" ) -var goodReadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) +var goodreadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) type BiblioReadsClient interface { Book(ctx context.Context, id string) (biblioreads.BookResponse, error) Reviews(ctx context.Context, resourceID string, legacyBookID int, nextPageToken, languageCode string) (biblioreads.ReviewsResponse, error) } -type GoodReadsIndex interface { - GoodReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) - StoreGoodReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error +type BiblioReadsIndex interface { + BiblioReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) + StoreBiblioReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error UpdateDataSourceLink(documentID, bookURL string) error } type BiblioReads struct { client BiblioReadsClient - index GoodReadsIndex + index BiblioReadsIndex } -func NewBiblioReads(client BiblioReadsClient, idx GoodReadsIndex) *BiblioReads { +func NewBiblioReads(client BiblioReadsClient, idx BiblioReadsIndex) *BiblioReads { return &BiblioReads{client: client, index: idx} } @@ -49,7 +49,7 @@ func (b *BiblioReads) Name() string { // AuthorLink returns the indexed author link without making a network request. func (b *BiblioReads) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { - return goodReadsDataSourceLink(author.DataSourceLinks["goodreads"], "/author/show/") + return goodreadsDataSourceLink(author.DataSourceLinks["goodreads"], "/author/show/") } // DocumentLink returns the indexed book link without making a network request. @@ -59,13 +59,13 @@ func (b *BiblioReads) DocumentLink(document index.Document) datasourcemodel.Data log.Println(err) return datasourcemodel.DataSourceLink{} } - return goodReadsDataSourceLink(data.URL, "/book/show/") + return goodreadsDataSourceLink(data.URL, "/book/show/") } -func goodReadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink { +func goodreadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink { link := datasourcemodel.DataSourceLink{Source: "Goodreads", Icon: "/images/goodreads.svg", IconInvertible: true} if id, ok := strings.CutPrefix(path, prefix); ok && id != "" { - link.URL = goodReadsBaseURL + path + link.URL = goodreadsBaseURL + path } return link } @@ -78,7 +78,7 @@ func (b *BiblioReads) StoredScore(document index.Document) (Score, error) { } return Score{ Source: b.Name(), Value: fmt.Sprintf("%.2f", document.DataSourceScoreValue), - Count: data.RatingCount, URL: goodReadsDataSourceLink(data.URL, goodReadsBookPrefix).URL, + Count: data.RatingCount, URL: goodreadsDataSourceLink(data.URL, goodreadsBookPrefix).URL, }, nil } @@ -89,7 +89,7 @@ func (b *BiblioReads) CanEditDocumentLink() bool { // UpdateDocumentLink saves a corrected Goodreads book link and clears stale score details. func (b *BiblioReads) UpdateDocumentLink(document index.Document, link string) error { - path, err := normalizeGoodReadsBookURL(link) + path, err := normalizeGoodreadsBookURL(link) if err != nil { return fmt.Errorf("%w: invalid Goodreads book URL", ErrInvalidDocumentLink) } @@ -102,19 +102,19 @@ func (b *BiblioReads) UpdateDocumentLink(document index.Document, link string) e return nil } -func normalizeGoodReadsBookURL(rawURL string) (string, error) { +func normalizeGoodreadsBookURL(rawURL string) (string, error) { bookURL, err := url.Parse(strings.TrimSpace(rawURL)) if err != nil || len(rawURL) > 2048 || bookURL.User != nil || (bookURL.Scheme != "https" && bookURL.Scheme != "http") || (bookURL.Host != "www.goodreads.com" && bookURL.Host != "goodreads.com") || - !goodReadsBookPath.MatchString(bookURL.Path) { + !goodreadsBookPath.MatchString(bookURL.Path) { return "", fmt.Errorf("invalid Goodreads book URL") } return strings.TrimSuffix(bookURL.Path, "/"), nil } func bookID(bookURL string) (string, error) { - id, ok := strings.CutPrefix(bookURL, goodReadsBookPrefix) + id, ok := strings.CutPrefix(bookURL, goodreadsBookPrefix) if !ok || id == "" { return "", fmt.Errorf("invalid Goodreads book URL %q", bookURL) } @@ -139,7 +139,7 @@ func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblio book.Rating = fmt.Sprintf("%.2f", document.DataSourceScoreValue) details.Rating = "" } - if err := b.index.StoreGoodReadsDetails(document, data.URL, details); err != nil { + if err := b.index.StoreBiblioReadsDetails(document, data.URL, details); err != nil { log.Printf("Error saving Goodreads details for %s: %s", document.ID, err) } return book, nil @@ -157,7 +157,7 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score return score, err } if data.URL == "" { - result, err := b.index.GoodReadsBook(ctx, document) + result, err := b.index.BiblioReadsBook(ctx, document) if err != nil { return score, err } @@ -219,7 +219,7 @@ func (b *BiblioReads) Reviews(ctx context.Context, document index.Document, lang } reviewURL := "" if review.ReviewURL != "" { - reviewURL = goodReadsBaseURL + review.ReviewURL + reviewURL = goodreadsBaseURL + review.ReviewURL } page.Reviews = append(page.Reviews, Review{ Reviewer: review.Reviewer.Name, Rating: review.Rating, Text: review.Text, diff --git a/internal/reviews/biblioreads_test.go b/internal/reviews/biblioreads_test.go index 588597c2..3f1411fe 100644 --- a/internal/reviews/biblioreads_test.go +++ b/internal/reviews/biblioreads_test.go @@ -10,7 +10,7 @@ import ( "github.com/svera/coreander/v5/internal/reviews" ) -func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { +func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { t.Helper() if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { t.Fatal(err) @@ -28,7 +28,7 @@ func TestBiblioReadsDataSourceLinks(t *testing.T) { t.Fatalf("unexpected Goodreads author link: %+v", author) } - document := provider.DocumentLink(withGoodReadsData(t, index.Document{}, biblioreads.DocumentData{URL: "/book/show/456.A_Book"})) + document := provider.DocumentLink(withBiblioReadsData(t, index.Document{}, biblioreads.DocumentData{URL: "/book/show/456.A_Book"})) if document.URL != "https://www.goodreads.com/book/show/456.A_Book" || document.Source != "Goodreads" || document.Icon != "/images/goodreads.svg" || !document.IconInvertible { t.Fatalf("unexpected Goodreads document link: %+v", document) @@ -50,11 +50,11 @@ type reviewLinkIndex struct { err error } -func (i *reviewLinkIndex) GoodReadsBook(context.Context, index.Document) (biblioreads.SearchResult, error) { +func (i *reviewLinkIndex) BiblioReadsBook(context.Context, index.Document) (biblioreads.SearchResult, error) { return biblioreads.SearchResult{}, nil } -func (i *reviewLinkIndex) StoreGoodReadsDetails(index.Document, string, biblioreads.BookResponse) error { +func (i *reviewLinkIndex) StoreBiblioReadsDetails(index.Document, string, biblioreads.BookResponse) error { return nil } diff --git a/internal/webserver/document_goodreads_test.go b/internal/webserver/document_biblioreads_test.go similarity index 89% rename from internal/webserver/document_goodreads_test.go rename to internal/webserver/document_biblioreads_test.go index 894d5214..b6e6dab5 100644 --- a/internal/webserver/document_goodreads_test.go +++ b/internal/webserver/document_biblioreads_test.go @@ -18,7 +18,7 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" - "github.com/svera/coreander/v5/internal/enrichment/goodreads" + biblioreadsenrichment "github.com/svera/coreander/v5/internal/enrichment/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -42,7 +42,7 @@ type fakeBookSource struct { reviewCalls []reviewCall } -func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { +func mustBiblioReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { t.Helper() data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) if err != nil { @@ -51,7 +51,7 @@ func mustGoodReadsData(t *testing.T, document index.Document) biblioreads.Docume return data } -func withGoodReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { +func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { t.Helper() if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { t.Fatal(err) @@ -63,7 +63,7 @@ func TestDocumentReviewsCanSelectSupportedLanguage(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "4", ResourceID: "work-id", LegacyBookID: 1, }} - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) req := mustGetRequest(t, "/documents/linked/reviews?language=fr") req.Header.Set("Accept-Language", "es") response, err := app.Test(req) @@ -102,7 +102,7 @@ func TestDocumentReviewsRejectUnsupportedLanguage(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "4", ResourceID: "work-id", LegacyBookID: 1, }} - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) response, err := app.Test(mustGetRequest(t, "/documents/linked/reviews?language=zz")) if err != nil { t.Fatal(err) @@ -165,7 +165,7 @@ func TestDocumentReviewsUILanguage(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "4", ResourceID: "work-id", LegacyBookID: 1, }} - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) req := mustGetRequest(t, "/documents/linked/reviews"+tt.query) if tt.header != "" { req.Header.Set("Accept-Language", tt.header) @@ -208,7 +208,7 @@ func TestDocumentAlternativeReviewProvider(t *testing.T) { } // This document has a cached Goodreads non-match; it must still use the chosen provider. detail := getBody(t, app, "/documents/unmatched") - if goodReadsPlaceholders(t, detail).Length() != 1 || len(provider.scoreSlugs) != 0 || len(provider.reviewSlugs) != 0 { + if biblioReadsPlaceholders(t, detail).Length() != 1 || len(provider.scoreSlugs) != 0 || len(provider.reviewSlugs) != 0 { t.Fatal("detail must defer lookups to the selected provider") } for _, path := range []string{"/documents/unmatched/score"} { @@ -305,7 +305,7 @@ func (f *fakeBookSource) Book(_ context.Context, id string) (biblioreads.BookRes return f.book, f.err } -func newGoodReadsTestApp(t *testing.T, source *fakeBookSource) (*fiber.App, *index.BleveIndexer) { +func newBiblioReadsTestApp(t *testing.T, source *fakeBookSource) (*fiber.App, *index.BleveIndexer) { t.Helper() return newReviewProviderTestApp(t, source, nil) } @@ -334,11 +334,11 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide } appFS := afero.NewMemMapFs() idxCfg := index.Config{} - var adapter *goodreads.Enricher + var adapter *biblioreadsenrichment.Enricher if source != nil { - adapter = goodreads.New(source) + adapter = biblioreadsenrichment.New(source) } else { - adapter = goodreads.New(nil) + adapter = biblioreadsenrichment.New(nil) } if source != nil { idxCfg.DocumentEnrichers = []index.DocumentEnricher{adapter} @@ -348,10 +348,10 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide if source != nil { provider = reviews.NewBiblioReads(source, adapter) } - goodReadsLinks := reviews.NewBiblioReads(nil, adapter) + biblioReadsLinks := reviews.NewBiblioReads(nil, adapter) t.Cleanup(func() { _ = idx.Close() }) - for _, doc := range []index.Document{withGoodReadsData(t, index.Document{ + for _, doc := range []index.Document{withBiblioReadsData(t, index.Document{ ID: "linked.epub", Slug: "linked", Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, @@ -360,7 +360,7 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide ID: "unlinked.epub", Slug: "unlinked", Metadata: metadata.Metadata{Title: "Unlinked", Authors: []string{"Jane Doe", "John Doe"}, Format: "EPUB"}, AuthorsSlugs: []string{"jane-doe", "john-doe"}, - }, withGoodReadsData(t, index.Document{ + }, withBiblioReadsData(t, index.Document{ ID: "unmatched.epub", Slug: "unmatched", Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, }, biblioreads.DocumentData{Enriched: true}), @@ -372,7 +372,7 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide cfg := defaultTestConfig() sender := &infrastructure.NoEmail{} - controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, wikidata.NewWikidataSource(wikidata.Gowikidata{}), provider, wikidata.NewWikidataSource(wikidata.Gowikidata{}), goodReadsLinks, dataSourceLinkProvider) + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, wikidata.NewWikidataSource(wikidata.Gowikidata{}), provider, wikidata.NewWikidataSource(wikidata.Gowikidata{}), biblioReadsLinks, dataSourceLinkProvider) app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) t.Cleanup(func() { _ = app.Shutdown() }) return app, idx @@ -397,7 +397,7 @@ func getBody(t *testing.T, app *fiber.App, path string) string { return string(body) } -func goodReadsPlaceholders(t *testing.T, html string) *goquery.Selection { +func biblioReadsPlaceholders(t *testing.T, html string) *goquery.Selection { t.Helper() page, err := goquery.NewDocumentFromReader(strings.NewReader(html)) if err != nil { @@ -406,16 +406,16 @@ func goodReadsPlaceholders(t *testing.T, html string) *goquery.Selection { return page.Find(`li[hx-get$="/score"]`) } -func TestDocumentDetailGoodReadsRating(t *testing.T) { +func TestDocumentDetailBiblioReadsRating(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "3.82", RatingCount: "2,285", ResourceID: "work-id", LegacyBookID: 12207444, }} - app, idx := newGoodReadsTestApp(t, source) + app, idx := newBiblioReadsTestApp(t, source) - placeholder := goodReadsPlaceholders(t, getBody(t, app, "/documents/linked")) + placeholder := biblioReadsPlaceholders(t, getBody(t, app, "/documents/linked")) if placeholder.Length() != 1 { - t.Fatalf("expected one GoodReads placeholder, got %d", placeholder.Length()) + t.Fatalf("expected one BiblioReads placeholder, got %d", placeholder.Length()) } if url, _ := placeholder.Attr("hx-get"); url != "/documents/linked/score" { t.Errorf("unexpected placeholder URL %q", url) @@ -448,14 +448,14 @@ func TestDocumentDetailGoodReadsRating(t *testing.T) { if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, stored).ResourceID != source.book.ResourceID || mustGoodReadsData(t, stored).LegacyBookID != source.book.LegacyBookID { + if mustBiblioReadsData(t, stored).ResourceID != source.book.ResourceID || mustBiblioReadsData(t, stored).LegacyBookID != source.book.LegacyBookID { t.Fatalf("expected book identifiers to be stored, got %+v", stored) } } -func TestDocumentGoodReadsEditor(t *testing.T) { +func TestDocumentBiblioReadsEditor(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{Rating: "3.82"}} - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) adminCookie, err := login(app, "admin@example.com", "admin", t) if err != nil { t.Fatal(err) @@ -519,11 +519,11 @@ func TestDocumentGoodReadsEditor(t *testing.T) { } } -func TestDocumentUpdateGoodReads(t *testing.T) { +func TestDocumentUpdateBiblioReads(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "3.82", ResourceID: "old-work", LegacyBookID: 12207444, }} - app, idx := newGoodReadsTestApp(t, source) + app, idx := newBiblioReadsTestApp(t, source) getBody(t, app, "/documents/linked/score") adminCookie, err := login(app, "admin@example.com", "admin", t) if err != nil { @@ -544,9 +544,9 @@ func TestDocumentUpdateGoodReads(t *testing.T) { if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, stored).URL != "/book/show/1368314.El_Cercano_Oriente" || - stored.DataSourceScoreValue != 0 || mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 || - !mustGoodReadsData(t, stored).Enriched { + if mustBiblioReadsData(t, stored).URL != "/book/show/1368314.El_Cercano_Oriente" || + stored.DataSourceScoreValue != 0 || mustBiblioReadsData(t, stored).ResourceID != "" || mustBiblioReadsData(t, stored).LegacyBookID != 0 || + !mustBiblioReadsData(t, stored).Enriched { t.Fatalf("expected correction persisted without stale rating/review identifiers: %+v", stored) } source.book = biblioreads.BookResponse{Rating: "4.2", ResourceID: "new-work", LegacyBookID: 1368314} @@ -562,8 +562,8 @@ func TestDocumentUpdateGoodReads(t *testing.T) { } } -func TestDocumentUpdateGoodReadsValidation(t *testing.T) { - app, idx := newGoodReadsTestApp(t, &fakeBookSource{}) +func TestDocumentUpdateBiblioReadsValidation(t *testing.T) { + app, idx := newBiblioReadsTestApp(t, &fakeBookSource{}) adminCookie, err := login(app, "admin@example.com", "admin", t) if err != nil { t.Fatal(err) @@ -587,7 +587,7 @@ func TestDocumentUpdateGoodReadsValidation(t *testing.T) { t.Fatalf("invalid input should be retained for correction, got %q", got) } stored, err := idx.Document("linked") - if err != nil || mustGoodReadsData(t, stored).URL != "/book/show/12207444-la-reina-dragon" || stored.DataSourceScoreValue != 3.82 { + if err != nil || mustBiblioReadsData(t, stored).URL != "/book/show/12207444-la-reina-dragon" || stored.DataSourceScoreValue != 3.82 { t.Fatalf("invalid input must not change the stored match: %+v, %v", stored, err) } } @@ -601,11 +601,11 @@ func TestDocumentUpdateGoodReadsValidation(t *testing.T) { mustReturnStatus(response, http.StatusNotFound, t) } -func TestDocumentDetailUpdatesStoredGoodReadsRating(t *testing.T) { +func TestDocumentDetailUpdatesStoredBiblioReadsRating(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "4.12", ResourceID: "work-id", LegacyBookID: 12207444, }} - app, idx := newGoodReadsTestApp(t, source) + app, idx := newBiblioReadsTestApp(t, source) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) if err != nil { @@ -628,11 +628,11 @@ func TestDocumentDetailUpdatesStoredGoodReadsRating(t *testing.T) { } } -func TestDocumentDetailKeepsStoredGoodReadsRatingWhenScraperReturnsZero(t *testing.T) { +func TestDocumentDetailKeepsStoredBiblioReadsRatingWhenScraperReturnsZero(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "0.00", RatingCount: "650", ResourceID: "work-id", LegacyBookID: 12207444, }} - app, idx := newGoodReadsTestApp(t, source) + app, idx := newBiblioReadsTestApp(t, source) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) if err != nil { @@ -651,14 +651,14 @@ func TestDocumentDetailKeepsStoredGoodReadsRatingWhenScraperReturnsZero(t *testi } } -func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { +func TestDocumentDetailBiblioReadsSearchesUnlinkedDocuments(t *testing.T) { source := &fakeBookSource{ book: biblioreads.BookResponse{Rating: "4.20", RatingCount: "10", ResourceID: "work-id", LegacyBookID: 1}, results: []biblioreads.SearchResult{{Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", AuthorURL: "/author/show/1.Jane_Doe", Rating: "4.10"}, {BookURL: "/book/show/2-other", Rating: "5.00"}}, } - app, idx := newGoodReadsTestApp(t, source) + app, idx := newBiblioReadsTestApp(t, source) - if got := goodReadsPlaceholders(t, getBody(t, app, "/documents/unlinked")).Length(); got != 1 { + if got := biblioReadsPlaceholders(t, getBody(t, app, "/documents/unlinked")).Length(); got != 1 { t.Fatalf("expected a placeholder for a document not looked up yet, got %d", got) } @@ -683,14 +683,14 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, stored).URL != "/book/show/1-unlinked" || !mustGoodReadsData(t, stored).Enriched { + if mustBiblioReadsData(t, stored).URL != "/book/show/1-unlinked" || !mustBiblioReadsData(t, stored).Enriched { t.Errorf("expected the found link to be stored, got %+v", stored) } author, err := idx.Author("jane-doe", "") if err != nil || author.DataSourceLinks["goodreads"] != "/author/show/1.Jane_Doe" || author.DocumentCount != 1 { t.Fatalf("expected missing author entry created with URL and document count, got %+v, %v", author, err) } - if mustGoodReadsData(t, stored).ResourceID != "" || mustGoodReadsData(t, stored).LegacyBookID != 0 { + if mustBiblioReadsData(t, stored).ResourceID != "" || mustBiblioReadsData(t, stored).LegacyBookID != 0 { t.Fatalf("search-only lookup must not populate scraper identifiers, got %+v", stored) } @@ -712,10 +712,10 @@ func TestDocumentDetailGoodReadsSearchesUnlinkedDocuments(t *testing.T) { } } -func TestDocumentDetailGoodReadsSkipsUnmatchedDocuments(t *testing.T) { +func TestDocumentDetailBiblioReadsSkipsUnmatchedDocuments(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{Rating: "3.82"}} - app, _ := newGoodReadsTestApp(t, source) - if got := goodReadsPlaceholders(t, getBody(t, app, "/documents/unmatched")).Length(); got != 1 { + app, _ := newBiblioReadsTestApp(t, source) + if got := biblioReadsPlaceholders(t, getBody(t, app, "/documents/unmatched")).Length(); got != 1 { t.Errorf("expected an async score placeholder, got %d", got) } for _, path := range []string{"/documents/unmatched/score"} { @@ -735,12 +735,12 @@ func TestDocumentDetailGoodReadsSkipsUnmatchedDocuments(t *testing.T) { } } -func TestDocumentDetailGoodReadsRatingRendersNothingWhenUnavailable(t *testing.T) { +func TestDocumentDetailBiblioReadsRatingRendersNothingWhenUnavailable(t *testing.T) { for name, source := range map[string]*fakeBookSource{ "error": {err: errors.New("unavailable")}, } { t.Run(name, func(t *testing.T) { - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) if body := strings.TrimSpace(getBody(t, app, "/documents/linked/score")); body != "" { t.Errorf("expected empty response, got %q", body) } @@ -748,9 +748,9 @@ func TestDocumentDetailGoodReadsRatingRendersNothingWhenUnavailable(t *testing.T } } -func TestDocumentDetailGoodReadsIdentifiersWithoutRating(t *testing.T) { +func TestDocumentDetailBiblioReadsIdentifiersWithoutRating(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}} - app, idx := newGoodReadsTestApp(t, source) + app, idx := newBiblioReadsTestApp(t, source) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) if err != nil { t.Fatal(err) @@ -762,7 +762,7 @@ func TestDocumentDetailGoodReadsIdentifiersWithoutRating(t *testing.T) { if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, stored).ResourceID != "work-id" || mustGoodReadsData(t, stored).LegacyBookID != 12207444 { + if mustBiblioReadsData(t, stored).ResourceID != "work-id" || mustBiblioReadsData(t, stored).LegacyBookID != 12207444 { t.Fatalf("expected identifiers even without a rating, got %+v", stored) } @@ -777,17 +777,17 @@ func TestDocumentDetailGoodReadsIdentifiersWithoutRating(t *testing.T) { if err != nil { t.Fatal(err) } - if mustGoodReadsData(t, stored).ResourceID != "work-id" || mustGoodReadsData(t, stored).LegacyBookID != 12207444 { + if mustBiblioReadsData(t, stored).ResourceID != "work-id" || mustBiblioReadsData(t, stored).LegacyBookID != 12207444 { t.Fatalf("incomplete response overwrote stored identifiers: %+v", stored) } } } -func TestDocumentDetailGoodReadsDisabled(t *testing.T) { - app, _ := newGoodReadsTestApp(t, nil) +func TestDocumentDetailBiblioReadsDisabled(t *testing.T) { + app, _ := newBiblioReadsTestApp(t, nil) for _, slug := range []string{"linked", "unlinked", "unmatched"} { - if got := goodReadsPlaceholders(t, getBody(t, app, "/documents/"+slug)).Length(); got != 0 { - t.Errorf("%s: expected no placeholder when GoodReads is disabled, got %d", slug, got) + if got := biblioReadsPlaceholders(t, getBody(t, app, "/documents/"+slug)).Length(); got != 0 { + t.Errorf("%s: expected no placeholder when BiblioReads is disabled, got %d", slug, got) } if body := strings.TrimSpace(getBody(t, app, "/documents/"+slug+"/score")); body != "" { t.Errorf("%s: expected empty response, got %q", slug, body) @@ -801,7 +801,7 @@ func TestDocumentDetailGoodReadsDisabled(t *testing.T) { } } -func TestDocumentDetailGoodReadsSearchDoesNotScrape(t *testing.T) { +func TestDocumentDetailBiblioReadsSearchDoesNotScrape(t *testing.T) { for _, tt := range []struct { name string results []biblioreads.SearchResult @@ -823,7 +823,7 @@ func TestDocumentDetailGoodReadsSearchDoesNotScrape(t *testing.T) { results: tt.results, searchErr: tt.err, book: biblioreads.BookResponse{Rating: "5.00"}, } - app, idx := newGoodReadsTestApp(t, source) + app, idx := newBiblioReadsTestApp(t, source) body := strings.TrimSpace(getBody(t, app, "/documents/unlinked/score")) if tt.noScore { page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) @@ -849,16 +849,16 @@ func TestDocumentDetailGoodReadsSearchDoesNotScrape(t *testing.T) { t.Errorf("expected %d searches and no book-scraper calls, got queries %q and ids %q", tt.queries, source.queries, source.ids) } stored, err := idx.Document("unlinked") - if err != nil || mustGoodReadsData(t, stored).Enriched != tt.enriched { + if err != nil || mustBiblioReadsData(t, stored).Enriched != tt.enriched { t.Fatalf("unexpected stored lookup status: %+v, %v", stored, err) } }) } } -func TestDocumentDetailGoodReadsNoScoreTranslations(t *testing.T) { +func TestDocumentDetailBiblioReadsNoScoreTranslations(t *testing.T) { source := &fakeBookSource{} - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) for _, locale := range []struct { lang string text string @@ -898,11 +898,11 @@ func TestDocumentDetailGoodReadsNoScoreTranslations(t *testing.T) { } } -func TestDocumentDetailGoodReadsReviewsLazyPlaceholder(t *testing.T) { +func TestDocumentDetailBiblioReadsReviewsLazyPlaceholder(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "3.82", ResourceID: "work-id", LegacyBookID: 12207444, }} - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked"))) if err != nil { t.Fatal(err) @@ -945,7 +945,7 @@ func TestDocumentDetailGoodReadsReviewsLazyPlaceholder(t *testing.T) { } } -func TestDocumentDetailGoodReadsReviewsHeader(t *testing.T) { +func TestDocumentDetailBiblioReadsReviewsHeader(t *testing.T) { for _, tt := range []struct { name string source *fakeBookSource @@ -960,7 +960,7 @@ func TestDocumentDetailGoodReadsReviewsHeader(t *testing.T) { {name: "not looked up", source: &fakeBookSource{}, slug: "unlinked"}, } { t.Run(tt.name, func(t *testing.T) { - app, _ := newGoodReadsTestApp(t, tt.source) + app, _ := newBiblioReadsTestApp(t, tt.source) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/"+tt.slug))) if err != nil { t.Fatal(err) @@ -982,7 +982,7 @@ func TestDocumentDetailGoodReadsReviewsHeader(t *testing.T) { } } -func TestDocumentDetailGoodReadsReviews(t *testing.T) { +func TestDocumentDetailBiblioReadsReviews(t *testing.T) { source := &fakeBookSource{ book: biblioreads.BookResponse{Rating: "4.20", ResourceID: "work-id", LegacyBookID: 12207444}, reviews: biblioreads.ReviewsResponse{Reviews: []biblioreads.Review{ @@ -992,7 +992,7 @@ func TestDocumentDetailGoodReadsReviews(t *testing.T) { {Reviewer: biblioreads.Reviewer{Name: "Spoiler reviewer"}, Text: "The ending", Spoiler: true}, }}, } - app, idx := newGoodReadsTestApp(t, source) + app, idx := newBiblioReadsTestApp(t, source) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews"))) if err != nil { t.Fatal(err) @@ -1035,7 +1035,7 @@ func TestDocumentDetailGoodReadsReviews(t *testing.T) { t.Fatal("expected review dates in ISO format for client-side localization") } stored, err := idx.Document("linked") - if err != nil || mustGoodReadsData(t, stored).ResourceID != "work-id" || mustGoodReadsData(t, stored).LegacyBookID != 12207444 { + if err != nil || mustBiblioReadsData(t, stored).ResourceID != "work-id" || mustBiblioReadsData(t, stored).LegacyBookID != 12207444 { t.Fatalf("expected review identifiers to be stored: %+v, %v", stored, err) } } @@ -1062,7 +1062,7 @@ func TestDocumentReviewFractionalStars(t *testing.T) { {Reviewer: biblioreads.Reviewer{Name: "Reader"}, Rating: tt.rating}, }}, } - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews"))) if err != nil { t.Fatal(err) @@ -1108,7 +1108,7 @@ func TestDocumentReviewNotScoredTranslations(t *testing.T) { {Reviewer: biblioreads.Reviewer{Name: "Reader"}, Text: "A review without a rating"}, }}, } - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) for _, tt := range []struct { lang string text string @@ -1131,7 +1131,7 @@ func TestDocumentReviewNotScoredTranslations(t *testing.T) { } } -func TestDocumentDetailGoodReadsReviewsAvailability(t *testing.T) { +func TestDocumentDetailBiblioReadsReviewsAvailability(t *testing.T) { for _, tt := range []struct { name string source *fakeBookSource @@ -1151,7 +1151,7 @@ func TestDocumentDetailGoodReadsReviewsAvailability(t *testing.T) { {name: "empty Spanish reviews", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}}, slug: "linked", status: http.StatusOK, calls: 1, language: "es", text: "No hay reseñas de usuarios disponibles en el idioma actual"}, } { t.Run(tt.name, func(t *testing.T) { - app, _ := newGoodReadsTestApp(t, tt.source) + app, _ := newBiblioReadsTestApp(t, tt.source) req := mustGetRequest(t, "/documents/"+tt.slug+"/reviews") req.Header.Set("Accept-Language", "en") if tt.language != "" { @@ -1181,12 +1181,12 @@ func TestDocumentDetailGoodReadsReviewsAvailability(t *testing.T) { } } -func TestDocumentDetailGoodReadsReviewsAfterSearch(t *testing.T) { +func TestDocumentDetailBiblioReadsReviewsAfterSearch(t *testing.T) { source := &fakeBookSource{ results: []biblioreads.SearchResult{{Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", Rating: "4"}}, book: biblioreads.BookResponse{Rating: "4", ResourceID: "work-id", LegacyBookID: 1}, } - app, _ := newGoodReadsTestApp(t, source) + app, _ := newBiblioReadsTestApp(t, source) getBody(t, app, "/documents/unlinked/score") if len(source.ids) != 0 || len(source.reviewCalls) != 0 { t.Fatal("search rating must not scrape the book or load reviews") diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index ea7c5d6d..9c790dd0 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -281,7 +281,7 @@ func TestSearchResultsShowStoredDataSourceScore(t *testing.T) { {score: 5, want: "5.00"}, } { t.Run(tt.want, func(t *testing.T) { - app, _ := newSearchResultsTestApp(t, []index.Document{withGoodReadsData(t, index.Document{ + app, _ := newSearchResultsTestApp(t, []index.Document{withBiblioReadsData(t, index.Document{ ID: "linked.epub", Slug: "linked", Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, @@ -304,15 +304,15 @@ func TestSearchResultsShowStoredDataSourceScore(t *testing.T) { } func TestSearchResultsSortByDataSourceScore(t *testing.T) { - app, _ := newSearchResultsTestApp(t, []index.Document{withGoodReadsData(t, index.Document{ + app, _ := newSearchResultsTestApp(t, []index.Document{withBiblioReadsData(t, index.Document{ ID: "linked.epub", Slug: "linked", Metadata: metadata.Metadata{Title: "Linked", Format: "EPUB"}, DataSourceScoreValue: 3.2, - }, biblioreads.DocumentData{URL: "/book/show/linked"}), withGoodReadsData(t, index.Document{ + }, biblioreads.DocumentData{URL: "/book/show/linked"}), withBiblioReadsData(t, index.Document{ ID: "unlinked.epub", Slug: "unlinked", Metadata: metadata.Metadata{Title: "Unlinked", Format: "EPUB"}, DataSourceScoreValue: 4.1, - }, biblioreads.DocumentData{URL: "/book/show/unlinked"}), withGoodReadsData(t, index.Document{ + }, biblioreads.DocumentData{URL: "/book/show/unlinked"}), withBiblioReadsData(t, index.Document{ ID: "unmatched.epub", Slug: "unmatched", Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, DataSourceScoreValue: 4.8, @@ -344,7 +344,7 @@ func TestSearchResultsSortByDataSourceScore(t *testing.T) { } func TestSearchResultsShowStoredZeroRatingAsNotScored(t *testing.T) { - app, _ := newSearchResultsTestApp(t, []index.Document{withGoodReadsData(t, index.Document{ + app, _ := newSearchResultsTestApp(t, []index.Document{withBiblioReadsData(t, index.Document{ ID: "unscored.epub", Slug: "unscored", Metadata: metadata.Metadata{Title: "Unscored", Format: "EPUB"}, }, biblioreads.DocumentData{URL: "/book/show/unscored"}), diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index a6c9c57d..b571c008 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -18,7 +18,7 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" - "github.com/svera/coreander/v5/internal/enrichment/goodreads" + biblioreadsenrichment "github.com/svera/coreander/v5/internal/enrichment/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -97,7 +97,7 @@ func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.F if err != nil { log.Fatal(err) } - adapter := goodreads.New(nil) + adapter := biblioreadsenrichment.New(nil) adapter.SetIndex(idx) controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, dataSource, reviews.NewBiblioReads(nil, adapter)) usersRepository := &model.UserRepository{DB: db} @@ -142,9 +142,9 @@ func newSearchResultsTestApp(t *testing.T, documents []index.Document) (*fiber.A // setReviewScore seeds a document's review score through the current provider-backed index storage. func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Document, score, count string) { t.Helper() - adapter := goodreads.New(nil) + adapter := biblioreadsenrichment.New(nil) adapter.SetIndex(idx) - bookURL := mustGoodReadsData(t, doc).URL + bookURL := mustBiblioReadsData(t, doc).URL if bookURL == "" { bookURL = "https://www.goodreads.com/book/show/1-" + doc.Slug if err := adapter.UpdateDataSourceLink(doc.ID, bookURL); err != nil { @@ -155,7 +155,7 @@ func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Documen if err != nil { t.Fatalf("reload %s: %v", doc.Slug, err) } - if err := adapter.StoreGoodReadsDetails(updated, mustGoodReadsData(t, updated).URL, biblioreads.BookResponse{ + if err := adapter.StoreBiblioReadsDetails(updated, mustBiblioReadsData(t, updated).URL, biblioreads.BookResponse{ Rating: score, RatingCount: count, }); err != nil { t.Fatalf("set review score for %s: %v", doc.Slug, err) diff --git a/main.go b/main.go index 3c6bb1c8..566af6ed 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,7 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" - "github.com/svera/coreander/v5/internal/enrichment/goodreads" + biblioreadsenrichment "github.com/svera/coreander/v5/internal/enrichment/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -104,14 +104,14 @@ func main() { var needsReindex bool documentsIndex, authorsIndex, needsReindex = getIndexes(appFs, input.IllustratedMinSize, input.MinOccurrenceRatio, maxTextRankWords) var documentEnrichers []index.DocumentEnricher - goodReadsAdapter := goodreads.New(nil) + biblioReadsAdapter := biblioreadsenrichment.New(nil) if !input.DisableBiblioreads { biblioreadsClient = biblioreads.NewClient( biblioreads.WithInstances(input.BiblioreadsInstances), biblioreads.WithCacheTTL(time.Duration(input.BiblioreadsCacheTTL)*time.Second), ) - goodReadsAdapter = goodreads.New(biblioreadsClient) - documentEnrichers = append(documentEnrichers, goodReadsAdapter) + biblioReadsAdapter = biblioreadsenrichment.New(biblioreadsClient) + documentEnrichers = append(documentEnrichers, biblioReadsAdapter) } idx = index.NewBleve(documentsIndex, authorsIndex, appFs, input.LibPath, metadataReaders, index.Config{ IllustratedMinAmount: input.IllustratedMinAmount, @@ -127,7 +127,7 @@ func main() { TextRankEnrichWorkers: resolvedIndexWorkers, DocumentEnrichers: documentEnrichers, }) - goodReadsAdapter.SetIndex(idx) + biblioReadsAdapter.SetIndex(idx) // If index was newly created or recreated, force reindexing if needsReindex { input.ForceIndexing = true @@ -209,7 +209,7 @@ func main() { dataSource := wikidata.NewWikidataSource(wikidata.Gowikidata{}) var reviewProvider document.ReviewProvider - biblioReads := reviews.NewBiblioReads(biblioreadsClient, goodReadsAdapter) + biblioReads := reviews.NewBiblioReads(biblioreadsClient, biblioReadsAdapter) if biblioreadsClient != nil { reviewProvider = biblioReads } From 107a0f8d0cc3c77ee8b49e199cad8b3230703ffb Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 11:12:44 +0100 Subject: [PATCH 29/50] Refactor --- ...data.go => bleve_document_data_sources.go} | 0 internal/index/bleve_document_read.go | 19 +- ...lds_test.go => data_source_fields_test.go} | 97 +++-- internal/index/document.go | 11 +- ..._test.go => document_data_sources_test.go} | 154 ++++++++ internal/index/document_data_test.go | 164 -------- internal/reviews/errors.go | 8 - internal/reviews/model.go | 6 + internal/webserver/controller.go | 4 +- internal/webserver/controller/search/parse.go | 12 +- .../webserver/document_biblioreads_test.go | 356 +---------------- internal/webserver/document_reviews_test.go | 363 ++++++++++++++++++ .../partials/document-metadata-filters.html | 9 +- .../document-score-filter-control.html | 6 +- internal/webserver/highlights_test.go | 27 +- .../webserver/model/highlight_repository.go | 13 +- internal/webserver/search_test.go | 61 ++- 17 files changed, 684 insertions(+), 626 deletions(-) rename internal/index/{bleve_document_data.go => bleve_document_data_sources.go} (100%) rename internal/index/{biblioreads_fields_test.go => data_source_fields_test.go} (73%) rename internal/index/{bleve_document_data_test.go => document_data_sources_test.go} (50%) delete mode 100644 internal/index/document_data_test.go delete mode 100644 internal/reviews/errors.go create mode 100644 internal/webserver/document_reviews_test.go diff --git a/internal/index/bleve_document_data.go b/internal/index/bleve_document_data_sources.go similarity index 100% rename from internal/index/bleve_document_data.go rename to internal/index/bleve_document_data_sources.go diff --git a/internal/index/bleve_document_read.go b/internal/index/bleve_document_read.go index 45d40bd2..32adbfd5 100644 --- a/internal/index/bleve_document_read.go +++ b/internal/index/bleve_document_read.go @@ -249,22 +249,9 @@ func (b *BleveIndexer) addFilters(searchFields SearchFields, filtersQuery *query bq.SetBoost(0) filtersQuery.AddQuery(bq) } - if searchFields.DataSourceScoreNone { - unscored := 0.0 - ratingQuery := newInclusiveNumericRangeQuery(&unscored, &unscored) - ratingQuery.SetField("DataSourceScoreValue") - ratingQuery.SetBoost(0) - filtersQuery.AddQuery(ratingQuery) - } else if searchFields.DataSourceScoreFrom > 0 || searchFields.DataSourceScoreTo > 0 { - min := math.SmallestNonzeroFloat64 - var max *float64 - if searchFields.DataSourceScoreFrom > 0 { - min = searchFields.DataSourceScoreFrom - } - if searchFields.DataSourceScoreTo > 0 { - max = &searchFields.DataSourceScoreTo - } - ratingQuery := newInclusiveNumericRangeQuery(&min, max) + if max := searchFields.DataSourceScoreUpperBound(); searchFields.DataSourceScoreFrom != 0 || max != 5 { + min := searchFields.DataSourceScoreFrom + ratingQuery := newInclusiveNumericRangeQuery(&min, &max) ratingQuery.SetField("DataSourceScoreValue") ratingQuery.SetBoost(0) filtersQuery.AddQuery(ratingQuery) diff --git a/internal/index/biblioreads_fields_test.go b/internal/index/data_source_fields_test.go similarity index 73% rename from internal/index/biblioreads_fields_test.go rename to internal/index/data_source_fields_test.go index 97419b69..07dd7083 100644 --- a/internal/index/biblioreads_fields_test.go +++ b/internal/index/data_source_fields_test.go @@ -6,31 +6,36 @@ import ( "testing" "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search/query" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" ) -func mustBiblioReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { +type storedProviderData struct { + URL string `json:"url"` + ScoreCount string `json:"scoreCount"` + ReferenceID string `json:"referenceId"` + RecordID int `json:"recordId"` +} + +func mustStoredProviderData(t *testing.T, document index.Document) storedProviderData { t.Helper() - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + data, err := index.DataSourceData[storedProviderData](document, "dummy") if err != nil { t.Fatal(err) } return data } -func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { +func withStoredProviderData(t *testing.T, document index.Document, data storedProviderData) index.Document { t.Helper() - if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + if err := index.SetDataSourceData(&document, "dummy", data); err != nil { t.Fatal(err) } return document } -func TestReviewScoreSorting(t *testing.T) { +func TestDataSourceScoreSorting(t *testing.T) { docs := []index.Document{ {ID: "high", Slug: "high", DataSourceScoreValue: 4.75}, {ID: "unrated", Slug: "unrated"}, @@ -103,7 +108,7 @@ func TestReviewScoreSorting(t *testing.T) { } } -func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) { +func TestDataSourceFieldsAreStoredButNotSearchable(t *testing.T) { documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) if err != nil { t.Fatal(err) @@ -116,25 +121,25 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) t.Cleanup(func() { _ = idx.Close() }) const ( - bookURL = "/book/show/12207444-zyxwvbook" - authorURL = "/author/show/151155.Zyxwv_Author" - resourceID = "kca://work/amzn1.gr.work.v1.zyxwvresource" - legacyBookID = 12207444 + bookURL = "https://example.org/books/12207444-zyxwvbook" + authorURL = "https://example.org/authors/151155.Zyxwv_Author" + referenceID = "opaque-zyxwvreference" + recordID = 12207444 ) for _, lang := range []string{"es", ""} { - doc := withBiblioReadsData(t, index.Document{ + doc := withStoredProviderData(t, index.Document{ ID: "doc-" + lang, Slug: "doc-" + lang, Metadata: metadata.Metadata{ - Title: "La Reina Dragon", - Authors: []string{"Alice Borchardt"}, + Title: "Example Book", + Authors: []string{"Example Writer"}, Language: lang, Format: "EPUB", }, - AuthorsSlugs: []string{"alice-borchardt"}, + AuthorsSlugs: []string{"example-writer"}, DataSourceScoreValue: 4.25, - }, biblioreads.DocumentData{URL: bookURL, RatingCount: "2285", ResourceID: resourceID, LegacyBookID: legacyBookID}) + }, storedProviderData{URL: bookURL, ScoreCount: "2285", ReferenceID: referenceID, RecordID: recordID}) if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) @@ -144,17 +149,17 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) if err != nil { t.Fatal(err) } - if mustBiblioReadsData(t, got).URL != bookURL { - t.Errorf("language %q: expected stored book URL, got %q", lang, mustBiblioReadsData(t, got).URL) + if mustStoredProviderData(t, got).URL != bookURL { + t.Errorf("language %q: expected stored book URL, got %q", lang, mustStoredProviderData(t, got).URL) } if got.DataSourceScoreValue != 4.25 { t.Errorf("language %q: expected stored rating, got %v", lang, got.DataSourceScoreValue) } - if mustBiblioReadsData(t, got).RatingCount != "2285" { - t.Errorf("language %q: expected stored rating count, got %q", lang, mustBiblioReadsData(t, got).RatingCount) + if mustStoredProviderData(t, got).ScoreCount != "2285" { + t.Errorf("language %q: expected stored rating count, got %q", lang, mustStoredProviderData(t, got).ScoreCount) } - if mustBiblioReadsData(t, got).ResourceID != resourceID || mustBiblioReadsData(t, got).LegacyBookID != legacyBookID { - t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, mustBiblioReadsData(t, got).ResourceID, mustBiblioReadsData(t, got).LegacyBookID) + if mustStoredProviderData(t, got).ReferenceID != referenceID || mustStoredProviderData(t, got).RecordID != recordID { + t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, mustStoredProviderData(t, got).ReferenceID, mustStoredProviderData(t, got).RecordID) } } if err := index.IndexDocument(documentsIndexMem, "unscored", index.Document{ @@ -168,7 +173,7 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) t.Fatal(err) } - for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155", "zyxwvresource", resourceID, "4.25", "2285"} { + for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155", "zyxwvreference", referenceID, "4.25", "2285"} { res, err := idx.Search(index.SearchFields{Keywords: keywords}, 1, 10) if err != nil { t.Fatal(err) @@ -177,25 +182,25 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) t.Errorf("expected no results searching keywords %q, got %d", keywords, res.TotalHits()) } } - filtered, err := idx.Search(index.SearchFields{DataSourceScoreFrom: 4.2, DataSourceScoreTo: 4.3}, 1, 10) + filtered, err := idx.Search(index.SearchFields{DataSourceScoreFrom: 4.2, DataSourceScoreTo: new(4.3)}, 1, 10) if err != nil || filtered.TotalHits() != 2 { - t.Errorf("expected both documents in the review score range, got %d results and error %v", filtered.TotalHits(), err) + t.Errorf("expected both documents in the data-source score range, got %d results and error %v", filtered.TotalHits(), err) } filtered, err = idx.Search(index.SearchFields{DataSourceScoreFrom: 4.3}, 1, 10) if err != nil || filtered.TotalHits() != 0 { - t.Errorf("expected no documents above the review score, got %d results and error %v", filtered.TotalHits(), err) + t.Errorf("expected no documents above the data-source score, got %d results and error %v", filtered.TotalHits(), err) } - filtered, err = idx.Search(index.SearchFields{DataSourceScoreNone: true}, 1, 10) + filtered, err = idx.Search(index.SearchFields{DataSourceScoreTo: new(0.0)}, 1, 10) if err != nil || filtered.TotalHits() != 1 || filtered.Hits()[0].DataSourceScoreValue != 0 { t.Errorf("expected only the unscored document, got %d results and error %v", filtered.TotalHits(), err) } queries := map[string]bleve.SearchRequest{ "provider data field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources", bookURL)), - "provider URL field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources.goodreads.url", bookURL)), - "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("GoodReadsAuthorURL", authorURL)), - "resource ID field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources.goodreads.resourceId", resourceID)), - "legacy ID numeric field": *bleve.NewSearchRequest(legacyBookIDQuery(legacyBookID)), + "provider URL field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources.dummy.url", bookURL)), + "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("DataSourceLinks.dummy", authorURL)), + "resource ID field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources.dummy.referenceId", referenceID)), + "legacy ID numeric field": *bleve.NewSearchRequest(providerRecordIDQuery(recordID)), "_all match": *bleve.NewSearchRequest(bleve.NewMatchQuery("zyxwvbook")), "_all term": *bleve.NewSearchRequest(bleve.NewTermQuery(bookURL)), "query string field": *bleve.NewSearchRequest(bleve.NewQueryStringQuery(`DataSources:"` + bookURL + `"`)), @@ -211,7 +216,7 @@ func TestReviewFieldsAndProviderMetadataAreStoredButNotSearchable(t *testing.T) } } -func TestReviewScoreSortingMissingFieldsAndTies(t *testing.T) { +func TestDataSourceScoreSortingMissingFieldsAndTies(t *testing.T) { documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) if err != nil { t.Fatal(err) @@ -263,10 +268,10 @@ func TestReviewScoreSortingMissingFieldsAndTies(t *testing.T) { } } -func legacyBookIDQuery(id int) *query.NumericRangeQuery { +func providerRecordIDQuery(id int) *query.NumericRangeQuery { value := float64(id) q := bleve.NewNumericRangeQuery(&value, &value) - q.SetField("DataSources.goodreads.legacyBookId") + q.SetField("DataSources.dummy.recordId") return q } @@ -277,12 +282,12 @@ func TestAuthorDataSourceLinksAreStoredButNotSearchable(t *testing.T) { } defer authors.Close() idx := index.NewBleve(nil, authors, nil, "", nil, index.Config{}) - const authorURL = "/author/show/151155.Zyxwv_Author" + const authorURL = "https://example.org/authors/151155.Zyxwv_Author" links := map[string]string{ - "goodreads": authorURL, - "other": "https://example.org/Zyxwv_Author", + "dummy": authorURL, + "other": "https://example.org/Zyxwv_Author", } - if err := idx.IndexAuthor(index.Author{Slug: "alice", Name: "Alice Borchardt", DataSourceLinks: links}); err != nil { + if err := idx.IndexAuthor(index.Author{Slug: "alice", Name: "Example Writer", DataSourceLinks: links}); err != nil { t.Fatal(err) } author, err := idx.Author("alice", "") @@ -294,7 +299,7 @@ func TestAuthorDataSourceLinksAreStoredButNotSearchable(t *testing.T) { } for _, q := range []query.Query{ fieldTermQuery("DataSourceLinks", authorURL), - fieldTermQuery("DataSourceLinks.goodreads", authorURL), + fieldTermQuery("DataSourceLinks.dummy", authorURL), fieldTermQuery("DataSourceLinks.other", links["other"]), bleve.NewTermQuery(authorURL), bleve.NewMatchQuery("Zyxwv_Author"), @@ -315,15 +320,3 @@ func fieldTermQuery(field, term string) *query.TermQuery { q.SetField(field) return q } - -func TestDocumentsMappingHasNoAuthorURL(t *testing.T) { - m := index.CreateDocumentsMapping().(*mapping.IndexMappingImpl) - for name, documentMapping := range m.TypeMapping { - if _, exists := documentMapping.Properties["GoodReadsAuthorURL"]; exists { - t.Errorf("mapping %q still contains an author URL field", name) - } - } - if _, exists := m.DefaultMapping.Properties["GoodReadsAuthorURL"]; exists { - t.Error("default mapping still contains an author URL field") - } -} diff --git a/internal/index/document.go b/internal/index/document.go index e9459cd3..24928eb5 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -35,12 +35,19 @@ type SearchFields struct { PagesFrom float64 PagesTo float64 DataSourceScoreFrom float64 - DataSourceScoreTo float64 - DataSourceScoreNone bool + DataSourceScoreTo *float64 IllustratedOnly bool SortBy []string } +// DataSourceScoreUpperBound defaults an unspecified upper bound to the maximum score. +func (s SearchFields) DataSourceScoreUpperBound() float64 { + if s.DataSourceScoreTo == nil { + return 5 + } + return *s.DataSourceScoreTo +} + type Document struct { metadata.Metadata ID string diff --git a/internal/index/bleve_document_data_test.go b/internal/index/document_data_sources_test.go similarity index 50% rename from internal/index/bleve_document_data_test.go rename to internal/index/document_data_sources_test.go index 039b6e31..f9433a8f 100644 --- a/internal/index/bleve_document_data_test.go +++ b/internal/index/document_data_sources_test.go @@ -10,6 +10,160 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) +func TestDocumentDataSourcesRoundTrip(t *testing.T) { + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + for _, language := range []string{"es", ""} { + document := withProviderData(t, Document{ + ID: "book-" + language, Slug: "book-" + language, + Metadata: metadata.Metadata{Title: "A book", Language: language, Format: "EPUB"}, + DataSourceScoreValue: 4.25, + }, testProviderData{URL: "/book/show/123", Enriched: true}) + other := map[string]any{ + "id": "opaqueprovideridentifier", "nested": map[string]any{"items": []any{"privatecatalogtoken", float64(42)}}, + } + if err := SetDataSourceData(&document, "other", other); err != nil { + t.Fatal(err) + } + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + got, err := idx.documentByIndexID(document.ID) + if err != nil { + t.Fatal(err) + } + decoded, err := DataSourceData[map[string]any](got, "other") + if err != nil || !reflect.DeepEqual(decoded, other) { + t.Fatalf("provider payload changed: %+v, %v", decoded, err) + } + if mustProviderData(t, got) != mustProviderData(t, document) || got.DataSourceScoreValue != 4.25 { + t.Fatalf("provider payload or score changed: %+v", got) + } + for _, term := range []string{"opaqueprovideridentifier", "privatecatalogtoken", "dummy", "other"} { + request := bleve.NewSearchRequest(bleve.NewMatchQuery(term)) + result, err := idx.documentsIdx.Search(request) + if err != nil || result.Total != 0 { + t.Fatalf("opaque data affected keyword search for %q: %+v, %v", term, result, err) + } + } + } + if _, total, err := idx.documentsNeedingProvider("dummy", 10); err != nil || total != 0 { + t.Fatalf("enriched payloads were queued again: %d, %v", total, err) + } + pending := Document{ID: "other-only", Slug: "other-only", Metadata: metadata.Metadata{Title: "Pending book"}} + if err := SetDataSourceData(&pending, "other", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + if err := IndexDocument(idx.documentsIdx, pending.ID, pending); err != nil { + t.Fatal(err) + } + if documents, total, err := idx.documentsNeedingProvider("dummy", 10); err != nil || total != 1 || documents[0].ID != pending.ID { + t.Fatalf("another provider marked provider as enriched: %+v, %d, %v", documents, total, err) + } +} + +func TestDocumentDataSourcesRejectInvalidData(t *testing.T) { + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} + for _, invalid := range []any{nil, "not an object", []int{1}, make(chan int)} { + if err := SetDataSourceData(&document, "invalid", invalid); err == nil { + t.Fatalf("accepted non-object provider payload: %#v", invalid) + } + if document.DataSources != nil { + t.Fatal("failed update mutated document data") + } + } + for _, raw := range []string{`{"url":`, `null`, `[]`} { + document.DataSources = map[string]json.RawMessage{"dummy": json.RawMessage(raw)} + if err := IndexDocument(idx.documentsIdx, document.ID, document); err == nil { + t.Fatalf("indexed invalid payload: %s", raw) + } + } +} + +func TestDocumentDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + if err := IndexDocument(idx.documentsIdx, "book", Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}}); err != nil { + t.Fatal(err) + } + stale, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + current := stale + other := map[string]any{"id": "external-id", "nested": map[string]any{"count": float64(3)}} + if err := SetDataSourceData(¤t, "other", other); err != nil { + t.Fatal(err) + } + if _, exists := stale.DataSources["other"]; exists { + t.Fatal("updating a provider mutated another document's map") + } + if err := IndexDocument(idx.documentsIdx, current.ID, current); err != nil { + t.Fatal(err) + } + if err := idx.StoreDocumentData(stale.ID, "dummy", testProviderData{URL: "/book/show/123", Enriched: true}); err != nil { + t.Fatal(err) + } + if err := idx.UpdateDocument(stale.ID, func(update *DocumentUpdate) (bool, error) { + update.DataSourceScoreValue = 4.5 + return true, SetDataSourceData(&update.Document, "dummy", testProviderData{URL: "/book/show/123", Enriched: true, RatingCount: "100", ResourceID: "work", LegacyBookID: 123}) + }); err != nil { + t.Fatal(err) + } + if err := idx.StoreDocumentData(stale.ID, "dummy", testProviderData{URL: "/book/show/456", Enriched: true}); err != nil { + t.Fatal(err) + } + stale.TextRankWords = []string{"updated"} + documents := []Document{stale} + idx.documentsMu.Lock() + err = idx.preserveDataSourcesLocked(documents) + if err == nil { + err = IndexDocument(idx.documentsIdx, stale.ID, documents[0]) + } + idx.documentsMu.Unlock() + if err != nil { + t.Fatal(err) + } + got, err := idx.documentByIndexID(stale.ID) + if err != nil { + t.Fatal(err) + } + decoded, err := DataSourceData[map[string]any](got, "other") + if err != nil || !reflect.DeepEqual(decoded, other) { + t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) + } + if data := mustProviderData(t, got); data != (testProviderData{URL: "/book/show/456", Enriched: true}) { + t.Fatalf("stale write restored old provider state: %+v", data) + } +} + +type testProviderData struct { + URL string `json:"url"` + Enriched bool `json:"enriched"` + RatingCount string `json:"ratingCount,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + LegacyBookID int `json:"legacyBookId,omitempty"` +} + +func mustProviderData(t *testing.T, document Document) testProviderData { + t.Helper() + data, err := DataSourceData[testProviderData](document, "dummy") + if err != nil { + t.Fatal(err) + } + return data +} + +func withProviderData(t *testing.T, document Document, data testProviderData) Document { + t.Helper() + if err := SetDataSourceData(&document, "dummy", data); err != nil { + t.Fatal(err) + } + return document +} + type documentUpdateBaseIndex interface { bleve.Index } diff --git a/internal/index/document_data_test.go b/internal/index/document_data_test.go deleted file mode 100644 index 0ebac9a8..00000000 --- a/internal/index/document_data_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package index - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/blevesearch/bleve/v2" - "github.com/svera/coreander/v5/internal/metadata" -) - -func TestDocumentDataSourcesRoundTrip(t *testing.T) { - idx, _ := newTestTextRankIndexer(t, Config{}) - t.Cleanup(func() { _ = idx.Close() }) - for _, language := range []string{"es", ""} { - document := withProviderData(t, Document{ - ID: "book-" + language, Slug: "book-" + language, - Metadata: metadata.Metadata{Title: "A book", Language: language, Format: "EPUB"}, - DataSourceScoreValue: 4.25, - }, testProviderData{URL: "/book/show/123", Enriched: true}) - other := map[string]any{ - "id": "opaqueprovideridentifier", "nested": map[string]any{"items": []any{"privatecatalogtoken", float64(42)}}, - } - if err := SetDataSourceData(&document, "other", other); err != nil { - t.Fatal(err) - } - if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { - t.Fatal(err) - } - got, err := idx.documentByIndexID(document.ID) - if err != nil { - t.Fatal(err) - } - decoded, err := DataSourceData[map[string]any](got, "other") - if err != nil || !reflect.DeepEqual(decoded, other) { - t.Fatalf("provider payload changed: %+v, %v", decoded, err) - } - if mustProviderData(t, got) != mustProviderData(t, document) || got.DataSourceScoreValue != 4.25 { - t.Fatalf("provider payload or score changed: %+v", got) - } - for _, term := range []string{"opaqueprovideridentifier", "privatecatalogtoken", "dummy", "other"} { - request := bleve.NewSearchRequest(bleve.NewMatchQuery(term)) - result, err := idx.documentsIdx.Search(request) - if err != nil || result.Total != 0 { - t.Fatalf("opaque data affected keyword search for %q: %+v, %v", term, result, err) - } - } - } - if _, total, err := idx.documentsNeedingProvider("dummy", 10); err != nil || total != 0 { - t.Fatalf("enriched payloads were queued again: %d, %v", total, err) - } - pending := Document{ID: "other-only", Slug: "other-only", Metadata: metadata.Metadata{Title: "Pending book"}} - if err := SetDataSourceData(&pending, "other", map[string]any{"enriched": true}); err != nil { - t.Fatal(err) - } - if err := IndexDocument(idx.documentsIdx, pending.ID, pending); err != nil { - t.Fatal(err) - } - if documents, total, err := idx.documentsNeedingProvider("dummy", 10); err != nil || total != 1 || documents[0].ID != pending.ID { - t.Fatalf("another provider marked provider as enriched: %+v, %d, %v", documents, total, err) - } -} - -func TestDocumentDataSourcesRejectInvalidData(t *testing.T) { - idx, _ := newTestTextRankIndexer(t, Config{}) - t.Cleanup(func() { _ = idx.Close() }) - document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} - for _, invalid := range []any{nil, "not an object", []int{1}, make(chan int)} { - if err := SetDataSourceData(&document, "invalid", invalid); err == nil { - t.Fatalf("accepted non-object provider payload: %#v", invalid) - } - if document.DataSources != nil { - t.Fatal("failed update mutated document data") - } - } - for _, raw := range []string{`{"url":`, `null`, `[]`} { - document.DataSources = map[string]json.RawMessage{"dummy": json.RawMessage(raw)} - if err := IndexDocument(idx.documentsIdx, document.ID, document); err == nil { - t.Fatalf("indexed invalid payload: %s", raw) - } - } -} - -func TestDocumentDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { - idx, _ := newTestTextRankIndexer(t, Config{}) - t.Cleanup(func() { _ = idx.Close() }) - if err := IndexDocument(idx.documentsIdx, "book", Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}}); err != nil { - t.Fatal(err) - } - stale, err := idx.documentByIndexID("book") - if err != nil { - t.Fatal(err) - } - current := stale - other := map[string]any{"id": "external-id", "nested": map[string]any{"count": float64(3)}} - if err := SetDataSourceData(¤t, "other", other); err != nil { - t.Fatal(err) - } - if _, exists := stale.DataSources["other"]; exists { - t.Fatal("updating a provider mutated another document's map") - } - if err := IndexDocument(idx.documentsIdx, current.ID, current); err != nil { - t.Fatal(err) - } - if err := idx.StoreDocumentData(stale.ID, "dummy", testProviderData{URL: "/book/show/123", Enriched: true}); err != nil { - t.Fatal(err) - } - if err := idx.UpdateDocument(stale.ID, func(update *DocumentUpdate) (bool, error) { - update.DataSourceScoreValue = 4.5 - return true, SetDataSourceData(&update.Document, "dummy", testProviderData{URL: "/book/show/123", Enriched: true, RatingCount: "100", ResourceID: "work", LegacyBookID: 123}) - }); err != nil { - t.Fatal(err) - } - if err := idx.StoreDocumentData(stale.ID, "dummy", testProviderData{URL: "/book/show/456", Enriched: true}); err != nil { - t.Fatal(err) - } - stale.TextRankWords = []string{"updated"} - documents := []Document{stale} - idx.documentsMu.Lock() - err = idx.preserveDataSourcesLocked(documents) - if err == nil { - err = IndexDocument(idx.documentsIdx, stale.ID, documents[0]) - } - idx.documentsMu.Unlock() - if err != nil { - t.Fatal(err) - } - got, err := idx.documentByIndexID(stale.ID) - if err != nil { - t.Fatal(err) - } - decoded, err := DataSourceData[map[string]any](got, "other") - if err != nil || !reflect.DeepEqual(decoded, other) { - t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) - } - if data := mustProviderData(t, got); data != (testProviderData{URL: "/book/show/456", Enriched: true}) { - t.Fatalf("stale write restored old provider state: %+v", data) - } -} - -type testProviderData struct { - URL string `json:"url"` - Enriched bool `json:"enriched"` - RatingCount string `json:"ratingCount,omitempty"` - ResourceID string `json:"resourceId,omitempty"` - LegacyBookID int `json:"legacyBookId,omitempty"` -} - -func mustProviderData(t *testing.T, document Document) testProviderData { - t.Helper() - data, err := DataSourceData[testProviderData](document, "dummy") - if err != nil { - t.Fatal(err) - } - return data -} - -func withProviderData(t *testing.T, document Document, data testProviderData) Document { - t.Helper() - if err := SetDataSourceData(&document, "dummy", data); err != nil { - t.Fatal(err) - } - return document -} diff --git a/internal/reviews/errors.go b/internal/reviews/errors.go deleted file mode 100644 index bccd3d72..00000000 --- a/internal/reviews/errors.go +++ /dev/null @@ -1,8 +0,0 @@ -package reviews - -import "errors" - -var ( - ErrInvalidDocumentLink = errors.New("invalid document link") - ErrDocumentNotFound = errors.New("document not found") -) diff --git a/internal/reviews/model.go b/internal/reviews/model.go index 6f042ed4..c35a59f4 100644 --- a/internal/reviews/model.go +++ b/internal/reviews/model.go @@ -2,11 +2,17 @@ package reviews import ( + "errors" "strconv" "strings" "time" ) +var ( + ErrInvalidDocumentLink = errors.New("invalid document link") + ErrDocumentNotFound = errors.New("document not found") +) + // Score has an empty URL for an unmatched document and an empty Value for an unavailable score. type Score struct { Source string diff --git a/internal/webserver/controller.go b/internal/webserver/controller.go index 416ade1a..acc8b0e5 100644 --- a/internal/webserver/controller.go +++ b/internal/webserver/controller.go @@ -30,7 +30,7 @@ type Controllers struct { Series *series.Controller } -func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender Sender, appFs afero.Fs, dataSource author.DataSource, reviewProvider document.ReviewProvider, dataSourceLinkProviders ...author.DataSourceLinkProvider) Controllers { +func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender Sender, appFs afero.Fs, authorDataSource author.DataSource, reviewProvider document.ReviewProvider, dataSourceLinkProviders ...author.DataSourceLinkProvider) Controllers { usersRepository := &model.UserRepository{DB: db} invitationsRepository := &model.InvitationRepository{DB: db} highlightsRepository := &model.HighlightRepository{DB: db, Idx: idx, IllustratedMinAmount: cfg.IllustratedMinAmount} @@ -102,7 +102,7 @@ func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender S LatestDocsLimit: 6, } - authorsController := author.NewController(highlightsRepository, readingRepository, sender, idx, authorsCfg, dataSource, appFs, imagesFS, dataSourceLinkProviders...) + authorsController := author.NewController(highlightsRepository, readingRepository, sender, idx, authorsCfg, authorDataSource, appFs, imagesFS, dataSourceLinkProviders...) go authorsController.MigrateJPEGsToWebP() return Controllers{ diff --git a/internal/webserver/controller/search/parse.go b/internal/webserver/controller/search/parse.go index 5da1bfd9..1f7eb1bb 100644 --- a/internal/webserver/controller/search/parse.go +++ b/internal/webserver/controller/search/parse.go @@ -26,13 +26,13 @@ func ParseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.Search PagesFrom: fiber.Query[float64](c, "pages-from", 0), PagesTo: fiber.Query[float64](c, "pages-to", 0), DataSourceScoreFrom: fiber.Query[float64](c, "review-score-from", 0), - DataSourceScoreTo: fiber.Query[float64](c, "review-score-to", 0), - DataSourceScoreNone: c.Query("review-score-from") == "none" || c.Query("review-score-to") == "none", + DataSourceScoreTo: new(fiber.Query[float64](c, "review-score-to", 5)), IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", } - if searchFields.DataSourceScoreNone { + // Preserve existing links using the former no-score query value. + if c.Query("review-score-from") == "none" || c.Query("review-score-to") == "none" { searchFields.DataSourceScoreFrom = 0 - searchFields.DataSourceScoreTo = 0 + searchFields.DataSourceScoreTo = new(0.0) } if c.Query("pub-date-from") != "" { @@ -63,8 +63,8 @@ func ParseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.Search searchFields.PagesFrom, searchFields.PagesTo = searchFields.PagesTo, searchFields.PagesFrom } - if !searchFields.DataSourceScoreNone && searchFields.DataSourceScoreTo != 0 && searchFields.DataSourceScoreFrom > searchFields.DataSourceScoreTo { - searchFields.DataSourceScoreFrom, searchFields.DataSourceScoreTo = searchFields.DataSourceScoreTo, searchFields.DataSourceScoreFrom + if searchFields.DataSourceScoreFrom > *searchFields.DataSourceScoreTo { + searchFields.DataSourceScoreFrom, *searchFields.DataSourceScoreTo = *searchFields.DataSourceScoreTo, searchFields.DataSourceScoreFrom } return searchFields, nil diff --git a/internal/webserver/document_biblioreads_test.go b/internal/webserver/document_biblioreads_test.go index b6e6dab5..6e76d120 100644 --- a/internal/webserver/document_biblioreads_test.go +++ b/internal/webserver/document_biblioreads_test.go @@ -3,14 +3,12 @@ package webserver_test import ( "context" "errors" - "fmt" "io" "net/http" "net/url" "strings" "sync" "testing" - "time" "github.com/PuerkitoBio/goquery" "github.com/blevesearch/bleve/v2" @@ -59,224 +57,6 @@ func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads return document } -func TestDocumentReviewsCanSelectSupportedLanguage(t *testing.T) { - source := &fakeBookSource{book: biblioreads.BookResponse{ - Rating: "4", ResourceID: "work-id", LegacyBookID: 1, - }} - app, _ := newBiblioReadsTestApp(t, source) - req := mustGetRequest(t, "/documents/linked/reviews?language=fr") - req.Header.Set("Accept-Language", "es") - response, err := app.Test(req) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - mustReturnStatus(response, http.StatusOK, t) - page, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - t.Fatal(err) - } - if len(source.reviewCalls) != 1 || source.reviewCalls[0].languageCode != "fr" { - t.Fatalf("expected Goodreads reviews in French, got %+v", source.reviewCalls) - } - if page.Find("h2").Text() != "Reseñas de usuarios de Goodreads" { - t.Fatalf("expected page language to remain Spanish while selecting French reviews, got %q", page.Find("h2").Text()) - } - selectBox := page.Find(`#review-language-linked`) - if selectBox.Length() != 1 || selectBox.Find(`option[value="fr"][selected]`).Length() != 1 { - html, _ := selectBox.Html() - t.Fatalf("expected the review-language selector to retain the selected language: %q", html) - } - if indicator, _ := selectBox.Attr("hx-indicator"); indicator != "#document-reviews-linked" || - page.Find(`#document-reviews-linked .document-reviews-loading > div`).Length() != 2 { - t.Fatal("expected a review skeleton indicator for language changes") - } - for _, language := range webserver.SupportedLanguages() { - if selectBox.Find(`option[value="`+language+`"]`).Length() != 1 { - t.Errorf("expected supported language %q in the review selector", language) - } - } -} - -func TestDocumentReviewsRejectUnsupportedLanguage(t *testing.T) { - source := &fakeBookSource{book: biblioreads.BookResponse{ - Rating: "4", ResourceID: "work-id", LegacyBookID: 1, - }} - app, _ := newBiblioReadsTestApp(t, source) - response, err := app.Test(mustGetRequest(t, "/documents/linked/reviews?language=zz")) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - mustReturnStatus(response, http.StatusBadRequest, t) - if len(source.reviewCalls) != 0 { - t.Fatalf("unsupported language must not be sent to Goodreads, got %+v", source.reviewCalls) - } -} - -type fakeReviewProvider struct { - score reviews.Score - page reviews.Page - scoreErr error - reviewsErr error - canEditDocumentLink bool - scoreSlugs []string - reviewSlugs []string - reviewLanguages []string -} - -func (f *fakeReviewProvider) CanEditDocumentLink() bool { - return f.canEditDocumentLink -} - -func (f *fakeReviewProvider) Name() string { - return f.score.Source -} - -func (f *fakeReviewProvider) UpdateDocumentLink(index.Document, string) error { - return nil -} - -func (f *fakeReviewProvider) Score(_ context.Context, document index.Document) (reviews.Score, error) { - f.scoreSlugs = append(f.scoreSlugs, document.Slug) - return f.score, f.scoreErr -} - -func (f *fakeReviewProvider) Reviews(_ context.Context, document index.Document, language string) (reviews.Page, error) { - f.reviewSlugs = append(f.reviewSlugs, document.Slug) - f.reviewLanguages = append(f.reviewLanguages, language) - return f.page, f.reviewsErr -} - -func TestDocumentReviewsUILanguage(t *testing.T) { - for _, tt := range []struct { - name string - query string - cookie string - header string - want string - }{ - {name: "unsupported header falls back to English", header: "zz", want: "en"}, - {name: "header", header: "fr", want: "fr"}, - {name: "cookie overrides header", cookie: "es", header: "en", want: "es"}, - {name: "query overrides cookie", query: "?l=de", cookie: "es", header: "en", want: "de"}, - } { - t.Run(tt.name, func(t *testing.T) { - source := &fakeBookSource{book: biblioreads.BookResponse{ - Rating: "4", ResourceID: "work-id", LegacyBookID: 1, - }} - app, _ := newBiblioReadsTestApp(t, source) - req := mustGetRequest(t, "/documents/linked/reviews"+tt.query) - if tt.header != "" { - req.Header.Set("Accept-Language", tt.header) - } - if tt.cookie != "" { - req.AddCookie(&http.Cookie{Name: "locale", Value: tt.cookie}) - } - response, err := app.Test(req) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - mustReturnStatus(response, http.StatusOK, t) - if len(source.reviewCalls) != 1 || source.reviewCalls[0].languageCode != tt.want { - t.Fatalf("expected BiblioReads language %q, got %+v", tt.want, source.reviewCalls) - } - }) - } -} - -func TestDocumentAlternativeReviewProvider(t *testing.T) { - provider := &fakeReviewProvider{ - score: reviews.Score{Source: "Another service", Value: "4.5", Count: "123", URL: "https://example.org/books/book-1"}, - page: reviews.Page{ - Source: "Another service", URL: "https://example.org/books/book-1", Available: true, - Reviews: []reviews.Review{{ - Reviewer: "Reader", Rating: 5, Text: "

        Excellent

        ", - CreatedAt: time.Date(2026, 10, 30, 0, 0, 0, 0, time.UTC), URL: "https://example.org/reviews/review-1", - }}, - }, - } - app, _ := newReviewProviderTestApp(t, nil, provider) - searchPage, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(searchPage.Find(`fieldset legend`).Text(), "Another service score") || - searchPage.Find(`#sort-by option[value="review-score-asc"]`).Text() != "Another service score (ascending)" { - t.Fatal("search filters and sort options must use the active review provider's name") - } - // This document has a cached Goodreads non-match; it must still use the chosen provider. - detail := getBody(t, app, "/documents/unmatched") - if biblioReadsPlaceholders(t, detail).Length() != 1 || len(provider.scoreSlugs) != 0 || len(provider.reviewSlugs) != 0 { - t.Fatal("detail must defer lookups to the selected provider") - } - for _, path := range []string{"/documents/unmatched/score"} { - body := getBody(t, app, path) - page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - link := page.Find(`li a[href="https://example.org/books/book-1"]`) - if link.Text() != "4.5" || !strings.Contains(body, "Another service rating") || strings.Contains(body, "goodreads.com") { - t.Fatalf("score must use the selected provider's labels, value and URL: %s", body) - } - } - body := getBody(t, app, "/documents/unmatched/reviews") - page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - if page.Find("h2").Text() != "Another service user reviews" || - page.Find(`a[href="https://example.org/reviews/review-1"] time[datetime="2026-10-30"]`).Length() != 1 || - page.Find("article strong").Text() != "Reader" || - page.Find("i.bi-star-fill").Length() != 5 || page.Find("script").Length() != 0 { - t.Fatalf("reviews must render normalized provider data safely: %s", body) - } - if len(provider.scoreSlugs) != 1 || len(provider.reviewSlugs) != 1 || provider.reviewSlugs[0] != "unmatched" { - t.Fatal("expected calls through the provider interface") - } - if len(provider.reviewLanguages) != 1 || provider.reviewLanguages[0] != "en" { - t.Fatalf("expected the current UI language to reach alternative providers, got %q", provider.reviewLanguages) - } -} - -func TestDocumentScoreMarksZeroWithoutValuationsAsNotScored(t *testing.T) { - for _, tt := range []struct { - name string - count string - wantScore string - wantScored bool - }{ - {name: "zero valuations", count: "0", wantScore: "Not scored"}, - {name: "valuations exist", count: "650", wantScore: "0.00", wantScored: true}, - } { - t.Run(tt.name, func(t *testing.T) { - provider := &fakeReviewProvider{ - score: reviews.Score{ - Source: "Goodreads", Value: "0.00", Count: tt.count, - URL: "https://www.goodreads.com/book/show/1", - }, - } - app, _ := newReviewProviderTestApp(t, nil, provider) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unmatched/score"))) - if err != nil { - t.Fatal(err) - } - row := page.Find("li") - if tt.wantScored { - if got := strings.TrimSpace(row.Find("a").Text()); got != tt.wantScore { - t.Fatalf("expected score %q, got %q", tt.wantScore, got) - } - return - } - if got := strings.TrimSpace(row.Text()); got != tt.wantScore || row.Find("a").Length() != 0 { - t.Fatalf("expected unlinked %q label, got %q", tt.wantScore, got) - } - }) - } -} - type reviewCall struct { resourceID string legacyBookID int @@ -307,12 +87,7 @@ func (f *fakeBookSource) Book(_ context.Context, id string) (biblioreads.BookRes func newBiblioReadsTestApp(t *testing.T, source *fakeBookSource) (*fiber.App, *index.BleveIndexer) { t.Helper() - return newReviewProviderTestApp(t, source, nil) -} - -func newReviewProviderTestApp(t *testing.T, source *fakeBookSource, provider document.ReviewProvider) (*fiber.App, *index.BleveIndexer) { - t.Helper() - return newDataSourceProvidersTestApp(t, source, provider, nil) + return newDataSourceProvidersTestApp(t, source, nil, nil) } func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provider document.ReviewProvider, dataSourceLinkProvider authorcontroller.DataSourceLinkProvider) (*fiber.App, *index.BleveIndexer) { @@ -378,34 +153,6 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide return app, idx } -func getBody(t *testing.T, app *fiber.App, path string) string { - t.Helper() - req := mustGetRequest(t, path) - req.Header.Set("Accept-Language", "en") - response, err := app.Test(req) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - t.Fatalf("%s: status = %d, want %d", path, response.StatusCode, http.StatusOK) - } - body, err := io.ReadAll(response.Body) - if err != nil { - t.Fatal(err) - } - return string(body) -} - -func biblioReadsPlaceholders(t *testing.T, html string) *goquery.Selection { - t.Helper() - page, err := goquery.NewDocumentFromReader(strings.NewReader(html)) - if err != nil { - t.Fatal(err) - } - return page.Find(`li[hx-get$="/score"]`) -} - func TestDocumentDetailBiblioReadsRating(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "3.82", RatingCount: "2,285", @@ -413,7 +160,7 @@ func TestDocumentDetailBiblioReadsRating(t *testing.T) { }} app, idx := newBiblioReadsTestApp(t, source) - placeholder := biblioReadsPlaceholders(t, getBody(t, app, "/documents/linked")) + placeholder := documentScorePlaceholders(t, getBody(t, app, "/documents/linked")) if placeholder.Length() != 1 { t.Fatalf("expected one BiblioReads placeholder, got %d", placeholder.Length()) } @@ -622,7 +369,7 @@ func TestDocumentDetailUpdatesStoredBiblioReadsRating(t *testing.T) { if stored.DataSourceScoreValue != 4.12 { t.Fatalf("expected latest rating and searchable value to be stored, got %+v", stored) } - results, err := idx.Search(index.SearchFields{DataSourceScoreFrom: 4.1, DataSourceScoreTo: 4.2}, 1, 10) + results, err := idx.Search(index.SearchFields{DataSourceScoreFrom: 4.1, DataSourceScoreTo: new(4.2)}, 1, 10) if err != nil || results.TotalHits() != 1 { t.Fatalf("expected document to match updated rating range, got %d results and error %v", results.TotalHits(), err) } @@ -658,7 +405,7 @@ func TestDocumentDetailBiblioReadsSearchesUnlinkedDocuments(t *testing.T) { } app, idx := newBiblioReadsTestApp(t, source) - if got := biblioReadsPlaceholders(t, getBody(t, app, "/documents/unlinked")).Length(); got != 1 { + if got := documentScorePlaceholders(t, getBody(t, app, "/documents/unlinked")).Length(); got != 1 { t.Fatalf("expected a placeholder for a document not looked up yet, got %d", got) } @@ -715,7 +462,7 @@ func TestDocumentDetailBiblioReadsSearchesUnlinkedDocuments(t *testing.T) { func TestDocumentDetailBiblioReadsSkipsUnmatchedDocuments(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{Rating: "3.82"}} app, _ := newBiblioReadsTestApp(t, source) - if got := biblioReadsPlaceholders(t, getBody(t, app, "/documents/unmatched")).Length(); got != 1 { + if got := documentScorePlaceholders(t, getBody(t, app, "/documents/unmatched")).Length(); got != 1 { t.Errorf("expected an async score placeholder, got %d", got) } for _, path := range []string{"/documents/unmatched/score"} { @@ -786,7 +533,7 @@ func TestDocumentDetailBiblioReadsIdentifiersWithoutRating(t *testing.T) { func TestDocumentDetailBiblioReadsDisabled(t *testing.T) { app, _ := newBiblioReadsTestApp(t, nil) for _, slug := range []string{"linked", "unlinked", "unmatched"} { - if got := biblioReadsPlaceholders(t, getBody(t, app, "/documents/"+slug)).Length(); got != 0 { + if got := documentScorePlaceholders(t, getBody(t, app, "/documents/"+slug)).Length(); got != 0 { t.Errorf("%s: expected no placeholder when BiblioReads is disabled, got %d", slug, got) } if body := strings.TrimSpace(getBody(t, app, "/documents/"+slug+"/score")); body != "" { @@ -1040,97 +787,6 @@ func TestDocumentDetailBiblioReadsReviews(t *testing.T) { } } -func TestDocumentReviewFractionalStars(t *testing.T) { - for _, tt := range []struct { - rating float64 - full int - empty int - width int - }{ - {rating: 0}, - {rating: 0.25, empty: 4, width: 25}, - {rating: 3.5, full: 3, empty: 1, width: 50}, - {rating: 4.25, full: 4, width: 25}, - {rating: 4.75, full: 4, width: 75}, - {rating: 4, full: 4, empty: 1}, - {rating: 5, full: 5}, - } { - t.Run(fmt.Sprint(tt.rating), func(t *testing.T) { - source := &fakeBookSource{ - book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}, - reviews: biblioreads.ReviewsResponse{Reviews: []biblioreads.Review{ - {Reviewer: biblioreads.Reviewer{Name: "Reader"}, Rating: tt.rating}, - }}, - } - app, _ := newBiblioReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews"))) - if err != nil { - t.Fatal(err) - } - stars := page.Find(`article span[role="img"]`) - if tt.rating == 0 { - if stars.Length() != 0 { - t.Fatal("unrated reviews must not display stars") - } - if page.Find("article header span.text-muted").Text() != "Not scored" { - t.Fatal("unrated reviews must display Not scored") - } - return - } - if page.Find("article header span.text-muted").Length() != 0 { - t.Fatal("scored reviews must not display the unrated label") - } - if label, _ := stars.Attr("aria-label"); label != fmt.Sprintf("%g/5", tt.rating) { - t.Fatalf("expected exact accessible rating, got %q", label) - } - if stars.ChildrenFiltered("i.bi-star-fill").Length() != tt.full || - stars.ChildrenFiltered("i.bi-star").Length() != tt.empty || - stars.Children().Length() != 5 { - t.Fatalf("unexpected full/empty star counts: %s", stars.Text()) - } - partial := stars.Find(".review-star-fill") - if tt.width == 0 { - if partial.Length() != 0 { - t.Fatal("whole ratings must not include a partial star") - } - } else if style, _ := partial.Attr("style"); style != fmt.Sprintf("width: %d%%", tt.width) || - stars.Find(`.review-star[aria-hidden="true"]`).Length() != 1 { - t.Fatalf("expected a %d%% decorative star fill, got %q", tt.width, style) - } - }) - } -} - -func TestDocumentReviewNotScoredTranslations(t *testing.T) { - source := &fakeBookSource{ - book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}, - reviews: biblioreads.ReviewsResponse{Reviews: []biblioreads.Review{ - {Reviewer: biblioreads.Reviewer{Name: "Reader"}, Text: "A review without a rating"}, - }}, - } - app, _ := newBiblioReadsTestApp(t, source) - for _, tt := range []struct { - lang string - text string - }{ - {"en", "Not scored"}, - {"de", "Nicht bewertet"}, - {"es", "Sin puntuar"}, - {"fr", "Non noté"}, - {"ru", "Не оценено"}, - } { - t.Run(tt.lang, func(t *testing.T) { - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews?l="+tt.lang))) - if err != nil { - t.Fatal(err) - } - if got := page.Find("article header span.text-muted").Text(); got != tt.text { - t.Fatalf("expected %q, got %q", tt.text, got) - } - }) - } -} - func TestDocumentDetailBiblioReadsReviewsAvailability(t *testing.T) { for _, tt := range []struct { name string diff --git a/internal/webserver/document_reviews_test.go b/internal/webserver/document_reviews_test.go new file mode 100644 index 00000000..9d8d5722 --- /dev/null +++ b/internal/webserver/document_reviews_test.go @@ -0,0 +1,363 @@ +package webserver_test + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/gofiber/fiber/v3" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/reviews" + "github.com/svera/coreander/v5/internal/webserver" + "github.com/svera/coreander/v5/internal/webserver/controller/document" +) + +func TestDocumentReviewsCanSelectSupportedLanguage(t *testing.T) { + provider := &fakeReviewProvider{ + score: reviews.Score{Source: "Another service"}, + page: reviews.Page{Source: "Another service", Available: true}, + } + app, _ := newReviewProviderTestApp(t, provider) + req := mustGetRequest(t, "/documents/linked/reviews?language=fr") + req.Header.Set("Accept-Language", "es") + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if len(provider.reviewLanguages) != 1 || provider.reviewLanguages[0] != "fr" { + t.Fatalf("expected reviews in French, got %+v", provider.reviewLanguages) + } + if page.Find("h2").Text() != "Reseñas de usuarios de Another service" { + t.Fatalf("expected page language to remain Spanish while selecting French reviews, got %q", page.Find("h2").Text()) + } + selectBox := page.Find(`#review-language-linked`) + if selectBox.Length() != 1 || selectBox.Find(`option[value="fr"][selected]`).Length() != 1 { + html, _ := selectBox.Html() + t.Fatalf("expected the review-language selector to retain the selected language: %q", html) + } + if indicator, _ := selectBox.Attr("hx-indicator"); indicator != "#document-reviews-linked" || + page.Find(`#document-reviews-linked .document-reviews-loading > div`).Length() != 2 { + t.Fatal("expected a review skeleton indicator for language changes") + } + for _, language := range webserver.SupportedLanguages() { + if selectBox.Find(`option[value="`+language+`"]`).Length() != 1 { + t.Errorf("expected supported language %q in the review selector", language) + } + } +} + +func TestDocumentReviewsRejectUnsupportedLanguage(t *testing.T) { + provider := &fakeReviewProvider{ + score: reviews.Score{Source: "Another service"}, + page: reviews.Page{Source: "Another service", Available: true}, + } + app, _ := newReviewProviderTestApp(t, provider) + response, err := app.Test(mustGetRequest(t, "/documents/linked/reviews?language=zz")) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusBadRequest, t) + if len(provider.reviewLanguages) != 0 { + t.Fatalf("unsupported language must not be sent to the provider, got %+v", provider.reviewLanguages) + } +} + +type fakeReviewProvider struct { + score reviews.Score + page reviews.Page + scoreErr error + reviewsErr error + canEditDocumentLink bool + scoreSlugs []string + reviewSlugs []string + reviewLanguages []string +} + +func (f *fakeReviewProvider) CanEditDocumentLink() bool { + return f.canEditDocumentLink +} + +func (f *fakeReviewProvider) Name() string { + return f.score.Source +} + +func (f *fakeReviewProvider) UpdateDocumentLink(index.Document, string) error { + return nil +} + +func (f *fakeReviewProvider) Score(_ context.Context, document index.Document) (reviews.Score, error) { + f.scoreSlugs = append(f.scoreSlugs, document.Slug) + return f.score, f.scoreErr +} + +func (f *fakeReviewProvider) Reviews(_ context.Context, document index.Document, language string) (reviews.Page, error) { + f.reviewSlugs = append(f.reviewSlugs, document.Slug) + f.reviewLanguages = append(f.reviewLanguages, language) + return f.page, f.reviewsErr +} + +func TestDocumentReviewsUILanguage(t *testing.T) { + for _, tt := range []struct { + name string + query string + cookie string + header string + want string + }{ + {name: "unsupported header falls back to English", header: "zz", want: "en"}, + {name: "header", header: "fr", want: "fr"}, + {name: "cookie overrides header", cookie: "es", header: "en", want: "es"}, + {name: "query overrides cookie", query: "?l=de", cookie: "es", header: "en", want: "de"}, + } { + t.Run(tt.name, func(t *testing.T) { + provider := &fakeReviewProvider{ + score: reviews.Score{Source: "Another service"}, + page: reviews.Page{Source: "Another service", Available: true}, + } + app, _ := newReviewProviderTestApp(t, provider) + req := mustGetRequest(t, "/documents/linked/reviews"+tt.query) + if tt.header != "" { + req.Header.Set("Accept-Language", tt.header) + } + if tt.cookie != "" { + req.AddCookie(&http.Cookie{Name: "locale", Value: tt.cookie}) + } + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + if len(provider.reviewLanguages) != 1 || provider.reviewLanguages[0] != tt.want { + t.Fatalf("expected review language %q, got %+v", tt.want, provider.reviewLanguages) + } + }) + } +} + +func TestDocumentAlternativeReviewProvider(t *testing.T) { + provider := &fakeReviewProvider{ + score: reviews.Score{Source: "Another service", Value: "4.5", Count: "123", URL: "https://example.org/books/book-1"}, + page: reviews.Page{ + Source: "Another service", URL: "https://example.org/books/book-1", Available: true, + Reviews: []reviews.Review{{ + Reviewer: "Reader", Rating: 5, Text: "

        Excellent

        ", + CreatedAt: time.Date(2026, 10, 30, 0, 0, 0, 0, time.UTC), URL: "https://example.org/reviews/review-1", + }}, + }, + } + app, _ := newReviewProviderTestApp(t, provider) + searchPage, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(searchPage.Find(`fieldset legend`).Text(), "Another service score") || + searchPage.Find(`#sort-by option[value="review-score-asc"]`).Text() != "Another service score (ascending)" { + t.Fatal("search filters and sort options must use the active review provider's name") + } + // This document has a cached Goodreads non-match; it must still use the chosen provider. + detail := getBody(t, app, "/documents/unmatched") + if documentScorePlaceholders(t, detail).Length() != 1 || len(provider.scoreSlugs) != 0 || len(provider.reviewSlugs) != 0 { + t.Fatal("detail must defer lookups to the selected provider") + } + for _, path := range []string{"/documents/unmatched/score"} { + body := getBody(t, app, path) + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + link := page.Find(`li a[href="https://example.org/books/book-1"]`) + if link.Text() != "4.5" || !strings.Contains(body, "Another service rating") || strings.Contains(body, "goodreads.com") { + t.Fatalf("score must use the selected provider's labels, value and URL: %s", body) + } + } + body := getBody(t, app, "/documents/unmatched/reviews") + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if page.Find("h2").Text() != "Another service user reviews" || + page.Find(`a[href="https://example.org/reviews/review-1"] time[datetime="2026-10-30"]`).Length() != 1 || + page.Find("article strong").Text() != "Reader" || + page.Find("i.bi-star-fill").Length() != 5 || page.Find("script").Length() != 0 { + t.Fatalf("reviews must render normalized provider data safely: %s", body) + } + if len(provider.scoreSlugs) != 1 || len(provider.reviewSlugs) != 1 || provider.reviewSlugs[0] != "unmatched" { + t.Fatal("expected calls through the provider interface") + } + if len(provider.reviewLanguages) != 1 || provider.reviewLanguages[0] != "en" { + t.Fatalf("expected the current UI language to reach alternative providers, got %q", provider.reviewLanguages) + } +} + +func TestDocumentScoreMarksZeroWithoutValuationsAsNotScored(t *testing.T) { + for _, tt := range []struct { + name string + count string + wantScore string + wantScored bool + }{ + {name: "zero valuations", count: "0", wantScore: "Not scored"}, + {name: "valuations exist", count: "650", wantScore: "0.00", wantScored: true}, + } { + t.Run(tt.name, func(t *testing.T) { + provider := &fakeReviewProvider{ + score: reviews.Score{ + Source: "Another service", Value: "0.00", Count: tt.count, + URL: "https://example.org/books/book-1", + }, + } + app, _ := newReviewProviderTestApp(t, provider) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unmatched/score"))) + if err != nil { + t.Fatal(err) + } + row := page.Find("li") + if tt.wantScored { + if got := strings.TrimSpace(row.Find("a").Text()); got != tt.wantScore { + t.Fatalf("expected score %q, got %q", tt.wantScore, got) + } + return + } + if got := strings.TrimSpace(row.Text()); got != tt.wantScore || row.Find("a").Length() != 0 { + t.Fatalf("expected unlinked %q label, got %q", tt.wantScore, got) + } + }) + } +} + +func newReviewProviderTestApp(t *testing.T, provider document.ReviewProvider) (*fiber.App, *index.BleveIndexer) { + t.Helper() + return newDataSourceProvidersTestApp(t, nil, provider, nil) +} + +func getBody(t *testing.T, app *fiber.App, path string) string { + t.Helper() + req := mustGetRequest(t, path) + req.Header.Set("Accept-Language", "en") + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("%s: status = %d, want %d", path, response.StatusCode, http.StatusOK) + } + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func documentScorePlaceholders(t *testing.T, html string) *goquery.Selection { + t.Helper() + page, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + if err != nil { + t.Fatal(err) + } + return page.Find(`li[hx-get$="/score"]`) +} + +func TestDocumentReviewFractionalStars(t *testing.T) { + for _, tt := range []struct { + rating float64 + full int + empty int + width int + }{ + {rating: 0}, + {rating: 0.25, empty: 4, width: 25}, + {rating: 3.5, full: 3, empty: 1, width: 50}, + {rating: 4.25, full: 4, width: 25}, + {rating: 4.75, full: 4, width: 75}, + {rating: 4, full: 4, empty: 1}, + {rating: 5, full: 5}, + } { + t.Run(fmt.Sprint(tt.rating), func(t *testing.T) { + provider := &fakeReviewProvider{ + score: reviews.Score{Source: "Another service"}, + page: reviews.Page{Source: "Another service", Available: true, Reviews: []reviews.Review{ + {Reviewer: "Reader", Rating: tt.rating}, + }}, + } + app, _ := newReviewProviderTestApp(t, provider) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews"))) + if err != nil { + t.Fatal(err) + } + stars := page.Find(`article span[role="img"]`) + if tt.rating == 0 { + if stars.Length() != 0 { + t.Fatal("unrated reviews must not display stars") + } + if page.Find("article header span.text-muted").Text() != "Not scored" { + t.Fatal("unrated reviews must display Not scored") + } + return + } + if page.Find("article header span.text-muted").Length() != 0 { + t.Fatal("scored reviews must not display the unrated label") + } + if label, _ := stars.Attr("aria-label"); label != fmt.Sprintf("%g/5", tt.rating) { + t.Fatalf("expected exact accessible rating, got %q", label) + } + if stars.ChildrenFiltered("i.bi-star-fill").Length() != tt.full || + stars.ChildrenFiltered("i.bi-star").Length() != tt.empty || + stars.Children().Length() != 5 { + t.Fatalf("unexpected full/empty star counts: %s", stars.Text()) + } + partial := stars.Find(".review-star-fill") + if tt.width == 0 { + if partial.Length() != 0 { + t.Fatal("whole ratings must not include a partial star") + } + } else if style, _ := partial.Attr("style"); style != fmt.Sprintf("width: %d%%", tt.width) || + stars.Find(`.review-star[aria-hidden="true"]`).Length() != 1 { + t.Fatalf("expected a %d%% decorative star fill, got %q", tt.width, style) + } + }) + } +} + +func TestDocumentReviewNotScoredTranslations(t *testing.T) { + provider := &fakeReviewProvider{ + score: reviews.Score{Source: "Another service"}, + page: reviews.Page{Source: "Another service", Available: true, Reviews: []reviews.Review{ + {Reviewer: "Reader", Text: "A review without a rating"}, + }}, + } + app, _ := newReviewProviderTestApp(t, provider) + for _, tt := range []struct { + lang string + text string + }{ + {"en", "Not scored"}, + {"de", "Nicht bewertet"}, + {"es", "Sin puntuar"}, + {"fr", "Non noté"}, + {"ru", "Не оценено"}, + } { + t.Run(tt.lang, func(t *testing.T) { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews?l="+tt.lang))) + if err != nil { + t.Fatal(err) + } + if got := page.Find("article header span.text-muted").Text(); got != tt.text { + t.Fatalf("expected %q, got %q", tt.text, got) + } + }) + } +} diff --git a/internal/webserver/embedded/views/partials/document-metadata-filters.html b/internal/webserver/embedded/views/partials/document-metadata-filters.html index 22f64d67..d9e9be8f 100644 --- a/internal/webserver/embedded/views/partials/document-metadata-filters.html +++ b/internal/webserver/embedded/views/partials/document-metadata-filters.html @@ -28,13 +28,12 @@ {{if .SearchFields}} {{$reviewScoreFrom = .SearchFields.DataSourceScoreFrom}} {{end}} - {{$reviewScoreTo := 0.0}} + {{$reviewScoreTo := 5.0}} {{if .SearchFields}} - {{$reviewScoreTo = .SearchFields.DataSourceScoreTo}} + {{$reviewScoreTo = .SearchFields.DataSourceScoreUpperBound}} {{end}} - {{$noScoreSelected := and .SearchFields .SearchFields.DataSourceScoreNone}} - {{template "partials/document-score-filter-control" dict "Lang" .Lang "IdPrefix" $idPrefix "Name" "review-score-from" "Label" "From" "Score" $reviewScoreFrom "NoScoreSelected" $noScoreSelected}} - {{template "partials/document-score-filter-control" dict "Lang" .Lang "IdPrefix" $idPrefix "Name" "review-score-to" "Label" "To" "Score" $reviewScoreTo "NoScoreSelected" $noScoreSelected "RoundedEnd" true}} + {{template "partials/document-score-filter-control" dict "Lang" .Lang "IdPrefix" $idPrefix "Name" "review-score-from" "Label" "From" "Score" $reviewScoreFrom "Default" 0.0}} + {{template "partials/document-score-filter-control" dict "Lang" .Lang "IdPrefix" $idPrefix "Name" "review-score-to" "Label" "To" "Score" $reviewScoreTo "Default" 5.0 "RoundedEnd" true}} diff --git a/internal/webserver/embedded/views/partials/document-score-filter-control.html b/internal/webserver/embedded/views/partials/document-score-filter-control.html index 6bd1a423..b9251974 100644 --- a/internal/webserver/embedded/views/partials/document-score-filter-control.html +++ b/internal/webserver/embedded/views/partials/document-score-filter-control.html @@ -4,13 +4,13 @@ {{end}}
        diff --git a/internal/webserver/highlights_test.go b/internal/webserver/highlights_test.go index b65c9778..2c876fda 100644 --- a/internal/webserver/highlights_test.go +++ b/internal/webserver/highlights_test.go @@ -8,6 +8,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v3" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/infrastructure" "github.com/svera/coreander/v5/internal/webserver/model" @@ -164,7 +165,12 @@ func TestHighlights(t *testing.T) { func TestHighlightsReviewScoreFilter(t *testing.T) { db := infrastructure.Connect(":memory:", 250) - app := bootstrapApp(db, &infrastructure.NoEmail{}, loadFilesInMemoryFs([]string{"testdata/library/metadata.epub"}), webserver.Config{}) + app, idx := bootstrapAppWithIndexer(db, &infrastructure.NoEmail{}, loadFilesInMemoryFs([]string{"testdata/library/metadata.epub"}), webserver.Config{}) + t.Cleanup(func() { _ = idx.Close() }) + document, err := idx.Document("john-doe-test-epub") + if err != nil { + t.Fatal(err) + } adminCookie, err := login(app, "admin@example.com", "admin", t) if err != nil { t.Fatalf("Unexpected error: %v", err) @@ -182,12 +188,29 @@ func TestHighlightsReviewScoreFilter(t *testing.T) { for _, test := range []struct { name string query string + score float64 expectedCount int }{ - {name: "no score", query: "review-score-from=none", expectedCount: 1}, + {name: "default range", expectedCount: 1}, + {name: "full range", query: "review-score-from=0&review-score-to=5", expectedCount: 1}, + {name: "zero lower bound", query: "review-score-from=0", expectedCount: 1}, + {name: "no score", query: "review-score-from=0&review-score-to=0", expectedCount: 1}, + {name: "legacy no score", query: "review-score-from=none", expectedCount: 1}, {name: "numeric score range", query: "review-score-from=1", expectedCount: 0}, + {name: "fractional score included", query: "review-score-to=1", score: 0.5, expectedCount: 1}, + {name: "lower boundary included", query: "review-score-from=1", score: 1, expectedCount: 1}, + {name: "upper boundary included", query: "review-score-from=1&review-score-to=5", score: 5, expectedCount: 1}, + {name: "rated excluded from unrated", query: "review-score-to=0", score: 1, expectedCount: 0}, + {name: "rated below range", query: "review-score-from=1", score: 0.5, expectedCount: 0}, + {name: "rated above range", query: "review-score-to=1", score: 5, expectedCount: 0}, } { t.Run(test.name, func(t *testing.T) { + if err := idx.UpdateDocument(document.ID, func(update *index.DocumentUpdate) (bool, error) { + update.DataSourceScoreValue = test.score + return true, nil + }); err != nil { + t.Fatal(err) + } response, err := getRequest(adminCookie, app, "/highlights?"+test.query, t) if err != nil { t.Fatalf("Unexpected error fetching highlights: %v", err) diff --git a/internal/webserver/model/highlight_repository.go b/internal/webserver/model/highlight_repository.go index 6e8eab5d..fb60655f 100644 --- a/internal/webserver/model/highlight_repository.go +++ b/internal/webserver/model/highlight_repository.go @@ -142,8 +142,7 @@ func hasSearchFields(sf index.SearchFields) bool { sf.PagesFrom != 0 || sf.PagesTo != 0 || sf.DataSourceScoreFrom != 0 || - sf.DataSourceScoreTo != 0 || - sf.DataSourceScoreNone || + sf.DataSourceScoreUpperBound() != 5 || sf.IllustratedOnly } @@ -194,14 +193,8 @@ func matchesSearchFields(doc AugmentedDocument, sf index.SearchFields, illustrat return false } - if sf.DataSourceScoreNone || sf.DataSourceScoreFrom > 0 || sf.DataSourceScoreTo > 0 { - score := doc.DataSourceScore() - if sf.DataSourceScoreNone && score != 0 { - return false - } - if !sf.DataSourceScoreNone && (score <= 0 || - (sf.DataSourceScoreFrom > 0 && score < sf.DataSourceScoreFrom) || - (sf.DataSourceScoreTo > 0 && score > sf.DataSourceScoreTo)) { + if max := sf.DataSourceScoreUpperBound(); sf.DataSourceScoreFrom != 0 || max != 5 { + if score := doc.DataSourceScore(); score < sf.DataSourceScoreFrom || score > max { return false } } diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index 9c790dd0..89774d9d 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -216,7 +216,7 @@ func TestReviewScoreFilter(t *testing.T) { value string }{ {name: "review-score-from", value: "4"}, - {name: "review-score-to", value: "5"}, + {name: "review-score-to", value: ""}, } { selects := doc.Find(`select[name="` + field.name + `"]`) if selects.Length() == 0 { @@ -231,7 +231,7 @@ func TestReviewScoreFilter(t *testing.T) { value, _ := option.Attr("value") wantLabel := "" switch value { - case "none": + case "0": if strings.TrimSpace(option.Text()) == "" { t.Errorf("Expected a label for %s no-score option", field.name) } @@ -244,7 +244,7 @@ func TestReviewScoreFilter(t *testing.T) { } return value }) - if strings.Join(options, ",") != ",none,1,2,3,4,5" { + if strings.Join(options, ",") != ",0,1,2,3,4,5" { t.Errorf("Expected blank, no-score, and scores 1 to 5 for %s, got %v", field.name, options) } }) @@ -254,7 +254,7 @@ func TestReviewScoreFilter(t *testing.T) { } - response, err = app.Test(mustGetRequest(t, "/search?type=documents&review-score-from=none")) + response, err = app.Test(mustGetRequest(t, "/search?type=documents&review-score-from=0&review-score-to=0")) if err != nil { t.Fatal(err) } @@ -265,8 +265,12 @@ func TestReviewScoreFilter(t *testing.T) { } for _, name := range []string{"review-score-from", "review-score-to"} { selected, _ := doc.Find(`select[name="` + name + `"] option[selected]`).Attr("value") - if selected != "none" { - t.Errorf("Expected no-score option selected for %s, got %q", name, selected) + want := "0" + if name == "review-score-from" { + want = "" + } + if selected != want { + t.Errorf("Expected %q selected for %s, got %q", want, name, selected) } } } @@ -343,6 +347,51 @@ func TestSearchResultsSortByDataSourceScore(t *testing.T) { } } +func TestSearchResultsFilterByInclusiveScoreRange(t *testing.T) { + documents := []index.Document{ + {ID: "unrated.epub", Slug: "unrated", DataSourceScoreValue: 0}, + {ID: "fractional.epub", Slug: "fractional", DataSourceScoreValue: 0.5}, + {ID: "one.epub", Slug: "one", DataSourceScoreValue: 1}, + {ID: "five.epub", Slug: "five", DataSourceScoreValue: 5}, + } + for i := range documents { + documents[i].Metadata = metadata.Metadata{Title: documents[i].Slug, Format: "EPUB"} + } + app, _ := newSearchResultsTestApp(t, documents) + for _, tt := range []struct { + name string + query string + want []string + }{ + {name: "default", want: []string{"five", "fractional", "one", "unrated"}}, + {name: "full range", query: "&review-score-from=0&review-score-to=5", want: []string{"five", "fractional", "one", "unrated"}}, + {name: "zero lower bound", query: "&review-score-from=0", want: []string{"five", "fractional", "one", "unrated"}}, + {name: "unrated", query: "&review-score-from=0&review-score-to=0", want: []string{"unrated"}}, + {name: "zero upper bound", query: "&review-score-to=0", want: []string{"unrated"}}, + {name: "positive range", query: "&review-score-from=1&review-score-to=5", want: []string{"five", "one"}}, + {name: "upper bound includes unrated", query: "&review-score-to=1", want: []string{"fractional", "one", "unrated"}}, + {name: "exact score", query: "&review-score-from=1&review-score-to=1", want: []string{"one"}}, + {name: "reversed bounds", query: "&review-score-from=5&review-score-to=1", want: []string{"five", "one"}}, + {name: "legacy unrated", query: "&review-score-from=none", want: []string{"unrated"}}, + } { + t.Run(tt.name, func(t *testing.T) { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents"+tt.query))) + if err != nil { + t.Fatal(err) + } + var got []string + page.Find("#list .list-group-item h2 a").Each(func(_ int, link *goquery.Selection) { + href, _ := link.Attr("href") + got = append(got, strings.TrimPrefix(href, "/documents/")) + }) + slices.Sort(got) + if !slices.Equal(got, tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} + func TestSearchResultsShowStoredZeroRatingAsNotScored(t *testing.T) { app, _ := newSearchResultsTestApp(t, []index.Document{withBiblioReadsData(t, index.Document{ ID: "unscored.epub", Slug: "unscored", From 8a132452c14016b1c87ade7909581bf1fc94c100 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 11:44:41 +0100 Subject: [PATCH 30/50] Refactor --- .../biblioreads/adapter.go | 53 +++-- .../biblioreads/adapter_test.go | 205 +++++++++--------- internal/datasource/biblioreads/client.go | 2 +- .../biblioreads/matching.go | 9 +- .../webserver/document_biblioreads_test.go | 7 +- internal/webserver/webserver_test.go | 5 +- main.go | 5 +- 7 files changed, 140 insertions(+), 146 deletions(-) rename internal/{enrichment => datasource}/biblioreads/adapter.go (70%) rename internal/{enrichment => datasource}/biblioreads/adapter_test.go (78%) rename internal/{enrichment => datasource}/biblioreads/matching.go (95%) diff --git a/internal/enrichment/biblioreads/adapter.go b/internal/datasource/biblioreads/adapter.go similarity index 70% rename from internal/enrichment/biblioreads/adapter.go rename to internal/datasource/biblioreads/adapter.go index 6c207df9..769a41ec 100644 --- a/internal/enrichment/biblioreads/adapter.go +++ b/internal/datasource/biblioreads/adapter.go @@ -1,4 +1,3 @@ -// Package biblioreads adapts BiblioReads matching and review storage to the generic index. package biblioreads import ( @@ -9,7 +8,6 @@ import ( "strings" "time" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" ) @@ -19,7 +17,7 @@ const ( ) type BiblioReadsSource interface { - SearchBooks(context.Context, string) (biblioreads.SearchBooksResponse, error) + SearchBooks(context.Context, string) (SearchBooksResponse, error) } type Enricher struct { @@ -28,7 +26,8 @@ type Enricher struct { slots chan struct{} } -func New(source BiblioReadsSource) *Enricher { +// NewEnricher adapts a BiblioReads source to the document-enrichment pipeline. +func NewEnricher(source BiblioReadsSource) *Enricher { return &Enricher{source: source, slots: make(chan struct{}, biblioReadsEnrichWorkers)} } @@ -37,10 +36,10 @@ func (p *Enricher) SetIndex(idx *index.BleveIndexer) { p.idx = idx } -func (*Enricher) Source() string { return biblioreads.Source } +func (*Enricher) Source() string { return Source } func (*Enricher) NeedsEnrichment(document index.Document) (bool, error) { - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + data, err := index.DataSourceData[DocumentData](document, Source) return !data.Enriched, err } @@ -50,10 +49,10 @@ func (p *Enricher) Enrich(ctx context.Context, idx *index.BleveIndexer, document } // BiblioReadsBook returns a stored match or performs an on-demand lookup. -func (p *Enricher) BiblioReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) { - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) +func (p *Enricher) BiblioReadsBook(ctx context.Context, document index.Document) (SearchResult, error) { + data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { - return biblioreads.SearchResult{}, err + return SearchResult{}, err } if data.URL != "" || data.Enriched || p.source == nil { return storedResult(document, data), nil @@ -61,8 +60,8 @@ func (p *Enricher) BiblioReadsBook(ctx context.Context, document index.Document) return p.enrich(ctx, p.idx, document) } -func storedResult(document index.Document, data biblioreads.DocumentData) biblioreads.SearchResult { - return biblioreads.SearchResult{ +func storedResult(document index.Document, data DocumentData) SearchResult { + return SearchResult{ BookURL: data.URL, Rating: fmt.Sprintf("%.2f", document.DataSourceScoreValue), RatingCount: data.RatingCount, } } @@ -73,18 +72,18 @@ func (p *Enricher) UpdateDataSourceLink(documentID, reviewLink string) error { return fmt.Errorf("BiblioReads adapter has no index") } return p.idx.UpdateDocument(documentID, func(current *index.DocumentUpdate) (bool, error) { - data, err := index.DataSourceData[biblioreads.DocumentData](current.Document, biblioreads.Source) + data, err := index.DataSourceData[DocumentData](current.Document, Source) if err != nil || data.URL == reviewLink { return false, err } current.DataSourceScoreValue = 0 - err = index.SetDataSourceData(¤t.Document, biblioreads.Source, biblioreads.DocumentData{URL: reviewLink, Enriched: true}) + err = index.SetDataSourceData(¤t.Document, Source, DocumentData{URL: reviewLink, Enriched: true}) return true, err }) } // StoreBiblioReadsDetails rejects responses for a match or metadata changed during the fetch. -func (p *Enricher) StoreBiblioReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error { +func (p *Enricher) StoreBiblioReadsDetails(document index.Document, bookURL string, book BookResponse) error { if p.idx == nil { return fmt.Errorf("BiblioReads adapter has no index") } @@ -96,7 +95,7 @@ func (p *Enricher) StoreBiblioReadsDetails(document index.Document, bookURL stri return nil } return p.idx.UpdateDocument(document.ID, func(current *index.DocumentUpdate) (bool, error) { - data, err := index.DataSourceData[biblioreads.DocumentData](current.Document, biblioreads.Source) + data, err := index.DataSourceData[DocumentData](current.Document, Source) if err != nil { return false, err } @@ -122,26 +121,26 @@ func (p *Enricher) StoreBiblioReadsDetails(document index.Document, bookURL stri if !changed { return false, nil } - return true, index.SetDataSourceData(¤t.Document, biblioreads.Source, data) + return true, index.SetDataSourceData(¤t.Document, Source, data) }) } -func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document index.Document) (biblioreads.SearchResult, error) { +func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document index.Document) (SearchResult, error) { if idx == nil || p.source == nil { - return biblioreads.SearchResult{}, fmt.Errorf("BiblioReads enrichment requires a source and index") + return SearchResult{}, fmt.Errorf("BiblioReads enrichment requires a source and index") } - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { - return biblioreads.SearchResult{}, err + return SearchResult{}, err } select { case p.slots <- struct{}{}: defer func() { <-p.slots }() case <-ctx.Done(): - return biblioreads.SearchResult{}, ctx.Err() + return SearchResult{}, ctx.Err() } q := biblioReadsQuery(document) - result := biblioreads.SearchResult{BookURL: data.URL} + result := SearchResult{BookURL: data.URL} if result.BookURL == "" && document.Title != "" { ctx, cancel := context.WithTimeout(ctx, biblioReadsRequestTimeout) defer cancel() @@ -152,8 +151,8 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document for _, query := range queries { response, err := p.source.SearchBooks(ctx, query) switch { - case errors.Is(err, biblioreads.ErrUnavailable), errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): - return biblioreads.SearchResult{}, fmt.Errorf("error searching %q on BiblioReads: %w", query, err) + case errors.Is(err, ErrUnavailable), errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): + return SearchResult{}, fmt.Errorf("error searching %q on BiblioReads: %w", query, err) case err != nil: log.Printf("Error searching %q on BiblioReads: %s", query, err) default: @@ -170,7 +169,7 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document if biblioReadsQuery(current.Document) != q { return false, nil } - data, err := index.DataSourceData[biblioreads.DocumentData](current.Document, biblioreads.Source) + data, err := index.DataSourceData[DocumentData](current.Document, Source) if err != nil { return false, err } @@ -189,11 +188,11 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document if result.AuthorURL != "" { if slug, name := biblioReadsAuthor(current.Document, result.Author); slug != "" { current.AuthorLinks = append(current.AuthorLinks, index.AuthorLink{ - Slug: slug, Name: name, Source: biblioreads.Source, URL: result.AuthorURL, + Slug: slug, Name: name, Source: Source, URL: result.AuthorURL, }) } } - return true, index.SetDataSourceData(¤t.Document, biblioreads.Source, data) + return true, index.SetDataSourceData(¤t.Document, Source, data) }) if errors.Is(err, index.ErrDocumentNotFound) { return result, nil diff --git a/internal/enrichment/biblioreads/adapter_test.go b/internal/datasource/biblioreads/adapter_test.go similarity index 78% rename from internal/enrichment/biblioreads/adapter_test.go rename to internal/datasource/biblioreads/adapter_test.go index 8c004b52..209bdb99 100644 --- a/internal/enrichment/biblioreads/adapter_test.go +++ b/internal/datasource/biblioreads/adapter_test.go @@ -14,29 +14,28 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/spf13/afero" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" ) -func mustBiblioReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { +func mustBiblioReadsData(t *testing.T, document index.Document) DocumentData { t.Helper() - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { t.Fatal(err) } return data } -func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { +func withBiblioReadsData(t *testing.T, document index.Document, data DocumentData) index.Document { t.Helper() - if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + if err := index.SetDataSourceData(&document, Source, data); err != nil { t.Fatal(err) } return document } -func updateBiblioReadsData(t *testing.T, document *index.Document, update func(*biblioreads.DocumentData)) { +func updateBiblioReadsData(t *testing.T, document *index.Document, update func(*DocumentData)) { t.Helper() data := mustBiblioReadsData(t, *document) update(&data) @@ -61,20 +60,20 @@ func assertNoDocumentAuthorURL(t *testing.T, documents bleve.Index) { type fakeBiblioReadsSource struct { mu sync.Mutex queries []string - results []biblioreads.SearchResult - byQuery map[string][]biblioreads.SearchResult + results []SearchResult + byQuery map[string][]SearchResult queryErrors map[string]error err error } -func (f *fakeBiblioReadsSource) SearchBooks(_ context.Context, query string) (biblioreads.SearchBooksResponse, error) { +func (f *fakeBiblioReadsSource) SearchBooks(_ context.Context, query string) (SearchBooksResponse, error) { f.mu.Lock() defer f.mu.Unlock() f.queries = append(f.queries, query) if f.byQuery != nil { - return biblioreads.SearchBooksResponse{Result: f.byQuery[query]}, f.queryErrors[query] + return SearchBooksResponse{Result: f.byQuery[query]}, f.queryErrors[query] } - return biblioreads.SearchBooksResponse{Result: f.results}, f.err + return SearchBooksResponse{Result: f.results}, f.err } func (f *fakeBiblioReadsSource) Queries() []string { @@ -93,7 +92,7 @@ func (biblioReadsTestReader) Cover(string, int) (image.Image, error) { return nil, nil } -var biblioReadsTestResults = []biblioreads.SearchResult{ +var biblioReadsTestResults = []SearchResult{ {Title: "La Reina Dragon", Author: "Alice Borchardt", BookURL: "/book/show/12207444-la-reina-dragon", AuthorURL: "/author/show/151155.Alice_Borchardt", Rating: "3.82", RatingCount: "2,285"}, {BookURL: "/book/show/2-other", AuthorURL: "/author/show/2.Other"}, } @@ -123,7 +122,7 @@ func newBiblioReadsTestIndexer(t *testing.T, source BiblioReadsSource) *biblioRe t.Fatal(err) } writes := &biblioReadsWriteCountingIndex{countedBiblioReadsIndex: documents} - adapter := New(source) + adapter := NewEnricher(source) var providers []index.DocumentEnricher if source != nil { providers = append(providers, adapter) @@ -190,8 +189,8 @@ func TestEnrichBiblioReadsMarksDocumentsWithoutMatch(t *testing.T) { for name, source := range map[string]*fakeBiblioReadsSource{ "no results": {}, "non-transient err": {err: errors.New("biblioreads: request failed with status code 504")}, - "wrong title": {results: []biblioreads.SearchResult{{Title: "Another book", Author: "Alice Borchardt", BookURL: "/book/show/1"}}}, - "wrong author": {results: []biblioreads.SearchResult{{Title: "La Reina Dragon", Author: "Another writer", BookURL: "/book/show/1"}}}, + "wrong title": {results: []SearchResult{{Title: "Another book", Author: "Alice Borchardt", BookURL: "/book/show/1"}}}, + "wrong author": {results: []SearchResult{{Title: "La Reina Dragon", Author: "Another writer", BookURL: "/book/show/1"}}}, } { t.Run(name, func(t *testing.T) { idx := newBiblioReadsTestIndexer(t, source) @@ -218,7 +217,7 @@ func TestEnrichBiblioReadsMarksDocumentsWithoutMatch(t *testing.T) { func TestEnrichBiblioReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) { const authorURL = "/author/show/2" - source := &fakeBiblioReadsSource{results: []biblioreads.SearchResult{ + source := &fakeBiblioReadsSource{results: []SearchResult{ {Title: "La Reina Dragon", Author: "Second Author", BookURL: wantDataSourceLink, AuthorURL: authorURL}, }} idx := newBiblioReadsTestIndexer(t, source) @@ -270,13 +269,13 @@ func TestEnrichBiblioReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing. } func TestEnrichBiblioReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { - source := &fakeBiblioReadsSource{err: biblioreads.ErrUnavailable} + source := &fakeBiblioReadsSource{err: ErrUnavailable} idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - if err := idx.EnrichDocuments(10); !errors.Is(err, biblioreads.ErrUnavailable) { + if err := idx.EnrichDocuments(10); !errors.Is(err, ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } doc, err := idx.DocumentByID(biblioReadsTestID) @@ -357,7 +356,7 @@ func TestEnrichBiblioReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if err != nil { t.Fatal(err) } - updateBiblioReadsData(t, &doc, func(data *biblioreads.DocumentData) { + updateBiblioReadsData(t, &doc, func(data *DocumentData) { data.URL = "/book/show/1-existing" }) if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { @@ -447,13 +446,13 @@ func TestParseBiblioReadsRating(t *testing.T) { } func TestBiblioReadsBookReturnsErrorWhenUnavailable(t *testing.T) { - source := &fakeBiblioReadsSource{err: biblioreads.ErrUnavailable} + source := &fakeBiblioReadsSource{err: ErrUnavailable} idx := newBiblioReadsTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } doc, _ := idx.DocumentByID(biblioReadsTestID) - if _, err := idx.BiblioReadsBook(context.Background(), doc); !errors.Is(err, biblioreads.ErrUnavailable) { + if _, err := idx.BiblioReadsBook(context.Background(), doc); !errors.Is(err, ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } if doc, _ = idx.DocumentByID(biblioReadsTestID); mustBiblioReadsData(t, doc).Enriched { @@ -491,23 +490,23 @@ func TestBiblioReadsTitleMatchScore(t *testing.T) { func TestPreferBiblioReadsBook(t *testing.T) { for _, tt := range []struct { name string - candidate, best biblioreads.SearchResult + candidate, best SearchResult score, bestScore int want bool }{ {name: "first match", score: 1, want: true}, - {name: "lower ID before rating for equal strength", candidate: biblioreads.SearchResult{BookURL: "/book/show/1"}, best: biblioreads.SearchResult{BookURL: "/book/show/2", Rating: "4"}, score: 3, bestScore: 3, want: true}, - {name: "higher ID loses despite rating", candidate: biblioreads.SearchResult{BookURL: "/book/show/2", Rating: "4"}, best: biblioreads.SearchResult{BookURL: "/book/show/1"}, score: 3, bestScore: 3}, - {name: "more ratings before ID", candidate: biblioreads.SearchResult{BookURL: "/book/show/2", RatingCount: "100"}, best: biblioreads.SearchResult{BookURL: "/book/show/1", RatingCount: "3"}, score: 3, bestScore: 3, want: true}, - {name: "fewer ratings loses", candidate: biblioreads.SearchResult{RatingCount: "3"}, best: biblioreads.SearchResult{RatingCount: "100"}, score: 3, bestScore: 3}, - {name: "result rank is not book ID", candidate: biblioreads.SearchResult{ID: 2, BookURL: "/book/show/1"}, best: biblioreads.SearchResult{ID: 1, BookURL: "/book/show/2"}, score: 3, bestScore: 3, want: true}, - {name: "rating before strength for different strength", candidate: biblioreads.SearchResult{ID: 2, Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 1, bestScore: 3, want: true}, - {name: "stronger unrated loses to rated", best: biblioreads.SearchResult{Rating: "4"}, score: 3, bestScore: 1}, + {name: "lower ID before rating for equal strength", candidate: SearchResult{BookURL: "/book/show/1"}, best: SearchResult{BookURL: "/book/show/2", Rating: "4"}, score: 3, bestScore: 3, want: true}, + {name: "higher ID loses despite rating", candidate: SearchResult{BookURL: "/book/show/2", Rating: "4"}, best: SearchResult{BookURL: "/book/show/1"}, score: 3, bestScore: 3}, + {name: "more ratings before ID", candidate: SearchResult{BookURL: "/book/show/2", RatingCount: "100"}, best: SearchResult{BookURL: "/book/show/1", RatingCount: "3"}, score: 3, bestScore: 3, want: true}, + {name: "fewer ratings loses", candidate: SearchResult{RatingCount: "3"}, best: SearchResult{RatingCount: "100"}, score: 3, bestScore: 3}, + {name: "result rank is not book ID", candidate: SearchResult{ID: 2, BookURL: "/book/show/1"}, best: SearchResult{ID: 1, BookURL: "/book/show/2"}, score: 3, bestScore: 3, want: true}, + {name: "rating before strength for different strength", candidate: SearchResult{ID: 2, Rating: "4"}, best: SearchResult{ID: 1}, score: 1, bestScore: 3, want: true}, + {name: "stronger unrated loses to rated", best: SearchResult{Rating: "4"}, score: 3, bestScore: 1}, {name: "strength with same rating status", score: 3, bestScore: 1, want: true}, {name: "weaker match loses with same rating status", score: 1, bestScore: 3}, - {name: "unknown ID falls back to rating", candidate: biblioreads.SearchResult{Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 3, bestScore: 3, want: true}, - {name: "same ID falls back to rating", candidate: biblioreads.SearchResult{ID: 1, Rating: "4"}, best: biblioreads.SearchResult{ID: 1}, score: 3, bestScore: 3, want: true}, - {name: "complete tie retains first match", candidate: biblioreads.SearchResult{ID: 1, Rating: "4"}, best: biblioreads.SearchResult{ID: 1, Rating: "3"}, score: 3, bestScore: 3}, + {name: "unknown ID falls back to rating", candidate: SearchResult{Rating: "4"}, best: SearchResult{ID: 1}, score: 3, bestScore: 3, want: true}, + {name: "same ID falls back to rating", candidate: SearchResult{ID: 1, Rating: "4"}, best: SearchResult{ID: 1}, score: 3, bestScore: 3, want: true}, + {name: "complete tie retains first match", candidate: SearchResult{ID: 1, Rating: "4"}, best: SearchResult{ID: 1, Rating: "3"}, score: 3, bestScore: 3}, } { t.Run(tt.name, func(t *testing.T) { if got := preferBiblioReadsBook(tt.candidate, tt.best, tt.score, tt.bestScore); got != tt.want { @@ -568,7 +567,7 @@ func TestBiblioReadsBookMatchesKennedyMiddleName(t *testing.T) { const title = "Auge y caída de las grandes potencias" const bookURL = "/book/show/27214" const authorURL = "/author/show/8390124.Paul_M_Kennedy" - source := &fakeBiblioReadsSource{results: []biblioreads.SearchResult{{ + source := &fakeBiblioReadsSource{results: []SearchResult{{ Title: title, Author: "Paul M. Kennedy", BookURL: bookURL, AuthorURL: authorURL, Rating: "4.14", }}} idx := newBiblioReadsTestIndexer(t, source) @@ -601,11 +600,11 @@ func TestBiblioReadsBookMatchesKennedyMiddleName(t *testing.T) { } func TestMatchingBiblioReadsBook(t *testing.T) { - correct := biblioreads.SearchResult{ + correct := SearchResult{ Title: "El señor de los anillos", Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810", AuthorURL: "/author/show/656983.J_R_R_Tolkien", } - parody := biblioreads.SearchResult{ + parody := SearchResult{ Title: "El sopor de los anillos", Author: "The Harvard Lampoon", BookURL: "/book/show/7297403-el-sopor-de-los-anillos", } @@ -613,45 +612,45 @@ func TestMatchingBiblioReadsBook(t *testing.T) { name string title string authors []string - results []biblioreads.SearchResult + results []SearchResult want string }{ - {name: "correct result after parody", results: []biblioreads.SearchResult{parody, correct}, want: correct.BookURL}, - {name: "parody rejected", results: []biblioreads.SearchResult{parody}}, - {name: "similar title with correct author rejected", results: []biblioreads.SearchResult{{Title: parody.Title, Author: correct.Author, BookURL: parody.BookURL}}}, - {name: "wrong author rejected", results: []biblioreads.SearchResult{{Title: correct.Title, Author: parody.Author, BookURL: parody.BookURL}}}, - {name: "normalization", title: " EL SEN\u0303OR DE LOS ANILLOS! ", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, - {name: "series suffix", results: []biblioreads.SearchResult{{Title: correct.Title + " (El señor de los anillos, #1)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, - {name: "document series suffix", title: correct.Title + " (Middle Earth, #1)", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, - {name: "omnibus range", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1-#3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, - {name: "omnibus range with dash", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1–3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, - {name: "document omnibus range", title: correct.Title + " (#1-#3)", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, - {name: "standalone volume retained", results: []biblioreads.SearchResult{{Title: correct.Title + " (#1)", Author: correct.Author, BookURL: correct.BookURL}}}, - {name: "series suffix ranks equally", results: []biblioreads.SearchResult{{Title: correct.Title + " (Middle Earth, #1)", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: "/book/show/2"}, - {name: "lower Goodreads ID preferred over rating for equal titles", results: []biblioreads.SearchResult{ + {name: "correct result after parody", results: []SearchResult{parody, correct}, want: correct.BookURL}, + {name: "parody rejected", results: []SearchResult{parody}}, + {name: "similar title with correct author rejected", results: []SearchResult{{Title: parody.Title, Author: correct.Author, BookURL: parody.BookURL}}}, + {name: "wrong author rejected", results: []SearchResult{{Title: correct.Title, Author: parody.Author, BookURL: parody.BookURL}}}, + {name: "normalization", title: " EL SEN\u0303OR DE LOS ANILLOS! ", results: []SearchResult{correct}, want: correct.BookURL}, + {name: "series suffix", results: []SearchResult{{Title: correct.Title + " (El señor de los anillos, #1)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "document series suffix", title: correct.Title + " (Middle Earth, #1)", results: []SearchResult{correct}, want: correct.BookURL}, + {name: "omnibus range", results: []SearchResult{{Title: correct.Title + " (#1-#3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "omnibus range with dash", results: []SearchResult{{Title: correct.Title + " (#1–3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "document omnibus range", title: correct.Title + " (#1-#3)", results: []SearchResult{correct}, want: correct.BookURL}, + {name: "standalone volume retained", results: []SearchResult{{Title: correct.Title + " (#1)", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "series suffix ranks equally", results: []SearchResult{{Title: correct.Title + " (Middle Earth, #1)", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: "/book/show/2"}, + {name: "lower Goodreads ID preferred over rating for equal titles", results: []SearchResult{ {ID: 1, Title: correct.Title, Author: correct.Author, BookURL: "/book/show/1-unrated", Rating: "0.00"}, {ID: 2, Title: correct.Title, Author: correct.Author, BookURL: "/book/show/2-rated", Rating: "4.05"}, }, want: "/book/show/1-unrated"}, - {name: "unrated exact match retained when no rated match exists", results: []biblioreads.SearchResult{ + {name: "unrated exact match retained when no rated match exists", results: []SearchResult{ {Title: correct.Title, Author: correct.Author, BookURL: "/book/show/1-unrated", Rating: "0.00"}, {Title: correct.Title, Author: correct.Author, BookURL: "/book/show/2"}, }, want: "/book/show/1-unrated"}, - {name: "colon subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, - {name: "period subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + ". Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, - {name: "dash subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + " — Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, - {name: "document subtitle", title: correct.Title + ": Una historia", results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, - {name: "different subtitles rejected", title: correct.Title + ": Primera historia", results: []biblioreads.SearchResult{{Title: correct.Title + ": Otra historia", Author: correct.Author, BookURL: correct.BookURL}}}, - {name: "subtitle wrong author rejected", results: []biblioreads.SearchResult{{Title: correct.Title + ": Una historia", Author: parody.Author, BookURL: correct.BookURL}}}, - {name: "exact preferred over subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: correct.BookURL}, - {name: "series preferred over subtitle", results: []biblioreads.SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: "/book/show/2"}, {Title: correct.Title + " (#1-#3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, - {name: "arbitrary prefix rejected", results: []biblioreads.SearchResult{{Title: correct.Title + " y otras historias", Author: correct.Author, BookURL: correct.BookURL}}}, - {name: "volume retained", results: []biblioreads.SearchResult{{Title: correct.Title + " 2", Author: correct.Author, BookURL: correct.BookURL}}}, - {name: "trailing parenthetical accepted as weaker match", results: []biblioreads.SearchResult{{Title: correct.Title + " (An edition)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, - {name: "missing result author", results: []biblioreads.SearchResult{{Title: correct.Title, BookURL: correct.BookURL}}}, - {name: "missing result title", results: []biblioreads.SearchResult{{Author: correct.Author, BookURL: correct.BookURL}}}, - {name: "missing URL", results: []biblioreads.SearchResult{{Title: correct.Title, Author: correct.Author}}}, - {name: "any document author", authors: []string{"Another Writer", "J. R. R. Tolkien"}, results: []biblioreads.SearchResult{correct}, want: correct.BookURL}, - {name: "no author", authors: []string{}, results: []biblioreads.SearchResult{correct}}, + {name: "colon subtitle", results: []SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "period subtitle", results: []SearchResult{{Title: correct.Title + ". Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "dash subtitle", results: []SearchResult{{Title: correct.Title + " — Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "document subtitle", title: correct.Title + ": Una historia", results: []SearchResult{correct}, want: correct.BookURL}, + {name: "different subtitles rejected", title: correct.Title + ": Primera historia", results: []SearchResult{{Title: correct.Title + ": Otra historia", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "subtitle wrong author rejected", results: []SearchResult{{Title: correct.Title + ": Una historia", Author: parody.Author, BookURL: correct.BookURL}}}, + {name: "exact preferred over subtitle", results: []SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: correct.BookURL}, + {name: "series preferred over subtitle", results: []SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: "/book/show/2"}, {Title: correct.Title + " (#1-#3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "arbitrary prefix rejected", results: []SearchResult{{Title: correct.Title + " y otras historias", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "volume retained", results: []SearchResult{{Title: correct.Title + " 2", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "trailing parenthetical accepted as weaker match", results: []SearchResult{{Title: correct.Title + " (An edition)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "missing result author", results: []SearchResult{{Title: correct.Title, BookURL: correct.BookURL}}}, + {name: "missing result title", results: []SearchResult{{Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "missing URL", results: []SearchResult{{Title: correct.Title, Author: correct.Author}}}, + {name: "any document author", authors: []string{"Another Writer", "J. R. R. Tolkien"}, results: []SearchResult{correct}, want: correct.BookURL}, + {name: "no author", authors: []string{}, results: []SearchResult{correct}}, {name: "no results"}, } for _, tt := range tests { @@ -675,22 +674,22 @@ func TestMatchingBiblioReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) document := index.Document{Metadata: metadata.Metadata{ Title: "Los asesinatos de Manhattan", Authors: []string{"Douglas Preston", "Lincoln Child"}, }} - popular := biblioreads.SearchResult{ + popular := SearchResult{ ID: 1, Title: "Los asesinatos de Manhattan (Pendergast, #3)", Author: "Douglas Preston", BookURL: "/book/show/67005.Los_asesinatos_de_Manhattan", Rating: "4.26", RatingCount: "57,469", } - obscure := biblioreads.SearchResult{ + obscure := SearchResult{ ID: 3, Title: document.Title, Author: "Douglas Preston", BookURL: "/book/show/152308668-los-asesinatos-de-manhattan", Rating: "3.67", RatingCount: "3", } - for _, results := range [][]biblioreads.SearchResult{{popular, obscure}, {obscure, popular}} { + for _, results := range [][]SearchResult{{popular, obscure}, {obscure, popular}} { match, ok := matchingBiblioReadsBook(document, results) if !ok || match.BookURL != popular.BookURL { t.Fatalf("expected popular series entry, got %+v, %v", match, ok) } } popular.Author = "Another author" - match, ok := matchingBiblioReadsBook(document, []biblioreads.SearchResult{popular, obscure}) + match, ok := matchingBiblioReadsBook(document, []SearchResult{popular, obscure}) if !ok || match.BookURL != obscure.BookURL { t.Fatalf("rating count must not bypass author matching, got %+v, %v", match, ok) } @@ -718,16 +717,16 @@ func TestMatchingBiblioReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { document := index.Document{Metadata: metadata.Metadata{ Title: "El cercano Oriente", Authors: []string{"Isaac Asimov"}, }} - older := biblioreads.SearchResult{ + older := SearchResult{ ID: 2, Title: "El Cercano Oriente", Author: "Isaac Asimov", BookURL: olderBookURL, Rating: "4.10", } - newer := biblioreads.SearchResult{ + newer := SearchResult{ ID: 1, Title: "El Cercano Oriente", Author: "Isaac Asimov", BookURL: newerBookURL, Rating: "4.26", } - for _, results := range [][]biblioreads.SearchResult{{newer, older}, {older, newer}} { + for _, results := range [][]SearchResult{{newer, older}, {older, newer}} { match, ok := matchingBiblioReadsBook(document, results) if !ok || match.BookURL != olderBookURL { t.Fatalf("expected lower Goodreads ID %q, got %+v, %v", olderBookURL, match, ok) @@ -737,22 +736,22 @@ func TestMatchingBiblioReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { func TestMatchingBiblioReadsBookPrefersRatedParentheticalTitle(t *testing.T) { doc := index.Document{Metadata: metadata.Metadata{Title: "La promesa", Authors: []string{"Pearl S. Buck"}}} - rated := biblioreads.SearchResult{ + rated := SearchResult{ Title: "La promesa (The Promise)", Author: "Pearl S. Buck", BookURL: "/book/show/7377104-la-promesa", Rating: "4.04", } - unrated := biblioreads.SearchResult{ + unrated := SearchResult{ Title: "La promesa", Author: "Pearl S. Buck", BookURL: "/book/show/210898619-la-promesa", Rating: "0.00", } - for _, results := range [][]biblioreads.SearchResult{{rated, unrated}, {unrated, rated}} { + for _, results := range [][]SearchResult{{rated, unrated}, {unrated, rated}} { match, ok := matchingBiblioReadsBook(doc, results) if !ok || match.BookURL != rated.BookURL || match.Rating != rated.Rating { t.Fatalf("expected rated translated-title entry, got %+v, %v", match, ok) } } rated.Author = "Another author" - match, ok := matchingBiblioReadsBook(doc, []biblioreads.SearchResult{rated, unrated}) + match, ok := matchingBiblioReadsBook(doc, []SearchResult{rated, unrated}) if !ok || match.BookURL != unrated.BookURL { t.Fatalf("rated entry by another author must not win, got %+v, %v", match, ok) } @@ -762,7 +761,7 @@ func TestBiblioReadsBookMatchesDolnickSubtitle(t *testing.T) { const title = "La escritura de los dioses" const author = "Edward Dolnick" const bookURL = "/book/show/208277727-la-escritura-de-los-dioses-descifrando-la-piedra-de-rosetta" - source := &fakeBiblioReadsSource{results: []biblioreads.SearchResult{{ + source := &fakeBiblioReadsSource{results: []SearchResult{{ Title: title + ". Descifrando la piedra de Rosetta", Author: author, BookURL: bookURL, Rating: "4.19", }}} @@ -789,15 +788,15 @@ func TestBiblioReadsBookMatchesDolnickSubtitle(t *testing.T) { func TestBiblioReadsTitleFallback(t *testing.T) { const title = "El señor de los anillos" - correct := biblioreads.SearchResult{Title: title, Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810"} - parody := biblioreads.SearchResult{Title: "El sopor de los anillos", Author: "The Harvard Lampoon", BookURL: "/book/show/7297403"} + correct := SearchResult{Title: title, Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810"} + parody := SearchResult{Title: "El sopor de los anillos", Author: "The Harvard Lampoon", BookURL: "/book/show/7297403"} for _, fallbackUnavailable := range []bool{false, true} { name := "matched" if fallbackUnavailable { name = "unavailable" } t.Run(name, func(t *testing.T) { - source := &fakeBiblioReadsSource{byQuery: map[string][]biblioreads.SearchResult{ + source := &fakeBiblioReadsSource{byQuery: map[string][]SearchResult{ title + " J. R. R. Tolkien": {parody}, title: {parody, correct}, }} @@ -807,11 +806,11 @@ func TestBiblioReadsTitleFallback(t *testing.T) { t.Fatal(err) } if fallbackUnavailable { - source.queryErrors = map[string]error{title: biblioreads.ErrUnavailable} + source.queryErrors = map[string]error{title: ErrUnavailable} } got, err := idx.BiblioReadsBook(context.Background(), doc) if fallbackUnavailable { - if !errors.Is(err, biblioreads.ErrUnavailable) || got.BookURL != "" { + if !errors.Is(err, ErrUnavailable) || got.BookURL != "" { t.Fatalf("expected unavailable error, got %+v, %v", got, err) } stored, readErr := idx.DocumentByID(doc.ID) @@ -839,14 +838,14 @@ func TestBiblioReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { const title = "El señor de los anillos" const wantURL = "/book/show/37910810-el-se-or-de-los-anillos" // Titles and ordering from BiblioReads' actual title-only search response. - results := []biblioreads.SearchResult{ + results := []SearchResult{ {Title: "La comunidad del anillo (El señor de los anillos, #1)", Author: "J.R.R. Tolkien", BookURL: "/book/show/222947.La_comunidad_del_anillo"}, {Title: "Las dos torres (El señor de los anillos, #2)", Author: "J.R.R. Tolkien", BookURL: "/book/show/15280.Las_dos_torres"}, {Title: "El retorno del rey (El señor de los anillos, #3)", Author: "J.R.R. Tolkien", BookURL: "/book/show/63358.El_retorno_del_rey"}, {Title: "El señor de los anillos (#1-#3)", Author: "J.R.R. Tolkien", BookURL: wantURL, AuthorURL: "/author/show/656983.J_R_R_Tolkien"}, {Title: "Apéndices (El Señor de los Anillos, #4)", Author: "J.R.R. Tolkien", BookURL: "/book/show/54694.Ap_ndices"}, } - source := &fakeBiblioReadsSource{byQuery: map[string][]biblioreads.SearchResult{title: results}} + source := &fakeBiblioReadsSource{byQuery: map[string][]SearchResult{title: results}} idx := newBiblioReadsTestIndexer(t, source) doc := index.Document{ID: biblioReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { @@ -882,7 +881,7 @@ func TestForceIndexingRefreshesBiblioReadsMatch(t *testing.T) { if err != nil { t.Fatal(err) } - updateBiblioReadsData(t, &doc, func(data *biblioreads.DocumentData) { + updateBiblioReadsData(t, &doc, func(data *DocumentData) { data.URL = "/book/show/1-wrong" data.ResourceID = "wrong-work" data.LegacyBookID = 1 @@ -918,20 +917,20 @@ func TestStoreBiblioReadsDetails(t *testing.T) { Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, TextRankWords: []string{"dragon"}, - }, biblioreads.DocumentData{URL: wantDataSourceLink, Enriched: true}) + }, DocumentData{URL: wantDataSourceLink, Enriched: true}) stale := doc resourceID := "work-id" switch scenario { case "newly linked": - updateBiblioReadsData(t, &stale, func(data *biblioreads.DocumentData) { + updateBiblioReadsData(t, &stale, func(data *DocumentData) { data.URL = "" data.Enriched = false }) case "changed metadata": doc.Title = "Another title" case "changed match": - updateBiblioReadsData(t, &doc, func(data *biblioreads.DocumentData) { + updateBiblioReadsData(t, &doc, func(data *DocumentData) { data.URL = "/book/show/2-other" }) case "partial identifiers": @@ -943,7 +942,7 @@ func TestStoreBiblioReadsDetails(t *testing.T) { } } - book := biblioreads.BookResponse{Rating: "4.12", ResourceID: resourceID, LegacyBookID: 12207444} + book := BookResponse{Rating: "4.12", ResourceID: resourceID, LegacyBookID: 12207444} writes := idx.documentsIdx writes.count.Store(0) err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, book) @@ -1011,26 +1010,26 @@ func (i *biblioReadsWriteCountingIndex) Index(id string, data any) error { func TestStoreBiblioReadsDetailsPreservesMissingFields(t *testing.T) { for _, tt := range []struct { name string - book biblioreads.BookResponse + book BookResponse wantRating string wantCount string wantWork string wantID int }{ - {name: "rating only", book: biblioreads.BookResponse{Rating: "4.12"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, - {name: "identifiers only", book: biblioreads.BookResponse{ResourceID: "new-work", LegacyBookID: 2}, wantRating: "3.82", wantWork: "new-work", wantID: 2}, - {name: "missing work", book: biblioreads.BookResponse{Rating: "4.12", LegacyBookID: 2}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, - {name: "missing edition", book: biblioreads.BookResponse{Rating: "4.12", ResourceID: "new-work"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "rating only", book: BookResponse{Rating: "4.12"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "identifiers only", book: BookResponse{ResourceID: "new-work", LegacyBookID: 2}, wantRating: "3.82", wantWork: "new-work", wantID: 2}, + {name: "missing work", book: BookResponse{Rating: "4.12", LegacyBookID: 2}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "missing edition", book: BookResponse{Rating: "4.12", ResourceID: "new-work"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, {name: "empty response", wantRating: "3.82", wantWork: "old-work", wantID: 1}, - {name: "count only", book: biblioreads.BookResponse{RatingCount: "650"}, wantRating: "3.82", wantCount: "650", wantWork: "old-work", wantID: 1}, - {name: "zero count", book: biblioreads.BookResponse{RatingCount: "0"}, wantRating: "3.82", wantCount: "0", wantWork: "old-work", wantID: 1}, + {name: "count only", book: BookResponse{RatingCount: "650"}, wantRating: "3.82", wantCount: "650", wantWork: "old-work", wantID: 1}, + {name: "zero count", book: BookResponse{RatingCount: "0"}, wantRating: "3.82", wantCount: "0", wantWork: "old-work", wantID: 1}, } { t.Run(tt.name, func(t *testing.T) { idx := newBiblioReadsTestIndexer(t, nil) doc := withBiblioReadsData(t, index.Document{ ID: biblioReadsTestID, DataSourceScoreValue: 3.82, - }, biblioreads.DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 1}) + }, DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 1}) if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) @@ -1064,7 +1063,7 @@ func TestUpdateDataSourceLink(t *testing.T) { TextRankWords: []string{"dragon"}, DataSourceScoreValue: 3.82, - }, biblioreads.DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 12207444, Enriched: true}) + }, DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 12207444, Enriched: true}) if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) @@ -1084,7 +1083,7 @@ func TestUpdateDataSourceLink(t *testing.T) { if stored.Title != doc.Title || stored.Slug != doc.Slug || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("review-link update changed unrelated document data: %+v", stored) } - if err := idx.StoreBiblioReadsDetails(doc, mustBiblioReadsData(t, doc).URL, biblioreads.BookResponse{Rating: "4.5"}); err == nil { + if err := idx.StoreBiblioReadsDetails(doc, mustBiblioReadsData(t, doc).URL, BookResponse{Rating: "4.5"}); err == nil { t.Fatal("stale score write must not overwrite the updated link") } } @@ -1119,7 +1118,7 @@ func TestBiblioReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { if err := idx.Enricher.Enrich(context.Background(), idx.BleveIndexer, stale); err != nil { t.Fatal(err) } - if err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, biblioreads.BookResponse{ + if err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, BookResponse{ Rating: "4.5", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, }); err != nil { t.Fatal(err) @@ -1142,14 +1141,14 @@ func TestBiblioReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { if err != nil || !reflect.DeepEqual(decoded, other) { t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) } - if data := mustBiblioReadsData(t, got); data != (biblioreads.DocumentData{URL: "/book/show/456", Enriched: true}) { + if data := mustBiblioReadsData(t, got); data != (DocumentData{URL: "/book/show/456", Enriched: true}) { t.Fatalf("stale write restored old Goodreads state: %+v", data) } } func TestBiblioReadsBookRejectsInvalidProviderData(t *testing.T) { idx := newBiblioReadsTestIndexer(t, nil) - document := index.Document{DataSources: map[string]json.RawMessage{biblioreads.Source: json.RawMessage(`{"url":42}`)}} + document := index.Document{DataSources: map[string]json.RawMessage{Source: json.RawMessage(`{"url":42}`)}} if _, err := idx.BiblioReadsBook(context.Background(), document); err == nil { t.Fatal("Goodreads accepted an incorrectly typed URL") } diff --git a/internal/datasource/biblioreads/client.go b/internal/datasource/biblioreads/client.go index bf6b8b1e..2dfa458f 100644 --- a/internal/datasource/biblioreads/client.go +++ b/internal/datasource/biblioreads/client.go @@ -1,4 +1,4 @@ -// Package biblioreads implements a client for the BiblioReads API +// Package biblioreads implements a client and index enrichment adapter for the BiblioReads API // (https://github.com/nesaku/biblioreads), an alternative Goodreads front-end. package biblioreads diff --git a/internal/enrichment/biblioreads/matching.go b/internal/datasource/biblioreads/matching.go similarity index 95% rename from internal/enrichment/biblioreads/matching.go rename to internal/datasource/biblioreads/matching.go index 2686e9db..67a526e6 100644 --- a/internal/enrichment/biblioreads/matching.go +++ b/internal/datasource/biblioreads/matching.go @@ -8,7 +8,6 @@ import ( "strings" "unicode" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/index" "golang.org/x/text/unicode/norm" ) @@ -112,7 +111,7 @@ func biblioReadsTitleMatchScore(candidate, title, mainTitle string) int { return 0 } -func preferBiblioReadsBook(candidate, best biblioreads.SearchResult, score, bestScore int) bool { +func preferBiblioReadsBook(candidate, best SearchResult, score, bestScore int) bool { if bestScore == 0 { return true } @@ -158,13 +157,13 @@ func goodreadsBookID(bookURL string) uint64 { return id } -func matchingBiblioReadsBook(document index.Document, results []biblioreads.SearchResult) (biblioreads.SearchResult, bool) { +func matchingBiblioReadsBook(document index.Document, results []SearchResult) (SearchResult, bool) { title := biblioReadsTitle(document.Title) if title == "" { - return biblioreads.SearchResult{}, false + return SearchResult{}, false } mainTitle := biblioReadsMainTitle(document.Title) - var best biblioreads.SearchResult + var best SearchResult bestScore := 0 for _, result := range results { if result.BookURL == "" { diff --git a/internal/webserver/document_biblioreads_test.go b/internal/webserver/document_biblioreads_test.go index 6e76d120..5ce9f2b8 100644 --- a/internal/webserver/document_biblioreads_test.go +++ b/internal/webserver/document_biblioreads_test.go @@ -16,7 +16,6 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" - biblioreadsenrichment "github.com/svera/coreander/v5/internal/enrichment/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -109,11 +108,11 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide } appFS := afero.NewMemMapFs() idxCfg := index.Config{} - var adapter *biblioreadsenrichment.Enricher + var adapter *biblioreads.Enricher if source != nil { - adapter = biblioreadsenrichment.New(source) + adapter = biblioreads.NewEnricher(source) } else { - adapter = biblioreadsenrichment.New(nil) + adapter = biblioreads.NewEnricher(nil) } if source != nil { idxCfg.DocumentEnrichers = []index.DocumentEnricher{adapter} diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index b571c008..a8f6524d 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -18,7 +18,6 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" - biblioreadsenrichment "github.com/svera/coreander/v5/internal/enrichment/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -97,7 +96,7 @@ func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.F if err != nil { log.Fatal(err) } - adapter := biblioreadsenrichment.New(nil) + adapter := biblioreads.NewEnricher(nil) adapter.SetIndex(idx) controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, dataSource, reviews.NewBiblioReads(nil, adapter)) usersRepository := &model.UserRepository{DB: db} @@ -142,7 +141,7 @@ func newSearchResultsTestApp(t *testing.T, documents []index.Document) (*fiber.A // setReviewScore seeds a document's review score through the current provider-backed index storage. func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Document, score, count string) { t.Helper() - adapter := biblioreadsenrichment.New(nil) + adapter := biblioreads.NewEnricher(nil) adapter.SetIndex(idx) bookURL := mustBiblioReadsData(t, doc).URL if bookURL == "" { diff --git a/main.go b/main.go index 566af6ed..ba528a27 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,6 @@ import ( "github.com/spf13/afero" "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" - biblioreadsenrichment "github.com/svera/coreander/v5/internal/enrichment/biblioreads" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/reviews" @@ -104,13 +103,13 @@ func main() { var needsReindex bool documentsIndex, authorsIndex, needsReindex = getIndexes(appFs, input.IllustratedMinSize, input.MinOccurrenceRatio, maxTextRankWords) var documentEnrichers []index.DocumentEnricher - biblioReadsAdapter := biblioreadsenrichment.New(nil) + biblioReadsAdapter := biblioreads.NewEnricher(nil) if !input.DisableBiblioreads { biblioreadsClient = biblioreads.NewClient( biblioreads.WithInstances(input.BiblioreadsInstances), biblioreads.WithCacheTTL(time.Duration(input.BiblioreadsCacheTTL)*time.Second), ) - biblioReadsAdapter = biblioreadsenrichment.New(biblioreadsClient) + biblioReadsAdapter = biblioreads.NewEnricher(biblioreadsClient) documentEnrichers = append(documentEnrichers, biblioReadsAdapter) } idx = index.NewBleve(documentsIndex, authorsIndex, appFs, input.LibPath, metadataReaders, index.Config{ From 93b7ab2ae93583c6310858c63ffab3488d1d7143 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 12:07:44 +0100 Subject: [PATCH 31/50] Refactor --- internal/datasource/biblioreads/adapter.go | 40 +- .../datasource/biblioreads/adapter_test.go | 356 +++++++++--------- internal/datasource/biblioreads/matching.go | 70 ++-- internal/reviews/biblioreads.go | 8 +- internal/reviews/biblioreads_test.go | 4 +- internal/webserver/webserver_test.go | 2 +- 6 files changed, 240 insertions(+), 240 deletions(-) diff --git a/internal/datasource/biblioreads/adapter.go b/internal/datasource/biblioreads/adapter.go index 769a41ec..af26703d 100644 --- a/internal/datasource/biblioreads/adapter.go +++ b/internal/datasource/biblioreads/adapter.go @@ -12,23 +12,23 @@ import ( ) const ( - biblioReadsEnrichWorkers = 4 - biblioReadsRequestTimeout = 2 * time.Minute + enrichWorkers = 4 + requestTimeout = 2 * time.Minute ) -type BiblioReadsSource interface { +type SearchSource interface { SearchBooks(context.Context, string) (SearchBooksResponse, error) } type Enricher struct { - source BiblioReadsSource + source SearchSource idx *index.BleveIndexer slots chan struct{} } -// NewEnricher adapts a BiblioReads source to the document-enrichment pipeline. -func NewEnricher(source BiblioReadsSource) *Enricher { - return &Enricher{source: source, slots: make(chan struct{}, biblioReadsEnrichWorkers)} +// NewEnricher adapts a source to the document-enrichment pipeline. +func NewEnricher(source SearchSource) *Enricher { + return &Enricher{source: source, slots: make(chan struct{}, enrichWorkers)} } // SetIndex binds the adapter before starting background or on-demand enrichment. @@ -48,8 +48,8 @@ func (p *Enricher) Enrich(ctx context.Context, idx *index.BleveIndexer, document return err } -// BiblioReadsBook returns a stored match or performs an on-demand lookup. -func (p *Enricher) BiblioReadsBook(ctx context.Context, document index.Document) (SearchResult, error) { +// Book returns a stored match or performs an on-demand lookup. +func (p *Enricher) Book(ctx context.Context, document index.Document) (SearchResult, error) { data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { return SearchResult{}, err @@ -66,7 +66,7 @@ func storedResult(document index.Document, data DocumentData) SearchResult { } } -// UpdateDataSourceLink clears cached BiblioReads data when an administrator corrects a match. +// UpdateDataSourceLink clears cached data when an administrator corrects a match. func (p *Enricher) UpdateDataSourceLink(documentID, reviewLink string) error { if p.idx == nil { return fmt.Errorf("BiblioReads adapter has no index") @@ -82,8 +82,8 @@ func (p *Enricher) UpdateDataSourceLink(documentID, reviewLink string) error { }) } -// StoreBiblioReadsDetails rejects responses for a match or metadata changed during the fetch. -func (p *Enricher) StoreBiblioReadsDetails(document index.Document, bookURL string, book BookResponse) error { +// StoreDetails rejects responses for a match or metadata changed during the fetch. +func (p *Enricher) StoreDetails(document index.Document, bookURL string, book BookResponse) error { if p.idx == nil { return fmt.Errorf("BiblioReads adapter has no index") } @@ -99,7 +99,7 @@ func (p *Enricher) StoreBiblioReadsDetails(document index.Document, bookURL stri if err != nil { return false, err } - if biblioReadsQuery(current.Document) != biblioReadsQuery(document) || data.URL != bookURL { + if searchQuery(current.Document) != searchQuery(document) || data.URL != bookURL { return false, fmt.Errorf("BiblioReads match changed for document %s", document.ID) } changed := false @@ -108,7 +108,7 @@ func (p *Enricher) StoreBiblioReadsDetails(document index.Document, bookURL stri changed = true } if book.Rating != "" { - value := parseBiblioReadsRating(book.Rating, document.ID) + value := parseRating(book.Rating, document.ID) if current.DataSourceScoreValue != value { current.DataSourceScoreValue = value changed = true @@ -139,10 +139,10 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document case <-ctx.Done(): return SearchResult{}, ctx.Err() } - q := biblioReadsQuery(document) + q := searchQuery(document) result := SearchResult{BookURL: data.URL} if result.BookURL == "" && document.Title != "" { - ctx, cancel := context.WithTimeout(ctx, biblioReadsRequestTimeout) + ctx, cancel := context.WithTimeout(ctx, requestTimeout) defer cancel() queries := []string{q} if title := strings.TrimSpace(document.Title); title != q { @@ -156,7 +156,7 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document case err != nil: log.Printf("Error searching %q on BiblioReads: %s", query, err) default: - if match, ok := matchingBiblioReadsBook(document, response.Result); ok { + if match, ok := matchingBook(document, response.Result); ok { result = match } } @@ -166,7 +166,7 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document } } err = idx.UpdateDocument(document.ID, func(current *index.DocumentUpdate) (bool, error) { - if biblioReadsQuery(current.Document) != q { + if searchQuery(current.Document) != q { return false, nil } data, err := index.DataSourceData[DocumentData](current.Document, Source) @@ -184,9 +184,9 @@ func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document if result.Rating == "" && current.DataSourceScoreValue > 0 { result.Rating = fmt.Sprintf("%.2f", current.DataSourceScoreValue) } - current.DataSourceScoreValue = parseBiblioReadsRating(result.Rating, current.ID) + current.DataSourceScoreValue = parseRating(result.Rating, current.ID) if result.AuthorURL != "" { - if slug, name := biblioReadsAuthor(current.Document, result.Author); slug != "" { + if slug, name := matchedAuthor(current.Document, result.Author); slug != "" { current.AuthorLinks = append(current.AuthorLinks, index.AuthorLink{ Slug: slug, Name: name, Source: Source, URL: result.AuthorURL, }) diff --git a/internal/datasource/biblioreads/adapter_test.go b/internal/datasource/biblioreads/adapter_test.go index 209bdb99..17ac217a 100644 --- a/internal/datasource/biblioreads/adapter_test.go +++ b/internal/datasource/biblioreads/adapter_test.go @@ -18,7 +18,7 @@ import ( "github.com/svera/coreander/v5/internal/metadata" ) -func mustBiblioReadsData(t *testing.T, document index.Document) DocumentData { +func mustData(t *testing.T, document index.Document) DocumentData { t.Helper() data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { @@ -27,7 +27,7 @@ func mustBiblioReadsData(t *testing.T, document index.Document) DocumentData { return data } -func withBiblioReadsData(t *testing.T, document index.Document, data DocumentData) index.Document { +func withData(t *testing.T, document index.Document, data DocumentData) index.Document { t.Helper() if err := index.SetDataSourceData(&document, Source, data); err != nil { t.Fatal(err) @@ -35,11 +35,11 @@ func withBiblioReadsData(t *testing.T, document index.Document, data DocumentDat return document } -func updateBiblioReadsData(t *testing.T, document *index.Document, update func(*DocumentData)) { +func updateData(t *testing.T, document *index.Document, update func(*DocumentData)) { t.Helper() - data := mustBiblioReadsData(t, *document) + data := mustData(t, *document) update(&data) - *document = withBiblioReadsData(t, *document, data) + *document = withData(t, *document, data) } func assertNoDocumentAuthorURL(t *testing.T, documents bleve.Index) { @@ -57,7 +57,7 @@ func assertNoDocumentAuthorURL(t *testing.T, documents bleve.Index) { } } -type fakeBiblioReadsSource struct { +type fakeSource struct { mu sync.Mutex queries []string results []SearchResult @@ -66,7 +66,7 @@ type fakeBiblioReadsSource struct { err error } -func (f *fakeBiblioReadsSource) SearchBooks(_ context.Context, query string) (SearchBooksResponse, error) { +func (f *fakeSource) SearchBooks(_ context.Context, query string) (SearchBooksResponse, error) { f.mu.Lock() defer f.mu.Unlock() f.queries = append(f.queries, query) @@ -76,42 +76,42 @@ func (f *fakeBiblioReadsSource) SearchBooks(_ context.Context, query string) (Se return SearchBooksResponse{Result: f.results}, f.err } -func (f *fakeBiblioReadsSource) Queries() []string { +func (f *fakeSource) Queries() []string { f.mu.Lock() defer f.mu.Unlock() return slices.Clone(f.queries) } -type biblioReadsTestReader struct{} +type testReader struct{} -func (biblioReadsTestReader) Metadata(string) (metadata.Metadata, error) { +func (testReader) Metadata(string) (metadata.Metadata, error) { return metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt", "Second Author"}, Format: "PDF"}, nil } -func (biblioReadsTestReader) Cover(string, int) (image.Image, error) { +func (testReader) Cover(string, int) (image.Image, error) { return nil, nil } -var biblioReadsTestResults = []SearchResult{ +var testResults = []SearchResult{ {Title: "La Reina Dragon", Author: "Alice Borchardt", BookURL: "/book/show/12207444-la-reina-dragon", AuthorURL: "/author/show/151155.Alice_Borchardt", Rating: "3.82", RatingCount: "2,285"}, {BookURL: "/book/show/2-other", AuthorURL: "/author/show/2.Other"}, } const ( - wantDataSourceLink = "/book/show/12207444-la-reina-dragon" - wantBiblioReadsAuthorURL = "/author/show/151155.Alice_Borchardt" - biblioReadsTestID = "book.pdf" + wantDataSourceLink = "/book/show/12207444-la-reina-dragon" + wantAuthorURL = "/author/show/151155.Alice_Borchardt" + testID = "book.pdf" ) -type biblioReadsTestIndexer struct { +type testIndexer struct { *index.BleveIndexer *Enricher - documentsIdx *biblioReadsWriteCountingIndex + documentsIdx *writeCountingIndex authorsIdx bleve.Index fs afero.Fs } -func newBiblioReadsTestIndexer(t *testing.T, source BiblioReadsSource) *biblioReadsTestIndexer { +func newTestIndexer(t *testing.T, source SearchSource) *testIndexer { t.Helper() documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) if err != nil { @@ -121,7 +121,7 @@ func newBiblioReadsTestIndexer(t *testing.T, source BiblioReadsSource) *biblioRe if err != nil { t.Fatal(err) } - writes := &biblioReadsWriteCountingIndex{countedBiblioReadsIndex: documents} + writes := &writeCountingIndex{countedIndex: documents} adapter := NewEnricher(source) var providers []index.DocumentEnricher if source != nil { @@ -131,15 +131,15 @@ func newBiblioReadsTestIndexer(t *testing.T, source BiblioReadsSource) *biblioRe if err := afero.WriteFile(appFS, "lib/book.pdf", []byte(""), 0644); err != nil { t.Fatal(err) } - idx := index.NewBleve(writes, authors, appFS, "lib", map[string]metadata.Reader{".pdf": biblioReadsTestReader{}}, index.Config{DocumentEnrichers: providers}) + idx := index.NewBleve(writes, authors, appFS, "lib", map[string]metadata.Reader{".pdf": testReader{}}, index.Config{DocumentEnrichers: providers}) adapter.SetIndex(idx) t.Cleanup(func() { _ = idx.Close() }) - return &biblioReadsTestIndexer{BleveIndexer: idx, Enricher: adapter, documentsIdx: writes, authorsIdx: authors, fs: appFS} + return &testIndexer{BleveIndexer: idx, Enricher: adapter, documentsIdx: writes, authorsIdx: authors, fs: appFS} } -func TestEnrichBiblioReadsStoresMatchingResult(t *testing.T) { - source := &fakeBiblioReadsSource{results: biblioReadsTestResults} - idx := newBiblioReadsTestIndexer(t, source) +func TestEnrichStoresMatchingResult(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -148,25 +148,25 @@ func TestEnrichBiblioReadsStoresMatchingResult(t *testing.T) { t.Fatal(err) } - doc, err := idx.DocumentByID(biblioReadsTestID) + doc, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - if mustBiblioReadsData(t, doc).URL != wantDataSourceLink { - t.Errorf("unexpected book URL %q", mustBiblioReadsData(t, doc).URL) + if mustData(t, doc).URL != wantDataSourceLink { + t.Errorf("unexpected book URL %q", mustData(t, doc).URL) } if doc.DataSourceScoreValue != 3.82 { t.Errorf("unexpected Goodreads rating %v", doc.DataSourceScoreValue) } - if mustBiblioReadsData(t, doc).RatingCount != "2,285" { - t.Errorf("unexpected Goodreads rating count %q", mustBiblioReadsData(t, doc).RatingCount) + if mustData(t, doc).RatingCount != "2,285" { + t.Errorf("unexpected Goodreads rating count %q", mustData(t, doc).RatingCount) } assertNoDocumentAuthorURL(t, idx.documentsIdx) - if !mustBiblioReadsData(t, doc).Enriched { + if !mustData(t, doc).Enriched { t.Error("expected document to be marked as BiblioReads-enriched") } author, err := idx.Author("alice-borchardt", "") - if err != nil || author.DataSourceLinks["goodreads"] != wantBiblioReadsAuthorURL { + if err != nil || author.DataSourceLinks["goodreads"] != wantAuthorURL { t.Fatalf("expected matching author URL stored, got %+v, %v", author, err) } other, err := idx.Author("second-author", "") @@ -185,15 +185,15 @@ func TestEnrichBiblioReadsStoresMatchingResult(t *testing.T) { } } -func TestEnrichBiblioReadsMarksDocumentsWithoutMatch(t *testing.T) { - for name, source := range map[string]*fakeBiblioReadsSource{ +func TestEnrichMarksDocumentsWithoutMatch(t *testing.T) { + for name, source := range map[string]*fakeSource{ "no results": {}, "non-transient err": {err: errors.New("biblioreads: request failed with status code 504")}, "wrong title": {results: []SearchResult{{Title: "Another book", Author: "Alice Borchardt", BookURL: "/book/show/1"}}}, "wrong author": {results: []SearchResult{{Title: "La Reina Dragon", Author: "Another writer", BookURL: "/book/show/1"}}}, } { t.Run(name, func(t *testing.T) { - idx := newBiblioReadsTestIndexer(t, source) + idx := newTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -201,11 +201,11 @@ func TestEnrichBiblioReadsMarksDocumentsWithoutMatch(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(biblioReadsTestID) + doc, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - if !mustBiblioReadsData(t, doc).Enriched || mustBiblioReadsData(t, doc).URL != "" { + if !mustData(t, doc).Enriched || mustData(t, doc).URL != "" { t.Errorf("expected document enriched with no URLs, got %+v", doc) } if want := []string{"La Reina Dragon Alice Borchardt", "La Reina Dragon"}; !slices.Equal(source.Queries(), want) { @@ -215,12 +215,12 @@ func TestEnrichBiblioReadsMarksDocumentsWithoutMatch(t *testing.T) { } } -func TestEnrichBiblioReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) { +func TestEnrichStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) { const authorURL = "/author/show/2" - source := &fakeBiblioReadsSource{results: []SearchResult{ + source := &fakeSource{results: []SearchResult{ {Title: "La Reina Dragon", Author: "Second Author", BookURL: wantDataSourceLink, AuthorURL: authorURL}, }} - idx := newBiblioReadsTestIndexer(t, source) + idx := newTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -268,9 +268,9 @@ func TestEnrichBiblioReadsStoresSecondAuthorURLAndPreservesItOnEdits(t *testing. } } -func TestEnrichBiblioReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { - source := &fakeBiblioReadsSource{err: ErrUnavailable} - idx := newBiblioReadsTestIndexer(t, source) +func TestEnrichLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { + source := &fakeSource{err: ErrUnavailable} + idx := newTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -278,29 +278,29 @@ func TestEnrichBiblioReadsLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { if err := idx.EnrichDocuments(10); !errors.Is(err, ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } - doc, err := idx.DocumentByID(biblioReadsTestID) + doc, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - if mustBiblioReadsData(t, doc).Enriched { + if mustData(t, doc).Enriched { t.Error("expected document not to be marked as BiblioReads-enriched") } source.mu.Lock() source.err = nil - source.results = biblioReadsTestResults + source.results = testResults source.mu.Unlock() if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - if doc, _ = idx.DocumentByID(biblioReadsTestID); mustBiblioReadsData(t, doc).URL != wantDataSourceLink { - t.Errorf("expected document to be enriched on retry, got %q", mustBiblioReadsData(t, doc).URL) + if doc, _ = idx.DocumentByID(testID); mustData(t, doc).URL != wantDataSourceLink { + t.Errorf("expected document to be enriched on retry, got %q", mustData(t, doc).URL) } } -func TestNewFileSchedulesBiblioReadsEnrichment(t *testing.T) { - source := &fakeBiblioReadsSource{results: biblioReadsTestResults} - idx := newBiblioReadsTestIndexer(t, source) +func TestNewFileSchedulesEnrichment(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) if _, err := idx.NewFile("new.pdf", []byte("")); err != nil { t.Fatal(err) @@ -312,9 +312,9 @@ func TestNewFileSchedulesBiblioReadsEnrichment(t *testing.T) { if err != nil { t.Fatal(err) } - if mustBiblioReadsData(t, doc).Enriched { - if mustBiblioReadsData(t, doc).URL != wantDataSourceLink { - t.Errorf("unexpected book URL %q", mustBiblioReadsData(t, doc).URL) + if mustData(t, doc).Enriched { + if mustData(t, doc).URL != wantDataSourceLink { + t.Errorf("unexpected book URL %q", mustData(t, doc).URL) } assertNoDocumentAuthorURL(t, idx.documentsIdx) return @@ -326,8 +326,8 @@ func TestNewFileSchedulesBiblioReadsEnrichment(t *testing.T) { } } -func TestEnrichBiblioReadsWithoutSourceIsNoop(t *testing.T) { - idx := newBiblioReadsTestIndexer(t, nil) +func TestEnrichWithoutSourceIsNoop(t *testing.T) { + idx := newTestIndexer(t, nil) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } @@ -337,26 +337,26 @@ func TestEnrichBiblioReadsWithoutSourceIsNoop(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(biblioReadsTestID) + doc, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - if mustBiblioReadsData(t, doc).Enriched { + if mustData(t, doc).Enriched { t.Error("expected document not to be BiblioReads-enriched when the source is disabled") } } -func TestEnrichBiblioReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { - source := &fakeBiblioReadsSource{results: biblioReadsTestResults} - idx := newBiblioReadsTestIndexer(t, source) +func TestEnrichSkipsSearchWhenDocumentHasLink(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(biblioReadsTestID) + doc, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - updateBiblioReadsData(t, &doc, func(data *DocumentData) { + updateData(t, &doc, func(data *DocumentData) { data.URL = "/book/show/1-existing" }) if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { @@ -370,42 +370,42 @@ func TestEnrichBiblioReadsSkipsSearchWhenDocumentHasLink(t *testing.T) { if len(source.Queries()) != 0 { t.Errorf("expected no search for a document with a link, got %q", source.Queries()) } - doc, err = idx.DocumentByID(biblioReadsTestID) + doc, err = idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - if !mustBiblioReadsData(t, doc).Enriched || mustBiblioReadsData(t, doc).URL != "/book/show/1-existing" { + if !mustData(t, doc).Enriched || mustData(t, doc).URL != "/book/show/1-existing" { t.Errorf("expected existing link kept and document marked enriched, got %+v", doc) } } -func TestBiblioReadsBook(t *testing.T) { - source := &fakeBiblioReadsSource{results: biblioReadsTestResults} - idx := newBiblioReadsTestIndexer(t, source) +func TestBook(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(biblioReadsTestID) + doc, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - got, err := idx.BiblioReadsBook(context.Background(), doc) + got, err := idx.Book(context.Background(), doc) if err != nil { t.Fatal(err) } if got.BookURL != wantDataSourceLink || got.Rating != "3.82" { t.Errorf("expected fresh book URL and search rating, got %+v", got) } - stored, err := idx.DocumentByID(biblioReadsTestID) + stored, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - if !mustBiblioReadsData(t, stored).Enriched || mustBiblioReadsData(t, stored).URL != wantDataSourceLink { + if !mustData(t, stored).Enriched || mustData(t, stored).URL != wantDataSourceLink { t.Errorf("expected the link to be persisted, got %+v", stored) } - if got, _ := idx.BiblioReadsBook(context.Background(), stored); got.BookURL != wantDataSourceLink || got.Rating != "3.82" { + if got, _ := idx.Book(context.Background(), stored); got.BookURL != wantDataSourceLink || got.Rating != "3.82" { t.Errorf("expected stored URL and rating, got %+v", got) } req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{stored.ID})) @@ -427,7 +427,7 @@ func TestBiblioReadsBook(t *testing.T) { } } -func TestParseBiblioReadsRating(t *testing.T) { +func TestParseRating(t *testing.T) { for _, tt := range []struct { rating string want float64 @@ -438,29 +438,29 @@ func TestParseBiblioReadsRating(t *testing.T) { {rating: "unavailable", want: 0}, } { t.Run(tt.rating, func(t *testing.T) { - if got := parseBiblioReadsRating(tt.rating, biblioReadsTestID); got != tt.want { + if got := parseRating(tt.rating, testID); got != tt.want { t.Errorf("parseBiblioReadsRating(%q) = %v, want %v", tt.rating, got, tt.want) } }) } } -func TestBiblioReadsBookReturnsErrorWhenUnavailable(t *testing.T) { - source := &fakeBiblioReadsSource{err: ErrUnavailable} - idx := newBiblioReadsTestIndexer(t, source) +func TestBookReturnsErrorWhenUnavailable(t *testing.T) { + source := &fakeSource{err: ErrUnavailable} + idx := newTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, _ := idx.DocumentByID(biblioReadsTestID) - if _, err := idx.BiblioReadsBook(context.Background(), doc); !errors.Is(err, ErrUnavailable) { + doc, _ := idx.DocumentByID(testID) + if _, err := idx.Book(context.Background(), doc); !errors.Is(err, ErrUnavailable) { t.Fatalf("expected ErrUnavailable, got %v", err) } - if doc, _ = idx.DocumentByID(biblioReadsTestID); mustBiblioReadsData(t, doc).Enriched { + if doc, _ = idx.DocumentByID(testID); mustData(t, doc).Enriched { t.Error("expected document to stay unenriched for a later retry") } } -func TestBiblioReadsTitleMatchScore(t *testing.T) { +func TestTitleMatchScore(t *testing.T) { for _, tt := range []struct { name string title string @@ -479,7 +479,7 @@ func TestBiblioReadsTitleMatchScore(t *testing.T) { {name: "empty candidate", title: "La promesa"}, } { t.Run(tt.name, func(t *testing.T) { - got := biblioReadsTitleMatchScore(tt.candidate, biblioReadsTitle(tt.title), biblioReadsMainTitle(tt.title)) + got := titleMatchScore(tt.candidate, normalizedTitle(tt.title), mainTitle(tt.title)) if got != tt.want { t.Fatalf("title match score = %d, want %d", got, tt.want) } @@ -487,7 +487,7 @@ func TestBiblioReadsTitleMatchScore(t *testing.T) { } } -func TestPreferBiblioReadsBook(t *testing.T) { +func TestPreferBook(t *testing.T) { for _, tt := range []struct { name string candidate, best SearchResult @@ -509,14 +509,14 @@ func TestPreferBiblioReadsBook(t *testing.T) { {name: "complete tie retains first match", candidate: SearchResult{ID: 1, Rating: "4"}, best: SearchResult{ID: 1, Rating: "3"}, score: 3, bestScore: 3}, } { t.Run(tt.name, func(t *testing.T) { - if got := preferBiblioReadsBook(tt.candidate, tt.best, tt.score, tt.bestScore); got != tt.want { + if got := preferBook(tt.candidate, tt.best, tt.score, tt.bestScore); got != tt.want { t.Fatalf("candidate preference = %v, want %v", got, tt.want) } }) } } -func TestBiblioReadsAuthorNameMatches(t *testing.T) { +func TestAuthorNameMatches(t *testing.T) { tests := []struct { name, candidate string want bool @@ -539,51 +539,51 @@ func TestBiblioReadsAuthorNameMatches(t *testing.T) { } for _, tt := range tests { t.Run(tt.name+"/"+tt.candidate, func(t *testing.T) { - if got := biblioReadsAuthorNameMatches(tt.name, tt.candidate); got != tt.want { + if got := authorNameMatches(tt.name, tt.candidate); got != tt.want { t.Errorf("match = %v, want %v", got, tt.want) } - if got := biblioReadsAuthorNameMatches(tt.candidate, tt.name); got != tt.want { + if got := authorNameMatches(tt.candidate, tt.name); got != tt.want { t.Errorf("reverse match = %v, want %v", got, tt.want) } }) } } -func TestBiblioReadsAuthorPrefersExactAndRejectsAmbiguousPartialMatches(t *testing.T) { +func TestAuthorPrefersExactAndRejectsAmbiguousPartialMatches(t *testing.T) { doc := index.Document{ Metadata: metadata.Metadata{Authors: []string{"Paul Kennedy", "Paul M. Kennedy"}}, AuthorsSlugs: []string{"paul-kennedy", "paul-m-kennedy"}, } - if slug, name := biblioReadsAuthor(doc, "Paul M. Kennedy"); slug != "paul-m-kennedy" || name != "Paul M. Kennedy" { + if slug, name := matchedAuthor(doc, "Paul M. Kennedy"); slug != "paul-m-kennedy" || name != "Paul M. Kennedy" { t.Fatalf("expected exact author, got %q, %q", slug, name) } doc.Authors = []string{"Paul M. Kennedy", "Paul A. Kennedy"} - if slug, name := biblioReadsAuthor(doc, "Paul Kennedy"); slug != "" || name != "" { + if slug, name := matchedAuthor(doc, "Paul Kennedy"); slug != "" || name != "" { t.Fatalf("ambiguous partial match must not select an author, got %q, %q", slug, name) } } -func TestBiblioReadsBookMatchesKennedyMiddleName(t *testing.T) { +func TestBookMatchesKennedyMiddleName(t *testing.T) { const title = "Auge y caída de las grandes potencias" const bookURL = "/book/show/27214" const authorURL = "/author/show/8390124.Paul_M_Kennedy" - source := &fakeBiblioReadsSource{results: []SearchResult{{ + source := &fakeSource{results: []SearchResult{{ Title: title, Author: "Paul M. Kennedy", BookURL: bookURL, AuthorURL: authorURL, Rating: "4.14", }}} - idx := newBiblioReadsTestIndexer(t, source) + idx := newTestIndexer(t, source) doc := index.Document{ - ID: biblioReadsTestID, AuthorsSlugs: []string{"paul-kennedy", "another-writer"}, + ID: testID, AuthorsSlugs: []string{"paul-kennedy", "another-writer"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"Paul Kennedy", "Another Writer"}}, } if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - got, err := idx.BiblioReadsBook(context.Background(), doc) + got, err := idx.Book(context.Background(), doc) if err != nil || got.BookURL != bookURL || got.Rating != "4.14" { t.Fatalf("expected Kennedy's book and rating, got %+v, %v", got, err) } stored, err := idx.DocumentByID(doc.ID) - if err != nil || !mustBiblioReadsData(t, stored).Enriched || mustBiblioReadsData(t, stored).URL != bookURL || stored.DataSourceScoreValue != 4.14 { + if err != nil || !mustData(t, stored).Enriched || mustData(t, stored).URL != bookURL || stored.DataSourceScoreValue != 4.14 { t.Fatalf("expected persisted book match and rating, got %+v, %v", stored, err) } author, err := idx.Author("paul-kennedy", "") @@ -599,7 +599,7 @@ func TestBiblioReadsBookMatchesKennedyMiddleName(t *testing.T) { } } -func TestMatchingBiblioReadsBook(t *testing.T) { +func TestMatchingBook(t *testing.T) { correct := SearchResult{ Title: "El señor de los anillos", Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810", AuthorURL: "/author/show/656983.J_R_R_Tolkien", @@ -662,7 +662,7 @@ func TestMatchingBiblioReadsBook(t *testing.T) { if authors == nil { authors = []string{"J. R. R. Tolkien"} } - match, ok := matchingBiblioReadsBook(index.Document{Metadata: metadata.Metadata{Title: title, Authors: authors}}, tt.results) + match, ok := matchingBook(index.Document{Metadata: metadata.Metadata{Title: title, Authors: authors}}, tt.results) if match.BookURL != tt.want || ok != (tt.want != "") { t.Fatalf("got %q, %v; want %q", match.BookURL, ok, tt.want) } @@ -670,7 +670,7 @@ func TestMatchingBiblioReadsBook(t *testing.T) { } } -func TestMatchingBiblioReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { +func TestMatchingBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { document := index.Document{Metadata: metadata.Metadata{ Title: "Los asesinatos de Manhattan", Authors: []string{"Douglas Preston", "Lincoln Child"}, }} @@ -683,13 +683,13 @@ func TestMatchingBiblioReadsBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) BookURL: "/book/show/152308668-los-asesinatos-de-manhattan", Rating: "3.67", RatingCount: "3", } for _, results := range [][]SearchResult{{popular, obscure}, {obscure, popular}} { - match, ok := matchingBiblioReadsBook(document, results) + match, ok := matchingBook(document, results) if !ok || match.BookURL != popular.BookURL { t.Fatalf("expected popular series entry, got %+v, %v", match, ok) } } popular.Author = "Another author" - match, ok := matchingBiblioReadsBook(document, []SearchResult{popular, obscure}) + match, ok := matchingBook(document, []SearchResult{popular, obscure}) if !ok || match.BookURL != obscure.BookURL { t.Fatalf("rating count must not bypass author matching, got %+v, %v", match, ok) } @@ -704,14 +704,14 @@ func TestDataSourceScoreCount(t *testing.T) { {"unknown", 0}, {"-1", 0}, {"3.5", 0}, {"18446744073709551616", 0}, } { t.Run(tt.value, func(t *testing.T) { - if got := biblioReadsRatingCount(tt.value); got != tt.want { + if got := ratingCount(tt.value); got != tt.want { t.Fatalf("rating count = %d, want %d", got, tt.want) } }) } } -func TestMatchingBiblioReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { +func TestMatchingBookPrefersLowerIDForEqualTitle(t *testing.T) { const olderBookURL = "/book/show/1368314.El_Cercano_Oriente" const newerBookURL = "/book/show/243175949-el-cercano-oriente" document := index.Document{Metadata: metadata.Metadata{ @@ -727,14 +727,14 @@ func TestMatchingBiblioReadsBookPrefersLowerIDForEqualTitle(t *testing.T) { } for _, results := range [][]SearchResult{{newer, older}, {older, newer}} { - match, ok := matchingBiblioReadsBook(document, results) + match, ok := matchingBook(document, results) if !ok || match.BookURL != olderBookURL { t.Fatalf("expected lower Goodreads ID %q, got %+v, %v", olderBookURL, match, ok) } } } -func TestMatchingBiblioReadsBookPrefersRatedParentheticalTitle(t *testing.T) { +func TestMatchingBookPrefersRatedParentheticalTitle(t *testing.T) { doc := index.Document{Metadata: metadata.Metadata{Title: "La promesa", Authors: []string{"Pearl S. Buck"}}} rated := SearchResult{ Title: "La promesa (The Promise)", Author: "Pearl S. Buck", @@ -745,40 +745,40 @@ func TestMatchingBiblioReadsBookPrefersRatedParentheticalTitle(t *testing.T) { BookURL: "/book/show/210898619-la-promesa", Rating: "0.00", } for _, results := range [][]SearchResult{{rated, unrated}, {unrated, rated}} { - match, ok := matchingBiblioReadsBook(doc, results) + match, ok := matchingBook(doc, results) if !ok || match.BookURL != rated.BookURL || match.Rating != rated.Rating { t.Fatalf("expected rated translated-title entry, got %+v, %v", match, ok) } } rated.Author = "Another author" - match, ok := matchingBiblioReadsBook(doc, []SearchResult{rated, unrated}) + match, ok := matchingBook(doc, []SearchResult{rated, unrated}) if !ok || match.BookURL != unrated.BookURL { t.Fatalf("rated entry by another author must not win, got %+v, %v", match, ok) } } -func TestBiblioReadsBookMatchesDolnickSubtitle(t *testing.T) { +func TestBookMatchesDolnickSubtitle(t *testing.T) { const title = "La escritura de los dioses" const author = "Edward Dolnick" const bookURL = "/book/show/208277727-la-escritura-de-los-dioses-descifrando-la-piedra-de-rosetta" - source := &fakeBiblioReadsSource{results: []SearchResult{{ + source := &fakeSource{results: []SearchResult{{ Title: title + ". Descifrando la piedra de Rosetta", Author: author, BookURL: bookURL, Rating: "4.19", }}} - idx := newBiblioReadsTestIndexer(t, source) + idx := newTestIndexer(t, source) doc := index.Document{ - ID: biblioReadsTestID, AuthorsSlugs: []string{"edward-dolnick"}, + ID: testID, AuthorsSlugs: []string{"edward-dolnick"}, Metadata: metadata.Metadata{Title: title, Authors: []string{author}}, } if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - got, err := idx.BiblioReadsBook(context.Background(), doc) + got, err := idx.Book(context.Background(), doc) if err != nil || got.BookURL != bookURL || got.Rating != "4.19" { t.Fatalf("expected Dolnick's book and score, got %+v, %v", got, err) } stored, err := idx.DocumentByID(doc.ID) - if err != nil || !mustBiblioReadsData(t, stored).Enriched || mustBiblioReadsData(t, stored).URL != bookURL { + if err != nil || !mustData(t, stored).Enriched || mustData(t, stored).URL != bookURL { t.Fatalf("expected subtitle match persisted, got %+v, %v", stored, err) } if want := []string{title + " " + author}; !slices.Equal(source.Queries(), want) { @@ -786,7 +786,7 @@ func TestBiblioReadsBookMatchesDolnickSubtitle(t *testing.T) { } } -func TestBiblioReadsTitleFallback(t *testing.T) { +func TestTitleFallback(t *testing.T) { const title = "El señor de los anillos" correct := SearchResult{Title: title, Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810"} parody := SearchResult{Title: "El sopor de los anillos", Author: "The Harvard Lampoon", BookURL: "/book/show/7297403"} @@ -796,29 +796,29 @@ func TestBiblioReadsTitleFallback(t *testing.T) { name = "unavailable" } t.Run(name, func(t *testing.T) { - source := &fakeBiblioReadsSource{byQuery: map[string][]SearchResult{ + source := &fakeSource{byQuery: map[string][]SearchResult{ title + " J. R. R. Tolkien": {parody}, title: {parody, correct}, }} - idx := newBiblioReadsTestIndexer(t, source) - doc := index.Document{ID: biblioReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + idx := newTestIndexer(t, source) + doc := index.Document{ID: testID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } if fallbackUnavailable { source.queryErrors = map[string]error{title: ErrUnavailable} } - got, err := idx.BiblioReadsBook(context.Background(), doc) + got, err := idx.Book(context.Background(), doc) if fallbackUnavailable { if !errors.Is(err, ErrUnavailable) || got.BookURL != "" { t.Fatalf("expected unavailable error, got %+v, %v", got, err) } stored, readErr := idx.DocumentByID(doc.ID) - if readErr != nil || mustBiblioReadsData(t, stored).Enriched { + if readErr != nil || mustData(t, stored).Enriched { t.Fatalf("expected document left for retry, got %+v, %v", stored, readErr) } source.queryErrors = nil - got, err = idx.BiblioReadsBook(context.Background(), stored) + got, err = idx.Book(context.Background(), stored) } if err != nil || got.BookURL != correct.BookURL { t.Fatalf("got %+v, %v; want %q", got, err, correct.BookURL) @@ -834,10 +834,10 @@ func TestBiblioReadsTitleFallback(t *testing.T) { } } -func TestBiblioReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { +func TestMatchesTolkienOmnibusSearchResults(t *testing.T) { const title = "El señor de los anillos" const wantURL = "/book/show/37910810-el-se-or-de-los-anillos" - // Titles and ordering from BiblioReads' actual title-only search response. + // Titles and ordering from ' actual title-only search response. results := []SearchResult{ {Title: "La comunidad del anillo (El señor de los anillos, #1)", Author: "J.R.R. Tolkien", BookURL: "/book/show/222947.La_comunidad_del_anillo"}, {Title: "Las dos torres (El señor de los anillos, #2)", Author: "J.R.R. Tolkien", BookURL: "/book/show/15280.Las_dos_torres"}, @@ -845,13 +845,13 @@ func TestBiblioReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { {Title: "El señor de los anillos (#1-#3)", Author: "J.R.R. Tolkien", BookURL: wantURL, AuthorURL: "/author/show/656983.J_R_R_Tolkien"}, {Title: "Apéndices (El Señor de los Anillos, #4)", Author: "J.R.R. Tolkien", BookURL: "/book/show/54694.Ap_ndices"}, } - source := &fakeBiblioReadsSource{byQuery: map[string][]SearchResult{title: results}} - idx := newBiblioReadsTestIndexer(t, source) - doc := index.Document{ID: biblioReadsTestID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + source := &fakeSource{byQuery: map[string][]SearchResult{title: results}} + idx := newTestIndexer(t, source) + doc := index.Document{ID: testID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - got, err := idx.BiblioReadsBook(context.Background(), doc) + got, err := idx.Book(context.Background(), doc) if err != nil || got.BookURL != wantURL { t.Fatalf("got %+v, %v; want %q", got, err, wantURL) } @@ -859,7 +859,7 @@ func TestBiblioReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { if err != nil { t.Fatal(err) } - if !mustBiblioReadsData(t, stored).Enriched || mustBiblioReadsData(t, stored).URL != wantURL { + if !mustData(t, stored).Enriched || mustData(t, stored).URL != wantURL { t.Fatalf("expected persisted omnibus match, got %+v", stored) } author, err := idx.Author("j-r-r-tolkien", "") @@ -871,17 +871,17 @@ func TestBiblioReadsMatchesTolkienOmnibusSearchResults(t *testing.T) { } } -func TestForceIndexingRefreshesBiblioReadsMatch(t *testing.T) { - source := &fakeBiblioReadsSource{results: biblioReadsTestResults} - idx := newBiblioReadsTestIndexer(t, source) +func TestForceIndexingRefreshesMatch(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - doc, err := idx.DocumentByID(biblioReadsTestID) + doc, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - updateBiblioReadsData(t, &doc, func(data *DocumentData) { + updateData(t, &doc, func(data *DocumentData) { data.URL = "/book/show/1-wrong" data.ResourceID = "wrong-work" data.LegacyBookID = 1 @@ -896,24 +896,24 @@ func TestForceIndexingRefreshesBiblioReadsMatch(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - doc, err = idx.DocumentByID(biblioReadsTestID) + doc, err = idx.DocumentByID(testID) if err != nil { t.Fatal(err) } - if mustBiblioReadsData(t, doc).URL != wantDataSourceLink { - t.Fatalf("expected refreshed book link, got %q", mustBiblioReadsData(t, doc).URL) + if mustData(t, doc).URL != wantDataSourceLink { + t.Fatalf("expected refreshed book link, got %q", mustData(t, doc).URL) } - if mustBiblioReadsData(t, doc).ResourceID != "" || mustBiblioReadsData(t, doc).LegacyBookID != 0 { - t.Fatalf("expected old identifiers to be cleared, got %q and %d", mustBiblioReadsData(t, doc).ResourceID, mustBiblioReadsData(t, doc).LegacyBookID) + if mustData(t, doc).ResourceID != "" || mustData(t, doc).LegacyBookID != 0 { + t.Fatalf("expected old identifiers to be cleared, got %q and %d", mustData(t, doc).ResourceID, mustData(t, doc).LegacyBookID) } } -func TestStoreBiblioReadsDetails(t *testing.T) { +func TestStoreDetails(t *testing.T) { for _, scenario := range []string{"linked", "newly linked", "changed metadata", "changed match", "deleted", "partial identifiers"} { t.Run(scenario, func(t *testing.T) { - idx := newBiblioReadsTestIndexer(t, nil) - doc := withBiblioReadsData(t, index.Document{ - ID: biblioReadsTestID, + idx := newTestIndexer(t, nil) + doc := withData(t, index.Document{ + ID: testID, Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, TextRankWords: []string{"dragon"}, @@ -923,14 +923,14 @@ func TestStoreBiblioReadsDetails(t *testing.T) { resourceID := "work-id" switch scenario { case "newly linked": - updateBiblioReadsData(t, &stale, func(data *DocumentData) { + updateData(t, &stale, func(data *DocumentData) { data.URL = "" data.Enriched = false }) case "changed metadata": doc.Title = "Another title" case "changed match": - updateBiblioReadsData(t, &doc, func(data *DocumentData) { + updateData(t, &doc, func(data *DocumentData) { data.URL = "/book/show/2-other" }) case "partial identifiers": @@ -945,10 +945,10 @@ func TestStoreBiblioReadsDetails(t *testing.T) { book := BookResponse{Rating: "4.12", ResourceID: resourceID, LegacyBookID: 12207444} writes := idx.documentsIdx writes.count.Store(0) - err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, book) + err := idx.StoreDetails(stale, wantDataSourceLink, book) wantError := scenario != "linked" && scenario != "newly linked" && scenario != "partial identifiers" if (err != nil) != wantError { - t.Fatalf("StoreBiblioReadsDetails error = %v, want error %v", err, wantError) + t.Fatalf("StoreDetails error = %v, want error %v", err, wantError) } if scenario == "deleted" { if !errors.Is(err, index.ErrDocumentNotFound) { @@ -964,7 +964,7 @@ func TestStoreBiblioReadsDetails(t *testing.T) { if writes.count.Load() != 0 { t.Fatal("stale details must not cause any index writes") } - if mustBiblioReadsData(t, stored).ResourceID != "" || mustBiblioReadsData(t, stored).LegacyBookID != 0 || stored.DataSourceScoreValue != 0 { + if mustData(t, stored).ResourceID != "" || mustData(t, stored).LegacyBookID != 0 || stored.DataSourceScoreValue != 0 { t.Fatalf("unexpected details persisted: %+v", stored) } return @@ -973,17 +973,17 @@ func TestStoreBiblioReadsDetails(t *testing.T) { if scenario == "partial identifiers" { wantLegacyBookID = 0 } - if mustBiblioReadsData(t, stored).ResourceID != resourceID || mustBiblioReadsData(t, stored).LegacyBookID != wantLegacyBookID { + if mustData(t, stored).ResourceID != resourceID || mustData(t, stored).LegacyBookID != wantLegacyBookID { t.Fatalf("identifiers not persisted: %+v", stored) } if stored.DataSourceScoreValue != 4.12 { t.Fatalf("rating not persisted with identifiers: %+v", stored) } - if !mustBiblioReadsData(t, stored).Enriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { + if !mustData(t, stored).Enriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("existing enrichment was lost: %+v", stored) } book.Rating = "4.120" - if err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, book); err != nil { + if err := idx.StoreDetails(stale, wantDataSourceLink, book); err != nil { t.Fatalf("repeated store failed: %v", err) } if writes.count.Load() != 1 { @@ -993,21 +993,21 @@ func TestStoreBiblioReadsDetails(t *testing.T) { } } -type countedBiblioReadsIndex interface { +type countedIndex interface { bleve.Index } -type biblioReadsWriteCountingIndex struct { - countedBiblioReadsIndex +type writeCountingIndex struct { + countedIndex count atomic.Int64 } -func (i *biblioReadsWriteCountingIndex) Index(id string, data any) error { +func (i *writeCountingIndex) Index(id string, data any) error { i.count.Add(1) - return i.countedBiblioReadsIndex.Index(id, data) + return i.countedIndex.Index(id, data) } -func TestStoreBiblioReadsDetailsPreservesMissingFields(t *testing.T) { +func TestStoreDetailsPreservesMissingFields(t *testing.T) { for _, tt := range []struct { name string book BookResponse @@ -1025,40 +1025,40 @@ func TestStoreBiblioReadsDetailsPreservesMissingFields(t *testing.T) { {name: "zero count", book: BookResponse{RatingCount: "0"}, wantRating: "3.82", wantCount: "0", wantWork: "old-work", wantID: 1}, } { t.Run(tt.name, func(t *testing.T) { - idx := newBiblioReadsTestIndexer(t, nil) - doc := withBiblioReadsData(t, index.Document{ - ID: biblioReadsTestID, + idx := newTestIndexer(t, nil) + doc := withData(t, index.Document{ + ID: testID, DataSourceScoreValue: 3.82, }, DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 1}) if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } - if err := idx.StoreBiblioReadsDetails(doc, mustBiblioReadsData(t, doc).URL, tt.book); err != nil { + if err := idx.StoreDetails(doc, mustData(t, doc).URL, tt.book); err != nil { t.Fatal(err) } stored, err := idx.DocumentByID(doc.ID) if err != nil { t.Fatal(err) } - if stored.DataSourceScoreValue != parseBiblioReadsRating(tt.wantRating, doc.ID) || mustBiblioReadsData(t, stored).ResourceID != tt.wantWork || mustBiblioReadsData(t, stored).LegacyBookID != tt.wantID { + if stored.DataSourceScoreValue != parseRating(tt.wantRating, doc.ID) || mustData(t, stored).ResourceID != tt.wantWork || mustData(t, stored).LegacyBookID != tt.wantID { t.Fatalf("partial details changed fields that were not supplied: %+v", stored) } wantCount := tt.wantCount if wantCount == "" { - wantCount = mustBiblioReadsData(t, doc).RatingCount + wantCount = mustData(t, doc).RatingCount } - if mustBiblioReadsData(t, stored).RatingCount != wantCount { - t.Fatalf("expected rating count %q, got %q", wantCount, mustBiblioReadsData(t, stored).RatingCount) + if mustData(t, stored).RatingCount != wantCount { + t.Fatalf("expected rating count %q, got %q", wantCount, mustData(t, stored).RatingCount) } }) } } func TestUpdateDataSourceLink(t *testing.T) { - idx := newBiblioReadsTestIndexer(t, nil) - doc := withBiblioReadsData(t, index.Document{ - ID: biblioReadsTestID, Slug: "la-reina-dragon", + idx := newTestIndexer(t, nil) + doc := withData(t, index.Document{ + ID: testID, Slug: "la-reina-dragon", Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, TextRankWords: []string{"dragon"}, @@ -1076,31 +1076,31 @@ func TestUpdateDataSourceLink(t *testing.T) { if err != nil { t.Fatal(err) } - if mustBiblioReadsData(t, stored).URL != newLink || !mustBiblioReadsData(t, stored).Enriched || - stored.DataSourceScoreValue != 0 || mustBiblioReadsData(t, stored).RatingCount != "" || mustBiblioReadsData(t, stored).ResourceID != "" || mustBiblioReadsData(t, stored).LegacyBookID != 0 { + if mustData(t, stored).URL != newLink || !mustData(t, stored).Enriched || + stored.DataSourceScoreValue != 0 || mustData(t, stored).RatingCount != "" || mustData(t, stored).ResourceID != "" || mustData(t, stored).LegacyBookID != 0 { t.Fatalf("updating review link must clear cached score and provider lookup data: %+v", stored) } if stored.Title != doc.Title || stored.Slug != doc.Slug || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { t.Fatalf("review-link update changed unrelated document data: %+v", stored) } - if err := idx.StoreBiblioReadsDetails(doc, mustBiblioReadsData(t, doc).URL, BookResponse{Rating: "4.5"}); err == nil { + if err := idx.StoreDetails(doc, mustData(t, doc).URL, BookResponse{Rating: "4.5"}); err == nil { t.Fatal("stale score write must not overwrite the updated link") } } func TestUpdateDataSourceLinkMissingDocument(t *testing.T) { - idx := newBiblioReadsTestIndexer(t, nil) + idx := newTestIndexer(t, nil) if err := idx.UpdateDataSourceLink("missing", "provider-specific-document-reference"); !errors.Is(err, index.ErrDocumentNotFound) { t.Fatalf("expected missing-document error, got %v", err) } } -func TestBiblioReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { - idx := newBiblioReadsTestIndexer(t, &fakeBiblioReadsSource{results: biblioReadsTestResults}) +func TestDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { + idx := newTestIndexer(t, &fakeSource{results: testResults}) if err := idx.AddLibrary(10, true, 1); err != nil { t.Fatal(err) } - stale, err := idx.DocumentByID(biblioReadsTestID) + stale, err := idx.DocumentByID(testID) if err != nil { t.Fatal(err) } @@ -1118,7 +1118,7 @@ func TestBiblioReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { if err := idx.Enricher.Enrich(context.Background(), idx.BleveIndexer, stale); err != nil { t.Fatal(err) } - if err := idx.StoreBiblioReadsDetails(stale, wantDataSourceLink, BookResponse{ + if err := idx.StoreDetails(stale, wantDataSourceLink, BookResponse{ Rating: "4.5", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, }); err != nil { t.Fatal(err) @@ -1141,15 +1141,15 @@ func TestBiblioReadsDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { if err != nil || !reflect.DeepEqual(decoded, other) { t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) } - if data := mustBiblioReadsData(t, got); data != (DocumentData{URL: "/book/show/456", Enriched: true}) { + if data := mustData(t, got); data != (DocumentData{URL: "/book/show/456", Enriched: true}) { t.Fatalf("stale write restored old Goodreads state: %+v", data) } } -func TestBiblioReadsBookRejectsInvalidProviderData(t *testing.T) { - idx := newBiblioReadsTestIndexer(t, nil) +func TestBookRejectsInvalidProviderData(t *testing.T) { + idx := newTestIndexer(t, nil) document := index.Document{DataSources: map[string]json.RawMessage{Source: json.RawMessage(`{"url":42}`)}} - if _, err := idx.BiblioReadsBook(context.Background(), document); err == nil { + if _, err := idx.Book(context.Background(), document); err == nil { t.Fatal("Goodreads accepted an incorrectly typed URL") } } diff --git a/internal/datasource/biblioreads/matching.go b/internal/datasource/biblioreads/matching.go index 67a526e6..149c6a56 100644 --- a/internal/datasource/biblioreads/matching.go +++ b/internal/datasource/biblioreads/matching.go @@ -13,15 +13,15 @@ import ( ) // Strip named series annotations and omnibus ranges, not standalone volume numbers. -var biblioReadsSeriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) +var seriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) -var biblioReadsSubtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+`) +var subtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+`) -var biblioReadsParentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) +var parentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) var goodreadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) -func normalizeBiblioReadsName(value string) string { +func normalizeName(value string) string { var result strings.Builder for _, r := range norm.NFD.String(strings.ToLower(value)) { switch { @@ -36,9 +36,9 @@ func normalizeBiblioReadsName(value string) string { return strings.Join(strings.Fields(result.String()), " ") } -func biblioReadsAuthorNameMatches(name, candidate string) bool { - name = normalizeBiblioReadsName(name) - candidate = normalizeBiblioReadsName(candidate) +func authorNameMatches(name, candidate string) bool { + name = normalizeName(name) + candidate = normalizeName(candidate) if name == "" || candidate == "" { return false } @@ -62,19 +62,19 @@ func biblioReadsAuthorNameMatches(name, candidate string) bool { return next == len(short)-1 } -func biblioReadsAuthor(document index.Document, matchedName string) (string, string) { - nameKey := strings.ReplaceAll(normalizeBiblioReadsName(matchedName), " ", "") +func matchedAuthor(document index.Document, matchedName string) (string, string) { + nameKey := strings.ReplaceAll(normalizeName(matchedName), " ", "") if nameKey == "" { return "", "" } for i, name := range document.Authors { - if i < len(document.AuthorsSlugs) && strings.ReplaceAll(normalizeBiblioReadsName(name), " ", "") == nameKey { + if i < len(document.AuthorsSlugs) && strings.ReplaceAll(normalizeName(name), " ", "") == nameKey { return document.AuthorsSlugs[i], name } } var slug, authorName string for i, name := range document.Authors { - if i < len(document.AuthorsSlugs) && biblioReadsAuthorNameMatches(name, matchedName) { + if i < len(document.AuthorsSlugs) && authorNameMatches(name, matchedName) { if slug != "" { return "", "" } @@ -84,39 +84,39 @@ func biblioReadsAuthor(document index.Document, matchedName string) (string, str return slug, authorName } -func biblioReadsTitle(value string) string { - return normalizeBiblioReadsName(biblioReadsSeriesSuffix.ReplaceAllString(value, "")) +func normalizedTitle(value string) string { + return normalizeName(seriesSuffix.ReplaceAllString(value, "")) } -func biblioReadsMainTitle(value string) string { - value = biblioReadsSeriesSuffix.ReplaceAllString(value, "") - if separator := biblioReadsSubtitleSeparator.FindStringIndex(value); separator != nil { +func mainTitle(value string) string { + value = seriesSuffix.ReplaceAllString(value, "") + if separator := subtitleSeparator.FindStringIndex(value); separator != nil { value = value[:separator[0]] } - return normalizeBiblioReadsName(value) + return normalizeName(value) } -func biblioReadsTitleMatchScore(candidate, title, mainTitle string) int { - candidateTitle := biblioReadsTitle(candidate) +func titleMatchScore(candidate, title, documentMainTitle string) int { + candidateTitle := normalizedTitle(candidate) if candidateTitle == "" { return 0 } if candidateTitle == title { return 3 } - if biblioReadsMainTitle(candidate) == title || mainTitle == candidateTitle || - biblioReadsTitle(biblioReadsParentheticalTitle.ReplaceAllString(candidate, "")) == title { + if mainTitle(candidate) == title || documentMainTitle == candidateTitle || + normalizedTitle(parentheticalTitle.ReplaceAllString(candidate, "")) == title { return 1 } return 0 } -func preferBiblioReadsBook(candidate, best SearchResult, score, bestScore int) bool { +func preferBook(candidate, best SearchResult, score, bestScore int) bool { if bestScore == 0 { return true } if score == bestScore { - count, bestCount := biblioReadsRatingCount(candidate.RatingCount), biblioReadsRatingCount(best.RatingCount) + count, bestCount := ratingCount(candidate.RatingCount), ratingCount(best.RatingCount) if count != bestCount { return count > bestCount } @@ -125,14 +125,14 @@ func preferBiblioReadsBook(candidate, best SearchResult, score, bestScore int) b return id < bestID } } - hasRating, bestHasRating := hasBiblioReadsRating(candidate.Rating), hasBiblioReadsRating(best.Rating) + hasRating, bestHasRating := hasRating(candidate.Rating), hasRating(best.Rating) if hasRating != bestHasRating { return hasRating } return score > bestScore } -func biblioReadsRatingCount(value string) uint64 { +func ratingCount(value string) uint64 { if strings.TrimSpace(value) == "" { return 0 } @@ -157,25 +157,25 @@ func goodreadsBookID(bookURL string) uint64 { return id } -func matchingBiblioReadsBook(document index.Document, results []SearchResult) (SearchResult, bool) { - title := biblioReadsTitle(document.Title) +func matchingBook(document index.Document, results []SearchResult) (SearchResult, bool) { + title := normalizedTitle(document.Title) if title == "" { return SearchResult{}, false } - mainTitle := biblioReadsMainTitle(document.Title) + mainTitle := mainTitle(document.Title) var best SearchResult bestScore := 0 for _, result := range results { if result.BookURL == "" { continue } - score := biblioReadsTitleMatchScore(result.Title, title, mainTitle) + score := titleMatchScore(result.Title, title, mainTitle) if score == 0 { continue } for _, author := range document.Authors { - if biblioReadsAuthorNameMatches(author, result.Author) { - if preferBiblioReadsBook(result, best, score, bestScore) { + if authorNameMatches(author, result.Author) { + if preferBook(result, best, score, bestScore) { best, bestScore = result, score } break @@ -185,14 +185,14 @@ func matchingBiblioReadsBook(document index.Document, results []SearchResult) (S return best, bestScore > 0 } -func hasBiblioReadsRating(rating string) bool { +func hasRating(rating string) bool { value, err := strconv.ParseFloat(rating, 64) return err == nil && value > 0 && value <= 5 && !math.IsNaN(value) && !math.IsInf(value, 0) } -// biblioReadsQuery returns the BiblioReads search string for document: its title +// searchQuery returns the search string for document: its title // followed by its first author, if any. -func biblioReadsQuery(document index.Document) string { +func searchQuery(document index.Document) string { q := strings.TrimSpace(document.Title) if len(document.Authors) > 0 { q = strings.TrimSpace(q + " " + document.Authors[0]) @@ -200,7 +200,7 @@ func biblioReadsQuery(document index.Document) string { return q } -func parseBiblioReadsRating(rating, documentID string) float64 { +func parseRating(rating, documentID string) float64 { if rating == "" { return 0 } diff --git a/internal/reviews/biblioreads.go b/internal/reviews/biblioreads.go index c0495a6d..8f79c7f3 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/reviews/biblioreads.go @@ -29,8 +29,8 @@ type BiblioReadsClient interface { } type BiblioReadsIndex interface { - BiblioReadsBook(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) - StoreBiblioReadsDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error + Book(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) + StoreDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error UpdateDataSourceLink(documentID, bookURL string) error } @@ -139,7 +139,7 @@ func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblio book.Rating = fmt.Sprintf("%.2f", document.DataSourceScoreValue) details.Rating = "" } - if err := b.index.StoreBiblioReadsDetails(document, data.URL, details); err != nil { + if err := b.index.StoreDetails(document, data.URL, details); err != nil { log.Printf("Error saving Goodreads details for %s: %s", document.ID, err) } return book, nil @@ -157,7 +157,7 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score return score, err } if data.URL == "" { - result, err := b.index.BiblioReadsBook(ctx, document) + result, err := b.index.Book(ctx, document) if err != nil { return score, err } diff --git a/internal/reviews/biblioreads_test.go b/internal/reviews/biblioreads_test.go index 3f1411fe..39fda6ae 100644 --- a/internal/reviews/biblioreads_test.go +++ b/internal/reviews/biblioreads_test.go @@ -50,11 +50,11 @@ type reviewLinkIndex struct { err error } -func (i *reviewLinkIndex) BiblioReadsBook(context.Context, index.Document) (biblioreads.SearchResult, error) { +func (i *reviewLinkIndex) Book(context.Context, index.Document) (biblioreads.SearchResult, error) { return biblioreads.SearchResult{}, nil } -func (i *reviewLinkIndex) StoreBiblioReadsDetails(index.Document, string, biblioreads.BookResponse) error { +func (i *reviewLinkIndex) StoreDetails(index.Document, string, biblioreads.BookResponse) error { return nil } diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index a8f6524d..2d3fbaef 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -154,7 +154,7 @@ func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Documen if err != nil { t.Fatalf("reload %s: %v", doc.Slug, err) } - if err := adapter.StoreBiblioReadsDetails(updated, mustBiblioReadsData(t, updated).URL, biblioreads.BookResponse{ + if err := adapter.StoreDetails(updated, mustBiblioReadsData(t, updated).URL, biblioreads.BookResponse{ Rating: score, RatingCount: count, }); err != nil { t.Fatalf("set review score for %s: %v", doc.Slug, err) From 42b3dc90359b9ba96ca86d5e05e27183e3afd3b1 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 12:27:35 +0100 Subject: [PATCH 32/50] Refactor --- .../biblioreads/reviews.go} | 94 +++++++++---------- .../biblioreads/reviews_test.go} | 14 +-- .../model.go => datasource/model/review.go} | 7 +- .../controller/document/controller.go | 6 +- .../webserver/controller/document/reviews.go | 4 +- .../document/update_document_link.go | 6 +- .../webserver/document_biblioreads_test.go | 5 +- internal/webserver/document_reviews_test.go | 38 ++++---- .../infrastructure/template_engine.go | 4 +- internal/webserver/webserver_test.go | 3 +- main.go | 3 +- 11 files changed, 87 insertions(+), 97 deletions(-) rename internal/{reviews/biblioreads.go => datasource/biblioreads/reviews.go} (62%) rename internal/{reviews/biblioreads_test.go => datasource/biblioreads/reviews_test.go} (91%) rename internal/{reviews/model.go => datasource/model/review.go} (84%) diff --git a/internal/reviews/biblioreads.go b/internal/datasource/biblioreads/reviews.go similarity index 62% rename from internal/reviews/biblioreads.go rename to internal/datasource/biblioreads/reviews.go index 8f79c7f3..dd9bbfad 100644 --- a/internal/reviews/biblioreads.go +++ b/internal/datasource/biblioreads/reviews.go @@ -1,4 +1,4 @@ -package reviews +package biblioreads import ( "context" @@ -6,55 +6,49 @@ import ( "fmt" "log" "net/url" - "regexp" "strconv" "strings" "time" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" ) -const ( - goodreadsBaseURL = "https://www.goodreads.com" - goodreadsBookPrefix = "/book/show/" -) - -var goodreadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) +const goodreadsBookPrefix = "/book/show/" -type BiblioReadsClient interface { - Book(ctx context.Context, id string) (biblioreads.BookResponse, error) - Reviews(ctx context.Context, resourceID string, legacyBookID int, nextPageToken, languageCode string) (biblioreads.ReviewsResponse, error) +type ReviewClient interface { + Book(ctx context.Context, id string) (BookResponse, error) + Reviews(ctx context.Context, resourceID string, legacyBookID int, nextPageToken, languageCode string) (ReviewsResponse, error) } -type BiblioReadsIndex interface { - Book(ctx context.Context, document index.Document) (biblioreads.SearchResult, error) - StoreDetails(document index.Document, bookURL string, book biblioreads.BookResponse) error +type ReviewIndex interface { + Book(ctx context.Context, document index.Document) (SearchResult, error) + StoreDetails(document index.Document, bookURL string, book BookResponse) error UpdateDataSourceLink(documentID, bookURL string) error } -type BiblioReads struct { - client BiblioReadsClient - index BiblioReadsIndex +type ReviewProvider struct { + client ReviewClient + index ReviewIndex } -func NewBiblioReads(client BiblioReadsClient, idx BiblioReadsIndex) *BiblioReads { - return &BiblioReads{client: client, index: idx} +// NewReviewProvider exposes BiblioReads data using the shared review models. +func NewReviewProvider(client ReviewClient, idx ReviewIndex) *ReviewProvider { + return &ReviewProvider{client: client, index: idx} } -func (b *BiblioReads) Name() string { +func (b *ReviewProvider) Name() string { return "Goodreads" } // AuthorLink returns the indexed author link without making a network request. -func (b *BiblioReads) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { +func (b *ReviewProvider) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { return goodreadsDataSourceLink(author.DataSourceLinks["goodreads"], "/author/show/") } // DocumentLink returns the indexed book link without making a network request. -func (b *BiblioReads) DocumentLink(document index.Document) datasourcemodel.DataSourceLink { - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) +func (b *ReviewProvider) DocumentLink(document index.Document) datasourcemodel.DataSourceLink { + data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { log.Println(err) return datasourcemodel.DataSourceLink{} @@ -71,31 +65,31 @@ func goodreadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink } // StoredScore builds a score from indexed data without making a network request. -func (b *BiblioReads) StoredScore(document index.Document) (Score, error) { - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) +func (b *ReviewProvider) StoredScore(document index.Document) (datasourcemodel.Score, error) { + data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { - return Score{}, err + return datasourcemodel.Score{}, err } - return Score{ + return datasourcemodel.Score{ Source: b.Name(), Value: fmt.Sprintf("%.2f", document.DataSourceScoreValue), Count: data.RatingCount, URL: goodreadsDataSourceLink(data.URL, goodreadsBookPrefix).URL, }, nil } // CanEditDocumentLink reports that Goodreads book links can be corrected. -func (b *BiblioReads) CanEditDocumentLink() bool { +func (b *ReviewProvider) CanEditDocumentLink() bool { return true } // UpdateDocumentLink saves a corrected Goodreads book link and clears stale score details. -func (b *BiblioReads) UpdateDocumentLink(document index.Document, link string) error { +func (b *ReviewProvider) UpdateDocumentLink(document index.Document, link string) error { path, err := normalizeGoodreadsBookURL(link) if err != nil { - return fmt.Errorf("%w: invalid Goodreads book URL", ErrInvalidDocumentLink) + return fmt.Errorf("%w: invalid Goodreads book URL", datasourcemodel.ErrInvalidDocumentLink) } if err := b.index.UpdateDataSourceLink(document.ID, path); err != nil { if errors.Is(err, index.ErrDocumentNotFound) { - return ErrDocumentNotFound + return datasourcemodel.ErrDocumentNotFound } return err } @@ -121,14 +115,14 @@ func bookID(bookURL string) (string, error) { return id, nil } -func (b *BiblioReads) book(ctx context.Context, document index.Document) (biblioreads.BookResponse, error) { - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) +func (b *ReviewProvider) book(ctx context.Context, document index.Document) (BookResponse, error) { + data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { - return biblioreads.BookResponse{}, err + return BookResponse{}, err } id, err := bookID(data.URL) if err != nil { - return biblioreads.BookResponse{}, err + return BookResponse{}, err } book, err := b.client.Book(ctx, id) if err != nil { @@ -150,9 +144,9 @@ func isZeroRating(rating string) bool { return err == nil && value == 0 } -func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score, error) { - score := Score{Source: "Goodreads"} - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) +func (b *ReviewProvider) Score(ctx context.Context, document index.Document) (datasourcemodel.Score, error) { + score := datasourcemodel.Score{Source: "Goodreads"} + data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { return score, err } @@ -168,7 +162,7 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score return score, err } data.URL = result.BookURL - if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + if err := index.SetDataSourceData(&document, Source, data); err != nil { return score, err } score.Value, score.Count, score.URL = result.Rating, result.RatingCount, b.DocumentLink(document).URL @@ -182,33 +176,33 @@ func (b *BiblioReads) Score(ctx context.Context, document index.Document) (Score return score, nil } -func (b *BiblioReads) Reviews(ctx context.Context, document index.Document, language string) (Page, error) { - data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) +func (b *ReviewProvider) Reviews(ctx context.Context, document index.Document, language string) (datasourcemodel.ReviewPage, error) { + data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { - return Page{}, err + return datasourcemodel.ReviewPage{}, err } if data.URL == "" { - return Page{}, nil + return datasourcemodel.ReviewPage{}, nil } book, err := b.book(ctx, document) if err != nil { - return Page{}, err + return datasourcemodel.ReviewPage{}, err } if book.Rating == "" { - return Page{}, nil + return datasourcemodel.ReviewPage{}, nil } resourceID, legacyBookID := data.ResourceID, data.LegacyBookID if book.ResourceID != "" && book.LegacyBookID > 0 { resourceID, legacyBookID = book.ResourceID, book.LegacyBookID } if resourceID == "" || legacyBookID <= 0 { - return Page{}, fmt.Errorf("missing Goodreads review identifiers for %s", document.ID) + return datasourcemodel.ReviewPage{}, fmt.Errorf("missing Goodreads review identifiers for %s", document.ID) } response, err := b.client.Reviews(ctx, resourceID, legacyBookID, "", language) if err != nil { - return Page{}, fmt.Errorf("retrieving Goodreads reviews for %s: %w", document.ID, err) + return datasourcemodel.ReviewPage{}, fmt.Errorf("retrieving Goodreads reviews for %s: %w", document.ID, err) } - page := Page{Source: "Goodreads", URL: b.DocumentLink(document).URL, Available: true} + page := datasourcemodel.ReviewPage{Source: "Goodreads", URL: b.DocumentLink(document).URL, Available: true} for _, review := range response.Reviews { var createdAt time.Time if review.CreatedAt != "" { @@ -221,7 +215,7 @@ func (b *BiblioReads) Reviews(ctx context.Context, document index.Document, lang if review.ReviewURL != "" { reviewURL = goodreadsBaseURL + review.ReviewURL } - page.Reviews = append(page.Reviews, Review{ + page.Reviews = append(page.Reviews, datasourcemodel.Review{ Reviewer: review.Reviewer.Name, Rating: review.Rating, Text: review.Text, CreatedAt: createdAt, Spoiler: review.Spoiler, URL: reviewURL, }) diff --git a/internal/reviews/biblioreads_test.go b/internal/datasource/biblioreads/reviews_test.go similarity index 91% rename from internal/reviews/biblioreads_test.go rename to internal/datasource/biblioreads/reviews_test.go index 39fda6ae..c53a758a 100644 --- a/internal/reviews/biblioreads_test.go +++ b/internal/datasource/biblioreads/reviews_test.go @@ -1,4 +1,4 @@ -package reviews_test +package biblioreads_test import ( "context" @@ -6,8 +6,8 @@ import ( "testing" "github.com/svera/coreander/v5/internal/datasource/biblioreads" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" - "github.com/svera/coreander/v5/internal/reviews" ) func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { @@ -19,7 +19,7 @@ func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads } func TestBiblioReadsDataSourceLinks(t *testing.T) { - provider := reviews.NewBiblioReads(nil, nil) + provider := biblioreads.NewReviewProvider(nil, nil) author := provider.AuthorLink(index.Author{ DataSourceLinks: map[string]string{"goodreads": "/author/show/123.Jane_Doe"}, }, "en") @@ -90,10 +90,10 @@ func TestBiblioReadsUpdateDocumentLink(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { idx := &reviewLinkIndex{} - provider := reviews.NewBiblioReads(nil, idx) + provider := biblioreads.NewReviewProvider(nil, idx) err := provider.UpdateDocumentLink(index.Document{ID: "doc-id"}, tt.url) if tt.want == "" { - if !errors.Is(err, reviews.ErrInvalidDocumentLink) { + if !errors.Is(err, datasourcemodel.ErrInvalidDocumentLink) { t.Fatalf("expected invalid-link error, got %v", err) } if idx.link != "" { @@ -113,9 +113,9 @@ func TestBiblioReadsUpdateDocumentLink(t *testing.T) { func TestBiblioReadsUpdateDocumentLinkMissingDocument(t *testing.T) { idx := &reviewLinkIndex{err: index.ErrDocumentNotFound} - provider := reviews.NewBiblioReads(nil, idx) + provider := biblioreads.NewReviewProvider(nil, idx) err := provider.UpdateDocumentLink(index.Document{ID: "missing"}, "https://www.goodreads.com/book/show/1368314") - if !errors.Is(err, reviews.ErrDocumentNotFound) { + if !errors.Is(err, datasourcemodel.ErrDocumentNotFound) { t.Fatalf("expected missing-document error, got %v", err) } } diff --git a/internal/reviews/model.go b/internal/datasource/model/review.go similarity index 84% rename from internal/reviews/model.go rename to internal/datasource/model/review.go index c35a59f4..08aa8be7 100644 --- a/internal/reviews/model.go +++ b/internal/datasource/model/review.go @@ -1,5 +1,4 @@ -// Package reviews provides service-independent review data and service adapters. -package reviews +package model import ( "errors" @@ -45,8 +44,8 @@ type Review struct { URL string } -// Page distinguishes an unavailable review section from an available, empty list. -type Page struct { +// ReviewPage distinguishes an unavailable review section from an available, empty list. +type ReviewPage struct { Source string URL string Available bool diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index c16cdf0d..60404a33 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -6,10 +6,10 @@ import ( "time" "github.com/spf13/afero" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/i18n" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/result" - "github.com/svera/coreander/v5/internal/reviews" authorcontroller "github.com/svera/coreander/v5/internal/webserver/controller/author" "github.com/svera/coreander/v5/internal/webserver/model" ) @@ -18,9 +18,9 @@ const relatedDocuments = 4 type ReviewProvider interface { Name() string - Score(ctx context.Context, document index.Document) (reviews.Score, error) + Score(ctx context.Context, document index.Document) (datasourcemodel.Score, error) // Reviews returns reviews in the current UI language (a two-letter ISO code). - Reviews(ctx context.Context, document index.Document, language string) (reviews.Page, error) + Reviews(ctx context.Context, document index.Document, language string) (datasourcemodel.ReviewPage, error) // CanEditDocumentLink reports whether this provider supports correcting its document link. CanEditDocumentLink() bool // UpdateDocumentLink saves an administrator's correction to the provider-specific link. diff --git a/internal/webserver/controller/document/reviews.go b/internal/webserver/controller/document/reviews.go index 14c05f73..53ad8459 100644 --- a/internal/webserver/controller/document/reviews.go +++ b/internal/webserver/controller/document/reviews.go @@ -8,13 +8,13 @@ import ( "time" "github.com/gofiber/fiber/v3" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/metadata" - "github.com/svera/coreander/v5/internal/reviews" "golang.org/x/exp/slices" ) type documentReview struct { - reviews.Review + datasourcemodel.Review HTML template.HTML CreatedOn string StarFills [5]int diff --git a/internal/webserver/controller/document/update_document_link.go b/internal/webserver/controller/document/update_document_link.go index 5197389f..3634bcef 100644 --- a/internal/webserver/controller/document/update_document_link.go +++ b/internal/webserver/controller/document/update_document_link.go @@ -5,7 +5,7 @@ import ( "log" "github.com/gofiber/fiber/v3" - "github.com/svera/coreander/v5/internal/reviews" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" ) func (d *Controller) UpdateDocumentLink(c fiber.Ctx) error { @@ -22,12 +22,12 @@ func (d *Controller) UpdateDocumentLink(c fiber.Ctx) error { } link := c.FormValue("bookURL") if err := d.reviewProvider.UpdateDocumentLink(document, link); err != nil { - if errors.Is(err, reviews.ErrInvalidDocumentLink) { + if errors.Is(err, datasourcemodel.ErrInvalidDocumentLink) { return c.Status(fiber.StatusUnprocessableEntity).Render("partials/document-review-link-form", fiber.Map{ "Slug": document.Slug, "BookURL": link, "InvalidURL": true, "Source": d.reviewProvider.Name(), }) } - if errors.Is(err, reviews.ErrDocumentNotFound) { + if errors.Is(err, datasourcemodel.ErrDocumentNotFound) { return fiber.ErrNotFound } log.Printf("Error correcting document link for %s: %s", document.ID, err) diff --git a/internal/webserver/document_biblioreads_test.go b/internal/webserver/document_biblioreads_test.go index 5ce9f2b8..77a1cf15 100644 --- a/internal/webserver/document_biblioreads_test.go +++ b/internal/webserver/document_biblioreads_test.go @@ -18,7 +18,6 @@ import ( "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" - "github.com/svera/coreander/v5/internal/reviews" "github.com/svera/coreander/v5/internal/webserver" authorcontroller "github.com/svera/coreander/v5/internal/webserver/controller/author" "github.com/svera/coreander/v5/internal/webserver/controller/document" @@ -120,9 +119,9 @@ func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provide idx := index.NewBleve(documents, authors, appFS, testLibraryDir, nil, idxCfg) adapter.SetIndex(idx) if source != nil { - provider = reviews.NewBiblioReads(source, adapter) + provider = biblioreads.NewReviewProvider(source, adapter) } - biblioReadsLinks := reviews.NewBiblioReads(nil, adapter) + biblioReadsLinks := biblioreads.NewReviewProvider(nil, adapter) t.Cleanup(func() { _ = idx.Close() }) for _, doc := range []index.Document{withBiblioReadsData(t, index.Document{ diff --git a/internal/webserver/document_reviews_test.go b/internal/webserver/document_reviews_test.go index 9d8d5722..db1516f1 100644 --- a/internal/webserver/document_reviews_test.go +++ b/internal/webserver/document_reviews_test.go @@ -11,16 +11,16 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v3" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" - "github.com/svera/coreander/v5/internal/reviews" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/controller/document" ) func TestDocumentReviewsCanSelectSupportedLanguage(t *testing.T) { provider := &fakeReviewProvider{ - score: reviews.Score{Source: "Another service"}, - page: reviews.Page{Source: "Another service", Available: true}, + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true}, } app, _ := newReviewProviderTestApp(t, provider) req := mustGetRequest(t, "/documents/linked/reviews?language=fr") @@ -59,8 +59,8 @@ func TestDocumentReviewsCanSelectSupportedLanguage(t *testing.T) { func TestDocumentReviewsRejectUnsupportedLanguage(t *testing.T) { provider := &fakeReviewProvider{ - score: reviews.Score{Source: "Another service"}, - page: reviews.Page{Source: "Another service", Available: true}, + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true}, } app, _ := newReviewProviderTestApp(t, provider) response, err := app.Test(mustGetRequest(t, "/documents/linked/reviews?language=zz")) @@ -75,8 +75,8 @@ func TestDocumentReviewsRejectUnsupportedLanguage(t *testing.T) { } type fakeReviewProvider struct { - score reviews.Score - page reviews.Page + score datasourcemodel.Score + page datasourcemodel.ReviewPage scoreErr error reviewsErr error canEditDocumentLink bool @@ -97,12 +97,12 @@ func (f *fakeReviewProvider) UpdateDocumentLink(index.Document, string) error { return nil } -func (f *fakeReviewProvider) Score(_ context.Context, document index.Document) (reviews.Score, error) { +func (f *fakeReviewProvider) Score(_ context.Context, document index.Document) (datasourcemodel.Score, error) { f.scoreSlugs = append(f.scoreSlugs, document.Slug) return f.score, f.scoreErr } -func (f *fakeReviewProvider) Reviews(_ context.Context, document index.Document, language string) (reviews.Page, error) { +func (f *fakeReviewProvider) Reviews(_ context.Context, document index.Document, language string) (datasourcemodel.ReviewPage, error) { f.reviewSlugs = append(f.reviewSlugs, document.Slug) f.reviewLanguages = append(f.reviewLanguages, language) return f.page, f.reviewsErr @@ -123,8 +123,8 @@ func TestDocumentReviewsUILanguage(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { provider := &fakeReviewProvider{ - score: reviews.Score{Source: "Another service"}, - page: reviews.Page{Source: "Another service", Available: true}, + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true}, } app, _ := newReviewProviderTestApp(t, provider) req := mustGetRequest(t, "/documents/linked/reviews"+tt.query) @@ -149,10 +149,10 @@ func TestDocumentReviewsUILanguage(t *testing.T) { func TestDocumentAlternativeReviewProvider(t *testing.T) { provider := &fakeReviewProvider{ - score: reviews.Score{Source: "Another service", Value: "4.5", Count: "123", URL: "https://example.org/books/book-1"}, - page: reviews.Page{ + score: datasourcemodel.Score{Source: "Another service", Value: "4.5", Count: "123", URL: "https://example.org/books/book-1"}, + page: datasourcemodel.ReviewPage{ Source: "Another service", URL: "https://example.org/books/book-1", Available: true, - Reviews: []reviews.Review{{ + Reviews: []datasourcemodel.Review{{ Reviewer: "Reader", Rating: 5, Text: "

        Excellent

        ", CreatedAt: time.Date(2026, 10, 30, 0, 0, 0, 0, time.UTC), URL: "https://example.org/reviews/review-1", }}, @@ -214,7 +214,7 @@ func TestDocumentScoreMarksZeroWithoutValuationsAsNotScored(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { provider := &fakeReviewProvider{ - score: reviews.Score{ + score: datasourcemodel.Score{ Source: "Another service", Value: "0.00", Count: tt.count, URL: "https://example.org/books/book-1", }, @@ -288,8 +288,8 @@ func TestDocumentReviewFractionalStars(t *testing.T) { } { t.Run(fmt.Sprint(tt.rating), func(t *testing.T) { provider := &fakeReviewProvider{ - score: reviews.Score{Source: "Another service"}, - page: reviews.Page{Source: "Another service", Available: true, Reviews: []reviews.Review{ + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true, Reviews: []datasourcemodel.Review{ {Reviewer: "Reader", Rating: tt.rating}, }}, } @@ -334,8 +334,8 @@ func TestDocumentReviewFractionalStars(t *testing.T) { func TestDocumentReviewNotScoredTranslations(t *testing.T) { provider := &fakeReviewProvider{ - score: reviews.Score{Source: "Another service"}, - page: reviews.Page{Source: "Another service", Available: true, Reviews: []reviews.Review{ + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true, Reviews: []datasourcemodel.Review{ {Reviewer: "Reader", Text: "A review without a rating"}, }}, } diff --git a/internal/webserver/infrastructure/template_engine.go b/internal/webserver/infrastructure/template_engine.go index 410b52df..55a7b416 100644 --- a/internal/webserver/infrastructure/template_engine.go +++ b/internal/webserver/infrastructure/template_engine.go @@ -2,6 +2,7 @@ package infrastructure import ( "fmt" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" "html/template" "io/fs" "net/http" @@ -12,7 +13,6 @@ import ( "github.com/gofiber/template/html/v3" "github.com/gosimple/slug" "github.com/svera/coreander/v5/internal/i18n" - "github.com/svera/coreander/v5/internal/reviews" ) // TemplateEngine builds the html/template engine. assetVersion is a cache-busting @@ -57,7 +57,7 @@ func TemplateEngine(viewsFS fs.FS, translator i18n.Translator, assetVersion stri }) engine.AddFunc("notLast", notLast[string]) - engine.AddFunc("storedDocumentScore", reviews.NewBiblioReads(nil, nil).StoredScore) + engine.AddFunc("storedDocumentScore", biblioreads.NewReviewProvider(nil, nil).StoredScore) engine.AddFunc("basename", func(path string) string { return filepath.Base(path) diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index 2d3fbaef..8fdf5824 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -20,7 +20,6 @@ import ( "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" - "github.com/svera/coreander/v5/internal/reviews" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/infrastructure" "github.com/svera/coreander/v5/internal/webserver/model" @@ -98,7 +97,7 @@ func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.F } adapter := biblioreads.NewEnricher(nil) adapter.SetIndex(idx) - controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, dataSource, reviews.NewBiblioReads(nil, adapter)) + controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, dataSource, biblioreads.NewReviewProvider(nil, adapter)) usersRepository := &model.UserRepository{DB: db} return webserver.New(webserverConfig, controllers, sender, idx, usersRepository), idx } diff --git a/main.go b/main.go index ba528a27..ff88d64f 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,6 @@ import ( "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" - "github.com/svera/coreander/v5/internal/reviews" "github.com/svera/coreander/v5/internal/versioncheck" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/controller/document" @@ -208,7 +207,7 @@ func main() { dataSource := wikidata.NewWikidataSource(wikidata.Gowikidata{}) var reviewProvider document.ReviewProvider - biblioReads := reviews.NewBiblioReads(biblioreadsClient, biblioReadsAdapter) + biblioReads := biblioreads.NewReviewProvider(biblioreadsClient, biblioReadsAdapter) if biblioreadsClient != nil { reviewProvider = biblioReads } From 39521ab77264f49ba6fd8729dd126e88d5aaa459 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 12:35:04 +0100 Subject: [PATCH 33/50] Refactor --- .../controller/document/controller.go | 19 +++++++++++++++++++ ...document_link.go => update_review_link.go} | 2 +- internal/webserver/document_reviews_test.go | 8 ++++++++ .../infrastructure/template_engine.go | 2 -- internal/webserver/routes.go | 2 +- internal/webserver/webserver.go | 1 + internal/webserver/webserver_test.go | 2 +- 7 files changed, 31 insertions(+), 5 deletions(-) rename internal/webserver/controller/document/{update_document_link.go => update_review_link.go} (95%) diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index 60404a33..e28e2667 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -130,3 +130,22 @@ func (d *Controller) DataSourceName() string { } return d.reviewProvider.Name() } + +// StoredScore reads cached scores without making requests to a review service. +func (d *Controller) StoredScore(document index.Document) (datasourcemodel.Score, error) { + type storedScoreProvider interface { + StoredScore(index.Document) (datasourcemodel.Score, error) + } + if d.reviewProvider != nil { + if provider, ok := d.reviewProvider.(storedScoreProvider); ok { + return provider.StoredScore(document) + } + return datasourcemodel.Score{}, nil + } + for _, linkProvider := range d.linkProviders { + if provider, ok := linkProvider.(storedScoreProvider); ok { + return provider.StoredScore(document) + } + } + return datasourcemodel.Score{}, nil +} diff --git a/internal/webserver/controller/document/update_document_link.go b/internal/webserver/controller/document/update_review_link.go similarity index 95% rename from internal/webserver/controller/document/update_document_link.go rename to internal/webserver/controller/document/update_review_link.go index 3634bcef..8785f1bd 100644 --- a/internal/webserver/controller/document/update_document_link.go +++ b/internal/webserver/controller/document/update_review_link.go @@ -8,7 +8,7 @@ import ( datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" ) -func (d *Controller) UpdateDocumentLink(c fiber.Ctx) error { +func (d *Controller) UpdateReviewLink(c fiber.Ctx) error { if d.reviewProvider == nil || !d.reviewProvider.CanEditDocumentLink() { return fiber.ErrNotFound } diff --git a/internal/webserver/document_reviews_test.go b/internal/webserver/document_reviews_test.go index db1516f1..9cb7303e 100644 --- a/internal/webserver/document_reviews_test.go +++ b/internal/webserver/document_reviews_test.go @@ -102,6 +102,10 @@ func (f *fakeReviewProvider) Score(_ context.Context, document index.Document) ( return f.score, f.scoreErr } +func (f *fakeReviewProvider) StoredScore(index.Document) (datasourcemodel.Score, error) { + return f.score, f.scoreErr +} + func (f *fakeReviewProvider) Reviews(_ context.Context, document index.Document, language string) (datasourcemodel.ReviewPage, error) { f.reviewSlugs = append(f.reviewSlugs, document.Slug) f.reviewLanguages = append(f.reviewLanguages, language) @@ -167,6 +171,10 @@ func TestDocumentAlternativeReviewProvider(t *testing.T) { searchPage.Find(`#sort-by option[value="review-score-asc"]`).Text() != "Another service score (ascending)" { t.Fatal("search filters and sort options must use the active review provider's name") } + if searchPage.Find(`#list a[href="https://example.org/books/book-1"]`).Length() != 1 || + searchPage.Find(`#list a[href^="https://www.goodreads.com/book/"]`).Length() != 0 { + t.Fatal("cached search-result scores must use the configured provider") + } // This document has a cached Goodreads non-match; it must still use the chosen provider. detail := getBody(t, app, "/documents/unmatched") if documentScorePlaceholders(t, detail).Length() != 1 || len(provider.scoreSlugs) != 0 || len(provider.reviewSlugs) != 0 { diff --git a/internal/webserver/infrastructure/template_engine.go b/internal/webserver/infrastructure/template_engine.go index 55a7b416..afc18664 100644 --- a/internal/webserver/infrastructure/template_engine.go +++ b/internal/webserver/infrastructure/template_engine.go @@ -2,7 +2,6 @@ package infrastructure import ( "fmt" - "github.com/svera/coreander/v5/internal/datasource/biblioreads" "html/template" "io/fs" "net/http" @@ -57,7 +56,6 @@ func TemplateEngine(viewsFS fs.FS, translator i18n.Translator, assetVersion stri }) engine.AddFunc("notLast", notLast[string]) - engine.AddFunc("storedDocumentScore", biblioreads.NewReviewProvider(nil, nil).StoredScore) engine.AddFunc("basename", func(path string) string { return filepath.Base(path) diff --git a/internal/webserver/routes.go b/internal/webserver/routes.go index 4cadc0bc..16012923 100644 --- a/internal/webserver/routes.go +++ b/internal/webserver/routes.go @@ -99,7 +99,7 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se app.Get("/upload", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UploadForm) docsGroup.Post("/", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Upload) docsGroup.Delete("/:slug", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Delete) - docsGroup.Put("/:slug/review-link", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UpdateDocumentLink) + docsGroup.Put("/:slug/review-link", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UpdateReviewLink) // Authentication requirement is configurable for all routes below this middleware app.Use(configurableAuthentication) diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go index 00d79d39..e893f0ee 100644 --- a/internal/webserver/webserver.go +++ b/internal/webserver/webserver.go @@ -135,6 +135,7 @@ func New(cfg Config, controllers Controllers, sender Sender, idx IndexInfo, user if err != nil { log.Fatal(err) } + engine.AddFunc("storedDocumentScore", controllers.Documents.StoredScore) // UploadDocumentMaxSize <= 0 means "unlimited", but fiber has no such // sentinel for BodyLimit - it coerces any value <= 0 to its own 4MB diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index 8fdf5824..00cb3c8d 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -131,7 +131,7 @@ func newSearchResultsTestApp(t *testing.T, documents []index.Document) (*fiber.A sender := &infrastructure.NoEmail{} dataSource := wikidata.NewWikidataSource(wikidata.Gowikidata{}) - controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, dataSource, nil, dataSource) + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, dataSource, nil, dataSource, biblioreads.NewReviewProvider(nil, nil)) app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) t.Cleanup(func() { _ = app.Shutdown() }) return app, idx From 1eceb35d2996b84b5f1b5bdbe63e691da3db83c6 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 12:47:40 +0100 Subject: [PATCH 34/50] Refactor --- README.md | 12 +++--- cli_input.go | 20 ++++----- cli_input_test.go | 72 +++++++++++++++++++++++++++------ internal/webserver/webserver.go | 2 +- main.go | 8 ++-- 5 files changed, 81 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index aa401507..5d4d856f 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,12 @@ Coreander looks up documents in the background after indexing and can also look By default, requests are spread across built-in public BiblioReads instances, with automatic failover when an instance is unavailable. To use your own instance or a different set of public instances, pass a comma-separated list: ```sh -coreander --biblioreads-instances=https://br.example.org,https://br2.example.org /absolute/path/to/library +coreander --review-source-instances=https://br.example.org,https://br2.example.org /absolute/path/to/library ``` -Alternatively, set `BIBLIOREADS_INSTANCES` to the same comma-separated list. The flag may also be repeated, overrides the environment variable, and replaces the built-in instance list. Instance URLs must use HTTP or HTTPS. +Alternatively, set `REVIEW_SOURCE_INSTANCES` to the same comma-separated list. The flag may also be repeated, overrides the environment variable, and replaces the built-in instance list. Instance URLs must use HTTP or HTTPS and serve the BiblioReads API. -Lookups send document titles and author names to the configured BiblioReads instances. To disable the integration, use `--disable-biblioreads` or set `DISABLE_BIBLIOREADS=true`. If no instance can serve a request, Goodreads information may be unavailable, but the rest of Coreander remains usable. +Lookups send document titles and author names to the configured BiblioReads instances. To disable the integration, use `--disable-reviews` or set `DISABLE_REVIEWS=true`. If no instance can serve a request, Goodreads information may be unavailable, but the rest of Coreander remains usable. ### Email @@ -192,9 +192,9 @@ In case both a flag and its equivalent environment variable are passed, flag tak |`--client-dynamic-image-cache-ttl` |`CLIENT_DYNAMIC_IMAGE_CACHE_TTL` | Client-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 86400 (24 hours). |`--server-static-cache-ttl` |`SERVER_STATIC_CACHE_TTL` | Server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 31536000 (1 year). |`--server-dynamic-image-cache-ttl` |`SERVER_DYNAMIC_IMAGE_CACHE_TTL` | Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 86400 (24 hours). -|`--disable-biblioreads` |`DISABLE_BIBLIOREADS` | Disable fetching Goodreads data (book and author links) from BiblioReads. Defaults to false. -|`--biblioreads-instances` |`BIBLIOREADS_INSTANCES` | Comma-separated HTTP or HTTPS BiblioReads instance URLs, replacing the built-in public instances. The flag may be repeated. For example: `--biblioreads-instances=https://br.example.org,https://br2.example.org`. Uses the built-in instances when omitted. -|`--biblioreads-cache-ttl` |`BIBLIOREADS_CACHE_TTL` | Duration in seconds for which BiblioReads responses are cached in memory. Set to 0 to disable caching. Defaults to 86400 (24 hours). +|`--disable-reviews` |`DISABLE_REVIEWS` | Disable fetching reviews, ratings, and book and author links from the review source. Defaults to false. +|`--review-source-instances` |`REVIEW_SOURCE_INSTANCES` | Comma-separated HTTP or HTTPS review-source URLs serving the BiblioReads API, replacing the built-in public instances. The flag may be repeated. For example: `--review-source-instances=https://br.example.org,https://br2.example.org`. Uses the built-in instances when omitted. +|`--review-source-cache-ttl` |`REVIEW_SOURCE_CACHE_TTL` | Duration in seconds for which review-source responses are cached in memory. Set to 0 to disable caching. Defaults to 86400 (24 hours). |`-f` or `--force-indexing` |`FORCE_INDEXING` | Whether to force indexing already indexed documents or not. Defaults to false. |`-s` or `--jwt-secret` |`JWT_SECRET` | String to use to sign JWTs. |`-a` or `--require-auth` |`REQUIRE_AUTH` | Require authentication to access the application if true. Defaults to false. diff --git a/cli_input.go b/cli_input.go index 1f0ed76f..cb4030ef 100644 --- a/cli_input.go +++ b/cli_input.go @@ -83,11 +83,11 @@ type CLIInput struct { ServerStaticCacheTTL int `env:"SERVER_STATIC_CACHE_TTL" default:"31536000" name:"server-static-cache-ttl" help:"Server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 1 year (31536000 seconds)."` // ServerDynamicImageCacheTTL defines the server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours. ServerDynamicImageCacheTTL int `env:"SERVER_DYNAMIC_IMAGE_CACHE_TTL" default:"86400" name:"server-dynamic-image-cache-ttl" help:"Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours (86400 seconds)."` - // DisableBiblioreads turns off fetching Goodreads data from BiblioReads. - DisableBiblioreads bool `env:"DISABLE_BIBLIOREADS" default:"false" name:"disable-biblioreads" help:"Disable fetching Goodreads data (book and author links) from BiblioReads."` - BiblioreadsInstances []string `env:"BIBLIOREADS_INSTANCES" name:"biblioreads-instances" sep:"," help:"Comma-separated BiblioReads instance URLs. Overrides the built-in public instances. May be repeated."` - // BiblioreadsCacheTTL defines how long responses from BiblioReads are cached in memory, in seconds. Defaults to 24 hours. - BiblioreadsCacheTTL int `env:"BIBLIOREADS_CACHE_TTL" default:"86400" name:"biblioreads-cache-ttl" help:"Duration in seconds for which BiblioReads responses are cached in memory. Set to 0 to disable caching. Defaults to 24 hours (86400 seconds)."` + // DisableReviews turns off review-source lookups, including ratings and book and author links. + DisableReviews bool `env:"DISABLE_REVIEWS" default:"false" name:"disable-reviews" help:"Disable fetching reviews, ratings, and book and author links from the review source."` + ReviewSourceInstances []string `env:"REVIEW_SOURCE_INSTANCES" name:"review-source-instances" sep:"," help:"Comma-separated HTTP or HTTPS instance URLs compatible with the review source. Replaces the built-in public instances. May be repeated."` + // ReviewSourceCacheTTL defines how long review-source responses are cached in memory, in seconds. Defaults to 24 hours. + ReviewSourceCacheTTL int `env:"REVIEW_SOURCE_CACHE_TTL" default:"86400" name:"review-source-cache-ttl" help:"Duration in seconds for which review-source responses are cached in memory. Set to 0 to disable caching. Defaults to 24 hours (86400 seconds)."` // ShareCommentMaxSize defines the maximum length for share comments in characters. Defaults to 280. ShareCommentMaxSize int `env:"SHARE_COMMENT_MAX_SIZE" short:"m" default:"280" name:"share-comment-max-size" help:"Maximum length for share comments in characters. Defaults to 280."` // ShareMaxRecipients defines the maximum number of recipients allowed when sharing a document. Defaults to 10. @@ -126,15 +126,15 @@ func (c CLIInput) Validate() error { if c.ShareCommentMaxSize < 0 { return fmt.Errorf("share-comment-max-size must not be negative, got %v", c.ShareCommentMaxSize) } - // BiblioreadsCacheTTL must not be negative: 0 is the documented way to + // ReviewSourceCacheTTL must not be negative: 0 is the documented way to // disable caching, so a negative value is most likely a typo. - if c.BiblioreadsCacheTTL < 0 { - return fmt.Errorf("biblioreads-cache-ttl must not be negative, got %v", c.BiblioreadsCacheTTL) + if c.ReviewSourceCacheTTL < 0 { + return fmt.Errorf("review-source-cache-ttl must not be negative, got %v", c.ReviewSourceCacheTTL) } - for _, instance := range c.BiblioreadsInstances { + for _, instance := range c.ReviewSourceInstances { u, err := url.Parse(instance) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" { - return fmt.Errorf("biblioreads-instances must contain absolute HTTP or HTTPS URLs, got %q", instance) + return fmt.Errorf("review-source-instances must contain absolute HTTP or HTTPS URLs, got %q", instance) } } // MinPasswordLength must be positive: 0 or negative disables the minimum diff --git a/cli_input_test.go b/cli_input_test.go index 0073ec69..796604ea 100644 --- a/cli_input_test.go +++ b/cli_input_test.go @@ -1,6 +1,7 @@ package main import ( + "os" "reflect" "strings" "testing" @@ -8,7 +9,7 @@ import ( "github.com/alecthomas/kong" ) -func TestBiblioreadsInstances(t *testing.T) { +func TestReviewSourceInstances(t *testing.T) { tests := []struct { name string args []string @@ -16,40 +17,87 @@ func TestBiblioreadsInstances(t *testing.T) { want []string }{ {name: "defaults"}, - {name: "single", args: []string{"--biblioreads-instances=https://br.example.org"}, want: []string{"https://br.example.org"}}, - {name: "comma separated", args: []string{"--biblioreads-instances=https://br.example.org,http://localhost:8080/"}, want: []string{"https://br.example.org", "http://localhost:8080/"}}, - {name: "repeated", args: []string{"--biblioreads-instances=https://br.example.org", "--biblioreads-instances=https://br2.example.org"}, want: []string{"https://br.example.org", "https://br2.example.org"}}, + {name: "single", args: []string{"--review-source-instances=https://br.example.org"}, want: []string{"https://br.example.org"}}, + {name: "comma separated", args: []string{"--review-source-instances=https://br.example.org,http://localhost:8080/"}, want: []string{"https://br.example.org", "http://localhost:8080/"}}, + {name: "repeated", args: []string{"--review-source-instances=https://br.example.org", "--review-source-instances=https://br2.example.org"}, want: []string{"https://br.example.org", "https://br2.example.org"}}, {name: "environment", env: "https://br.example.org,https://br2.example.org", want: []string{"https://br.example.org", "https://br2.example.org"}}, - {name: "flag overrides environment", env: "https://env.example.org", args: []string{"--biblioreads-instances=https://cli.example.org"}, want: []string{"https://cli.example.org"}}, + {name: "flag overrides environment", env: "https://env.example.org", args: []string{"--review-source-instances=https://cli.example.org"}, want: []string{"https://cli.example.org"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Setenv("BIBLIOREADS_INSTANCES", tt.env) + t.Setenv("REVIEW_SOURCE_INSTANCES", tt.env) var input CLIInput parser := newCLIParser(t, &input) if _, err := parser.Parse(append(tt.args, t.TempDir())); err != nil { t.Fatal(err) } - if !reflect.DeepEqual(input.BiblioreadsInstances, tt.want) { - t.Errorf("instances = %v, want %v", input.BiblioreadsInstances, tt.want) + if !reflect.DeepEqual(input.ReviewSourceInstances, tt.want) { + t.Errorf("instances = %v, want %v", input.ReviewSourceInstances, tt.want) } }) } } -func TestBiblioreadsInstancesRejectInvalidURLs(t *testing.T) { +func TestReviewSourceInstancesRejectInvalidURLs(t *testing.T) { for _, instance := range []string{"br.example.org", "/relative", "ftp://br.example.org", "https://", "https://bad host", "https://br.example.org,invalid"} { t.Run(instance, func(t *testing.T) { var input CLIInput parser := newCLIParser(t, &input) - _, err := parser.Parse([]string{"--biblioreads-instances=" + instance, t.TempDir()}) - if err == nil || !strings.Contains(err.Error(), "biblioreads-instances") { - t.Fatalf("expected biblioreads-instances validation error, got %v", err) + _, err := parser.Parse([]string{"--review-source-instances=" + instance, t.TempDir()}) + if err == nil || !strings.Contains(err.Error(), "review-source-instances") { + t.Fatalf("expected review-source-instances validation error, got %v", err) } }) } } +func TestReviewSourceSettings(t *testing.T) { + for _, tt := range []struct { + name string + args []string + disabled string + ttl string + wantOff bool + wantTTL int + }{ + {name: "defaults", wantTTL: 86400}, + {name: "flags", args: []string{"--disable-reviews", "--review-source-cache-ttl=120"}, wantOff: true, wantTTL: 120}, + {name: "environment", disabled: "true", ttl: "300", wantOff: true, wantTTL: 300}, + {name: "flags override environment", disabled: "true", ttl: "300", args: []string{"--disable-reviews=false", "--review-source-cache-ttl=0"}}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("DISABLE_REVIEWS", tt.disabled) + t.Setenv("REVIEW_SOURCE_CACHE_TTL", tt.ttl) + for key, value := range map[string]string{ + "DISABLE_REVIEWS": tt.disabled, "REVIEW_SOURCE_CACHE_TTL": tt.ttl, + } { + if value == "" { + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + var input CLIInput + parser := newCLIParser(t, &input) + if _, err := parser.Parse(append(tt.args, t.TempDir())); err != nil { + t.Fatal(err) + } + if input.DisableReviews != tt.wantOff || input.ReviewSourceCacheTTL != tt.wantTTL { + t.Fatalf("disabled/TTL = %v/%d, want %v/%d", input.DisableReviews, input.ReviewSourceCacheTTL, tt.wantOff, tt.wantTTL) + } + }) + } +} + +func TestReviewSourceCacheTTLRejectsNegative(t *testing.T) { + var input CLIInput + parser := newCLIParser(t, &input) + _, err := parser.Parse([]string{"--review-source-cache-ttl=-1", t.TempDir()}) + if err == nil || !strings.Contains(err.Error(), "review-source-cache-ttl") { + t.Fatalf("expected review-source-cache-ttl validation error, got %v", err) + } +} + func newCLIParser(t *testing.T, input *CLIInput) *kong.Kong { t.Helper() parser, err := kong.New(input, kong.Vars{ diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go index e893f0ee..be881c40 100644 --- a/internal/webserver/webserver.go +++ b/internal/webserver/webserver.go @@ -57,7 +57,7 @@ type Config struct { ClientDynamicImageCacheTTL int ServerStaticCacheTTL int ServerDynamicImageCacheTTL int - BiblioreadsCacheTTL int + ReviewSourceCacheTTL int ShareCommentMaxSize int ShareMaxRecipients int IllustratedMinAmount int diff --git a/main.go b/main.go index ff88d64f..6ffc6829 100644 --- a/main.go +++ b/main.go @@ -103,10 +103,10 @@ func main() { documentsIndex, authorsIndex, needsReindex = getIndexes(appFs, input.IllustratedMinSize, input.MinOccurrenceRatio, maxTextRankWords) var documentEnrichers []index.DocumentEnricher biblioReadsAdapter := biblioreads.NewEnricher(nil) - if !input.DisableBiblioreads { + if !input.DisableReviews { biblioreadsClient = biblioreads.NewClient( - biblioreads.WithInstances(input.BiblioreadsInstances), - biblioreads.WithCacheTTL(time.Duration(input.BiblioreadsCacheTTL)*time.Second), + biblioreads.WithInstances(input.ReviewSourceInstances), + biblioreads.WithCacheTTL(time.Duration(input.ReviewSourceCacheTTL)*time.Second), ) biblioReadsAdapter = biblioreads.NewEnricher(biblioreadsClient) documentEnrichers = append(documentEnrichers, biblioReadsAdapter) @@ -167,7 +167,7 @@ func main() { ClientDynamicImageCacheTTL: input.ClientDynamicImageCacheTTL, ServerStaticCacheTTL: input.ServerStaticCacheTTL, ServerDynamicImageCacheTTL: input.ServerDynamicImageCacheTTL, - BiblioreadsCacheTTL: input.BiblioreadsCacheTTL, + ReviewSourceCacheTTL: input.ReviewSourceCacheTTL, ShareCommentMaxSize: input.ShareCommentMaxSize, ShareMaxRecipients: input.ShareMaxRecipients, IllustratedMinAmount: input.IllustratedMinAmount, From 31df2048753867cadd77dd8c265cb436e2fc9ca2 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 12:52:07 +0100 Subject: [PATCH 35/50] Refactor --- .../webserver/document_biblioreads_test.go | 263 +----------------- internal/webserver/document_reviews_test.go | 253 ++++++++++++++++- 2 files changed, 246 insertions(+), 270 deletions(-) diff --git a/internal/webserver/document_biblioreads_test.go b/internal/webserver/document_biblioreads_test.go index 77a1cf15..f98b3c3a 100644 --- a/internal/webserver/document_biblioreads_test.go +++ b/internal/webserver/document_biblioreads_test.go @@ -169,10 +169,7 @@ func TestDocumentDetailBiblioReadsRating(t *testing.T) { t.Error("expected the placeholder inside the document metadata list") } - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, "/documents/linked/score") link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) if page.Find("li i.bi-star-fill").Length() != 1 { t.Error("expected the rating row to retain its star icon") @@ -198,72 +195,6 @@ func TestDocumentDetailBiblioReadsRating(t *testing.T) { } } -func TestDocumentBiblioReadsEditor(t *testing.T) { - source := &fakeBookSource{book: biblioreads.BookResponse{Rating: "3.82"}} - app, _ := newBiblioReadsTestApp(t, source) - adminCookie, err := login(app, "admin@example.com", "admin", t) - if err != nil { - t.Fatal(err) - } - for _, tt := range []struct { - name string - cookie *http.Cookie - want int - }{ - {name: "anonymous", cookie: &http.Cookie{}}, - {name: "administrator", cookie: adminCookie, want: 1}, - } { - t.Run(tt.name, func(t *testing.T) { - response, err := getRequest(tt.cookie, app, "/documents/linked/score", t) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - mustReturnStatus(response, http.StatusOK, t) - page, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - t.Fatal(err) - } - control := page.Find(`li a[data-bs-toggle="collapse"][href="#edit-review-link-linked"]`) - if control.Length() != tt.want { - t.Fatalf("expected %d edit links, got %d", tt.want, control.Length()) - } - if tt.want == 0 { - if page.Find(`input[name="bookURL"]`).Length() != 0 { - t.Fatal("anonymous readers must not receive the editor") - } - return - } - if control.Text() != "" || - control.Find(`i.bi-pencil[alt="Edit Goodreads link"]`).Length() != 1 || - control.Filter(`[title="Edit Goodreads link"][aria-label="Edit Goodreads link"]`).Length() != 1 || - page.Find(`#edit-review-link-linked.collapse:not(.show)`).Length() != 1 || - page.Find(`form[hx-put="/documents/linked/review-link"][hx-target="this"]`).Length() != 1 || - page.Find(`label[for="document-book-url-linked"]`).Length() != 1 { - t.Fatal("expected accessible, initially collapsed Goodreads editor beside the score") - } - if value, _ := page.Find(`input[name="bookURL"]`).Attr("value"); value != "https://www.goodreads.com/book/show/12207444-la-reina-dragon" { - t.Fatalf("editor must be prefilled with the current link, got %q", value) - } - }) - } - - source.err = biblioreads.ErrUnavailable - response, err := getRequest(adminCookie, app, "/documents/linked/score", t) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - page, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(page.Find("li").Text(), "Unable to load Goodreads score") || - page.Find(`input[name="bookURL"]`).Length() != 1 { - t.Fatal("an unavailable score must not prevent administrators from correcting the link") - } -} - func TestDocumentUpdateBiblioReads(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ Rating: "3.82", ResourceID: "old-work", LegacyBookID: 12207444, @@ -352,10 +283,7 @@ func TestDocumentDetailUpdatesStoredBiblioReadsRating(t *testing.T) { }} app, idx := newBiblioReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, "/documents/linked/score") link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) if got := strings.TrimSpace(link.Text()); got != "4.12" { t.Fatalf("expected latest Goodreads rating 4.12, got %q", got) @@ -379,10 +307,7 @@ func TestDocumentDetailKeepsStoredBiblioReadsRatingWhenScraperReturnsZero(t *tes }} app, idx := newBiblioReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, "/documents/linked/score") link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) if got := strings.TrimSpace(link.Text()); got != "3.82" { t.Fatalf("expected known Goodreads rating to be retained, got %q", got) @@ -407,10 +332,7 @@ func TestDocumentDetailBiblioReadsSearchesUnlinkedDocuments(t *testing.T) { t.Fatalf("expected a placeholder for a document not looked up yet, got %d", got) } - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unlinked/score"))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, "/documents/unlinked/score") if got := strings.TrimSpace(page.Find(`li a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.10" { t.Errorf("expected rating link with 4.10, got %q", got) } @@ -442,10 +364,7 @@ func TestDocumentDetailBiblioReadsSearchesUnlinkedDocuments(t *testing.T) { if err := idx.EnrichDocuments(10); err != nil { t.Fatal(err) } - page, err = goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unlinked/score"))) - if err != nil { - t.Fatal(err) - } + page = getPage(t, app, "/documents/unlinked/score") if got := strings.TrimSpace(page.Find(`li a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.20" { t.Errorf("expected later request to use book-scraper rating, got %q", got) } @@ -464,10 +383,7 @@ func TestDocumentDetailBiblioReadsSkipsUnmatchedDocuments(t *testing.T) { t.Errorf("expected an async score placeholder, got %d", got) } for _, path := range []string{"/documents/unmatched/score"} { - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, path) score := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { return item.Find("i.bi-star-fill").Length() == 1 }) @@ -480,26 +396,10 @@ func TestDocumentDetailBiblioReadsSkipsUnmatchedDocuments(t *testing.T) { } } -func TestDocumentDetailBiblioReadsRatingRendersNothingWhenUnavailable(t *testing.T) { - for name, source := range map[string]*fakeBookSource{ - "error": {err: errors.New("unavailable")}, - } { - t.Run(name, func(t *testing.T) { - app, _ := newBiblioReadsTestApp(t, source) - if body := strings.TrimSpace(getBody(t, app, "/documents/linked/score")); body != "" { - t.Errorf("expected empty response, got %q", body) - } - }) - } -} - func TestDocumentDetailBiblioReadsIdentifiersWithoutRating(t *testing.T) { source := &fakeBookSource{book: biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}} app, idx := newBiblioReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, "/documents/linked/score") if strings.TrimSpace(page.Find("li").Text()) != "Not scored" || page.Find("h2").Text() != "Goodreads user reviews" { t.Fatal("expected no score and a reviews heading for a matched book without a rating") } @@ -528,24 +428,6 @@ func TestDocumentDetailBiblioReadsIdentifiersWithoutRating(t *testing.T) { } } -func TestDocumentDetailBiblioReadsDisabled(t *testing.T) { - app, _ := newBiblioReadsTestApp(t, nil) - for _, slug := range []string{"linked", "unlinked", "unmatched"} { - if got := documentScorePlaceholders(t, getBody(t, app, "/documents/"+slug)).Length(); got != 0 { - t.Errorf("%s: expected no placeholder when BiblioReads is disabled, got %d", slug, got) - } - if body := strings.TrimSpace(getBody(t, app, "/documents/"+slug+"/score")); body != "" { - t.Errorf("%s: expected empty response, got %q", slug, body) - } - if body := getBody(t, app, "/documents/"+slug); strings.Contains(body, "bi-star-fill") { - t.Errorf("%s: expected no score row when integration is disabled", slug) - } - if body := getBody(t, app, "/documents/"+slug); strings.Contains(body, `id="document-reviews-`) { - t.Errorf("%s: expected no reviews target when integration is disabled", slug) - } - } -} - func TestDocumentDetailBiblioReadsSearchDoesNotScrape(t *testing.T) { for _, tt := range []struct { name string @@ -601,132 +483,6 @@ func TestDocumentDetailBiblioReadsSearchDoesNotScrape(t *testing.T) { } } -func TestDocumentDetailBiblioReadsNoScoreTranslations(t *testing.T) { - source := &fakeBookSource{} - app, _ := newBiblioReadsTestApp(t, source) - for _, locale := range []struct { - lang string - text string - }{ - {"en", "Not scored"}, - {"es", "Sin puntuar"}, - {"de", "Nicht bewertet"}, - {"fr", "Non noté"}, - {"ru", "Не оценено"}, - } { - for _, path := range []string{"/documents/unmatched/score"} { - t.Run(locale.lang+path, func(t *testing.T) { - req := mustGetRequest(t, path) - req.Header.Set("Accept-Language", locale.lang) - response, err := app.Test(req) - if err != nil { - t.Fatal(err) - } - - defer response.Body.Close() - mustReturnStatus(response, http.StatusOK, t) - page, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - t.Fatal(err) - } - score := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { - return item.Find("i.bi-star-fill").Length() == 1 - }) - if strings.TrimSpace(score.Text()) != locale.text || score.Find("a").Length() != 0 { - t.Fatalf("expected localized no-score row %q, got %q", locale.text, score.Text()) - } - }) - } - } - if len(source.queries) != 0 || len(source.ids) != 0 { - t.Fatal("cached unmatched documents must not trigger Goodreads requests") - } -} - -func TestDocumentDetailBiblioReadsReviewsLazyPlaceholder(t *testing.T) { - source := &fakeBookSource{book: biblioreads.BookResponse{ - Rating: "3.82", ResourceID: "work-id", LegacyBookID: 12207444, - }} - app, _ := newBiblioReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked"))) - if err != nil { - t.Fatal(err) - } - target := page.Find("#document-reviews-linked") - if target.Length() != 1 || target.Is("[hx-get]") { - t.Fatal("expected an inactive reviews target before the rating loads") - } - if target.Find("h2").Text() != "Goodreads user reviews" { - t.Fatal("expected the reviews heading before the score loads") - } - if !target.Is(".document-detail-content > :last-child") { - t.Fatal("reviews must appear at the bottom of the document content") - } - page, err = goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/score"))) - if err != nil { - t.Fatal(err) - } - placeholder := page.Find(`#document-reviews-linked[hx-get="/documents/linked/reviews"][hx-trigger="intersect once"][hx-swap-oob="outerHTML"]`) - if placeholder.Length() != 1 { - t.Fatal("expected successful rating to enable reviews on intersection only") - } - if placeholder.Find("h2").Text() != "Goodreads user reviews" || placeholder.Find(`[aria-hidden="true"] h2`).Length() != 0 { - t.Fatal("the reviews heading must remain visible and accessible while reviews load") - } - skeletons := placeholder.Find(".border-top") - if skeletons.Length() != 2 || placeholder.Find(`div[aria-hidden="true"]`).Length() != 1 { - t.Fatal("expected two decorative review-shaped skeletons") - } - skeletons.Each(func(i int, skeleton *goquery.Selection) { - if skeleton.Find(".fs-5 .placeholder").Length() != 1 || - skeleton.Find("i.bi-star").Length() != 5 || - skeleton.Find(".placeholder.ms-auto").Length() != 1 || - skeleton.ChildrenFiltered("div").Length() != 6 { - t.Errorf("skeleton %d must include a name, five stars, a date and five text lines", i) - } - }) - if len(source.reviewCalls) != 0 { - t.Fatal("page and rating requests must not fetch reviews") - } -} - -func TestDocumentDetailBiblioReadsReviewsHeader(t *testing.T) { - for _, tt := range []struct { - name string - source *fakeBookSource - slug string - want int - }{ - {name: "matched", source: &fakeBookSource{}, slug: "linked", want: 1}, - {name: "zero score", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "0.00"}}, slug: "linked", want: 1}, - {name: "unavailable", source: &fakeBookSource{err: biblioreads.ErrUnavailable}, slug: "linked", want: 1}, - {name: "disabled", slug: "linked"}, - {name: "unmatched", source: &fakeBookSource{}, slug: "unmatched"}, - {name: "not looked up", source: &fakeBookSource{}, slug: "unlinked"}, - } { - t.Run(tt.name, func(t *testing.T) { - app, _ := newBiblioReadsTestApp(t, tt.source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/"+tt.slug))) - if err != nil { - t.Fatal(err) - } - header := page.Find(`#document-reviews-` + tt.slug + ` h2 a`) - if header.Length() != tt.want { - t.Fatalf("expected %d reviews headings, got %d", tt.want, header.Length()) - } - if tt.want == 1 { - url, _ := header.Attr("href") - if header.Text() != "Goodreads user reviews" || url != "https://www.goodreads.com/book/show/12207444-la-reina-dragon" { - t.Fatal("expected a linked Goodreads reviews heading") - } - } - if tt.source != nil && (len(tt.source.ids) != 0 || len(tt.source.reviewCalls) != 0 || len(tt.source.queries) != 0) { - t.Fatal("rendering the heading must not make any API calls") - } - }) - } -} - func TestDocumentDetailBiblioReadsReviews(t *testing.T) { source := &fakeBookSource{ book: biblioreads.BookResponse{Rating: "4.20", ResourceID: "work-id", LegacyBookID: 12207444}, @@ -738,10 +494,7 @@ func TestDocumentDetailBiblioReadsReviews(t *testing.T) { }}, } app, idx := newBiblioReadsTestApp(t, source) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews"))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, "/documents/linked/reviews") if len(source.reviewCalls) != 1 || source.reviewCalls[0] != (reviewCall{"work-id", 12207444, "", "en"}) { t.Fatalf("unexpected review calls: %+v", source.reviewCalls) } diff --git a/internal/webserver/document_reviews_test.go b/internal/webserver/document_reviews_test.go index 9cb7303e..8f7f0c59 100644 --- a/internal/webserver/document_reviews_test.go +++ b/internal/webserver/document_reviews_test.go @@ -2,6 +2,7 @@ package webserver_test import ( "context" + "errors" "fmt" "io" "net/http" @@ -83,6 +84,7 @@ type fakeReviewProvider struct { scoreSlugs []string reviewSlugs []string reviewLanguages []string + documentURL string } func (f *fakeReviewProvider) CanEditDocumentLink() bool { @@ -93,6 +95,10 @@ func (f *fakeReviewProvider) Name() string { return f.score.Source } +func (f *fakeReviewProvider) DocumentLink(index.Document) datasourcemodel.DataSourceLink { + return datasourcemodel.DataSourceLink{Source: f.Name(), URL: f.documentURL} +} + func (f *fakeReviewProvider) UpdateDocumentLink(index.Document, string) error { return nil } @@ -163,10 +169,7 @@ func TestDocumentAlternativeReviewProvider(t *testing.T) { }, } app, _ := newReviewProviderTestApp(t, provider) - searchPage, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) - if err != nil { - t.Fatal(err) - } + searchPage := getPage(t, app, "/search?type=documents&search=La") if !strings.Contains(searchPage.Find(`fieldset legend`).Text(), "Another service score") || searchPage.Find(`#sort-by option[value="review-score-asc"]`).Text() != "Another service score (ascending)" { t.Fatal("search filters and sort options must use the active review provider's name") @@ -228,10 +231,7 @@ func TestDocumentScoreMarksZeroWithoutValuationsAsNotScored(t *testing.T) { }, } app, _ := newReviewProviderTestApp(t, provider) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/unmatched/score"))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, "/documents/unmatched/score") row := page.Find("li") if tt.wantScored { if got := strings.TrimSpace(row.Find("a").Text()); got != tt.wantScore { @@ -270,6 +270,15 @@ func getBody(t *testing.T, app *fiber.App, path string) string { return string(body) } +func getPage(t *testing.T, app *fiber.App, path string) *goquery.Document { + t.Helper() + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) + if err != nil { + t.Fatal(err) + } + return page +} + func documentScorePlaceholders(t *testing.T, html string) *goquery.Selection { t.Helper() page, err := goquery.NewDocumentFromReader(strings.NewReader(html)) @@ -302,10 +311,7 @@ func TestDocumentReviewFractionalStars(t *testing.T) { }}, } app, _ := newReviewProviderTestApp(t, provider) - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews"))) - if err != nil { - t.Fatal(err) - } + page := getPage(t, app, "/documents/linked/reviews") stars := page.Find(`article span[role="img"]`) if tt.rating == 0 { if stars.Length() != 0 { @@ -359,13 +365,230 @@ func TestDocumentReviewNotScoredTranslations(t *testing.T) { {"ru", "Не оценено"}, } { t.Run(tt.lang, func(t *testing.T) { - page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/documents/linked/reviews?l="+tt.lang))) + page := getPage(t, app, "/documents/linked/reviews?l="+tt.lang) + if got := page.Find("article header span.text-muted").Text(); got != tt.text { + t.Fatalf("expected %q, got %q", tt.text, got) + } + }) + } +} + +func TestDocumentScoreNoScoreTranslations(t *testing.T) { + provider := &fakeReviewProvider{score: datasourcemodel.Score{Source: "Another service"}} + app, _ := newReviewProviderTestApp(t, provider) + for _, locale := range []struct { + lang string + text string + }{ + {"en", "Not scored"}, + {"es", "Sin puntuar"}, + {"de", "Nicht bewertet"}, + {"fr", "Non noté"}, + {"ru", "Не оценено"}, + } { + for _, path := range []string{"/documents/unmatched/score"} { + t.Run(locale.lang+path, func(t *testing.T) { + req := mustGetRequest(t, path) + req.Header.Set("Accept-Language", locale.lang) + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + score := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { + return item.Find("i.bi-star-fill").Length() == 1 + }) + if strings.TrimSpace(score.Text()) != locale.text || score.Find("a").Length() != 0 { + t.Fatalf("expected localized no-score row %q, got %q", locale.text, score.Text()) + } + }) + } + } + if len(provider.reviewSlugs) != 0 { + t.Fatal("score requests must not fetch reviews") + } +} + +func TestDocumentReviewsLazyPlaceholder(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service", Value: "3.82", URL: "https://example.org/books/book-1"}, + documentURL: "https://example.org/books/book-1", + } + app, _ := newReviewProviderTestApp(t, provider) + page := getPage(t, app, "/documents/linked") + target := page.Find("#document-reviews-linked") + if target.Length() != 1 || target.Is("[hx-get]") { + t.Fatal("expected an inactive reviews target before the rating loads") + } + if target.Find("h2").Text() != "Another service user reviews" { + t.Fatal("expected the reviews heading before the score loads") + } + if !target.Is(".document-detail-content > :last-child") { + t.Fatal("reviews must appear at the bottom of the document content") + } + page = getPage(t, app, "/documents/linked/score") + placeholder := page.Find(`#document-reviews-linked[hx-get="/documents/linked/reviews"][hx-trigger="intersect once"][hx-swap-oob="outerHTML"]`) + if placeholder.Length() != 1 { + t.Fatal("expected successful rating to enable reviews on intersection only") + } + if placeholder.Find("h2").Text() != "Another service user reviews" || placeholder.Find(`[aria-hidden="true"] h2`).Length() != 0 { + t.Fatal("the reviews heading must remain visible and accessible while reviews load") + } + skeletons := placeholder.Find(".border-top") + if skeletons.Length() != 2 || placeholder.Find(`div[aria-hidden="true"]`).Length() != 1 { + t.Fatal("expected two decorative review-shaped skeletons") + } + skeletons.Each(func(i int, skeleton *goquery.Selection) { + if skeleton.Find(".fs-5 .placeholder").Length() != 1 || + skeleton.Find("i.bi-star").Length() != 5 || + skeleton.Find(".placeholder.ms-auto").Length() != 1 || + skeleton.ChildrenFiltered("div").Length() != 6 { + t.Errorf("skeleton %d must include a name, five stars, a date and five text lines", i) + } + }) + if len(provider.reviewSlugs) != 0 { + t.Fatal("page and rating requests must not fetch reviews") + } +} + +func TestDocumentReviewsHeader(t *testing.T) { + for _, tt := range []struct { + name string + provider *fakeReviewProvider + slug string + want int + }{ + {name: "matched", provider: &fakeReviewProvider{documentURL: "https://example.org/books/book-1"}, slug: "linked", want: 1}, + {name: "zero score", provider: &fakeReviewProvider{score: datasourcemodel.Score{Value: "0.00"}, documentURL: "https://example.org/books/book-1"}, slug: "linked", want: 1}, + {name: "unavailable", provider: &fakeReviewProvider{scoreErr: errors.New("unavailable"), documentURL: "https://example.org/books/book-1"}, slug: "linked", want: 1}, + {name: "disabled", slug: "linked"}, + {name: "unmatched", provider: &fakeReviewProvider{}, slug: "unmatched"}, + {name: "not looked up", provider: &fakeReviewProvider{}, slug: "unlinked"}, + } { + t.Run(tt.name, func(t *testing.T) { + var provider document.ReviewProvider + if tt.provider != nil { + tt.provider.score.Source = "Another service" + provider = tt.provider + } + app, _ := newReviewProviderTestApp(t, provider) + page := getPage(t, app, "/documents/"+tt.slug) + header := page.Find(`#document-reviews-` + tt.slug + ` h2 a`) + if header.Length() != tt.want { + t.Fatalf("expected %d reviews headings, got %d", tt.want, header.Length()) + } + if tt.want == 1 { + url, _ := header.Attr("href") + if header.Text() != "Another service user reviews" || url != "https://example.org/books/book-1" { + t.Fatal("expected a linked provider reviews heading") + } + } + if tt.provider != nil && (len(tt.provider.scoreSlugs) != 0 || len(tt.provider.reviewSlugs) != 0) { + t.Fatal("rendering the heading must not make any API calls") + } + }) + } +} + +func TestDocumentReviewsDisabled(t *testing.T) { + app, _ := newReviewProviderTestApp(t, nil) + for _, slug := range []string{"linked", "unlinked", "unmatched"} { + body := getBody(t, app, "/documents/"+slug) + if got := documentScorePlaceholders(t, body).Length(); got != 0 { + t.Errorf("%s: expected no placeholder when reviews are disabled, got %d", slug, got) + } + if body := strings.TrimSpace(getBody(t, app, "/documents/"+slug+"/score")); body != "" { + t.Errorf("%s: expected empty response, got %q", slug, body) + } + if strings.Contains(body, "bi-star-fill") { + t.Errorf("%s: expected no score row when integration is disabled", slug) + } + if strings.Contains(body, `id="document-reviews-`) { + t.Errorf("%s: expected no reviews target when integration is disabled", slug) + } + } +} + +func TestDocumentScoreRendersNothingWhenUnavailable(t *testing.T) { + provider := &fakeReviewProvider{scoreErr: errors.New("unavailable")} + app, _ := newReviewProviderTestApp(t, provider) + if body := strings.TrimSpace(getBody(t, app, "/documents/linked/score")); body != "" { + t.Errorf("expected empty response, got %q", body) + } +} + +func TestDocumentReviewLinkEditor(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service", Value: "3.82", URL: "https://example.org/books/book-1"}, + documentURL: "https://example.org/books/book-1", + canEditDocumentLink: true, + } + app, _ := newReviewProviderTestApp(t, provider) + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + cookie *http.Cookie + want int + }{ + {name: "anonymous", cookie: &http.Cookie{}}, + {name: "administrator", cookie: adminCookie, want: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + response, err := getRequest(tt.cookie, app, "/documents/linked/score", t) if err != nil { t.Fatal(err) } - if got := page.Find("article header span.text-muted").Text(); got != tt.text { - t.Fatalf("expected %q, got %q", tt.text, got) + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + control := page.Find(`li a[data-bs-toggle="collapse"][href="#edit-review-link-linked"]`) + if control.Length() != tt.want { + t.Fatalf("expected %d edit links, got %d", tt.want, control.Length()) + } + if tt.want == 0 { + if page.Find(`input[name="bookURL"]`).Length() != 0 { + t.Fatal("anonymous readers must not receive the editor") + } + return + } + if control.Text() != "" || + control.Find(`i.bi-pencil[alt="Edit Another service link"]`).Length() != 1 || + control.Filter(`[title="Edit Another service link"][aria-label="Edit Another service link"]`).Length() != 1 || + page.Find(`#edit-review-link-linked.collapse:not(.show)`).Length() != 1 || + page.Find(`form[hx-put="/documents/linked/review-link"][hx-target="this"]`).Length() != 1 || + page.Find(`label[for="document-book-url-linked"]`).Length() != 1 { + t.Fatal("expected accessible, initially collapsed Another service editor beside the score") + } + if value, _ := page.Find(`input[name="bookURL"]`).Attr("value"); value != "https://example.org/books/book-1" { + t.Fatalf("editor must be prefilled with the current link, got %q", value) } }) } + + provider.scoreErr = errors.New("unavailable") + response, err := getRequest(adminCookie, app, "/documents/linked/score", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(page.Find("li").Text(), "Unable to load Another service score") || + page.Find(`input[name="bookURL"]`).Length() != 1 { + t.Fatal("an unavailable score must not prevent administrators from correcting the link") + } } From ddf41cd3e13b8f4f964a9dd8f1e0dc577ad32da8 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Nov 2026 15:33:24 +0100 Subject: [PATCH 36/50] Added progress bar for biblioreads --- internal/datasource/biblioreads/adapter.go | 2 + internal/datasource/biblioreads/document.go | 3 + internal/datasource/biblioreads/reviews.go | 8 +- internal/index/bleve.go | 13 ++ internal/index/bleve_document_enrich.go | 22 +++- internal/index/bleve_document_enrich_test.go | 111 +++++++++++++++++- internal/index/bleve_document_write.go | 2 +- internal/index/progress.go | 66 ++++++++++- .../webserver/document_biblioreads_test.go | 51 ++++++++ internal/webserver/document_reviews_test.go | 3 + .../webserver/embedded/translations/de.yml | 5 +- .../webserver/embedded/translations/es.yml | 5 +- .../webserver/embedded/translations/fr.yml | 5 +- .../webserver/embedded/translations/ru.yml | 5 +- internal/webserver/embedded/views/layout.html | 2 +- .../views/partials/background-progress.html | 35 ++++++ .../embedded/views/partials/main.html | 25 +--- internal/webserver/middleware.go | 10 +- internal/webserver/routes.go | 5 + internal/webserver/webserver.go | 5 + 20 files changed, 336 insertions(+), 47 deletions(-) create mode 100644 internal/webserver/embedded/views/partials/background-progress.html diff --git a/internal/datasource/biblioreads/adapter.go b/internal/datasource/biblioreads/adapter.go index af26703d..37639c75 100644 --- a/internal/datasource/biblioreads/adapter.go +++ b/internal/datasource/biblioreads/adapter.go @@ -38,6 +38,8 @@ func (p *Enricher) SetIndex(idx *index.BleveIndexer) { func (*Enricher) Source() string { return Source } +func (*Enricher) Name() string { return Name } + func (*Enricher) NeedsEnrichment(document index.Document) (bool, error) { data, err := index.DataSourceData[DocumentData](document, Source) return !data.Enriched, err diff --git a/internal/datasource/biblioreads/document.go b/internal/datasource/biblioreads/document.go index 23fce34f..ceeee0d8 100644 --- a/internal/datasource/biblioreads/document.go +++ b/internal/datasource/biblioreads/document.go @@ -3,6 +3,9 @@ package biblioreads // Source retains the persisted key for Goodreads data fetched through BiblioReads. const Source = "goodreads" +// Name is the user-facing name of the data source. +const Name = "Goodreads" + // DocumentData is the persisted Goodreads state used by BiblioReads. type DocumentData struct { URL string `json:"url,omitempty"` diff --git a/internal/datasource/biblioreads/reviews.go b/internal/datasource/biblioreads/reviews.go index dd9bbfad..16f79770 100644 --- a/internal/datasource/biblioreads/reviews.go +++ b/internal/datasource/biblioreads/reviews.go @@ -38,7 +38,7 @@ func NewReviewProvider(client ReviewClient, idx ReviewIndex) *ReviewProvider { } func (b *ReviewProvider) Name() string { - return "Goodreads" + return Name } // AuthorLink returns the indexed author link without making a network request. @@ -57,7 +57,7 @@ func (b *ReviewProvider) DocumentLink(document index.Document) datasourcemodel.D } func goodreadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink { - link := datasourcemodel.DataSourceLink{Source: "Goodreads", Icon: "/images/goodreads.svg", IconInvertible: true} + link := datasourcemodel.DataSourceLink{Source: Name, Icon: "/images/goodreads.svg", IconInvertible: true} if id, ok := strings.CutPrefix(path, prefix); ok && id != "" { link.URL = goodreadsBaseURL + path } @@ -145,7 +145,7 @@ func isZeroRating(rating string) bool { } func (b *ReviewProvider) Score(ctx context.Context, document index.Document) (datasourcemodel.Score, error) { - score := datasourcemodel.Score{Source: "Goodreads"} + score := datasourcemodel.Score{Source: Name} data, err := index.DataSourceData[DocumentData](document, Source) if err != nil { return score, err @@ -202,7 +202,7 @@ func (b *ReviewProvider) Reviews(ctx context.Context, document index.Document, l if err != nil { return datasourcemodel.ReviewPage{}, fmt.Errorf("retrieving Goodreads reviews for %s: %w", document.ID, err) } - page := datasourcemodel.ReviewPage{Source: "Goodreads", URL: b.DocumentLink(document).URL, Available: true} + page := datasourcemodel.ReviewPage{Source: Name, URL: b.DocumentLink(document).URL, Available: true} for _, review := range response.Reviews { var createdAt time.Time if review.CreatedAt != "" { diff --git a/internal/index/bleve.go b/internal/index/bleve.go index bccc04d6..1b357476 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -219,6 +219,8 @@ type BleveIndexer struct { textRankEnrichJobs chan Document documentEnrichJobs chan string documentEnrichers []DocumentEnricher + // documentEnrichProgress measures each provider's current run rate, by enricher index. + documentEnrichProgress []progressTracker // fileLocks holds a *sync.Mutex per document ID, serializing concurrent // indexFile calls for the same file (upload vs. file watcher reacting to // the same write). @@ -265,6 +267,16 @@ func (p *progressTracker) recordN(n int) { p.processed.Add(uint64(n)) } +// remainingTime estimates how long the given remaining entries take at the current run's rate. +func (p *progressTracker) remainingTime(remaining uint64) time.Duration { + start, processed := p.startNanos.Load(), p.processed.Load() + if start == 0 || processed == 0 || remaining == 0 { + return 0 + } + elapsed := float64(time.Now().UnixNano() - start) + return time.Duration(elapsed * float64(remaining) / float64(processed)) +} + // NewBleve creates a new BleveIndexer instance using the passed parameters func NewBleve(documentsIndex bleve.Index, authorsIndex bleve.Index, fs afero.Fs, libraryPath string, read map[string]metadata.Reader, cfg Config) *BleveIndexer { maxSimilarityCandidates := cfg.MaxSimilarityCandidates @@ -324,6 +336,7 @@ func NewBleve(documentsIndex bleve.Index, authorsIndex bleve.Index, fs afero.Fs, pruneChangeTriggerRatio: pruneChangeTriggerRatio, textRankEnrichJobs: make(chan Document, textRankEnrichWorkers), documentEnrichers: slices.Clone(cfg.DocumentEnrichers), + documentEnrichProgress: make([]progressTracker, len(cfg.DocumentEnrichers)), } for range textRankEnrichWorkers { diff --git a/internal/index/bleve_document_enrich.go b/internal/index/bleve_document_enrich.go index 4661415f..5780c204 100644 --- a/internal/index/bleve_document_enrich.go +++ b/internal/index/bleve_document_enrich.go @@ -14,7 +14,9 @@ const documentEnrichWorkers = 4 // DocumentEnricher owns a provider's lookup policy and persisted payload. type DocumentEnricher interface { + // Source is the persisted storage key; Name is shown to users. Source() string + Name() string NeedsEnrichment(Document) (bool, error) // Enrich persists an enriched=true payload for completed lookups, including no matches. // Transient failures must leave the document pending. @@ -49,19 +51,28 @@ func (b *BleveIndexer) EnrichDocuments(batchSize int) error { return fmt.Errorf("document enrichment batch size must be positive") } var failures []error - for _, provider := range b.documentEnrichers { - if err := b.enrichProvider(provider, batchSize); err != nil { + for i, provider := range b.documentEnrichers { + if err := b.enrichProvider(provider, b.documentEnrichTracker(i), batchSize); err != nil { failures = append(failures, fmt.Errorf("enriching documents with %s: %w", provider.Source(), err)) } } return errors.Join(failures...) } -func (b *BleveIndexer) enrichProvider(provider DocumentEnricher, batchSize int) error { +func (b *BleveIndexer) documentEnrichTracker(i int) *progressTracker { + if i < len(b.documentEnrichProgress) { + return &b.documentEnrichProgress[i] + } + return &progressTracker{} +} + +func (b *BleveIndexer) enrichProvider(provider DocumentEnricher, tracker *progressTracker, batchSize int) error { _, total, err := b.documentsNeedingProvider(provider.Source(), 0) if err != nil || total == 0 { return err } + tracker.begin(int(total)) + defer tracker.end() log.Printf("Enriching %d documents with %s data", total, provider.Source()) for iteration := 0; iteration < int(total)+10; iteration++ { documents, _, err := b.documentsNeedingProvider(provider.Source(), batchSize) @@ -71,14 +82,14 @@ func (b *BleveIndexer) enrichProvider(provider DocumentEnricher, batchSize int) if len(documents) == 0 { return nil } - if err := b.enrichProviderBatch(provider, documents); err != nil { + if err := b.enrichProviderBatch(provider, tracker, documents); err != nil { return err } } return fmt.Errorf("document enrichment did not finish; some documents remain pending") } -func (b *BleveIndexer) enrichProviderBatch(provider DocumentEnricher, documents []Document) error { +func (b *BleveIndexer) enrichProviderBatch(provider DocumentEnricher, tracker *progressTracker, documents []Document) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() jobs := make(chan Document, len(documents)) @@ -103,6 +114,7 @@ func (b *BleveIndexer) enrichProviderBatch(provider DocumentEnricher, documents } return } + tracker.record() } }) } diff --git a/internal/index/bleve_document_enrich_test.go b/internal/index/bleve_document_enrich_test.go index 409c6717..04fe16e8 100644 --- a/internal/index/bleve_document_enrich_test.go +++ b/internal/index/bleve_document_enrich_test.go @@ -3,6 +3,7 @@ package index import ( "context" "errors" + "fmt" "sync/atomic" "testing" "time" @@ -23,6 +24,8 @@ type blockingDocumentEnricher struct { func (*blockingDocumentEnricher) Source() string { return "blocking" } +func (*blockingDocumentEnricher) Name() string { return "Blocking" } + func (*blockingDocumentEnricher) NeedsEnrichment(Document) (bool, error) { return true, nil } @@ -46,7 +49,7 @@ func TestDocumentEnrichmentUsesBoundedConcurrency(t *testing.T) { idx, _ := newTestTextRankIndexer(t, Config{}) documents := make([]Document, documentCount) done := make(chan error, 1) - go func() { done <- idx.enrichProviderBatch(provider, documents) }() + go func() { done <- idx.enrichProviderBatch(provider, &progressTracker{}, documents) }() defer func() { close(provider.release) if err := <-done; err != nil { @@ -67,8 +70,114 @@ func TestDocumentEnrichmentUsesBoundedConcurrency(t *testing.T) { } } +func TestBackgroundProgressTracksPersistedProviderLookups(t *testing.T) { + first := &testDocumentEnricher{source: "first", failure: errors.New("unavailable")} + second := &testDocumentEnricher{source: "second"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{first, second}}) + t.Cleanup(func() { _ = idx.Close() }) + for _, id := range []string{"matched", "unmatched", "pending"} { + if err := IndexDocument(idx.documentsIdx, id, Document{ID: id}); err != nil { + t.Fatal(err) + } + } + // A completed lookup without a score is still processed. + for _, id := range []string{"matched", "unmatched"} { + if err := idx.StoreDocumentData(id, "first", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + } + idx.indexProgress.begin(3) + idx.indexProgress.record() + idx.textRankEnrichProgress.begin(3) + idx.textRankEnrichProgress.record() + idx.authorEnrichProgress.begin(3) + idx.authorEnrichProgress.record() + idx.pruneProgress.begin(3) + idx.pruneProgress.record() + progress, err := idx.BackgroundProgress() + if err != nil { + t.Fatal(err) + } + if len(progress) != 6 || progress[0].Kind != ProgressDocuments || progress[1].Kind != ProgressAuthors || + progress[2].Kind != ProgressTextRank || progress[3].Kind != ProgressPruning || + progress[4].Source != "first" || progress[4].Processed != 2 || progress[4].Total != 3 || progress[4].Percentage != 66 || + progress[5].Source != "second" || progress[5].Processed != 0 { + t.Fatalf("incorrect simultaneous progress: %+v", progress) + } + if progress[4].RemainingTime != 0 { + t.Fatalf("estimated time without an active lookup run: %v", progress[4].RemainingTime) + } + // One lookup in this run took a minute, so the remaining one should take about a minute. + run := &idx.documentEnrichProgress[0] + run.begin(1) + run.startNanos.Store(time.Now().Add(-time.Minute).UnixNano()) + run.record() + progress, err = idx.BackgroundProgress() + if err != nil || progress[4].RemainingTime < time.Minute || progress[4].RemainingTime > time.Minute+time.Second { + t.Fatalf("incorrect review lookup estimate: %+v, %v", progress, err) + } + run.end() + idx.indexProgress.end() + idx.textRankEnrichProgress.end() + idx.authorEnrichProgress.end() + idx.pruneProgress.end() + if err := idx.EnrichDocuments(1); err == nil { + t.Fatal("expected unavailable provider to fail") + } + progress, err = idx.BackgroundProgress() + if err != nil || len(progress) != 1 || progress[0].Source != "first" || progress[0].Processed != 2 { + t.Fatalf("failed lookups must remain visible; successful providers must disappear: %+v, %v", progress, err) + } + // Progress is reconstructed from stored data, not a running worker counter. + resumed := &BleveIndexer{documentsIdx: idx.documentsIdx, documentEnrichers: []DocumentEnricher{first}} + progress, err = resumed.BackgroundProgress() + if err != nil || len(progress) != 1 || progress[0].Processed != 2 { + t.Fatalf("persisted progress lost on restart: %+v, %v", progress, err) + } + if err := idx.StoreDocumentData("pending", "first", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + progress, err = idx.BackgroundProgress() + if err != nil || len(progress) != 0 { + t.Fatalf("completed lookups still reported as pending: %+v, %v", progress, err) + } +} + +func TestBackgroundProgressNeverRoundsPendingLookupsToComplete(t *testing.T) { + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{&testDocumentEnricher{source: "test"}}}) + t.Cleanup(func() { _ = idx.Close() }) + for i := range 201 { + document := Document{ID: fmt.Sprintf("book-%d", i)} + if i < 200 { + if err := SetDataSourceData(&document, "test", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + } + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + } + progress, err := idx.BackgroundProgress() + if err != nil || len(progress) != 1 || progress[0].Percentage != 99 || progress[0].Processed != 200 { + t.Fatalf("unfinished lookups must not display 100%%: %+v, %v", progress, err) + } +} + +func TestBackgroundProgressWithoutPendingProviders(t *testing.T) { + for _, config := range []Config{{}, {DocumentEnrichers: []DocumentEnricher{&testDocumentEnricher{source: "test"}}}} { + idx, _ := newTestTextRankIndexer(t, config) + t.Cleanup(func() { _ = idx.Close() }) + progress, err := idx.BackgroundProgress() + if err != nil || len(progress) != 0 { + t.Fatalf("empty or disabled provider should have no progress: %+v, %v", progress, err) + } + } +} + func (p *testDocumentEnricher) Source() string { return p.source } +func (p *testDocumentEnricher) Name() string { return p.source } + func (p *testDocumentEnricher) NeedsEnrichment(document Document) (bool, error) { data, err := DataSourceData[struct{ Enriched bool }](document, p.source) return !data.Enriched, err diff --git a/internal/index/bleve_document_write.go b/internal/index/bleve_document_write.go index 0a2d57e7..610fb96d 100644 --- a/internal/index/bleve_document_write.go +++ b/internal/index/bleve_document_write.go @@ -42,7 +42,7 @@ func (b *BleveIndexer) IndexingProgress() (Progress, error) { } func (b *BleveIndexer) progressFrom(kind ProgressKind, startNanos int64, processed, total uint64) Progress { - progress := Progress{Kind: kind, InProgress: true} + progress := Progress{Kind: kind, InProgress: true, Processed: processed, Total: total} if total > 0 { progress.Percentage = math.Round(100 * float64(processed) / float64(total)) if progress.Percentage > 100 { diff --git a/internal/index/progress.go b/internal/index/progress.go index 87bd467a..e96a51d0 100644 --- a/internal/index/progress.go +++ b/internal/index/progress.go @@ -1,14 +1,21 @@ package index -import "time" +import ( + "fmt" + "math" + "time" + + "github.com/blevesearch/bleve/v2" +) type ProgressKind string const ( - ProgressDocuments ProgressKind = "documents" - ProgressAuthors ProgressKind = "authors" - ProgressTextRank ProgressKind = "textrank" - ProgressPruning ProgressKind = "pruning" + ProgressDocuments ProgressKind = "documents" + ProgressAuthors ProgressKind = "authors" + ProgressTextRank ProgressKind = "textrank" + ProgressPruning ProgressKind = "pruning" + ProgressDataSources ProgressKind = "data-sources" ) type Progress struct { @@ -16,4 +23,53 @@ type Progress struct { InProgress bool RemainingTime time.Duration Percentage float64 + Processed uint64 + Total uint64 + Source string + Name string +} + +// BackgroundProgress includes concurrent tasks and persisted pending provider lookups. +func (b *BleveIndexer) BackgroundProgress() ([]Progress, error) { + var progress []Progress + for _, task := range []struct { + kind ProgressKind + tracker *progressTracker + }{ + {ProgressDocuments, &b.indexProgress}, + {ProgressAuthors, &b.authorEnrichProgress}, + {ProgressTextRank, &b.textRankEnrichProgress}, + {ProgressPruning, &b.pruneProgress}, + } { + if start := task.tracker.startNanos.Load(); start != 0 { + progress = append(progress, b.progressFrom(task.kind, start, task.tracker.processed.Load(), task.tracker.total.Load())) + } + } + if len(b.documentEnrichers) == 0 { + return progress, nil + } + b.documentsMu.RLock() + defer b.documentsMu.RUnlock() + total, err := b.documentsIdx.DocCount() + if err != nil { + return nil, fmt.Errorf("counting documents for enrichment progress: %w", err) + } + for i, provider := range b.documentEnrichers { + query := bleve.NewTermQuery(provider.Source()) + query.SetField("EnrichedDataSources") + result, err := b.documentsIdx.Search(bleve.NewSearchRequestOptions(query, 0, 0, false)) + if err != nil { + return nil, fmt.Errorf("reading %s enrichment progress: %w", provider.Source(), err) + } + if result.Total < total { + // Do not round pending work up to 100%. + progress = append(progress, Progress{ + Kind: ProgressDataSources, InProgress: true, Source: provider.Source(), Name: provider.Name(), + Processed: result.Total, Total: total, + Percentage: math.Floor(100 * float64(result.Total) / float64(total)), + RemainingTime: b.documentEnrichTracker(i).remainingTime(total - result.Total), + }) + } + } + return progress, nil } diff --git a/internal/webserver/document_biblioreads_test.go b/internal/webserver/document_biblioreads_test.go index f98b3c3a..e3cdbd78 100644 --- a/internal/webserver/document_biblioreads_test.go +++ b/internal/webserver/document_biblioreads_test.go @@ -88,6 +88,57 @@ func newBiblioReadsTestApp(t *testing.T, source *fakeBookSource) (*fiber.App, *i return newDataSourceProvidersTestApp(t, source, nil, nil) } +func TestReviewEnrichmentProgressBanner(t *testing.T) { + source := &fakeBookSource{searchErr: biblioreads.ErrUnavailable} + app, idx := newBiblioReadsTestApp(t, source) + idx.BeginIndexing() + page := getPage(t, app, "/documents/linked") + banner := page.Find("#background-progress") + if banner.Find(`[data-progress-kind="documents"]`).Length() != 1 || + banner.Find(`[data-progress-kind="data-sources"]`).Length() != 1 { + t.Fatal("indexing and review progress must share a banner") + } + if banner.AttrOr("hx-trigger", "") != "every 5s" || + !strings.Contains(banner.Text(), "Score filters may omit documents") { + t.Fatal("expected live polling and score-filter warning") + } + checkPending := func() { + t.Helper() + page := getPage(t, app, "/background-progress?l=en") + reviews := page.Find(`[data-progress-kind="data-sources"]`) + if !strings.Contains(reviews.Text(), "(Goodreads)") || + reviews.Find(`[role="progressbar"]`).AttrOr("aria-valuenow", "") != "66" { + t.Fatalf("incorrect persisted lookup progress: %s", reviews.Text()) + } + } + checkPending() + spanish := getPage(t, app, "/background-progress?l=es") + if !strings.Contains(spanish.Find("#background-progress").Text(), "La información de valoraciones está incompleta") { + t.Fatal("live progress must retain the selected language") + } + if err := idx.EnrichDocuments(1); err == nil { + t.Fatal("expected unavailable source to fail") + } + checkPending() + source.searchErr = nil + if err := idx.EnrichDocuments(1); err != nil { + t.Fatal(err) + } + page = getPage(t, app, "/background-progress?l=en") + if page.Find(`[data-progress-kind="data-sources"]`).Length() != 0 || + page.Find(`[data-progress-kind="documents"]`).Length() != 1 { + t.Fatal("completion must remove only the review bar, including no-match lookups") + } + response, err := app.Test(mustGetRequest(t, "/background-progress")) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.Header.Get("Cache-Control") != "no-store" { + t.Fatal("live progress responses must not be cached") + } +} + func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provider document.ReviewProvider, dataSourceLinkProvider authorcontroller.DataSourceLinkProvider) (*fiber.App, *index.BleveIndexer) { t.Helper() db := infrastructure.Connect(":memory:", 250) diff --git a/internal/webserver/document_reviews_test.go b/internal/webserver/document_reviews_test.go index 8f7f0c59..f26f840c 100644 --- a/internal/webserver/document_reviews_test.go +++ b/internal/webserver/document_reviews_test.go @@ -498,6 +498,9 @@ func TestDocumentReviewsHeader(t *testing.T) { func TestDocumentReviewsDisabled(t *testing.T) { app, _ := newReviewProviderTestApp(t, nil) + if getPage(t, app, "/background-progress").Find(".alert").Length() != 0 { + t.Fatal("disabled providers must not show incomplete review progress") + } for _, slug := range []string{"linked", "unlinked", "unmatched"} { body := getBody(t, app, "/documents/"+slug) if got := documentScorePlaceholders(t, body).Length(); got != 0 { diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index 38dd5e5d..fbcc0e6b 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -179,7 +179,7 @@ "Keyword clean-up progress": "Fortschritt der Schlüsselwortbereinigung" "Indexing progress": "Indexierungsfortschritt" "Remaining time": "Verbleibende Zeit" -"Remaining time: %s": "Verbleibende Zeit: %s" +"%s left": "noch %s" "There was an error deleting the user please try again later": "Beim Löschen des Benutzers ist ein Fehler aufgetreten bitte versuchen Sie es später erneut." "A user with this username already exists": "Ein Benutzer mit diesem Benutzernamen existiert bereits." "A user with this email address already exists": "Ein Benutzer mit dieser E-Mail-Adresse existiert bereits." @@ -426,3 +426,6 @@ "No score": "Keine Bewertung" "Not scored": "Nicht bewertet" "%s ratings": "%s Bewertungen" +"Review information progress": "Fortschritt der Bewertungsinformationen" +"Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved.": "Bewertungsinformationen sind unvollständig. Bewertungsfilter können Dokumente auslassen, deren Bewertungen noch nicht abgerufen wurden." +"Unable to retrieve background progress.": "Der Fortschritt der Hintergrundaufgaben konnte nicht abgerufen werden." diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index 8a17fc25..a5f81aea 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -175,7 +175,7 @@ "Cleaning up overly common keywords, search results and recommendations may not be accurate yet.": "Limpiando palabras clave demasiado comunes, los resultados de búsqueda y las recomendaciones pueden no ser precisos todavía." "Keyword clean-up progress": "Progreso de la limpieza de palabras clave" "Indexing progress": "Progreso de indexación" -"Remaining time: %s": "Tiempo restante: %s" +"%s left": "quedan %s" "There was an error deleting the user, please try again later": "Hubo un error al borrar el usuario, por favor, vuelva a intentarlo más tarde" "A user with this username already exists": "Ya existe un usuario con ese nombre de usuario" "A user with this email address already exists": "Ya existe un usuario con esa dirección de correo electrónico" @@ -422,3 +422,6 @@ "No score": "Sin puntuación" "Not scored": "Sin puntuar" "%s ratings": "%s valoraciones" +"Review information progress": "Progreso de la información de valoraciones" +"Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved.": "La información de valoraciones está incompleta. Los filtros de puntuación pueden omitir documentos cuyas valoraciones aún no se han obtenido." +"Unable to retrieve background progress.": "No se pudo obtener el progreso de las tareas en segundo plano." diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index 31bb9748..1b90bd41 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -176,7 +176,7 @@ "Cleaning up overly common keywords, search results and recommendations may not be accurate yet.": "Nettoyage des mots-clés trop courants, les résultats de recherche et les recommandations peuvent ne pas être précis pour le moment." "Keyword clean-up progress": "Progression du nettoyage des mots-clés" "Indexing progress": "Progression de l'indexation" -"Remaining time: %s": "Temps restant: %s" +"%s left": "%s restant" "There was an error deleting the user, please try again later": "Une erreur s'est produite lors de supprimer l'utilisateur, veuillez réessayer ultérieurement" "A user with this username already exists": "Un utilisateur avec ce nom d'utilisateur existe déjà" "A user with this email address already exists": "Un utilisateur avec cette adresse e-mail existe déjà" @@ -424,3 +424,6 @@ "No score": "Aucune note" "Not scored": "Non noté" "%s ratings": "%s notes" +"Review information progress": "Progression des informations d'évaluation" +"Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved.": "Les informations d'évaluation sont incomplètes. Les filtres de note peuvent omettre les documents dont les évaluations n'ont pas encore été récupérées." +"Unable to retrieve background progress.": "Impossible de récupérer la progression des tâches en arrière-plan." diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index 24d8ad5c..4d3085eb 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -177,7 +177,7 @@ "Cleaning up overly common keywords, search results and recommendations may not be accurate yet.": "Очистка слишком часто встречающихся ключевых слов; результаты поиска и рекомендации могут быть пока неточными." "Keyword clean-up progress": "Прогресс очистки ключевых слов" "Indexing progress": "Прогресс индексации" -"Remaining time: %s": "Оставшееся время: %s" +"%s left": "осталось %s" "There was an error deleting the user, please try again later": "Ошибка при удалении пользователя. Повторите попытку позже." "A user with this username already exists": "Пользователь с таким именем уже существует" "A user with this email address already exists": "Пользователь с таким e‑mail уже существует" @@ -418,3 +418,6 @@ "No score": "Нет оценки" "Not scored": "Не оценено" "%s ratings": "Оценок: %s" +"Review information progress": "Ход получения информации об оценках" +"Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved.": "Информация об оценках неполная. Фильтры по оценкам могут пропускать документы, оценки которых ещё не получены." +"Unable to retrieve background progress.": "Не удалось получить состояние фоновых задач." diff --git a/internal/webserver/embedded/views/layout.html b/internal/webserver/embedded/views/layout.html index 027cd25d..025a2632 100644 --- a/internal/webserver/embedded/views/layout.html +++ b/internal/webserver/embedded/views/layout.html @@ -34,7 +34,7 @@ {{template "partials/navbar" .}}
        - {{template "partials/main" dict "Lang" .Lang "IndexingInProgress" .IndexingInProgress "IndexingProgressKind" .IndexingProgressKind "RemainingIndexingTime" .RemainingIndexingTime "IndexingProgressPercentage" .IndexingProgressPercentage "Error" .Error "Warning" .Warning "Success" .Success "Embed" .Embed}} + {{template "partials/main" dict "BackgroundProgress" .BackgroundProgress "BackgroundProgressError" .BackgroundProgressError "Lang" .Lang "IndexingInProgress" .IndexingInProgress "IndexingProgressKind" .IndexingProgressKind "RemainingIndexingTime" .RemainingIndexingTime "IndexingProgressPercentage" .IndexingProgressPercentage "Error" .Error "Warning" .Warning "Success" .Success "Embed" .Embed}}
        {{template "partials/share-modal" .}} {{template "partials/keyboard-shortcuts-modal" .}} diff --git a/internal/webserver/embedded/views/partials/background-progress.html b/internal/webserver/embedded/views/partials/background-progress.html new file mode 100644 index 00000000..f0243ca1 --- /dev/null +++ b/internal/webserver/embedded/views/partials/background-progress.html @@ -0,0 +1,35 @@ +
        + {{if .BackgroundProgressError}} + + {{end}} + {{if .BackgroundProgress}} +
        + {{range .BackgroundProgress}} + {{$label := "Indexing progress"}} + {{if eq .Kind "authors"}}{{$label = "Author information progress"}} + {{else if eq .Kind "textrank"}}{{$label = "Related content analysis progress"}} + {{else if eq .Kind "pruning"}}{{$label = "Keyword clean-up progress"}} + {{else if eq .Kind "data-sources"}}{{$label = "Review information progress"}} + {{end}} +
        +

        {{t $.Lang $label}}{{if .Name}} ({{.Name}}){{end}}

        + {{$remaining := ""}}{{if gt .RemainingTime 0}}{{$remaining = t $.Lang "%s left" (remainingTime .RemainingTime)}}{{end}} +
        +
        {{.Percentage}}%{{if $remaining}} ({{$remaining}}){{end}}
        +
        + {{if eq .Kind "data-sources"}} +

        {{t $.Lang "Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved."}}

        + {{else if eq .Kind "authors"}} +

        {{t $.Lang "Retrieving author information, author pages may not be complete yet."}}

        + {{else if eq .Kind "textrank"}} +

        {{t $.Lang "Analyzing documents, search results and recommendations may not be accurate yet."}}

        + {{else if eq .Kind "pruning"}} +

        {{t $.Lang "Cleaning up overly common keywords, search results and recommendations may not be accurate yet."}}

        + {{else}} +

        {{t $.Lang "Indexing in progress, search results may not be accurate."}}

        + {{end}} +
        + {{end}} +
        + {{end}} +
        diff --git a/internal/webserver/embedded/views/partials/main.html b/internal/webserver/embedded/views/partials/main.html index 364d559d..95eefb9a 100644 --- a/internal/webserver/embedded/views/partials/main.html +++ b/internal/webserver/embedded/views/partials/main.html @@ -1,28 +1,5 @@
        - {{if .IndexingInProgress}} -
        -
      - {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "FilterIdPrefix" "sidebar" "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" true "HideSearch" true}} + {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "ReviewSourceConfigured" .ReviewSourceConfigured "FilterIdPrefix" "sidebar" "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" true "HideSearch" true}}
      {{if not .DocumentsOnly}}
      diff --git a/internal/webserver/embedded/views/partials/searchbox.html b/internal/webserver/embedded/views/partials/searchbox.html index 4d50be5e..d45dbcbd 100644 --- a/internal/webserver/embedded/views/partials/searchbox.html +++ b/internal/webserver/embedded/views/partials/searchbox.html @@ -32,7 +32,7 @@
    - {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "HomeSearch" true "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} + {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "ReviewSourceConfigured" .ReviewSourceConfigured "HomeSearch" true "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}}
    {{template "partials/author-search-filters" dict "Lang" .Lang "HomeSearch" true}} diff --git a/internal/webserver/highlights_test.go b/internal/webserver/highlights_test.go index 2c876fda..f43d0390 100644 --- a/internal/webserver/highlights_test.go +++ b/internal/webserver/highlights_test.go @@ -185,6 +185,19 @@ func TestHighlightsReviewScoreFilter(t *testing.T) { t.Fatalf("Expected status %d, received %d", fiber.StatusOK, response.StatusCode) } + response, err = getRequest(adminCookie, app, "/highlights", t) + if err != nil { + t.Fatal(err) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + response.Body.Close() + if err != nil { + t.Fatal(err) + } + if doc.Find(`select[name="review-score-from"], select[name="review-score-to"]`).Length() != 0 { + t.Fatal("review score filter must be hidden on highlights when no review source is configured") + } + for _, test := range []struct { name string query string diff --git a/internal/webserver/middleware.go b/internal/webserver/middleware.go index 3332ae03..a8e06a12 100644 --- a/internal/webserver/middleware.go +++ b/internal/webserver/middleware.go @@ -42,6 +42,7 @@ func SetConfigLocals(cfg Config) func(fiber.Ctx) error { c.Locals("ShareCommentMaxSize", cfg.ShareCommentMaxSize) c.Locals("ShareMaxRecipients", cfg.ShareMaxRecipients) c.Locals("IllustratedMinAmount", cfg.IllustratedMinAmount) + c.Locals("ReviewSourceConfigured", cfg.ReviewSourceConfigured) return c.Next() } } diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index 89774d9d..c6635222 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -197,7 +197,22 @@ func TestDataSourceScoreSortOptions(t *testing.T) { func TestReviewScoreFilter(t *testing.T) { db := infrastructure.Connect(":memory:", 250) - app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), webserver.Config{}) + cfg := defaultTestConfig() + cfg.ReviewSourceConfigured = true + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), cfg) + + homeResponse, err := app.Test(mustGetRequest(t, "/")) + if err != nil { + t.Fatal(err) + } + defer homeResponse.Body.Close() + homeDoc, err := goquery.NewDocumentFromReader(homeResponse.Body) + if err != nil { + t.Fatal(err) + } + if homeDoc.Find(`select[name="review-score-from"]`).Length() == 0 { + t.Fatal("review score filter must be visible on home search when a review source is configured") + } response, err := app.Test(mustGetRequest(t, "/search?type=documents&review-score-from=4&review-score-to=5")) if err != nil { @@ -275,6 +290,29 @@ func TestReviewScoreFilter(t *testing.T) { } } +func TestReviewScoreFilterHiddenWithoutConfiguredInstances(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), webserver.Config{}) + + for _, path := range []string{"/", "/search?type=documents"} { + response, err := app.Test(mustGetRequest(t, path)) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("Expected status %d for %s, received %d", http.StatusOK, path, response.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if doc.Find(`select[name="review-score-from"], select[name="review-score-to"]`).Length() != 0 { + t.Fatalf("review score filter must be hidden for %s when no review source is configured", path) + } + } +} + func TestSearchResultsShowStoredDataSourceScore(t *testing.T) { for _, tt := range []struct { score float64 diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go index e88d00af..6939d14b 100644 --- a/internal/webserver/webserver.go +++ b/internal/webserver/webserver.go @@ -57,6 +57,7 @@ type Config struct { ClientDynamicImageCacheTTL int ServerStaticCacheTTL int ServerDynamicImageCacheTTL int + ReviewSourceConfigured bool ReviewSourceCacheTTL int ShareCommentMaxSize int ShareMaxRecipients int diff --git a/main.go b/main.go index c2702383..ffaffb42 100644 --- a/main.go +++ b/main.go @@ -170,6 +170,7 @@ func main() { ClientDynamicImageCacheTTL: input.ClientDynamicImageCacheTTL, ServerStaticCacheTTL: input.ServerStaticCacheTTL, ServerDynamicImageCacheTTL: input.ServerDynamicImageCacheTTL, + ReviewSourceConfigured: len(input.ReviewSourceInstances) > 0, ReviewSourceCacheTTL: input.ReviewSourceCacheTTL, ShareCommentMaxSize: input.ShareCommentMaxSize, ShareMaxRecipients: input.ShareMaxRecipients, From d8761f050a4679667f3c06270f8d444327460858 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Thu, 5 Nov 2026 14:50:54 +0100 Subject: [PATCH 45/50] Consolidate file --- .../webserver/background_progress_test.go | 9 +++++--- .../embedded/js/background-progress.js | 16 -------------- internal/webserver/embedded/js/feedback.js | 22 +++++++++++++++++++ internal/webserver/embedded/views/layout.html | 1 - 4 files changed, 28 insertions(+), 20 deletions(-) delete mode 100644 internal/webserver/embedded/js/background-progress.js diff --git a/internal/webserver/background_progress_test.go b/internal/webserver/background_progress_test.go index 0d913fec..00190ae4 100644 --- a/internal/webserver/background_progress_test.go +++ b/internal/webserver/background_progress_test.go @@ -199,7 +199,7 @@ func TestBackgroundProgressPollingAccess(t *testing.T) { } } -func TestBackgroundProgressErrorHandlerIsLoaded(t *testing.T) { +func TestBackgroundProgressErrorHandlerUsesFeedbackScript(t *testing.T) { db := infrastructure.Connect(":memory:", 250) app := bootstrapApp(db, &infrastructure.NoEmail{}, afero.NewMemMapFs(), webserver.Config{}) response, err := app.Test(mustGetRequest(t, "/")) @@ -211,7 +211,10 @@ func TestBackgroundProgressErrorHandlerIsLoaded(t *testing.T) { if err != nil { t.Fatal(err) } - if page.Find(`script[src^="/js/background-progress.js"]`).Length() != 1 { - t.Fatal("expected background progress error handler to be loaded") + if page.Find(`script[src^="/js/feedback.js"]`).Length() != 1 { + t.Fatal("expected feedback script to be loaded") + } + if page.Find(`script[src^="/js/background-progress.js"]`).Length() != 0 { + t.Fatal("expected no standalone background progress script") } } diff --git a/internal/webserver/embedded/js/background-progress.js b/internal/webserver/embedded/js/background-progress.js deleted file mode 100644 index 5d0d1eaa..00000000 --- a/internal/webserver/embedded/js/background-progress.js +++ /dev/null @@ -1,16 +0,0 @@ -function showBackgroundProgressError(event) { - const progress = event.detail.elt; - if (!(progress instanceof HTMLElement) || progress.id !== 'background-progress') { - return; - } - - const alert = document.createElement('div'); - alert.className = 'alert alert-warning'; - alert.setAttribute('role', 'alert'); - alert.textContent = progress.dataset.errorMessage; - progress.replaceChildren(alert); -} - -document.body.addEventListener('htmx:sendError', showBackgroundProgressError); -document.body.addEventListener('htmx:responseError', showBackgroundProgressError); -document.body.addEventListener('htmx:timeout', showBackgroundProgressError); diff --git a/internal/webserver/embedded/js/feedback.js b/internal/webserver/embedded/js/feedback.js index 466dbcd1..1ac58ff9 100644 --- a/internal/webserver/embedded/js/feedback.js +++ b/internal/webserver/embedded/js/feedback.js @@ -115,6 +115,28 @@ htmx.on('htmx:afterRequest', (evt) => { } }); +function showBackgroundProgressError(evt) { + const progress = evt.detail?.elt + if (!(progress instanceof HTMLElement) || progress.id !== 'background-progress') { + return + } + + const message = progress.dataset.errorMessage + if (!message) { + return + } + + const alert = document.createElement('div') + alert.className = 'alert alert-warning' + alert.setAttribute('role', 'alert') + alert.textContent = message + progress.replaceChildren(alert) +} + +htmx.on('htmx:sendError', showBackgroundProgressError) +htmx.on('htmx:responseError', showBackgroundProgressError) +htmx.on('htmx:timeout', showBackgroundProgressError) + const setUpWarningsListener = () => { document.querySelectorAll('[data-warning-once]').forEach((el) => { el.addEventListener('click', () => { diff --git a/internal/webserver/embedded/views/layout.html b/internal/webserver/embedded/views/layout.html index 693e89d2..8d2c1d5e 100644 --- a/internal/webserver/embedded/views/layout.html +++ b/internal/webserver/embedded/views/layout.html @@ -60,7 +60,6 @@ document.body.addEventListener('htmx:afterSettle', initReadingTimePopovers); - From 6beb9c4ef6d784eb3291e18525c916902cd51bfa Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Thu, 5 Nov 2026 15:27:53 +0100 Subject: [PATCH 46/50] WIP --- internal/webserver/embedded/translations/de.yml | 2 +- internal/webserver/embedded/translations/es.yml | 2 +- internal/webserver/embedded/translations/fr.yml | 2 +- internal/webserver/embedded/translations/ru.yml | 2 +- .../webserver/embedded/views/partials/background-progress.html | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index fbcc0e6b..5e32fd4b 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -171,7 +171,7 @@ "Click to upload a custom image": "Klicken Sie, um ein benutzerdefiniertes Bild hochzuladen" "Indexing in progress search results may not be accurate.": "Indexierung läuft die Suchergebnisse sind möglicherweise nicht genau." "Indexing in progress, search results may not be accurate.": "Indexierung läuft, Suchergebnisse sind möglicherweise nicht genau." -"Retrieving author information, author pages may not be complete yet.": "Autoreninformationen werden abgerufen, Autorenseiten sind möglicherweise noch unvollständig." +"Retrieving author information. Author pages may be incomplete, and author filters may not work yet.": "Autoreninformationen werden abgerufen. Autorenseiten sind möglicherweise unvollständig, und Autorenfilter funktionieren möglicherweise noch nicht." "Analyzing documents, search results and recommendations may not be accurate yet.": "Dokumente werden analysiert, Suchergebnisse und Empfehlungen sind möglicherweise noch nicht genau." "Author information progress": "Fortschritt der Autoreninformationen" "Related content analysis progress": "Fortschritt der Analyse verwandter Inhalte" diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index a5f81aea..bcfea15c 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -168,7 +168,7 @@ "An error occurred while uploading the image": "Ocurrió un error al subir la imagen" "Click to upload a custom image": "Haz clic para subir una imagen personalizada" "Indexing in progress, search results may not be accurate.": "Indexando documentos, los resultados de búsqueda pueden no ser precisos." -"Retrieving author information, author pages may not be complete yet.": "Recuperando información de autores, las páginas de autor pueden estar incompletas." +"Retrieving author information. Author pages may be incomplete, and author filters may not work yet.": "Recuperando información de autores. Las páginas de autor pueden estar incompletas y los filtros de autor quizá no funcionen todavía." "Analyzing documents, search results and recommendations may not be accurate yet.": "Analizando documentos, los resultados de búsqueda y las recomendaciones pueden no ser precisos todavía." "Author information progress": "Progreso de información de autores" "Related content analysis progress": "Progreso del análisis de contenido relacionado" diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index 1b90bd41..1346ad3f 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -169,7 +169,7 @@ "An error occurred while uploading the image": "Une erreur s'est produite lors du téléchargement de l'image" "Click to upload a custom image": "Cliquez pour télécharger une image personnalisée" "Indexing in progress, search results may not be accurate.": "Indexation en cours, les résultats de la recherche peuvent ne pas être précis." -"Retrieving author information, author pages may not be complete yet.": "Récupération des informations sur les auteurs, les pages auteur peuvent être incomplètes." +"Retrieving author information. Author pages may be incomplete, and author filters may not work yet.": "Récupération des informations sur les auteurs. Les pages auteur peuvent être incomplètes et les filtres par auteur peuvent ne pas encore fonctionner." "Analyzing documents, search results and recommendations may not be accurate yet.": "Analyse des documents, les résultats de recherche et les recommandations peuvent ne pas être précis pour le moment." "Author information progress": "Progression des informations sur les auteurs" "Related content analysis progress": "Progression de l'analyse du contenu associé" diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index 4d3085eb..7fd89c59 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -170,7 +170,7 @@ "An error occurred while uploading the image": "Произошла ошибка при загрузке изображения" "Click to upload a custom image": "Нажмите, чтобы загрузить пользовательское изображение" "Indexing in progress, search results may not be accurate.": "Идёт индексация; результаты поиска могут быть неточными." -"Retrieving author information, author pages may not be complete yet.": "Получение информации об авторах; страницы авторов могут быть неполными." +"Retrieving author information. Author pages may be incomplete, and author filters may not work yet.": "Получение информации об авторах. Страницы авторов могут быть неполными, а фильтры по авторам могут пока не работать." "Analyzing documents, search results and recommendations may not be accurate yet.": "Анализ документов; результаты поиска и рекомендации могут быть пока неточными." "Author information progress": "Прогресс получения информации об авторах" "Related content analysis progress": "Прогресс анализа похожего контента" diff --git a/internal/webserver/embedded/views/partials/background-progress.html b/internal/webserver/embedded/views/partials/background-progress.html index 90407ea4..45ee5747 100644 --- a/internal/webserver/embedded/views/partials/background-progress.html +++ b/internal/webserver/embedded/views/partials/background-progress.html @@ -22,7 +22,7 @@ {{if eq .Kind "data-sources"}}

    {{t $.Lang "Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved."}}

    {{else if eq .Kind "authors"}} -

    {{t $.Lang "Retrieving author information, author pages may not be complete yet."}}

    +

    {{t $.Lang "Retrieving author information. Author pages may be incomplete, and author filters may not work yet."}}

    {{else if eq .Kind "textrank"}}

    {{t $.Lang "Analyzing documents, search results and recommendations may not be accurate yet."}}

    {{else if eq .Kind "pruning"}} From 4af575870534f93e0678d89aefaa09c11f2ed5d5 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Thu, 5 Nov 2026 15:34:12 +0100 Subject: [PATCH 47/50] Hide score sorting --- internal/webserver/controller.go | 16 ++--- .../webserver/controller/author/controller.go | 13 ++-- .../webserver/controller/author/documents.go | 63 ++++++++++++------- .../webserver/controller/search/search.go | 18 ++++-- .../webserver/controller/series/controller.go | 3 +- .../webserver/controller/series/documents.go | 63 ++++++++++++------- internal/webserver/search_test.go | 44 ++++++++++++- 7 files changed, 152 insertions(+), 68 deletions(-) diff --git a/internal/webserver/controller.go b/internal/webserver/controller.go index 43fd3f69..3b4363d9 100644 --- a/internal/webserver/controller.go +++ b/internal/webserver/controller.go @@ -80,16 +80,18 @@ func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender S } authorsCfg := author.Config{ - WordsPerMinute: cfg.WordsPerMinute, - CacheDir: cfg.CacheDir, - CacheMaxSize: cfg.CacheMaxSize, - AuthorImageMaxWidth: cfg.AuthorImageMaxWidth, - ClientImageCacheTTL: cfg.ClientDynamicImageCacheTTL, - ServerImageCacheTTL: cfg.ServerDynamicImageCacheTTL, + WordsPerMinute: cfg.WordsPerMinute, + CacheDir: cfg.CacheDir, + CacheMaxSize: cfg.CacheMaxSize, + AuthorImageMaxWidth: cfg.AuthorImageMaxWidth, + ClientImageCacheTTL: cfg.ClientDynamicImageCacheTTL, + ServerImageCacheTTL: cfg.ServerDynamicImageCacheTTL, + ReviewSourceConfigured: cfg.ReviewSourceConfigured, } seriesCfg := series.Config{ - WordsPerMinute: cfg.WordsPerMinute, + WordsPerMinute: cfg.WordsPerMinute, + ReviewSourceConfigured: cfg.ReviewSourceConfigured, } searchCfg := search.Config{ diff --git a/internal/webserver/controller/author/controller.go b/internal/webserver/controller/author/controller.go index b05f29ba..d44a0eac 100644 --- a/internal/webserver/controller/author/controller.go +++ b/internal/webserver/controller/author/controller.go @@ -36,12 +36,13 @@ type readingRepository interface { } type Config struct { - WordsPerMinute float64 - CacheDir string - CacheMaxSize int - AuthorImageMaxWidth int - ClientImageCacheTTL int - ServerImageCacheTTL int + WordsPerMinute float64 + CacheDir string + CacheMaxSize int + AuthorImageMaxWidth int + ClientImageCacheTTL int + ServerImageCacheTTL int + ReviewSourceConfigured bool } type Controller struct { diff --git a/internal/webserver/controller/author/documents.go b/internal/webserver/controller/author/documents.go index 2104eaef..d24e0115 100644 --- a/internal/webserver/controller/author/documents.go +++ b/internal/webserver/controller/author/documents.go @@ -65,30 +65,20 @@ func (a *Controller) Documents(c fiber.Ctx) error { } templateVars := fiber.Map{ - "Author": author, - "DataSourceLinks": a.authorDataSourceLinks(author, lang), - "ImageVersion": a.getImageVersion(author.Slug), - "DocumentSearchFields": searchFields, - "DocumentsOnly": true, - "Results": searchResults, - "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), - "Title": author.Name, - "EmailFrom": a.sender.From(), - "WordsPerMinute": a.config.WordsPerMinute, - "URL": view.URL(c), - "SortURL": view.BaseURLWithout(c, "sort-by", "page"), - "SortBy": c.Query("sort-by"), - "AdditionalSortOptions": []struct { - Key string - Value string - }{ - {"pub-date-older-first", "older"}, - {"pub-date-newer-first", "newer"}, - {"est-read-time-shorter-first", "shorter"}, - {"est-read-time-longer-first", "longer"}, - {"review-score-asc", "review-score-asc"}, - {"review-score-desc", "review-score-desc"}, - }, + "Author": author, + "DataSourceLinks": a.authorDataSourceLinks(author, lang), + "ImageVersion": a.getImageVersion(author.Slug), + "DocumentSearchFields": searchFields, + "DocumentsOnly": true, + "Results": searchResults, + "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), + "Title": author.Name, + "EmailFrom": a.sender.From(), + "WordsPerMinute": a.config.WordsPerMinute, + "URL": view.URL(c), + "SortURL": view.BaseURLWithout(c, "sort-by", "page"), + "SortBy": c.Query("sort-by"), + "AdditionalSortOptions": authorDocumentSortOptions(a.config.ReviewSourceConfigured), } if c.Get("hx-request") == "true" { @@ -105,3 +95,28 @@ func (a *Controller) Documents(c fiber.Ctx) error { } return nil } + +func authorDocumentSortOptions(reviewSourceConfigured bool) []struct { + Key string + Value string +} { + options := []struct { + Key string + Value string + }{ + {"pub-date-older-first", "older"}, + {"pub-date-newer-first", "newer"}, + {"est-read-time-shorter-first", "shorter"}, + {"est-read-time-longer-first", "longer"}, + } + if reviewSourceConfigured { + options = append(options, []struct { + Key string + Value string + }{ + {"review-score-asc", "review-score-asc"}, + {"review-score-desc", "review-score-desc"}, + }...) + } + return options +} diff --git a/internal/webserver/controller/search/search.go b/internal/webserver/controller/search/search.go index 2faecfae..f618edc1 100644 --- a/internal/webserver/controller/search/search.go +++ b/internal/webserver/controller/search/search.go @@ -87,7 +87,7 @@ func (s *Controller) renderDocumentSearch(c fiber.Ctx, session model.Session, pa templateVars["Paginator"] = view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()) templateVars["Title"] = "Search results" templateVars["WordsPerMinute"] = wordsPerMinute - templateVars["AdditionalSortOptions"] = documentSortOptions() + templateVars["AdditionalSortOptions"] = documentSortOptions(s.config.ReviewSourceConfigured) templateVars["DocumentsTotalHits"] = docCount templateVars["AuthorsTotalHits"] = authorCount templateVars["SimilarCandidatesCapped"] = documentResults.Candidates.Capped() @@ -210,11 +210,11 @@ func (s *Controller) tabCounts(docFields index.SearchFields, authorFields index. return docResults, authorCount, nil } -func documentSortOptions() []struct { +func documentSortOptions(reviewSourceConfigured bool) []struct { Key string Value string } { - return []struct { + options := []struct { Key string Value string }{ @@ -223,9 +223,17 @@ func documentSortOptions() []struct { {"pub-date-newer-first", "newer"}, {"est-read-time-shorter-first", "shorter"}, {"est-read-time-longer-first", "longer"}, - {"review-score-asc", "review-score-asc"}, - {"review-score-desc", "review-score-desc"}, } + if reviewSourceConfigured { + options = append(options, []struct { + Key string + Value string + }{ + {"review-score-asc", "review-score-asc"}, + {"review-score-desc", "review-score-desc"}, + }...) + } + return options } func authorSortOptions() []struct { diff --git a/internal/webserver/controller/series/controller.go b/internal/webserver/controller/series/controller.go index 6c42d561..cbbd0449 100644 --- a/internal/webserver/controller/series/controller.go +++ b/internal/webserver/controller/series/controller.go @@ -26,7 +26,8 @@ type readingRepository interface { } type Config struct { - WordsPerMinute float64 + WordsPerMinute float64 + ReviewSourceConfigured bool } type Controller struct { diff --git a/internal/webserver/controller/series/documents.go b/internal/webserver/controller/series/documents.go index a2986bf3..61e06f8a 100644 --- a/internal/webserver/controller/series/documents.go +++ b/internal/webserver/controller/series/documents.go @@ -72,30 +72,18 @@ func (a *Controller) Documents(c fiber.Ctx) error { } templateVars := fiber.Map{ - "SeriesTitle": title, - "DocumentSearchFields": searchFields, - "DocumentsOnly": true, - "Results": searchResults, - "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), - "Title": title, - "EmailFrom": a.sender.From(), - "WordsPerMinute": a.config.WordsPerMinute, - "URL": view.URL(c), - "SortURL": view.BaseURLWithout(c, "sort-by", "page"), - "SortBy": c.Query("sort-by"), - "AdditionalSortOptions": []struct { - Key string - Value string - }{ - {"number", "series number"}, - {"number-desc", "series number (descending)"}, - {"pub-date-older-first", "older"}, - {"pub-date-newer-first", "newer"}, - {"est-read-time-shorter-first", "shorter"}, - {"est-read-time-longer-first", "longer"}, - {"review-score-asc", "review-score-asc"}, - {"review-score-desc", "review-score-desc"}, - }, + "SeriesTitle": title, + "DocumentSearchFields": searchFields, + "DocumentsOnly": true, + "Results": searchResults, + "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), + "Title": title, + "EmailFrom": a.sender.From(), + "WordsPerMinute": a.config.WordsPerMinute, + "URL": view.URL(c), + "SortURL": view.BaseURLWithout(c, "sort-by", "page"), + "SortBy": c.Query("sort-by"), + "AdditionalSortOptions": seriesDocumentSortOptions(a.config.ReviewSourceConfigured), } if c.Get("hx-request") == "true" { @@ -113,6 +101,33 @@ func (a *Controller) Documents(c fiber.Ctx) error { return nil } +func seriesDocumentSortOptions(reviewSourceConfigured bool) []struct { + Key string + Value string +} { + options := []struct { + Key string + Value string + }{ + {"number", "series number"}, + {"number-desc", "series number (descending)"}, + {"pub-date-older-first", "older"}, + {"pub-date-newer-first", "newer"}, + {"est-read-time-shorter-first", "shorter"}, + {"est-read-time-longer-first", "longer"}, + } + if reviewSourceConfigured { + options = append(options, []struct { + Key string + Value string + }{ + {"review-score-asc", "review-score-asc"}, + {"review-score-desc", "review-score-desc"}, + }...) + } + return options +} + func (d *Controller) parseSortBy(c fiber.Ctx) []string { if c.Query("sort-by") != "" { switch c.Query("sort-by") { diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index c6635222..d0cba89b 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -150,8 +150,10 @@ func TestUnifiedSearch(t *testing.T) { } func TestDataSourceScoreSortOptions(t *testing.T) { + cfg := defaultTestConfig() + cfg.ReviewSourceConfigured = true app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, - loadDirInMemoryFs("testdata/library"), webserver.Config{}) + loadDirInMemoryFs("testdata/library"), cfg) for _, path := range []string{ "/search?type=documents&", "/authors/john-doe?", @@ -195,6 +197,46 @@ func TestDataSourceScoreSortOptions(t *testing.T) { } } +func TestDataSourceScoreSortOptionsHiddenWithoutReviewSources(t *testing.T) { + app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, + loadDirInMemoryFs("testdata/library"), defaultTestConfig()) + for _, path := range []string{ + "/search?type=documents&", + "/authors/john-doe?", + "/series/the-lord-of-the-rings?", + } { + for _, fragment := range []bool{false, true} { + mode := "/full" + if fragment { + mode = "/fragment" + } + t.Run(path+mode, func(t *testing.T) { + req := mustGetRequest(t, path) + if fragment { + req.Header.Set("HX-Request", "true") + } + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("unexpected status %d", response.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"review-score-asc", "review-score-desc"} { + if doc.Find(`#sort-by option[value="`+value+`"]`).Length() != 0 { + t.Errorf("unexpected score sort option %q", value) + } + } + }) + } + } +} + func TestReviewScoreFilter(t *testing.T) { db := infrastructure.Connect(":memory:", 250) cfg := defaultTestConfig() From 35c83f475c085e8b5b6e582790dd63b1489d6c94 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Thu, 5 Nov 2026 15:45:54 +0100 Subject: [PATCH 48/50] Consolidation --- README.md | 4 +- cli_input.go | 2 +- internal/webserver/search_test.go | 152 ++++++++++++++---------------- 3 files changed, 75 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 05662745..df9110ef 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ When using a reverse proxy, include its client-facing port in `FQDN`, for exampl Coreander can use [BiblioReads](https://github.com/nesaku/biblioreads), an alternative Goodreads front-end, to find Goodreads book and author links and display book ratings on document detail pages. Lookups are disabled by default and require explicitly configured endpoints. BiblioReads asks API and automation users to host their own instance; configure an instance you host, or one whose operator explicitly permits automated library-wide requests. No API key or Goodreads account is required. **You can find a list of public endpoints at https://github.com/nesaku/BiblioReads#instances**. -To enable lookups, pass your permitted endpoints as a comma-separated list. Requests are spread across only these endpoints, with automatic failover when one is unavailable. There are no built-in public endpoints or public-instance fallback: +To enable lookups, pass your permitted endpoints as a comma-separated list. Requests are spread across only these endpoints, with automatic failover when one is unavailable. Configuring multiple permitted instances can improve lookup throughput by spreading requests across them; actual performance depends on each instance's availability and capacity. There are no built-in public endpoints or public-instance fallback: ```sh coreander --review-source-instances=https://br.example.org,https://br2.example.org /absolute/path/to/library @@ -190,7 +190,7 @@ In case both a flag and its equivalent environment variable are passed, flag tak |`--client-dynamic-image-cache-ttl` |`CLIENT_DYNAMIC_IMAGE_CACHE_TTL` | Client-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 86400 (24 hours). |`--server-static-cache-ttl` |`SERVER_STATIC_CACHE_TTL` | Server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 31536000 (1 year). |`--server-dynamic-image-cache-ttl` |`SERVER_DYNAMIC_IMAGE_CACHE_TTL` | Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 86400 (24 hours). -|`--review-source-instances` |`REVIEW_SOURCE_INSTANCES` | Comma-separated HTTP or HTTPS BiblioReads endpoints you host or whose operators permit automation. Required to enable lookups; defaults to none. Omit and unset the environment variable to disable lookups. The flag may be repeated. For example: `--review-source-instances=http://localhost:8080`. +|`--review-source-instances` |`REVIEW_SOURCE_INSTANCES` | Comma-separated HTTP or HTTPS BiblioReads endpoints you host or whose operators permit automation. Multiple instances can improve lookup throughput. Required to enable lookups; defaults to none. Omit and unset the environment variable to disable lookups. The flag may be repeated. For example: `--review-source-instances=http://localhost:8080`. |`--review-source-cache-ttl` |`REVIEW_SOURCE_CACHE_TTL` | Duration in seconds for which review-source responses are cached in memory. Set to 0 to disable caching. Defaults to 86400 (24 hours). |`-f` or `--force-indexing` |`FORCE_INDEXING` | Whether to force indexing already indexed documents or not. Defaults to false. |`-s` or `--jwt-secret` |`JWT_SECRET` | String to use to sign JWTs. diff --git a/cli_input.go b/cli_input.go index a5190ade..a07e633e 100644 --- a/cli_input.go +++ b/cli_input.go @@ -83,7 +83,7 @@ type CLIInput struct { ServerStaticCacheTTL int `env:"SERVER_STATIC_CACHE_TTL" default:"31536000" name:"server-static-cache-ttl" help:"Server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 1 year (31536000 seconds)."` // ServerDynamicImageCacheTTL defines the server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours. ServerDynamicImageCacheTTL int `env:"SERVER_DYNAMIC_IMAGE_CACHE_TTL" default:"86400" name:"server-dynamic-image-cache-ttl" help:"Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours (86400 seconds)."` - ReviewSourceInstances []string `env:"REVIEW_SOURCE_INSTANCES" name:"review-source-instances" sep:"," help:"Comma-separated HTTP or HTTPS BiblioReads endpoints you host or whose operators permit automation. Required to enable review lookups; omit to disable. May be repeated."` + ReviewSourceInstances []string `env:"REVIEW_SOURCE_INSTANCES" name:"review-source-instances" sep:"," help:"Comma-separated HTTP or HTTPS BiblioReads endpoints you host or whose operators permit automation. Multiple instances can improve lookup throughput. Required to enable review lookups; omit to disable. May be repeated."` // ReviewSourceCacheTTL defines how long review-source responses are cached in memory, in seconds. Defaults to 24 hours. ReviewSourceCacheTTL int `env:"REVIEW_SOURCE_CACHE_TTL" default:"86400" name:"review-source-cache-ttl" help:"Duration in seconds for which review-source responses are cached in memory. Set to 0 to disable caching. Defaults to 24 hours (86400 seconds)."` // ShareCommentMaxSize defines the maximum length for share comments in characters. Defaults to 280. diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index d0cba89b..86dc4d36 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -150,90 +150,82 @@ func TestUnifiedSearch(t *testing.T) { } func TestDataSourceScoreSortOptions(t *testing.T) { - cfg := defaultTestConfig() - cfg.ReviewSourceConfigured = true - app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, - loadDirInMemoryFs("testdata/library"), cfg) - for _, path := range []string{ - "/search?type=documents&", - "/authors/john-doe?", - "/series/the-lord-of-the-rings?", - } { - for _, direction := range []string{"asc", "desc"} { - for _, fragment := range []bool{false, true} { - mode := "/full" - if fragment { - mode = "/fragment" - } - t.Run(path+direction+mode, func(t *testing.T) { - selected := "review-score-" + direction - req := mustGetRequest(t, path+"sort-by="+selected) - if fragment { - req.Header.Set("HX-Request", "true") - } - response, err := app.Test(req) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - t.Fatalf("unexpected status %d", response.StatusCode) - } - doc, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - t.Fatal(err) - } - for _, value := range []string{"review-score-asc", "review-score-desc"} { - if doc.Find(`#sort-by option[value="`+value+`"]`).Length() != 1 { - t.Errorf("missing sort option %q", value) - } - } - if doc.Find(`#sort-by option[value="`+selected+`"][selected]`).Length() != 1 { - t.Errorf("expected selected option %q", selected) - } - }) - } - } + paths := []struct { + name string + url string + }{ + {name: "document search", url: "/search?type=documents&"}, + {name: "author documents", url: "/authors/john-doe?"}, + {name: "series documents", url: "/series/the-lord-of-the-rings?"}, + } + modes := []struct { + name string + htmx bool + }{ + {name: "full page"}, + {name: "fragment", htmx: true}, } -} -func TestDataSourceScoreSortOptionsHiddenWithoutReviewSources(t *testing.T) { - app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, - loadDirInMemoryFs("testdata/library"), defaultTestConfig()) - for _, path := range []string{ - "/search?type=documents&", - "/authors/john-doe?", - "/series/the-lord-of-the-rings?", - } { - for _, fragment := range []bool{false, true} { - mode := "/full" - if fragment { - mode = "/fragment" - } - t.Run(path+mode, func(t *testing.T) { - req := mustGetRequest(t, path) - if fragment { - req.Header.Set("HX-Request", "true") - } - response, err := app.Test(req) - if err != nil { - t.Fatal(err) - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - t.Fatalf("unexpected status %d", response.StatusCode) - } - doc, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - t.Fatal(err) - } - for _, value := range []string{"review-score-asc", "review-score-desc"} { - if doc.Find(`#sort-by option[value="`+value+`"]`).Length() != 0 { - t.Errorf("unexpected score sort option %q", value) + for _, configured := range []bool{false, true} { + name := "without review sources" + directions := []string{""} + if configured { + name = "with review sources" + directions = []string{"asc", "desc"} + } + t.Run(name, func(t *testing.T) { + cfg := defaultTestConfig() + cfg.ReviewSourceConfigured = configured + app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, + loadDirInMemoryFs("testdata/library"), cfg) + + for _, path := range paths { + for _, mode := range modes { + for _, direction := range directions { + name := path.name + "/" + mode.name + selected := "" + if direction != "" { + selected = "review-score-" + direction + name += "/" + direction + } + t.Run(name, func(t *testing.T) { + url := path.url + if selected != "" { + url += "sort-by=" + selected + } + req := mustGetRequest(t, url) + if mode.htmx { + req.Header.Set("HX-Request", "true") + } + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("unexpected status %d", response.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + wantCount := 0 + if configured { + wantCount = 1 + } + for _, value := range []string{"review-score-asc", "review-score-desc"} { + if got := doc.Find(`#sort-by option[value="` + value + `"]`).Length(); got != wantCount { + t.Errorf("score sort option %q count = %d, want %d", value, got, wantCount) + } + } + if selected != "" && doc.Find(`#sort-by option[value="`+selected+`"][selected]`).Length() != 1 { + t.Errorf("expected selected option %q", selected) + } + }) } } - }) - } + } + }) } } From 6c07ad194b76cf05483a4897900bd83078cfd16d Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Thu, 5 Nov 2026 16:21:16 +0100 Subject: [PATCH 49/50] A failed document lookup no longer cancels the batch --- internal/index/bleve_document_enrich.go | 53 ++++++++++--------- internal/index/bleve_document_enrich_test.go | 51 +++++++++++++++--- .../index/bleve_provider_preservation_test.go | 2 +- internal/index/document_data_sources_test.go | 4 +- 4 files changed, 76 insertions(+), 34 deletions(-) diff --git a/internal/index/bleve_document_enrich.go b/internal/index/bleve_document_enrich.go index 381105e2..df19a292 100644 --- a/internal/index/bleve_document_enrich.go +++ b/internal/index/bleve_document_enrich.go @@ -26,12 +26,19 @@ type DocumentEnricher interface { Enrich(context.Context, *BleveIndexer, Document) error } -func (b *BleveIndexer) documentsNeedingProvider(source string, size int) ([]Document, uint64, error) { +func (b *BleveIndexer) documentsNeedingProvider(source string, size int, excluded map[string]struct{}) ([]Document, uint64, error) { enriched := bleve.NewTermQuery(source) enriched.SetField("EnrichedDataSources") q := bleve.NewBooleanQuery() q.AddMust(bleve.NewMatchAllQuery()) q.AddMustNot(enriched) + if len(excluded) > 0 { + ids := make([]string, 0, len(excluded)) + for id := range excluded { + ids = append(ids, id) + } + q.AddMustNot(bleve.NewDocIDQuery(ids)) + } req := bleve.NewSearchRequestOptions(q, size, 0, false) req.Fields = []string{"*"} b.documentsMu.RLock() @@ -70,64 +77,62 @@ func (b *BleveIndexer) documentEnrichTracker(i int) *progressTracker { } func (b *BleveIndexer) enrichProvider(provider DocumentEnricher, tracker *progressTracker, batchSize int) error { - _, total, err := b.documentsNeedingProvider(provider.Source(), 0) + _, total, err := b.documentsNeedingProvider(provider.Source(), 0, nil) if err != nil || total == 0 { return err } tracker.begin(int(total)) defer tracker.end() log.Printf("Enriching %d documents with %s data", total, provider.Source()) + var failures []error + failedDocuments := make(map[string]struct{}) for iteration := 0; iteration < int(total)+10; iteration++ { - documents, _, err := b.documentsNeedingProvider(provider.Source(), batchSize) + documents, _, err := b.documentsNeedingProvider(provider.Source(), batchSize, failedDocuments) if err != nil { - return err + failures = append(failures, err) + break } if len(documents) == 0 { - return nil + return errors.Join(failures...) } - if err := b.enrichProviderBatch(provider, tracker, documents); err != nil { - return err + for id, err := range b.enrichProviderBatch(provider, tracker, documents) { + failedDocuments[id] = struct{}{} + failures = append(failures, err) } } + if len(failures) > 0 { + return errors.Join(failures...) + } return fmt.Errorf("document enrichment did not finish; some documents remain pending") } -func (b *BleveIndexer) enrichProviderBatch(provider DocumentEnricher, tracker *progressTracker, documents []Document) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +func (b *BleveIndexer) enrichProviderBatch(provider DocumentEnricher, tracker *progressTracker, documents []Document) map[string]error { jobs := make(chan Document, len(documents)) for _, document := range documents { jobs <- document } close(jobs) var workers sync.WaitGroup - failures := make(chan error, 1) + var failureMu sync.Mutex + failures := make(map[string]error) for range min(documentEnrichWorkers, len(documents)) { workers.Go(func() { for document := range jobs { needed, err := provider.NeedsEnrichment(document) if err == nil && needed { - err = provider.Enrich(ctx, b, document) + err = provider.Enrich(context.Background(), b, document) } if err != nil { - select { - case failures <- err: - cancel() - default: - } - return + failureMu.Lock() + failures[document.ID] = fmt.Errorf("enriching document %s: %w", document.ID, err) + failureMu.Unlock() } tracker.record() } }) } workers.Wait() - select { - case err := <-failures: - return err - default: - return nil - } + return failures } func (b *BleveIndexer) runDocumentEnrichWorker() { diff --git a/internal/index/bleve_document_enrich_test.go b/internal/index/bleve_document_enrich_test.go index 3a9088fe..69fcc838 100644 --- a/internal/index/bleve_document_enrich_test.go +++ b/internal/index/bleve_document_enrich_test.go @@ -12,9 +12,10 @@ import ( ) type testDocumentEnricher struct { - source string - calls atomic.Int32 - failure error + source string + calls atomic.Int32 + failure error + failureID string } type blockingDocumentEnricher struct { @@ -48,12 +49,12 @@ func TestDocumentEnrichmentUsesBoundedConcurrency(t *testing.T) { } idx, _ := newTestTextRankIndexer(t, Config{}) documents := make([]Document, documentCount) - done := make(chan error, 1) + done := make(chan map[string]error, 1) go func() { done <- idx.enrichProviderBatch(provider, &progressTracker{}, documents) }() defer func() { close(provider.release) - if err := <-done; err != nil { - t.Errorf("enrichment failed: %v", err) + if failures := <-done; len(failures) != 0 { + t.Errorf("enrichment failed: %v", failures) } }() for range documentEnrichWorkers { @@ -185,7 +186,7 @@ func (p *testDocumentEnricher) NeedsEnrichment(document Document) (bool, error) func (p *testDocumentEnricher) Enrich(_ context.Context, idx *BleveIndexer, document Document) error { p.calls.Add(1) - if p.failure != nil { + if p.failure != nil && (p.failureID == "" || p.failureID == document.ID) { return p.failure } return idx.StoreDocumentData(document.ID, p.source, map[string]any{ @@ -193,6 +194,42 @@ func (p *testDocumentEnricher) Enrich(_ context.Context, idx *BleveIndexer, docu }) } +func TestDocumentEnrichmentContinuesAfterDocumentFailure(t *testing.T) { + unavailable := errors.New("document unavailable") + provider := &testDocumentEnricher{source: "first", failure: unavailable, failureID: "book-2"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{provider}}) + t.Cleanup(func() { _ = idx.Close() }) + for _, id := range []string{"book-1", "book-2", "book-3"} { + if err := IndexDocument(idx.documentsIdx, id, Document{ID: id}); err != nil { + t.Fatal(err) + } + } + + if err := idx.EnrichDocuments(1); !errors.Is(err, unavailable) { + t.Fatalf("expected the individual lookup error, got %v", err) + } + if provider.calls.Load() != 3 { + t.Fatalf("expected all three documents to be attempted, got %d", provider.calls.Load()) + } + for _, id := range []string{"book-1", "book-3"} { + document, err := idx.documentByIndexID(id) + if err != nil { + t.Fatal(err) + } + data, err := DataSourceData[map[string]any](document, "first") + if err != nil || data["enriched"] != true { + t.Errorf("%s was not enriched despite another document failing: %+v, %v", id, data, err) + } + } + pending, _, err := idx.documentsNeedingProvider("first", 10, nil) + if err != nil { + t.Fatal(err) + } + if len(pending) != 1 || pending[0].ID != "book-2" { + t.Fatalf("failed document should remain pending for a later retry: %+v", pending) + } +} + func TestDocumentEnrichmentSupportsIndependentProviders(t *testing.T) { first := &testDocumentEnricher{source: "first"} second := &testDocumentEnricher{source: "second"} diff --git a/internal/index/bleve_provider_preservation_test.go b/internal/index/bleve_provider_preservation_test.go index d89fbd58..6b0105ca 100644 --- a/internal/index/bleve_provider_preservation_test.go +++ b/internal/index/bleve_provider_preservation_test.go @@ -59,7 +59,7 @@ func TestTextRankWritesPreserveProviderRemovals(t *testing.T) { if !got.TextRankEnriched { t.Fatal("TextRank result must still be persisted") } - pending, total, err := idx.documentsNeedingProvider("dummy", 10) + pending, total, err := idx.documentsNeedingProvider("dummy", 10, nil) if err != nil || total != 1 || len(pending) != 1 || pending[0].ID != "book" { t.Fatalf("reset provider lookup must remain pending: %+v, %d, %v", pending, total, err) } diff --git a/internal/index/document_data_sources_test.go b/internal/index/document_data_sources_test.go index f9433a8f..8e0f7b67 100644 --- a/internal/index/document_data_sources_test.go +++ b/internal/index/document_data_sources_test.go @@ -47,7 +47,7 @@ func TestDocumentDataSourcesRoundTrip(t *testing.T) { } } } - if _, total, err := idx.documentsNeedingProvider("dummy", 10); err != nil || total != 0 { + if _, total, err := idx.documentsNeedingProvider("dummy", 10, nil); err != nil || total != 0 { t.Fatalf("enriched payloads were queued again: %d, %v", total, err) } pending := Document{ID: "other-only", Slug: "other-only", Metadata: metadata.Metadata{Title: "Pending book"}} @@ -57,7 +57,7 @@ func TestDocumentDataSourcesRoundTrip(t *testing.T) { if err := IndexDocument(idx.documentsIdx, pending.ID, pending); err != nil { t.Fatal(err) } - if documents, total, err := idx.documentsNeedingProvider("dummy", 10); err != nil || total != 1 || documents[0].ID != pending.ID { + if documents, total, err := idx.documentsNeedingProvider("dummy", 10, nil); err != nil || total != 1 || documents[0].ID != pending.ID { t.Fatalf("another provider marked provider as enriched: %+v, %d, %v", documents, total, err) } } From ddeeb563462f5458a6bcae9703d481b1772d5ca0 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Thu, 5 Nov 2026 16:35:04 +0100 Subject: [PATCH 50/50] Changed readme --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index df9110ef..191b0100 100644 --- a/README.md +++ b/README.md @@ -96,9 +96,7 @@ To enable lookups, pass your permitted endpoints as a comma-separated list. Requ coreander --review-source-instances=https://br.example.org,https://br2.example.org /absolute/path/to/library ``` -Alternatively, set `REVIEW_SOURCE_INSTANCES` to the same comma-separated list. The flag may also be repeated and overrides the environment variable. Instance URLs must use HTTP or HTTPS and serve the BiblioReads API. Existing installations must configure permitted endpoints to resume automated lookups after upgrading. - -Lookups send document titles and author names to the configured BiblioReads instances. To disable lookups, omit `--review-source-instances` and unset `REVIEW_SOURCE_INSTANCES`. The former `--disable-reviews` flag and `DISABLE_REVIEWS` environment variable have been removed; endpoint configuration alone controls whether lookups are enabled. Previously stored ratings and links remain available without enabling network lookups. If no configured instance can serve a request, Goodreads information may be unavailable, but the rest of Coreander remains usable. +Alternatively, set `REVIEW_SOURCE_INSTANCES` to the same comma-separated list. The flag may also be repeated and overrides the environment variable. Instance URLs must use HTTP or HTTPS and serve the BiblioReads API. ### Email