diff --git a/README.md b/README.md index a68a9569..191b0100 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ A personal documents server, Coreander indexes the documents (EPUBs and PDFs wit * Upload documents through the web interface. * Download as kepub (epub for Kobo devices) converted on the fly thanks to [Kepubify](https://github.com/pgaskin/kepubify). * Gather information about authors from [Wikidata](https://wikidata.org). +* Show Goodreads ratings and book and author links through [BiblioReads](#biblioreads-integration). ## Installation @@ -85,6 +86,18 @@ Refer to [Caddy documentation](https://caddyserver.com/docs) for more informatio When using a reverse proxy, include its client-facing port in `FQDN`, for example `FQDN=coreander.example.com:443` for the HTTPS setup above (or `:80` for HTTP), so generated links do not use Coreander's internal listening port. +### BiblioReads integration + +Coreander can use [BiblioReads](https://github.com/nesaku/biblioreads), an alternative Goodreads front-end, to find Goodreads book and author links and display book ratings on document detail pages. Lookups are disabled by default and require explicitly configured endpoints. BiblioReads asks API and automation users to host their own instance; configure an instance you host, or one whose operator explicitly permits automated library-wide requests. No API key or Goodreads account is required. **You can find a list of public endpoints at https://github.com/nesaku/BiblioReads#instances**. + +To enable lookups, pass your permitted endpoints as a comma-separated list. Requests are spread across only these endpoints, with automatic failover when one is unavailable. Configuring multiple permitted instances can improve lookup throughput by spreading requests across them; actual performance depends on each instance's availability and capacity. There are no built-in public endpoints or public-instance fallback: + +```sh +coreander --review-source-instances=https://br.example.org,https://br2.example.org /absolute/path/to/library +``` + +Alternatively, set `REVIEW_SOURCE_INSTANCES` to the same comma-separated list. The flag may also be repeated and overrides the environment variable. Instance URLs must use HTTP or HTTPS and serve the BiblioReads API. + ### Email Some features rely on having an SMTP email service set up, and won't be available otherwise: @@ -175,6 +188,8 @@ In case both a flag and its equivalent environment variable are passed, flag tak |`--client-dynamic-image-cache-ttl` |`CLIENT_DYNAMIC_IMAGE_CACHE_TTL` | Client-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 86400 (24 hours). |`--server-static-cache-ttl` |`SERVER_STATIC_CACHE_TTL` | Server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 31536000 (1 year). |`--server-dynamic-image-cache-ttl` |`SERVER_DYNAMIC_IMAGE_CACHE_TTL` | Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 86400 (24 hours). +|`--review-source-instances` |`REVIEW_SOURCE_INSTANCES` | Comma-separated HTTP or HTTPS BiblioReads endpoints you host or whose operators permit automation. Multiple instances can improve lookup throughput. Required to enable lookups; defaults to none. Omit and unset the environment variable to disable lookups. The flag may be repeated. For example: `--review-source-instances=http://localhost:8080`. +|`--review-source-cache-ttl` |`REVIEW_SOURCE_CACHE_TTL` | Duration in seconds for which review-source responses are cached in memory. Set to 0 to disable caching. Defaults to 86400 (24 hours). |`-f` or `--force-indexing` |`FORCE_INDEXING` | Whether to force indexing already indexed documents or not. Defaults to false. |`-s` or `--jwt-secret` |`JWT_SECRET` | String to use to sign JWTs. |`-a` or `--require-auth` |`REQUIRE_AUTH` | Require authentication to access the application if true. Defaults to false. diff --git a/biblioreads.json b/biblioreads.json new file mode 100644 index 00000000..9b9694b2 --- /dev/null +++ b/biblioreads.json @@ -0,0 +1,765 @@ +//POST https://biblioreads.eu.org/api/search/books +//PAYLOAD +//{queryURL: "https://www.goodreads.com/search?q=La%20Reina%20Drag%C3%B3n%20Alice%20Borchardt"} +//RESPONSE +{ + "status": "Received", + "source": "https://github.com/nesaku/biblioreads", + "apiURL": "https://www.goodreads.com/book/auto_complete?format=json&q=La%20Reina%20Drag%C3%B3n%20Alice%20Borchardt", + "searchType": "books", + "numberOfResults": "1 results", + "result": [ + { + "id": 1, + "cover": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/books/1332599049i/12207444._SX50_.jpg", + "title": "La Reina Dragon (La leyenda de Guinevere, #1)", + "bookURL": "/book/show/12207444-la-reina-dragon", + "author": "Alice Borchardt", + "authorURL": "/author/show/151155.Alice_Borchardt", + "rating": "3.82" + } + ], + "lastScraped": "2026-10-28T17:45:31.333Z" +} + +//POST https://biblioreads.eu.org/api/book-scraper +// PAYLOAD +// {queryURL: "https://www.goodreads.com/book/show/12207444-la-reina-dragon"} +// RESPONSE +{ + "status": "Received", + "statusCode": 200, + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + "legacyBookID": 12207444, + "bookID": "kca://book/amzn1.gr.book.v1.4FDu7eHal9NDfA3jGNhOXw", + "resourceID": "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", + "cover": "https://m.media-amazon.com/images/S/compressed.photo.goodreads.com/books/1332599049i/12207444.jpg", + "series": "Tales of Guinevere", + "seriesURL": "/series/50072-tales-of-guinevere", + "seriesPosition": "1", + "workURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + "title": "La Reina Dragon", + "author": [ + { + "id": 1, + "name": "Alice Borchardt", + "role": "Author", + "url": "/author/show/151155.Alice_Borchardt" + } + ], + "rating": "3.82", + "ratingCount": "2,285", + "reviewsCount": "115", + "desc": "Arturo se dio la vuelta y vino resueltamente hacia nosotros. Estaba espléndido, y nunca olvidaré que en ese momento comencé a amarlo. Y ya nunca dejé de hacerlo. Nunca nadie me ha provocado tanta pena, dolor y alegría como él. Ni siquiera por mis hijos he sentido el amor que le profeso todavía ahora. Y creo que aunque hubiera sabido lo que nos deparaba el futuro (preocupaciones, tormentos, batallas y penas), aun así levaría entregado mi corazón tal como hice entonces.\r\n\r\nAlice Brondchardt entra en el reino de la leyenda artúrica y lo hace suyo. La reina Dragón es el primer volumen de la trilogía que recrea el mundo de Camelot con la reina Guinevere convertida en una guerrea sagaz y obstinada.\r\n\r\nNacida en un entorno en el que los enfrentamientos son constantes y las armas están siempre presentes en la vida de hombres y mujeres, Guinevere, hija de una poderosa reina pagana, es una amenaza para su gente y una preciada presa para el temible mago Merlín.\r\n\r\nEnviada a u lugar secreto, crece bajo la protección de un hombre-lobo y de un druida de agrio carácter. Pero ni siquiera en las remotas costas de Escocia, donde los dragones velan por ella, está a salvo del todopoderoso mago. Merlín conoce el destino de la bella joven y no se detendrá ante nada para evitar el cumplimiento de la profecía, puesto que si Guinevere y Arturo llegan a reinar, llevarán un paz al país que convertirá a Merlín, hambriento de poder, en un personaje secundario.", + "genres": [ + "Fantasy", + "Arthurian", + "Fiction", + "Historical Fiction", + "Dragons", + "Historical", + "Mythology", + "Magic", + "Science Fiction Fantasy", + "Retellings" + ], + "bookEdition": "452 pages, Hardcover", + "publishDate": "Wed Jan 01 2003", + "publisher": "Timun Mas", + "language": "Spanish; Castilian", + "asin": "8448033183", + "isbn": "8448033183", + "isbn13": "9788448033187", + "related": [], + "reviews": [], + "reviewBreakdown": { + "rating1": "72", + "rating2": "170", + "rating3": "586", + "rating4": "731", + "rating5": "726" + }, + "quotes": "4", + "quotesURL": "/work/quotes/2440721", + "questions": "1", + "questionsURL": "/book/258666/questions", + "lastScraped": "2026-10-28T18:34:27.525Z" +} + +// POST https://biblioreads.eu.org/api/reviews-scraper +// PAYLOAD +// {resourceID: "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", legacyBookID: 12207444,…} +// RESPONSE +{ + "status": "Received", + "statusCode": 200, + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/book/show/12207444/reviews", + "totalCount": 115, + "nextPageToken": "NTgsMTcwNTAxNjc4NjQzNA", + "prevPageToken": "MTgwMSwxMjE0MTUxODU5MDAw", + "reviews": [ + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.gH7Sf4_G4iz72wvI_yVVyg", + "reviewer": { + "id": 1259454, + "name": "Shawna Astley", + "url": "/user/show/1259454-shawna-astley", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1214150161i/1259454._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 42, + "followerCount": 0 + }, + "rating": 5, + "text": "Women fighting men with swords - whats not to like? ", + "createdAt": "Sun Jun 22 2008", + "updatedAt": "Tue Jan 07 2025", + "spoiler": false, + "likeCount": 18, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/25118938" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.be94OXB9ezWIlFZvdGMKvA", + "reviewer": { + "id": 2478293, + "name": "Zeek", + "url": "/user/show/2478293-zeek", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1339154824i/2478293.jpg", + "isAuthor": false, + "reviewCount": 939, + "followerCount": 149 + }, + "rating": 3, + "text": "Although I liked this tale overall, there were pieces of it that bogged the story down for me.

The Dragon Queen is yet another entry into the tales of Arthur and The Knights of the Round Table. For those who love their Arthurian legends with Merlin as the big bad- then this book's for you. For those who don't-then Run away!

TDQ centers around Guinevere, the warrior queen of Arthur and in Borchardt's version she is indeed a warrior. Guinevere, has the beauty of previous Arthurian tales, but in TDQ she's not the sweet little misguided star-crossed lover of Lancelot that we usually see her portrayed to be. Courageous and bold she has power in her right hand where ferocious fire emanates (grand super power that!), and healing in her left. Moreover, arrows flyout her bow as if an extension of her arm, she communicates with dragons- as well with the dead- displays prophetic abilities, and audaciously defeats asundry of evil beasts that Merlin flings at her.

The story begins with Maeniel, Guinevere's soon to be foster father. Maeniel, otherwise knows as the Gray Watcher, is a shapeshifting man/wolf who has personally felt the destructive powers of Merlin. Deciding that being a wolf was much preferred to living with the humans, he escapes to the wilds and begins a family. It is during his time with the wolves that Guinevere falls into his care, and the Gray Watcher, along with the sorcerer Dugald- who Merlin chased into hiding long ago- take up the charge of raising her. The Gray Watcher's wolf mate nurses
Guinevere from the time they find her left on a cliff and later a woman joins them who will teach her the ways of her people- a people she never knew.

As a child, Guinevere helps her village to defeat a marauding group of seafarers- and it is here where we see her potential for greatness begin to come forth. Guinevere grows in stature, wisdom and power and, though still only a young teen through much of the story, she quickly finds herself thrust into peril. Merlin wants her dead, and though the author details many things in TDQ, I never quite got the gist of his motive for seeking to destroy her so passionately. The only thing I can figure is that she must be the one who either precipitates or actually carries out Merlin's eventual destruction.

Many other side characters show up to aid or engage her in battle. There are all kinds of fantasy creatures- including mythological gods- that she meets up with and, eventually, she meets her future in the boy of Arthur.

Arthur is just as strong as she- and closer to her age then what I've seen portrayed before. Set long before Camalot, he's young and only discovering who and what he's eventually going to be. Much of the middle of the book is from his POV, where he meets the Lady of the Lake. I was disappointed however, that at the end of TDQ, he and Guinevere do not meet up again. (I suppose that will come in later installments.)

The jumping around of POV's was rather confusing for me and the author lost me a bit until it began to pick up again toward the end. The book is full of detailed descriptions of the world Borchardt has created and some of it became overwhelming. To me her descriptions dragged the story down and lost me for most of the middle of the book. But, as I said before, by the end I was back with her.

This is definitely sequel bait- the story is far from over. I'm interested to see where the author- who was Anne Rice's sister, btw- will take it.", + "createdAt": "Thu May 20 2010", + "updatedAt": "Sun May 24 2026", + "spoiler": false, + "likeCount": 10, + "commentCount": 4, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/103546143" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.sC0IQDlAXY4LbkJ3LTJHvA", + "reviewer": { + "id": 80386185, + "name": "Rial", + "url": "/user/show/80386185-rial", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1545932910i/80386185._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 30, + "followerCount": 0 + }, + "rating": 1, + "text": "I did not like this book. However, it is my first 1 Star, so it's got that going for it.

Why did I read this? It was selected for my wife and roommate's book club, and I parasitically piggyback a lot of their books (gives me something to talk about...). It's Arthurian! It's from Guinivere's perspective! Magic! Celtic! Post 'Fall of the Western Empire'! These things and more!

No, it's terrible. Why did I keep reading it? It did take me on a journey: I experienced all the 'Stages of Grief' (some of them more than once.

Denial: Ha, it's so weird, but it'll get better! The chapters are long, but that's okay! Weird, the perspective shifted...?
Anger: I liked the other character; this one is dumb. She's a Mary Sue! WTF? That just worked out for her with zero effort! Okay, she needs to show some growth...
Bargaining: ...Okay, the parts with Arthur aren't bad, maybe?
Anger: Why are they having revenge sex in the forest?
Denial: Arthur is cool, yes, he's very cool... :|
Depression: I really shouldn't be reading this anymore; I think it's actively hurting me.
Acceptance: There is only book. Oh yeah, I guess your shoes can do that. Yep, you're definitely super fucking hot, and everyone wants to be or be with you. Sure why not? I don't have enough brain power to think it should be any different. After all, you're the best, Gwen!

But the real joy of this book is in not starting it. If you made that mistake and you're too masochistic to give up, then finishing it is an exultant experience

The contents are one thing, but they aren't even well served by the writing. Transitions are abrupt and jarring. I back-read multiple times to try and figure out where the hell I was, only to realize there wasn't any explanation; it just got there and it was fine (it wasn't fine).

This thing had such potential: Arthurian set in the 'proper' time frame, the Christianization of Britannia and the clashes with pagan beliefs that ensue, Celtic and Brythonic myth woven throughout. But it all reads so poorly in the end. The author is far too verbose and floral with her language. First person POV ruins any sense of actual accomplishment because we only see the character succeeding; no real growth or change is experienced. I find Divine Right to be problematic, and this book is very much into the romanticism of a destined ruler (both for Arthur and for Gwen).

Oh yeah, Merlin is evil and helps Arthur's mom torment Arthur, I guess? Like, if you hate the kid so much just kill him? Oh wait, 'destiny'...

Tedious tripe, glorified garbage, wretched rubbish. What a waste of time.", + "createdAt": "Mon Nov 05 2018", + "updatedAt": "Fri Jun 10 2022", + "spoiler": false, + "likeCount": 6, + "commentCount": 2, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2578561727" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.rwxYpuQEynxP2f5dSLFmPQ", + "reviewer": { + "id": 90733221, + "name": "Monique Mihalitsianos", + "url": "/user/show/90733221-monique-mihalitsianos", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1773156601i/90733221._UX200_CR0,29,200,200_.jpg", + "isAuthor": true, + "reviewCount": 88, + "followerCount": 9 + }, + "rating": 3, + "text": "This book was splendid in the story-telling sense. The story was epic and magical and engaging. Also, this is, by far, the best Guinevere I have encountered in modern fiction so far. She was strong but not brass, brave but not stupid, powerful yet humble, compassionate but not weak, feminine yet full of agency. Perhaps what I liked the most about her was that she was in communion with animals and gods alike and knew her place in the wheel of existence. She was wise like that, but never overbearingly so. I loved her. I wanted to *be* her.

Arthur I also liked very much, and I liked that this was a tortured Arthur who had suffered child abuse at the hands of her horrible queen mother Igrane (*the* worst evil queen ever) but was still good and noble and could not lie and never feared danger. I liked his relationship with his father, Uther, the High King, which was refreshingly healthy and loving. Merlin I hated. This is a book with a bad Merlin in it, which I don't like as a trope. I much prefer my Merlins to be good and Gandalf-like, but in this case it worked to have him be so evil and act as the villain. The black magic he wrought was one of the sickest and most horrifying ones I've ever read in a story.

Alice Borchardt seems to know her history well, so much so in fact that it's almost like if she lived in this period of time (Roman - Saxon) in a past life. I, for one, appreciated all of the extraneous detail relating to the customs and mores of the time because it made the world-building that much better.

The magic system in this book was refreshingly deep and believable. So many magic systems in recent fantasy books are light and superficial, as if they had been thought up in brainstorming sessions instead of being fully inspired. Not so with Borchardt's story; the magic here seemed inspired by the author's own psyche and thus, was multilayered and poliphacetic, changeable and malleable but also essentially true to its own core.

I also liked that an existencial and historical struggle was hinted at: the convergence between paganism and Christianity, if there can be a convergence at all. It was only hinted at here but I wouldn't mind if this theme in particular were elaborated more in the second book (The Raven Warrior) because it is of particular interest to me. On some level I think it was also of particular interest to Borchardt, as well. I am a huge fan of Anne Rice's (Alice Borchardt's sister) Vampire Chronicles, and Rice definitely waxes philosophical about her own struggles with different faith systems and beliefs in her own fiction, much more so than Borchardt, who chose to focus more on the story. But I wouldn't have minded if she had expanded on this in her fiction, just like her sister did. But then again, I am of a philosophical bent and like these kinds of tangents.

Why the three-star rating, then, if I liked the book so much? Well, the story was golden, but the writing was confusing at times, especially at the beginning and end. Sometimes it was almost stream-of-consciousness like, and I got lost and confused. At other times, when a character referred to somebody else as 'she', for example, I didn't know who they were talking about, because there were several 'she's' they could have been referring to. So some more specificity in this and other similar instances would have been appreciated. Sometimes, also, the writing got too dream-like, too airy fairy and dreamy...and perhaps more concrete writing would have served the book better overall. I also suspect this book could have had tighter editing. I know for a fact Anne Rice is not a fan of people editing her writing, and I don't know if Alice Borchardt felt the same as her sister does in this matter, but I for one think good editing is absolutely crucial in any book.

Putting this aside, though, I can say that I thoroughly enjoyed this book and am looking forward to reading the second and final installment. This was meant to be a trilogy, but unfortunately, the author died from illness before she could complete the third book, so two books is all we have. I will read her other books as well, though. She's a gem of a story-teller, at least in my estimation.", + "createdAt": "Wed Mar 18 2020", + "updatedAt": "Thu Nov 30 2023", + "spoiler": false, + "likeCount": 3, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3208179930" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.XvL2nHk7Bt4tULNXUE2yaA", + "reviewer": { + "id": 2207539, + "name": "Steen", + "url": "/user/show/2207539-steen", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1408254648i/2207539._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 473, + "followerCount": 4 + }, + "rating": 1, + "text": "This was a did not finish book for me.

So after reading about 185 pages of The Dragon Queen by Alice Borchardt I have decided to thrown it down and stop reading. I thought I would like it being about Quinevere and all but I hate it. It's not like the Arthur legends I've ever known. But that's not really even it though. I could handle a different take on the stories ( I still have yet to read any books about Arthur that are out there that are more well known) it's just the way she has written it. She jumps from one place to another and it's like when did that happen? Also sometimes when there are a few people talking 'she says...' your like who is talking? Which one is saying what. UGH. I hate not finishing a book but I really can't continue to read it anymore.

Having said that I really have no urge to read Bequiled which is also by her. I read the first in that series and while it was better written then The Dragon Queen I really don't love the characters in the first book to continue to read the second one anymore. Not after trying to get through The Dragon Queen and hating it. It's sad but yeah. I love Anne Rice's books and hoped that I would equally like her sister's books. But I guess they are just not for me.

I've read reviews on these two books and it sounds like her werewolf books are pretty good but I think I may wait a while to even attempt to try those ones.

So now its off to the library today to pick up some more to read.", + "createdAt": "Sat Apr 11 2009", + "updatedAt": "Wed Jan 23 2019", + "spoiler": false, + "likeCount": 3, + "commentCount": 3, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/52264098" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.rZ5icbmFfLHPKpEQOm2kNw", + "reviewer": { + "id": 41814516, + "name": "Alberto García", + "url": "/user/show/41814516-alberto-garc-a", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1508706260i/41814516._UX200_CR0,33,200,200_.jpg", + "isAuthor": true, + "reviewCount": 45, + "followerCount": 40 + }, + "rating": 4, + "text": "Voto de confianza... esta m¡¡bien escrito y la revisión de la leyenda artúrica es muy interesante. Al final, en algunos capítulos, se le va un poco la pinza con salidas algo paranoicas que me han dejado bastante descolocado... lo que es seguro es que no me lo esperaba... pero tampoco comprendo demasiado la razón para ello. Veremos cómo acaba y si se queda en una anécdota.", + "createdAt": "Fri Jun 21 2019", + "updatedAt": "Mon Sep 14 2026", + "spoiler": false, + "likeCount": 3, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2796875885" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.G3-a2b8-oopoAbpH6RmbsA", + "reviewer": { + "id": 73527602, + "name": "Rachel", + "url": "/user/show/73527602-rachel", + "avatar": "", + "isAuthor": false, + "reviewCount": 13, + "followerCount": 0 + }, + "rating": 3, + "text": "This was like reading someone's dream. The ideas were all loosely connected but the transitions were weird and random or completely non-existent. However, the characters and concepts were fun and interesting and I did like this book.", + "createdAt": "Fri Jan 04 2019", + "updatedAt": "Mon Nov 17 2025", + "spoiler": false, + "likeCount": 4, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2582029737" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.juoLRbi8R_urzbj6jgRAww", + "reviewer": { + "id": 101256289, + "name": "Malischa Nijenhuis", + "url": "/user/show/101256289-malischa-nijenhuis", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1773261867i/101256289._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 1222, + "followerCount": 32 + }, + "rating": 5, + "text": "Eigenlijks lees ik nu een boek van mijn man maar wauw wat is dit een gave uitgave en boek. Nu op naar boek 2", + "createdAt": "Sat Mar 21 2020", + "updatedAt": "Mon Mar 23 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3238259324" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.GLjLEKHIsU-RrmRdU73hGQ", + "reviewer": { + "id": 3504881, + "name": "Suzanne", + "url": "/user/show/3504881-suzanne", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1294105241i/3504881.jpg", + "isAuthor": false, + "reviewCount": 546, + "followerCount": 12 + }, + "rating": 2, + "text": "I'm very sorry to say, I could not finish this book. I REALLY wanted to be in love with this story. I even purchased the 2nd book from Thrift Books soon after I started this one. I figured they would become part of my Arthurian obsession, and I would need to own them. It started out with a great concept, and I liked the differences in the legend that I hadn't read before. I wanted to know more about the different characters, but it was so choppy, from one point of view to another back and forth, I couldn't keep track of which character I was reading about.

It just dragged on and on and on and on....it was trial after trial after trial. The characters would accomplish one win and suddenly face another trial. I felt like I was reading a bad rendition of The Odyssey. I REALLY wanted to love it. But reading became a chore, and I fell into a reading slump. The book sat in one spot for a couple weeks, and I felt it hanging over my head. I finally decided enough was enough. I wanted to be eager to read again. So I took out the bookmark, and this is going on my \"couldn't finish\" list.

I will donate the set to a Free Little Library somewhere, and I hope someone else will get some pleasure out of them.", + "createdAt": "Sun Oct 14 2018", + "updatedAt": "Fri Oct 30 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 2, + "recommendFor": null, + "shelf": "did-not-finish", + "reviewURL": "/review/show/2522667985" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.73ge1ENF_VjVNC1CSJb8iw", + "reviewer": { + "id": 67537582, + "name": "Kaylee", + "url": "/user/show/67537582-kaylee", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1627453497i/67537582._UX200_CR0,1,200,200_.jpg", + "isAuthor": false, + "reviewCount": 793, + "followerCount": 33 + }, + "rating": 1, + "text": "I do have to start by saying that this is the first Arthurian legend I can recall reading. I don't think that really affected my POV but...

Extremely hard to get through from the beginning. Kept hoping it would get better but nope. Not only was the switching between third and first POV annoying but it was made worse by the main character. Did not want her voice at all. It was also hard to tell what was going on/who was speaking at times (during the third POV of course). In the beginning I went back to the and re-read things but quickly learned there was no point. Sometimes things arrived randomly and others I realized I had just spaced out because of all the needless words.  You could take out half the words in this book and be all the better off for it...

Also if I had to read \"As I said...\" one more time...!

I can't remember the last time I didn't finish a book and if I hadn't had to read this because of my book club I don't think I would have finished. Says everything in a nutshell.

So much more but I'll just leave it at that. Extremely glad to be done with this book!", + "createdAt": "Fri Nov 09 2018", + "updatedAt": "Fri Oct 30 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 2, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2577596943" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.6kd8QVi2sBrNhUslyz1aUQ", + "reviewer": { + "id": 54738977, + "name": "TigerLily ", + "url": "/user/show/54738977-tigerlily", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1460164836i/54738977.jpg", + "isAuthor": false, + "reviewCount": 133, + "followerCount": 3 + }, + "rating": 5, + "text": "Really enjoyable, a page Turner that I didn't want to be finished. I am looking forward to reading the 2nd book.", + "createdAt": "Sun Jan 10 2021", + "updatedAt": "Sun Mar 21 2021", + "spoiler": false, + "likeCount": 2, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3732026445" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.njxZJZUHk2BVK8mcTJp2aQ", + "reviewer": { + "id": 40251914, + "name": "Elizabeth Sanders", + "url": "/user/show/40251914-elizabeth-sanders", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1530132382i/40251914.jpg", + "isAuthor": false, + "reviewCount": 38, + "followerCount": 0 + }, + "rating": 4, + "text": "It's an interesting retelling of Guinevere and Arthur. I loved every bit of it and can't wait to read the sequel.", + "createdAt": "Thu Jul 04 2019", + "updatedAt": "Thu Jul 11 2019", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2871843590" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.HegnVxYZnLTsNGxJeVV2cA", + "reviewer": { + "id": 11384547, + "name": "Lyndsay MacDonald", + "url": "/user/show/11384547-lyndsay-macdonald", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1371265927i/11384547.jpg", + "isAuthor": false, + "reviewCount": 38, + "followerCount": 0 + }, + "rating": 2, + "text": "This book was weird. I think it might have been tolerable if the author didn't try to weave it into the Arthurian legend (ie. if she made up original characters), but as it is I just couldn't get into it. There were some parts/scenes that I did enjoy, and others that I didn't... but overall I found it very confusing, and often found myself asking \"What is happening right now?\" and \"Why?\"

The story line flip-flopped around a lot... the characters were not wholly believable (Guinevere did NOT seem like a 13 year old girl, by any stretch of the imagination).. and I found parts of the plot silly and unrelated.

I won't be reading any of the other books in this sequence.", + "createdAt": "Wed Jan 22 2014", + "updatedAt": "Fri Aug 12 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/831334767" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.oazwOZoGnu8yIQJ_48FVMw", + "reviewer": { + "id": 27373770, + "name": "Andrea Krivak", + "url": "/user/show/27373770-andrea-krivak", + "avatar": "", + "isAuthor": false, + "reviewCount": 26, + "followerCount": 1 + }, + "rating": 2, + "text": "I think you have to really like arthurian legends to read this book. I liked it in the beginning, it started telling the story of Geunivere as a young girl and the \"what if\" this was her upbringing. There were wizards, men who turned into wolves and of course, merlin and morgana. About halfway through she meets Arthur, and then the book divides into the telling of each of their lives and they each stray into their own mystic journeys. To me, it spent too much time on these mystic journeys and by the end, i was ready to be done with this whole story. Not interesting enough ending for me to read into the sequel.", + "createdAt": "Mon Mar 24 2014", + "updatedAt": "Fri Aug 12 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/890822661" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.IxTTRac3W8gbXoHFdfTu5g", + "reviewer": { + "id": 1009204, + "name": "Cindy", + "url": "/user/show/1009204-cindy", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1228465282i/1009204._UX200_CR0,28,200,200_.jpg", + "isAuthor": false, + "reviewCount": 856, + "followerCount": 105 + }, + "rating": 1, + "text": "Maybe some day I'll come back to this book, but I just don't think I can read this book. I find myself wondering if I'm not following something of if this book is just beyond my reading. The writing is in a way that makes it seems like, wow we want the read to think this author is just so descriptive and great. Only it becomes confusing and bored. If you aren't familiar with the story of Guinevere then there is no point in reading it because nothing is explained and you are just sitting there wondering what the heck is going on. ", + "createdAt": "Wed Aug 27 2008", + "updatedAt": "Thu Jan 25 2018", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/31374042" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1._BHlRCbTwy-DJRq5mCTQnw", + "reviewer": { + "id": 4804714, + "name": "Angela", + "url": "/user/show/4804714-angela", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1294946776i/4804714.jpg", + "isAuthor": false, + "reviewCount": 129, + "followerCount": 6 + }, + "rating": 3, + "text": "I enjoyed this book up until the last third and then it got way too much involved with Arthur. It didn't quite wrap up the way I had expected and I would suggest not reading it unless you want to continue on with the series. I may read the sequel at a later time but I need a break from the acid trip like sections with Arthur.", + "createdAt": "Thu Mar 28 2013", + "updatedAt": "Tue Aug 09 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/572964102" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.P4s-Ysv7_6bKQAupxDvCwA", + "reviewer": { + "id": 4018173, + "name": "Misty", + "url": "/user/show/4018173-misty", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1286250306i/4018173._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 307, + "followerCount": 0 + }, + "rating": 2, + "text": "I really wanted to like this book.

About halfway through, I got bored and tried to push through it, but ultimately gave up. There was so much description and not enough happening.

Also, considering this was titled the Dragon Queen, I expected more dragons. Their lack of presence was disappointing.", + "createdAt": "Tue Nov 26 2013", + "updatedAt": "Thu Jun 23 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "could-not-finish", + "reviewURL": "/review/show/774467120" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.2E0pSS7MyztIPHdoIbH4Nw", + "reviewer": { + "id": 120860, + "name": "Lassarina Aoibhell", + "url": "/user/show/120860-lassarina-aoibhell", + "avatar": "", + "isAuthor": false, + "reviewCount": 487, + "followerCount": 5 + }, + "rating": 1, + "text": "I am DNFing this book on page 84 and we're going to go on a wild ride about why.

NB: If you are a scholar of Arthuriana, history, religion, or....well, almost anything, you might want to put down your drink. I'm not responsible for the state of your monitor.

We begin with the cover copy: Daughter of a mighty pagan queen, Guinevere has become a threat to her people and a prize to the dreaded sorcerer Merlin. Sent into hiding, she grows up under the protection of a shapeshifting man-wolf and an ornery Druid. But even on the remote coast of Scotland, where dragons feed and watch over her, she is not safe. Merlin knows the young beauty's destiny--and he will stop at nothing to prevent what has been foretold...

I also need to strongly highlight one of the blurbs. Like, really strongly. \"In addition to being exceptionally well written, [The Dragon Queen] has enormous integrity: All of the stories--of Arthur's parentage, of Merlin, of the characters' daily struggles--are well grounded in the history, politics, and religions of the time.....This is a trilogy to watch.\" --San Jose Mercury News

I want you to look at that quote carefully, and then come on a ride with me.

I will acknowledge up front my own reading comprehension fail, because I had forgotten that the shapeshifter was actually in the blurb and was yelling about \"what the fuck.\". I'm still saying \"what the fuck,\" just about plenty of other things.

Let's start with: It takes until page 48 for Guinevere to show up in any real sense. 500 page book; the main character doesn't actually appear until 10% in. Don't like that. Instead we are following around Dugald, the druid, and Maeniel, the shapeshifter. He's allegedly from Gaul. We begin at Tintagel in the year 470 CE, where Merlin and some of his cronies murder Vortigen. Some kind of magical snake is involved. (Aside: absolutely no problem with Merlin as the villain, particularly in anything allegedly centering the women of this tale.) Maeniel the shifter teleports??? out of the carnage, finds himself a she-wolf who has recently given birth, and shacks up with her. They encounter Dugald and the toddler Guinevere, who is currently disguised as a boy. For some reason, although everyone complains about disliking the Romans, we re-enact the founding of Rome, with Maeniel's mate suckling little Guinevere. Why does the she-wolf agree to this? Who can say. Dugald, meanwhile, talks about the influence of St. Patrick. Yes, that one. \"We don't call ourselves [druids] any longer. Not since Patrick came to Ireland bearing Christ's message.\"

....ten pages ago we called Merlin a chief druid. Also. It's the year 470. We don't have exact dates of the saint's life, but we know it was roughly in that century. More importantly, it was the Romans who banned druidic practices, fucking 400 years earlier (Augustus had no chill.) So there is no way that Patrick's actions in Ireland are affecting the order in England. Remember that blurb I highlighted? HMMMM. ALSO WHY ARE YOU, A GUY IN CORNWALL, TALKING ABOUT PATRICK. why. what is even your deal.

Also also, in this variation of the story (?), apparently, Dugald had a philosophical disagreement with Merlin and \"lost,\" so he was expected by the rules of his order to commit suicide because he lost.

Me: I do not remember reading this in any of the books I consumed during my obsessive period about Celtic histories and cultures and archaeology. (Wiki and checking with knowledgeable friends suggests I am not alone in not remembering this.)

If nothing else, that seems like a giant fucking waste of a 20 year apprenticeship, man.

(At this point I also stopped to verify my memory that Dugald and Maeniel do not appear in other Arthurian sources. I seem to be correct. And like, yeah, we add characters to stories all the time, I'm just. why are you making me deal with men in a book about Guinevere!)

There's also a point where young Guinevere, Dugald, and Maeniel are involved in helping to fight off a ship of slavers, which conveniently results in there being a woman to join the household and teach Guinevere How To Be A Girl. I should specify that the captain of said slaver's ship was apparently named Cymry. Why is he literally named \"Welsh\"? fuck if I know.

There's also a particular kind of ambient misogyny that I find tiresome, and like, I know we're in the fifth century and the Romans were assholes, and also I am aware that Celtic cultures were not paragons of equality and virtue, but the available evidence both of Romans being bitchy about how ass-backward Celtic societies were (omg they let their women do things!!!!) and the remnants of brehon law allocating specific property to women after divorce and categorizing a type of marriage as \"she married down and gets to keep all her stuff\" [wildly paraphrased] I'm just like

what. what are you doing, ma'am

and once again I'm going to refer back to that blurb. About being well grounded in the history, politics, and religions of the time.

I do not recommend actually reading this unless you want to play a drinking game, in which case, may I suggest that the drink you use should perhaps be water?", + "createdAt": "Thu Apr 09 2026", + "updatedAt": "Thu Apr 09 2026", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "did-not-finish", + "reviewURL": "/review/show/1761732484" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.9X0QSJx7BxHSTHSctKMkwg", + "reviewer": { + "id": 11348002, + "name": "Laural", + "url": "/user/show/11348002-laural", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1349329030i/11348002._UX200_CR0,5,200,200_.jpg", + "isAuthor": false, + "reviewCount": 36, + "followerCount": 5 + }, + "rating": 2, + "text": "I got about halfway through this book. Someday I'll give it another chance, or maybe try to figure out what exactly the author did wrong. A sword weilding Gweneviere? It could have been great. I thought it would be after reading the description on the back of the gorgeous cover. ", + "createdAt": "Thu Oct 04 2012", + "updatedAt": "Thu Jun 23 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/427412066" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.p0mSKsHft_BTv2vScLHOsQ", + "reviewer": { + "id": 1250008, + "name": "Giuila", + "url": "/user/show/1250008-giuila", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1244141968i/1250008._UY200_CR2,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 61, + "followerCount": 0 + }, + "rating": 0, + "text": "... t read", + "createdAt": "Thu Jul 17 2008", + "updatedAt": "Fri Mar 24 2017", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "to-read", + "reviewURL": "/review/show/27520982" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.QMO0-GkHiX0UEYcsOaO7lw", + "reviewer": { + "id": 1431136, + "name": "Matt", + "url": "/user/show/1431136-matt", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1218856549i/1431136._UY200_CR25,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 6, + "followerCount": 0 + }, + "rating": 5, + "text": "an overall awesome book", + "createdAt": "Sat Aug 16 2008", + "updatedAt": "Fri Mar 24 2017", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/30279062" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.qOdgrbVKU-XMQR5DSbHfDQ", + "reviewer": { + "id": 16979552, + "name": "Sarah Garner", + "url": "/user/show/16979552-sarah-garner", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1408284741i/16979552._UX200_CR0,53,200,200_.jpg", + "isAuthor": false, + "reviewCount": 84, + "followerCount": 18 + }, + "rating": 2, + "text": "I struggled with this book and I couldn't finish it, although it was a great idea, I didn't love this book.", + "createdAt": "Sat Sep 20 2014", + "updatedAt": "Thu Aug 20 2020", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/1059961768" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.JjlmKD1eCF28HlpY4KmCGw", + "reviewer": { + "id": 34744007, + "name": "Michael D.", + "url": "/user/show/34744007-michael-d", + "avatar": "", + "isAuthor": false, + "reviewCount": 81, + "followerCount": 1 + }, + "rating": 2, + "text": "I had this book about 20 years ago, tried to read it, but I just couldn't get into it. I got another copy used and gave it another try. While not as bad as my first experience in the first 20 pages, I found in the ensuing pages that the narrative is quite disjointed, and timelines are fluid and unclear to me. Perhaps that was Borchardt's intent, but I was confused by it. Additionally, it seems that characters are magically transported to strange places without any transition that I could discern. This narrative was hard to follow.

The book begins in a scene in 470 in Tintigal (not Tintagil, the famous castle in Cornwall where Arthur was conceived and born in the traditional romance). Ostensibly, the main character is a wolfman (a wolf who changes into a man when he chooses to) named Maeniel, and he is a friend to the British king Vortigen, who is not the villain he is normally portrayed as in history and the romances. Merlin shows up with a contingent of Saxons, and a battle ensues in which Vortigen is slain and Maeniel is knocked unconscious by an unknown cudgel. In the next scene, he wakes up on a hillside far away, presumably the far north, near Hadrian's Wall, and he becomes the mate of a she-wolf, who bears him a son, Black Leg (who will become Lancelot in the second novel The Raven Warrior). Sometime later, we find that 70 years have passed since Vortigen's death. Oddly, there is a baby girl in the nearby village, Guenevere (Borchardt spells her name differently depending on who is addressing her), who is one of the major characters of this series, the eponymous destined Dragon Queen. When Guenevere is 15, we are then told that 100 years have passed since Vortigen died. Confusion abounds.

This story had occasional high points, with modern fantastical treatments of the transformation duels, but I really had a hard time following the flow of Borchardt's narrative. I am a pretty astute reader, so I find it difficult to believe this was my fault. Borchardt seemed to be playing coy with the unfolding of events, and that was annoying. She was the sister of the better-known Anne Rice, author of the Vampire Chronicles, so I tried hard to give this a chance.

Borchardt intended to make this a trilogy, but the passed away in 2007 before the third book in the trilogy could be published. I don't know if anyone picked up her notes with the intent to complete her work, and I can't imagine that there was a demand for it. I do intend to slog through The Raven Warrior, as it is my rule to read books through and evaluate them honestly. If they're bad, I point out why. The lack of comprehensible transitions in time and space are my primary complaint with this novel. I hope that The Raven Warrior is more cognizant of walking me through how characters get from Point A to Point B than this one was.
", + "createdAt": "Thu Nov 09 2023", + "updatedAt": "Thu Nov 09 2023", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5944805120" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v2.2025072400.a7efbaf3-dd61-4567-b4e1-d3f344fabb0c", + "reviewer": { + "id": 42506425, + "name": "Michelle M", + "url": "/user/show/42506425-michelle-m", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1715615833i/42506425._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 380, + "followerCount": 3 + }, + "rating": 5, + "text": "Honestly? 5/6 or 4.75/6
I really loved this book! It's hard to describe what exactly I loved about it though lol. It had a very ethereal quality to it... The magic in this world is just a fact of life, some peple have it, while others don't, and yet, it didn't feel fantastical. Everyone knows about magic and most peoples still believe in the gods and things even though \"The Church\" (christianity) is present on the British Isles.

Borchardt grounds the story in the time right after Rome leaves the Isles, though they are still very much an influence on the peoples. It was a very mystical vibe to the whole story, kind of like there is a shroud of fog that is the magic of the world. IDK, like I said, hard to describe lol.

I loved Guinevere, and I liked how different characters referred to her by various spellings/pronouciations thoughout the story, as if they were speaking different languages. It is a very different take on the Arthur/Guinevere story, and very much an action oriented one, with Guinevere being the focus of the action and not just a passive bystander.

I though the portrayal of the other major characters interesting, and they were given roles that they are not usual given in the story. Arthur is \"good\", Merlin and Ingrane are the \"evil\" ones, Guinevere is strong willed and knows what she wants and sets out to get it. She doesn't seem like the type of person who will betray Arthur with Lancelot, but we'll just have to find out! (I hope she doesn't). The portrayal of these characters (and other named characters) made me do some research to try to get a better picture of if they had ever been portayed like this before. My search was unsucessful lol, as they were very diffefrent from \"traditional\" stories, but I haven't really read many retellings, so I cant be sure if they are exclusive to this story.

My only compaint with this book is that I found out it was supposed to be a trilogy!!!! And YET there are only two books! T.T *cries* I know the author can't help that she passed away, but book 2 came out in 2003 and she passed in 2007! That seems like a decent amount of time to finish the third book (I'm being very ungrateful, I know I am, I just lament what never was). Also apparently Anne Rice of vampire fame is Borchardt's sister?!?!?! She doesn't have like an early draft of her sister's just laying around??? T.T

I will def be reading the next one! (this one took me so long because I paused to participate in a read a thon and then got distracted by other books lol and then I got busy with work so I didn't have as much time to read)", + "createdAt": "Thu Jul 24 2025", + "updatedAt": "Thu Jul 24 2025", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/7714101862" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.HXGbQg0020aQJjD9G6pW2A", + "reviewer": { + "id": 163241100, + "name": "Alex", + "url": "/user/show/163241100-alex", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1763011908i/163241100._UY200_CR11,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 98, + "followerCount": 0 + }, + "rating": 2, + "text": "I enjoyed Guinevere’s character early on in the book. It was interesting watching her being raised alongside her foster brother, Black Leg, by a wolf shifter (Maniel) and a she-wolf. I do appreciate her relationship with the She Wolf, who Guinevere calls Mother. The thing that really kept me going through this book was Guinevere’s interactions with the dragons of the land and her connection to the land. I was also intrigued by some of the same conversations about religion in this book since Guinevere is raised in nature and has regular interactions with a Druid and some interactions with Christiany.
All this being said I struggled with the writing. There are huge chunks of it that are confusing since it’s not always clear who is being talked about when he, she or they are mentioned considering there might be multiple people mentioned previously in the scene. Transitions between moments are also rough. I was questioning when things changed and what was being talked about. The long descriptions of things really didn’t help either and me zoning out as I read.
It was also a bit rough to read this because of how things were worded/certain sciences being really uncomfortable for seemingly no reason out of nowhere. The dynamics between Igraine and Merlin’s relationship being one them. Scenes with them were sexual with zero warning and it wasn’t just limited to them. It happened a lot with various characters. Things were also made pretty uncomfortable with minors even if it was a scene that wasn’t intended to uncomfortably sexual, it was made that way by directly saying things like they “were far too young for even a dream of passion.” With how confusing the writing was I couldn’t even tell if there was a reason for why these sort of scenes were written in detail outside of making the audience feel uncomfortable and actively dislike the characters who are at fault for these behaviors.", + "createdAt": "Sun May 24 2026", + "updatedAt": "Sun May 24 2026", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5900548383" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.VoNZbtbhrTyL1Fhg43jpGw", + "reviewer": { + "id": 42064438, + "name": "Briana", + "url": "/user/show/42064438-briana", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1715717411i/42064438._UY200_CR5,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 161, + "followerCount": 1 + }, + "rating": 2, + "text": "This book was not for me. I had high hopes, the summary sounded interesting. It took me much longer than usual to finish this book as I struggled to enjoy it and I will not be picking up the second installment. I almost DNF'd this book several times but decided to stick with it.

The first half of the book felt like it was the author's freewriting development of the book, which she never bothered to go back and actually edit into a cohesive and understandable story. It was like being is some kind of dream where everything was hazy and only loosely connected. Sometimes you ended up in an entirely different location or time without even realizing it. It was also almost impossible to understand who was being discussed as the author tends to avoid using names and simply use pronouns leaving the reader to try and figure out (usually without success) who the \"she\" is that is being discussed. It was also very interesting that the author (or editor) chose to spell Guinevere's name differently throughout the book.

The second half was a little better and is the only reason I didn't give this book a one star. I don't mind Merlin as the antagonist as some other reviewers do and I found his character interesting. However, I did not really like the relationship between Arthur and Guinevere. I'm not sure how I am supposed to be convinced that this was some fated love story when they met for all of maybe an hour total. It seemed more insta-lust than anything else and there was no relationship development at all after that.

Overall I spent my time either being very confused and a little bored or kind of confused and mildly interested.", + "createdAt": "Thu Aug 15 2024", + "updatedAt": "Thu Aug 15 2024", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/6728900786" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.IoZW54HX8Qad9w5IgbQB9A", + "reviewer": { + "id": 6424389, + "name": "Joy Pouros", + "url": "/user/show/6424389-joy-pouros", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1567889312i/6424389._UY200_CR47,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 125, + "followerCount": 2 + }, + "rating": 3, + "text": "2.5 stars.

I REALLY like the character of Guinevere, but there were other parts of the story I didn't enjoy.

The writing style can be very winding and even abstract in some ways. There are whole parts you can skim without missing much. There aren't a ton of people, but enough to get them confused, especially a couple with similar names.

I'm always very frustrated with stories that try to have strong female characters but still have an overly sexually charged world. There are so many wonderfully creative parts of this world and then we default back to several tired tropes. There's just no need.

Some chapters are 30 pages, some are 5. It doesn't appear based on POV or time, so why is this so inconsistent?

And finally... I understand this is the first book of the trilogy, but this book just felt like a long prologue. Other series the first book is still a worthwhile read and entertaining story. This is more of an origin story before you fully appreciate the main story. So there are very mixed feelings about continuing the series because while I love Guinevere, there's an awful lot to overlook. ", + "createdAt": "Tue Aug 30 2022", + "updatedAt": "Tue Aug 30 2022", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/4955253716" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.ahVbGwMWksFfbjRS9c6Vbg", + "reviewer": { + "id": 29405146, + "name": "Anita", + "url": "/user/show/29405146-anita", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1407269712i/29405146._CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 614, + "followerCount": 3 + }, + "rating": 1, + "text": "First of all, I love Arthurian legends. Second, I always finish books, constantly hopeful that the story will improve, but I must confess that I skimmed this book to the end. I just cannot read any more!
This story has no cohesion, no main narrator and skips from one location to another mid-chapter. Plus there are men suddenly shapeshifting into wolves, and back again; no reasons given; no context for the intrigue and killings taking place; and a young girl child, supposedly Guinevere of Arthurian legend, being suckled by a she-wolf after being kidnapped.
Imaginative description is great in a fantasy story; confusion is not. This book is one long puzzle - a maze with apparently no direction, interspersed with some flowery descriptions of sunsets and misty mornings; visions of ravens and dragons (fishing); and a cast of hundreds of unrelated and irrelevant characters. Reading this book feels akin to experiencing one of those dreams in which you are lost and desperately trying to find your way back home.
Need I add that I did not enjoy this book, and will not be reading the sequel?", + "createdAt": "Fri Oct 30 2020", + "updatedAt": "Fri Oct 30 2020", + "spoiler": true, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3618732305" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.zo7-0bYwOUS0LlPXyqeuXQ", + "reviewer": { + "id": 3198878, + "name": "Stevie", + "url": "/user/show/3198878-stevie", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1560478233i/3198878._UX200_CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 71, + "followerCount": 0 + }, + "rating": 2, + "text": "Rating this book was a challenge. It was an impulse grab. I suspected it would be wonderfully terrible or terribly wonderful and there was only one way to find out.

What I liked: Arthurian legend from Guin's point of view. Cool. That awkward time period in history when they were both Christian but still observed many pagan rites. It had all the names I knew, and some of the tropes I knew: Vortigen, the witch in the well, so on and so forth from various movies I've seen.

What I disliked: the writing. Some of the dialogue wasn't needed. The descriptions could be too much in places and entirely lacking in others. Who is speaking? What do they look like? They were a human in a tower and now they are a fish? When? How? Transitions don't exist. This makes it hard to follow.

It was a wild ride, and I hung on until the end because I was curious as to how it would turn out. Turns out there's a sequel...I don't know that I have that in me...", + "createdAt": "Sat Mar 21 2020", + "updatedAt": "Sat Mar 21 2020", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3240571912" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.E7XotAu3DvsavlUw1u62eA", + "reviewer": { + "id": 107938758, + "name": "Danai", + "url": "/user/show/107938758-danai", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1705239736i/107938758._UX200_CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 133, + "followerCount": 0 + }, + "rating": 3, + "text": "I didn't love it, nor did I hate it.

Everything was magical, mystical, and mysterious; Borchardt painted an imaginative world with interesting and diverse characters. She evidently has a deep knowledge of the British Isles, its history, and its lore.

But the writing was, at times, completely chaotic and confusing. As if someone was just spilling their string of thoughts without sufficient context or details - or too many of the wrong details. More than once, I found myself confused by abrupt changes in the situation, only to learn - through narration - that a memory was being shared.

This book didn't capture me nor engross me by any means, but it did beckon me to come back and finish it. If I come across the next book in the series, I will read it simply out of curiosity - what happens next for our heroes and villains? But I won't go hunting for a copy of it anytime soon...", + "createdAt": "Thu Jan 11 2024", + "updatedAt": "Thu Jan 11 2024", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5638508977" + } + ], + "lastScraped": "2026-10-28T18:38:19.808Z" +} \ No newline at end of file diff --git a/cli_input.go b/cli_input.go index f707b116..a07e633e 100644 --- a/cli_input.go +++ b/cli_input.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "net/url" "github.com/alecthomas/kong" ) @@ -81,7 +82,10 @@ type CLIInput struct { // ServerStaticCacheTTL defines the server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 1 year. ServerStaticCacheTTL int `env:"SERVER_STATIC_CACHE_TTL" default:"31536000" name:"server-static-cache-ttl" help:"Server-side cache duration for static assets (CSS, JS, images) in seconds. Defaults to 1 year (31536000 seconds)."` // ServerDynamicImageCacheTTL defines the server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours. - ServerDynamicImageCacheTTL int `env:"SERVER_DYNAMIC_IMAGE_CACHE_TTL" default:"86400" name:"server-dynamic-image-cache-ttl" help:"Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours (86400 seconds)."` + ServerDynamicImageCacheTTL int `env:"SERVER_DYNAMIC_IMAGE_CACHE_TTL" default:"86400" name:"server-dynamic-image-cache-ttl" help:"Server-side cache duration for dynamically generated images (covers, author images) in seconds. Defaults to 24 hours (86400 seconds)."` + ReviewSourceInstances []string `env:"REVIEW_SOURCE_INSTANCES" name:"review-source-instances" sep:"," help:"Comma-separated HTTP or HTTPS BiblioReads endpoints you host or whose operators permit automation. Multiple instances can improve lookup throughput. Required to enable review lookups; omit to disable. May be repeated."` + // ReviewSourceCacheTTL defines how long review-source responses are cached in memory, in seconds. Defaults to 24 hours. + ReviewSourceCacheTTL int `env:"REVIEW_SOURCE_CACHE_TTL" default:"86400" name:"review-source-cache-ttl" help:"Duration in seconds for which review-source responses are cached in memory. Set to 0 to disable caching. Defaults to 24 hours (86400 seconds)."` // ShareCommentMaxSize defines the maximum length for share comments in characters. Defaults to 280. ShareCommentMaxSize int `env:"SHARE_COMMENT_MAX_SIZE" short:"m" default:"280" name:"share-comment-max-size" help:"Maximum length for share comments in characters. Defaults to 280."` // ShareMaxRecipients defines the maximum number of recipients allowed when sharing a document. Defaults to 10. @@ -120,6 +124,17 @@ func (c CLIInput) Validate() error { if c.ShareCommentMaxSize < 0 { return fmt.Errorf("share-comment-max-size must not be negative, got %v", c.ShareCommentMaxSize) } + // ReviewSourceCacheTTL must not be negative: 0 is the documented way to + // disable caching, so a negative value is most likely a typo. + if c.ReviewSourceCacheTTL < 0 { + return fmt.Errorf("review-source-cache-ttl must not be negative, got %v", c.ReviewSourceCacheTTL) + } + for _, instance := range c.ReviewSourceInstances { + u, err := url.Parse(instance) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" { + return fmt.Errorf("review-source-instances must contain absolute HTTP or HTTPS URLs, got %q", instance) + } + } // MinPasswordLength must be positive: 0 or negative disables the minimum // length check entirely, silently allowing empty passwords. if c.MinPasswordLength <= 0 { diff --git a/cli_input_test.go b/cli_input_test.go new file mode 100644 index 00000000..2ce5f4f2 --- /dev/null +++ b/cli_input_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "os" + "reflect" + "strings" + "testing" + + "github.com/alecthomas/kong" +) + +func TestReviewSourceInstances(t *testing.T) { + tests := []struct { + name string + args []string + env string + want []string + }{ + {name: "defaults"}, + {name: "single", args: []string{"--review-source-instances=https://br.example.org"}, want: []string{"https://br.example.org"}}, + {name: "comma separated", args: []string{"--review-source-instances=https://br.example.org,http://localhost:8080/"}, want: []string{"https://br.example.org", "http://localhost:8080/"}}, + {name: "repeated", args: []string{"--review-source-instances=https://br.example.org", "--review-source-instances=https://br2.example.org"}, want: []string{"https://br.example.org", "https://br2.example.org"}}, + {name: "environment", env: "https://br.example.org,https://br2.example.org", want: []string{"https://br.example.org", "https://br2.example.org"}}, + {name: "flag overrides environment", env: "https://env.example.org", args: []string{"--review-source-instances=https://cli.example.org"}, want: []string{"https://cli.example.org"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("REVIEW_SOURCE_INSTANCES", tt.env) + var input CLIInput + parser := newCLIParser(t, &input) + if _, err := parser.Parse(append(tt.args, t.TempDir())); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(input.ReviewSourceInstances, tt.want) { + t.Errorf("instances = %v, want %v", input.ReviewSourceInstances, tt.want) + } + }) + } +} + +func TestReviewSourceInstancesRejectInvalidURLs(t *testing.T) { + for _, instance := range []string{"br.example.org", "/relative", "ftp://br.example.org", "https://", "https://bad host", "https://br.example.org,invalid"} { + t.Run(instance, func(t *testing.T) { + var input CLIInput + parser := newCLIParser(t, &input) + _, err := parser.Parse([]string{"--review-source-instances=" + instance, t.TempDir()}) + if err == nil || !strings.Contains(err.Error(), "review-source-instances") { + t.Fatalf("expected review-source-instances validation error, got %v", err) + } + }) + } +} + +func TestReviewSourceSettings(t *testing.T) { + for _, tt := range []struct { + name string + args []string + ttl string + wantTTL int + }{ + {name: "defaults", wantTTL: 86400}, + {name: "flags", args: []string{"--review-source-cache-ttl=120"}, wantTTL: 120}, + {name: "environment", ttl: "300", wantTTL: 300}, + {name: "flags override environment", ttl: "300", args: []string{"--review-source-cache-ttl=0"}}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("REVIEW_SOURCE_CACHE_TTL", tt.ttl) + for key, value := range map[string]string{ + "REVIEW_SOURCE_CACHE_TTL": tt.ttl, + } { + if value == "" { + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + var input CLIInput + parser := newCLIParser(t, &input) + if _, err := parser.Parse(append(tt.args, t.TempDir())); err != nil { + t.Fatal(err) + } + if input.ReviewSourceCacheTTL != tt.wantTTL { + t.Fatalf("TTL = %d, want %d", input.ReviewSourceCacheTTL, tt.wantTTL) + } + }) + } +} + +func TestDisableReviewsFlagRemoved(t *testing.T) { + var input CLIInput + parser := newCLIParser(t, &input) + if _, err := parser.Parse([]string{"--disable-reviews", t.TempDir()}); err == nil || + !strings.Contains(err.Error(), "unknown flag") { + t.Fatalf("expected removed flag to be rejected, got %v", err) + } +} + +func TestReviewSourceCacheTTLRejectsNegative(t *testing.T) { + var input CLIInput + parser := newCLIParser(t, &input) + _, err := parser.Parse([]string{"--review-source-cache-ttl=-1", t.TempDir()}) + if err == nil || !strings.Contains(err.Error(), "review-source-cache-ttl") { + t.Fatalf("expected review-source-cache-ttl validation error, got %v", err) + } +} + +func newCLIParser(t *testing.T, input *CLIInput) *kong.Kong { + t.Helper() + parser, err := kong.New(input, kong.Vars{ + "defaultMaxSimilarityCandidates": "0", + "defaultMinSimilarityScoreRatio": "0", + "defaultMaxSimilarityPhrases": "0", + "defaultCommonTextRankEntryRatio": "0", + "defaultMinCommonTextRankAbsoluteCount": "0", + "defaultPruneChangeTriggerRatio": "0", + }) + if err != nil { + t.Fatal(err) + } + return parser +} diff --git a/index_migration_test.go b/index_migration_test.go new file mode 100644 index 00000000..ac92b868 --- /dev/null +++ b/index_migration_test.go @@ -0,0 +1,314 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" +) + +type migrationReader struct{ metadata.PdfReader } + +func (migrationReader) Metadata(string) (metadata.Metadata, error) { + return metadata.Metadata{Title: "Rebuilt book", Authors: []string{"Existing Author"}, Format: "PDF"}, nil +} + +type legacyV13Document struct { + metadata.Metadata + ID string + Slug string + AuthorsSlugs []string + IllustratorsSlugs []string + SeriesSlug string + SubjectsSlugs []string + AddedOn time.Time + TextRankPhrases []string + TextRankWords []string + TextRankEnriched bool +} + +func seedDocumentIndex(t *testing.T, version string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(homeDir, ".coreander"), 0755); err != nil { + t.Fatal(err) + } + documentsMapping := index.CreateDocumentsMapping().(*mapping.IndexMappingImpl) + delete(documentsMapping.DefaultMapping.Properties, "DataSources") + delete(documentsMapping.DefaultMapping.Properties, "EnrichedDataSources") + delete(documentsMapping.DefaultMapping.Properties, "DataSourceScoreValue") + for _, documentMapping := range documentsMapping.TypeMapping { + delete(documentMapping.Properties, "DataSources") + delete(documentMapping.Properties, "EnrichedDataSources") + delete(documentMapping.Properties, "DataSourceScoreValue") + } + legacyURLMapping := bleve.NewKeywordFieldMapping() + legacyURLMapping.Index = false + documentsMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsBookURL", legacyURLMapping) + documentsMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsAuthorURL", legacyURLMapping) + documentsMapping.DefaultMapping.AddFieldMappingsAt("GoodReadsEnriched", bleve.NewBooleanFieldMapping()) + documents, err := bleve.New(homeDir+documentsIndexPath, documentsMapping) + if err != nil { + t.Fatal(err) + } + for key, value := range map[string]string{ + "version": version, "illustrated_min_size": "0.25", + "min_occurrence_ratio": "0.1", "max_textrank_words": "10000", + } { + if err := documents.SetInternal([]byte(key), []byte(value)); err != nil { + t.Fatal(err) + } + } + if err := documents.Index("legacy-book", legacyV13Document{ + ID: "legacy-book", Slug: "legacy-book", AuthorsSlugs: []string{"existing-author"}, + Metadata: metadata.Metadata{Title: "Legacy book", Authors: []string{"Existing Author"}}, + }); err != nil { + t.Fatal(err) + } + if err := documents.Close(); err != nil { + t.Fatal(err) + } +} + +func TestGetIndexesRebuildsOlderDocuments(t *testing.T) { + for number := 1; number <= 13; number++ { + for _, force := range []bool{false, true} { + version := fmt.Sprintf("v%d", number) + t.Run(fmt.Sprintf("%s/force=%t", version, force), func(t *testing.T) { + previousHome, previousInput := homeDir, input + t.Cleanup(func() { homeDir, input = previousHome, previousInput }) + homeDir = t.TempDir() + input.BatchSize, input.ForceIndexing = 1, force + seedDocumentIndex(t, version) + authors := index.CreateAuthorsIndex(homeDir + authorsIndexPath) + retrieved := time.Now().UTC().Truncate(time.Second) + if err := authors.Index("existing-author", index.Author{ + Slug: "existing-author", Name: "Existing Author", DataSourceID: "Q1", RetrievedOn: retrieved, + }); err != nil { + t.Fatal(err) + } + if err := authors.Close(); err != nil { + t.Fatal(err) + } + + fs := afero.NewOsFs() + lib := filepath.Join(homeDir, "library") + if err := fs.MkdirAll(lib, 0755); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(fs, filepath.Join(lib, "book.pdf"), []byte("book"), 0644); err != nil { + t.Fatal(err) + } + documents, authors, needsReindex := getIndexes(fs, 0.25, 0.1, 10000) + if version == "v13" && !force { + if needsReindex { + t.Fatal("v13 migration must not request library reindexing") + } + if count, err := documents.DocCount(); err != nil || count != 1 { + t.Fatalf("v13 migration must preserve documents: count=%d, err=%v", count, err) + } + if migratedVersion, err := documents.GetInternal([]byte("version")); err != nil || string(migratedVersion) != index.DocumentVersion { + t.Fatalf("expected migrated index version %q, got %q, %v", index.DocumentVersion, migratedVersion, err) + } + idx := index.NewBleve(documents, authors, fs, lib, nil, index.Config{}) + document, err := idx.Document("legacy-book") + if err != nil || document.Title != "Legacy book" { + t.Fatalf("v13 migration must preserve indexed document: %+v, %v", document, err) + } + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil || data.URL != "" || data.Enriched { + t.Fatalf("v13 migration must not introduce Goodreads data: %+v, %v", data, err) + } + if err := idx.Close(); err != nil { + t.Fatal(err) + } + return + } + if !needsReindex { + t.Fatal("older index upgrade must request forced library indexing") + } + if count, err := documents.DocCount(); err != nil || count != 0 { + t.Fatalf("older documents must be rebuilt: count=%d, err=%v", count, err) + } + idx := index.NewBleve(documents, authors, fs, lib, + map[string]metadata.Reader{".pdf": migrationReader{}}, + index.Config{IllustratedMinSize: 0.25, MinOccurrenceRatio: 0.1, MaxTextRankWords: 10000}) + if err := idx.AddLibrary(1, needsReindex || input.ForceIndexing, 1); err != nil { + t.Fatal(err) + } + book, err := idx.Document("existing-author-rebuilt-book") + if err != nil || book.Title != "Rebuilt book" { + t.Fatalf("library must be searchable after rebuilding: %+v, %v", book, err) + } + author, err := idx.Author("existing-author", "") + if err != nil || author.DataSourceID != "Q1" || !author.RetrievedOn.Equal(retrieved) { + t.Fatalf("rebuilding must preserve author metadata: %+v, %v", author, err) + } + if err := idx.Close(); err != nil { + t.Fatal(err) + } + + documents, authors, needsReindex = getIndexes(fs, 0.25, 0.1, 10000) + defer documents.Close() + defer authors.Close() + if needsReindex { + t.Fatal("completed upgrade must not rebuild again on restart") + } + if version, err := documents.GetInternal([]byte("version")); err != nil || string(version) != index.DocumentVersion { + t.Fatalf("expected persisted current version, got %q, %v", version, err) + } + if count, err := documents.DocCount(); err != nil || count != 1 { + t.Fatalf("restart must retain rebuilt documents: count=%d, err=%v", count, err) + } + }) + } + } +} + +func TestGetIndexesRejectsUnsupportedDocuments(t *testing.T) { + if childHome := os.Getenv("COREANDER_TEST_UNSUPPORTED_INDEX_HOME"); childHome != "" { + homeDir = childHome + input.BatchSize, input.ForceIndexing = 1, true + getIndexes(afero.NewOsFs(), 0.25, 0.1, 10000) + return + } + for _, version := range []string{"", "v0", "v15", "v99", "v01", "v-1", "13", "unknown"} { + t.Run(version, func(t *testing.T) { + previousHome := homeDir + t.Cleanup(func() { homeDir = previousHome }) + homeDir = t.TempDir() + seedDocumentIndex(t, version) + command := exec.Command(os.Args[0], "-test.run=^TestGetIndexesRejectsUnsupportedDocuments$") + command.Env = append(os.Environ(), "COREANDER_TEST_UNSUPPORTED_INDEX_HOME="+homeDir) + output, err := command.CombinedOutput() + if err == nil || !strings.Contains(string(output), "Unsupported documents index version") { + t.Fatalf("expected startup refusal, got %v: %s", err, output) + } + documents, err := bleve.Open(homeDir + documentsIndexPath) + if err != nil { + t.Fatal(err) + } + defer documents.Close() + if stored, err := documents.GetInternal([]byte("version")); err != nil || string(stored) != version { + t.Fatalf("unsupported version must remain untouched: %q, %v", stored, err) + } + if count, err := documents.DocCount(); err != nil || count != 1 { + t.Fatalf("unsupported documents must remain untouched: count=%d, err=%v", count, err) + } + }) + } +} + +func TestGetIndexesMigratesV13WithoutReindexWhenAuthorsIndexIsMissing(t *testing.T) { + previousHome, previousInput := homeDir, input + t.Cleanup(func() { homeDir, input = previousHome, previousInput }) + homeDir = t.TempDir() + input.BatchSize, input.ForceIndexing = 1, false + seedDocumentIndex(t, "v13") + + documents, authors, needsReindex := getIndexes(afero.NewOsFs(), 0.25, 0.1, 10000) + defer documents.Close() + defer authors.Close() + if needsReindex { + t.Fatal("creating the missing authors index must not force document reindexing") + } + if count, err := documents.DocCount(); err != nil || count != 1 { + t.Fatalf("v13 migration must preserve documents: count=%d, err=%v", count, err) + } + if count, err := authors.DocCount(); err != nil || count != 1 { + t.Fatalf("authors index must be rebuilt from migrated documents: count=%d, err=%v", count, err) + } +} + +func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + t.Fatal(err) + } + return document +} + +func TestGetIndexesMigratesAuthorsWithoutReindexingDocuments(t *testing.T) { + previousHome, previousInput := homeDir, input + t.Cleanup(func() { homeDir, input = previousHome, previousInput }) + homeDir = t.TempDir() + input.BatchSize = 1 + fs := afero.NewOsFs() + if err := fs.MkdirAll(filepath.Join(homeDir, ".coreander"), 0755); err != nil { + t.Fatal(err) + } + documents := index.CreateDocumentsIndex(homeDir + documentsIndexPath) + for key, value := range map[string]string{ + "version": index.DocumentVersion, + "illustrated_min_size": "0.25", + "min_occurrence_ratio": "0.1", + "max_textrank_words": "10000", + } { + if err := documents.SetInternal([]byte(key), []byte(value)); err != nil { + t.Fatal(err) + } + } + doc := withBiblioReadsData(t, index.Document{ + ID: "book", Slug: "book", AuthorsSlugs: []string{"alice-borchardt"}, + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + }, biblioreads.DocumentData{Enriched: true}) + + if err := index.IndexDocument(documents, doc.ID, doc); err != nil { + t.Fatal(err) + } + if err := documents.Close(); err != nil { + t.Fatal(err) + } + legacyMapping := index.CreateAuthorsMapping().(*mapping.IndexMappingImpl) + delete(legacyMapping.DefaultMapping.Properties, "DataSourceLinks") + authors, err := bleve.New(homeDir+authorsIndexPath, legacyMapping) + if err != nil { + t.Fatal(err) + } + retrieved := time.Now().UTC().Truncate(time.Second) + if err := authors.Index("alice-borchardt", index.Author{ + Slug: "alice-borchardt", Name: "Alice Borchardt", DataSourceID: "Q1", RetrievedOn: retrieved, + }); err != nil { + t.Fatal(err) + } + if err := authors.SetInternal([]byte("version"), []byte("2")); err != nil { + t.Fatal(err) + } + if err := authors.Close(); err != nil { + t.Fatal(err) + } + + documents, authors, needsReindex := getIndexes(fs, 0.25, 0.1, 10000) + idx := index.NewBleve(documents, authors, nil, "", nil, index.Config{}) + defer idx.Close() + if needsReindex { + t.Fatal("author migration must not require document reindexing") + } + if count, err := documents.DocCount(); err != nil || count != 1 { + t.Fatalf("document index was not preserved: count=%d, err=%v", count, err) + } + stored, err := idx.Document("book") + if err != nil { + t.Fatal(err) + } + data, err := index.DataSourceData[biblioreads.DocumentData](stored, biblioreads.Source) + if err != nil || !data.Enriched { + t.Fatalf("author migration changed document enrichment state: %+v, %v", data, err) + } + author, err := idx.Author("alice-borchardt", "") + if err != nil || author.DataSourceID != "Q1" || !author.RetrievedOn.Equal(retrieved) { + t.Fatalf("expected existing author metadata preserved: %+v, %v", author, err) + } + if pending, err := idx.AuthorsWithoutInfo(); err != nil || len(pending) != 0 { + t.Fatalf("index migrations queued enriched authors again: %+v, %v", pending, err) + } +} diff --git a/internal/datasource/biblioreads/adapter.go b/internal/datasource/biblioreads/adapter.go new file mode 100644 index 00000000..96274ad1 --- /dev/null +++ b/internal/datasource/biblioreads/adapter.go @@ -0,0 +1,200 @@ +package biblioreads + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/svera/coreander/v5/internal/index" +) + +const ( + // maxConcurrentLookups bounds lookups across every enrichment caller, so they do not pile up + // waiting on the client. Client request spacing and cooldowns protect each instance. + maxConcurrentLookups = 4 + requestTimeout = 2 * time.Minute +) + +type SearchSource interface { + SearchBooks(context.Context, string) (SearchBooksResponse, error) +} + +type Enricher struct { + source SearchSource + idx *index.BleveIndexer + lookupSlots chan struct{} +} + +// NewEnricher adapts a source to the document-enrichment pipeline. +func NewEnricher(source SearchSource) *Enricher { + return &Enricher{source: source, lookupSlots: make(chan struct{}, maxConcurrentLookups)} +} + +// SetIndex binds the adapter before starting background or on-demand enrichment. +func (p *Enricher) SetIndex(idx *index.BleveIndexer) { + p.idx = idx +} + +func (*Enricher) Source() string { return Source } + +func (*Enricher) Name() string { return Name } + +func (*Enricher) NeedsEnrichment(document index.Document) (bool, error) { + data, err := index.DataSourceData[DocumentData](document, Source) + return !data.Enriched, err +} + +func (p *Enricher) Enrich(ctx context.Context, idx *index.BleveIndexer, document index.Document) error { + _, err := p.enrich(ctx, idx, document) + return err +} + +// Book returns a stored match or performs an on-demand lookup. +func (p *Enricher) Book(ctx context.Context, document index.Document) (SearchResult, error) { + data, err := index.DataSourceData[DocumentData](document, Source) + if err != nil { + return SearchResult{}, err + } + if data.URL != "" || data.Enriched || p.source == nil { + return storedResult(document, data), nil + } + return p.enrich(ctx, p.idx, document) +} + +func storedResult(document index.Document, data DocumentData) SearchResult { + return SearchResult{ + BookURL: data.URL, Rating: fmt.Sprintf("%.2f", document.DataSourceScoreValue), RatingCount: data.RatingCount, + } +} + +// UpdateDataSourceLink clears cached data when an administrator corrects a match. +func (p *Enricher) UpdateDataSourceLink(documentID, reviewLink string) error { + if p.idx == nil { + return fmt.Errorf("BiblioReads adapter has no index") + } + return p.idx.UpdateDocument(documentID, func(current *index.DocumentUpdate) (bool, error) { + data, err := index.DataSourceData[DocumentData](current.Document, Source) + if err != nil || data.URL == reviewLink { + return false, err + } + current.DataSourceScoreValue = 0 + err = index.SetDataSourceData(¤t.Document, Source, DocumentData{URL: reviewLink, Enriched: true}) + return true, err + }) +} + +// StoreDetails rejects responses for a match or metadata changed during the fetch. +func (p *Enricher) StoreDetails(document index.Document, bookURL string, book BookResponse) error { + if p.idx == nil { + return fmt.Errorf("BiblioReads adapter has no index") + } + if bookURL == "" { + return fmt.Errorf("invalid Goodreads book URL for document %s", document.ID) + } + hasIDs := book.ResourceID != "" && book.LegacyBookID > 0 + if book.Rating == "" && book.RatingCount == "" && !hasIDs { + return nil + } + return p.idx.UpdateDocument(document.ID, func(current *index.DocumentUpdate) (bool, error) { + data, err := index.DataSourceData[DocumentData](current.Document, Source) + if err != nil { + return false, err + } + if searchQuery(current.Document) != searchQuery(document) || data.URL != bookURL { + return false, fmt.Errorf("BiblioReads match changed for document %s", document.ID) + } + changed := false + if book.RatingCount != "" && data.RatingCount != book.RatingCount { + data.RatingCount = book.RatingCount + changed = true + } + if book.Rating != "" { + value := parseRating(book.Rating, document.ID) + if current.DataSourceScoreValue != value { + current.DataSourceScoreValue = value + changed = true + } + } + if hasIDs && (data.ResourceID != book.ResourceID || data.LegacyBookID != book.LegacyBookID) { + data.ResourceID, data.LegacyBookID = book.ResourceID, book.LegacyBookID + changed = true + } + if !changed { + return false, nil + } + return true, index.SetDataSourceData(¤t.Document, Source, data) + }) +} + +func (p *Enricher) enrich(ctx context.Context, idx *index.BleveIndexer, document index.Document) (SearchResult, error) { + if idx == nil || p.source == nil { + return SearchResult{}, fmt.Errorf("BiblioReads enrichment requires a source and index") + } + data, err := index.DataSourceData[DocumentData](document, Source) + if err != nil { + return SearchResult{}, err + } + select { + case p.lookupSlots <- struct{}{}: + defer func() { <-p.lookupSlots }() + case <-ctx.Done(): + return SearchResult{}, ctx.Err() + } + q := searchQuery(document) + result := SearchResult{BookURL: data.URL} + if result.BookURL == "" && document.Title != "" { + ctx, cancel := context.WithTimeout(ctx, requestTimeout) + defer cancel() + queries := []string{q} + if title := strings.TrimSpace(document.Title); title != q { + queries = append(queries, title) + } + for _, query := range queries { + response, err := p.source.SearchBooks(ctx, query) + if err != nil { + return SearchResult{}, fmt.Errorf("error searching %q on BiblioReads: %w", query, err) + } + if match, ok := matchingBook(document, response.Result); ok { + result = match + } + if result.BookURL != "" { + break + } + } + } + err = idx.UpdateDocument(document.ID, func(current *index.DocumentUpdate) (bool, error) { + if searchQuery(current.Document) != q { + return false, nil + } + data, err := index.DataSourceData[DocumentData](current.Document, Source) + if err != nil { + return false, err + } + if data.Enriched { + result = storedResult(current.Document, data) + return false, nil + } + if data.URL != "" && data.URL != result.BookURL { + result = storedResult(current.Document, data) + } + data.URL, data.RatingCount, data.Enriched = result.BookURL, result.RatingCount, true + if result.Rating == "" && current.DataSourceScoreValue > 0 { + result.Rating = fmt.Sprintf("%.2f", current.DataSourceScoreValue) + } + current.DataSourceScoreValue = parseRating(result.Rating, current.ID) + if result.AuthorURL != "" { + if slug, name := matchedAuthor(current.Document, result.Author); slug != "" { + current.AuthorLinks = append(current.AuthorLinks, index.AuthorLink{ + Slug: slug, Name: name, Source: Source, URL: result.AuthorURL, + }) + } + } + return true, index.SetDataSourceData(¤t.Document, Source, data) + }) + if errors.Is(err, index.ErrDocumentNotFound) { + return result, nil + } + return result, err +} diff --git a/internal/datasource/biblioreads/adapter_test.go b/internal/datasource/biblioreads/adapter_test.go new file mode 100644 index 00000000..f1758a68 --- /dev/null +++ b/internal/datasource/biblioreads/adapter_test.go @@ -0,0 +1,1240 @@ +package biblioreads + +import ( + "context" + "encoding/json" + "errors" + "image" + "reflect" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" +) + +func mustData(t *testing.T, document index.Document) DocumentData { + t.Helper() + data, err := index.DataSourceData[DocumentData](document, Source) + if err != nil { + t.Fatal(err) + } + return data +} + +func withData(t *testing.T, document index.Document, data DocumentData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, Source, data); err != nil { + t.Fatal(err) + } + return document +} + +func updateData(t *testing.T, document *index.Document, update func(*DocumentData)) { + t.Helper() + data := mustData(t, *document) + update(&data) + *document = withData(t, *document, data) +} + +func assertNoDocumentAuthorURL(t *testing.T, documents bleve.Index) { + t.Helper() + req := bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), 100, 0, false) + req.Fields = []string{"*"} + result, err := documents.Search(req) + if err != nil { + t.Fatal(err) + } + for _, hit := range result.Hits { + if _, exists := hit.Fields["GoodReadsAuthorURL"]; exists { + t.Errorf("document %s still stores an author URL", hit.ID) + } + } +} + +type fakeSource struct { + mu sync.Mutex + queries []string + results []SearchResult + byQuery map[string][]SearchResult + queryErrors map[string]error + err error +} + +func (f *fakeSource) SearchBooks(_ context.Context, query string) (SearchBooksResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.queries = append(f.queries, query) + if f.byQuery != nil { + return SearchBooksResponse{Result: f.byQuery[query]}, f.queryErrors[query] + } + return SearchBooksResponse{Result: f.results}, f.err +} + +func (f *fakeSource) Queries() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.queries) +} + +type testReader struct{} + +func (testReader) Metadata(string) (metadata.Metadata, error) { + return metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt", "Second Author"}, Format: "PDF"}, nil +} + +func (testReader) Cover(string, int) (image.Image, error) { + return nil, nil +} + +var testResults = []SearchResult{ + {Title: "La Reina Dragon", Author: "Alice Borchardt", BookURL: "/book/show/12207444-la-reina-dragon", AuthorURL: "/author/show/151155.Alice_Borchardt", Rating: "3.82", RatingCount: "2,285"}, + {BookURL: "/book/show/2-other", AuthorURL: "/author/show/2.Other"}, +} + +const ( + wantDataSourceLink = "/book/show/12207444-la-reina-dragon" + wantAuthorURL = "/author/show/151155.Alice_Borchardt" + testID = "book.pdf" +) + +type testIndexer struct { + *index.BleveIndexer + *Enricher + documentsIdx *writeCountingIndex + authorsIdx bleve.Index + fs afero.Fs +} + +func newTestIndexer(t *testing.T, source SearchSource) *testIndexer { + t.Helper() + documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + writes := &writeCountingIndex{countedIndex: documents} + adapter := NewEnricher(source) + var providers []index.DocumentEnricher + if source != nil { + providers = append(providers, adapter) + } + appFS := afero.NewMemMapFs() + if err := afero.WriteFile(appFS, "lib/book.pdf", []byte(""), 0644); err != nil { + t.Fatal(err) + } + idx := index.NewBleve(writes, authors, appFS, "lib", map[string]metadata.Reader{".pdf": testReader{}}, index.Config{DocumentEnrichers: providers}) + adapter.SetIndex(idx) + t.Cleanup(func() { _ = idx.Close() }) + return &testIndexer{BleveIndexer: idx, Enricher: adapter, documentsIdx: writes, authorsIdx: authors, fs: appFS} +} + +func TestEnrichStoresMatchingResult(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + + doc, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + if mustData(t, doc).URL != wantDataSourceLink { + t.Errorf("unexpected book URL %q", mustData(t, doc).URL) + } + if doc.DataSourceScoreValue != 3.82 { + t.Errorf("unexpected Goodreads rating %v", doc.DataSourceScoreValue) + } + if mustData(t, doc).RatingCount != "2,285" { + t.Errorf("unexpected Goodreads rating count %q", mustData(t, doc).RatingCount) + } + assertNoDocumentAuthorURL(t, idx.documentsIdx) + if !mustData(t, doc).Enriched { + t.Error("expected document to be marked as BiblioReads-enriched") + } + author, err := idx.Author("alice-borchardt", "") + if err != nil || author.DataSourceLinks["goodreads"] != wantAuthorURL { + t.Fatalf("expected matching author URL stored, got %+v, %v", author, err) + } + other, err := idx.Author("second-author", "") + if err != nil || other.DataSourceLinks["goodreads"] != "" { + t.Fatalf("unmatched coauthor must not receive the URL, got %+v, %v", other, err) + } + if want := []string{"La Reina Dragon Alice Borchardt"}; !slices.Equal(source.Queries(), want) { + t.Errorf("expected queries %q, got %q", want, source.Queries()) + } + + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + if len(source.Queries()) != 1 { + t.Errorf("expected enriched documents not to be searched again, got queries %q", source.Queries()) + } +} + +func TestEnrichMarksDocumentsWithoutMatch(t *testing.T) { + for name, source := range map[string]*fakeSource{ + "no results": {}, + "wrong title": {results: []SearchResult{{Title: "Another book", Author: "Alice Borchardt", BookURL: "/book/show/1"}}}, + "wrong author": {results: []SearchResult{{Title: "La Reina Dragon", Author: "Another writer", BookURL: "/book/show/1"}}}, + } { + t.Run(name, func(t *testing.T) { + idx := newTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + doc, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + if !mustData(t, doc).Enriched || mustData(t, doc).URL != "" { + t.Errorf("expected document enriched with no URLs, got %+v", doc) + } + if want := []string{"La Reina Dragon Alice Borchardt", "La Reina Dragon"}; !slices.Equal(source.Queries(), want) { + t.Errorf("expected both searches before giving up, got %q", source.Queries()) + } + }) + } +} + +func TestEnrichStoresSecondAuthorURLAndPreservesItOnEdits(t *testing.T) { + const authorURL = "/author/show/2" + source := &fakeSource{results: []SearchResult{ + {Title: "La Reina Dragon", Author: "Second Author", BookURL: wantDataSourceLink, AuthorURL: authorURL}, + }} + idx := newTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + stale, err := idx.Author("second-author", "") + if err != nil { + t.Fatal(err) + } + stale.DataSourceLinks = map[string]string{"other": "https://example.org/second-author"} + if err := idx.IndexAuthor(stale); err != nil { + t.Fatal(err) + } + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + author, err := idx.Author("second-author", "") + if err != nil || author.DataSourceLinks["goodreads"] != authorURL || + author.DataSourceLinks["other"] != "https://example.org/second-author" { + t.Fatalf("expected second author URL stored, got %+v, %v", author, err) + } + first, err := idx.Author("alice-borchardt", "") + if err != nil || first.DataSourceLinks["goodreads"] != "" { + t.Fatalf("first author must not receive coauthor URL, got %+v, %v", first, err) + } + stale.DataSourceID = "Q2" + stale.RetrievedOn = time.Now().UTC() + stale.DataSourceLinks["goodreads"] = "/author/show/999.Stale" + if err := idx.IndexAuthor(stale); err != nil { + t.Fatal(err) + } + if stale.DataSourceLinks["goodreads"] != "/author/show/999.Stale" { + t.Fatal("indexing an author mutated the caller's links") + } + author, err = idx.Author("second-author", "") + if err != nil || author.DataSourceLinks["goodreads"] != authorURL || + author.DataSourceLinks["other"] != "https://example.org/second-author" || + author.DataSourceID != "Q2" || author.DocumentCount != 1 { + t.Fatalf("stale author edit lost URL or metadata: %+v, %v", author, err) + } + if _, err := idx.NewFile("second.pdf", []byte("")); err != nil { + t.Fatal(err) + } + author, err = idx.Author("second-author", "") + if err != nil || author.DataSourceLinks["goodreads"] != authorURL || author.DocumentCount != 2 { + t.Fatalf("incrementing document count lost author URL: %+v, %v", author, err) + } +} + +func TestSearchErrorsLeaveLookupPending(t *testing.T) { + const title = "El señor de los anillos" + const author = "J. R. R. Tolkien" + searchError := errors.New("BiblioReads returned HTTP 404: upstream fetch or JSON failure") + for _, failedQuery := range []string{title + " " + author, title} { + t.Run(failedQuery, func(t *testing.T) { + source := &fakeSource{ + byQuery: map[string][]SearchResult{}, + queryErrors: map[string]error{failedQuery: searchError}, + } + idx := newTestIndexer(t, source) + document := index.Document{ + ID: testID, Metadata: metadata.Metadata{Title: title, Authors: []string{author}}, + } + if err := index.IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + if _, err := idx.Book(context.Background(), document); !errors.Is(err, searchError) { + t.Fatalf("expected wrapped search error, got %v", err) + } + stored, err := idx.DocumentByID(document.ID) + if err != nil { + t.Fatal(err) + } + if data := mustData(t, stored); data.Enriched || data.URL != "" { + t.Fatalf("failed search must not persist a completed lookup: %+v", data) + } + source.queryErrors = nil + source.byQuery[title] = []SearchResult{ + {Title: title, Author: author, BookURL: "/book/show/37910810"}, + } + result, err := idx.Book(context.Background(), stored) + if err != nil || result.BookURL != "/book/show/37910810" { + t.Fatalf("expected successful retry, got %+v, %v", result, err) + } + stored, err = idx.DocumentByID(document.ID) + if err != nil || !mustData(t, stored).Enriched { + t.Fatalf("successful retry must persist completion: %+v, %v", stored, err) + } + }) + } +} + +func TestEnrichLeavesDocumentsForLaterWhenUnavailable(t *testing.T) { + source := &fakeSource{err: ErrUnavailable} + idx := newTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + if err := idx.EnrichDocuments(10); !errors.Is(err, ErrUnavailable) { + t.Fatalf("expected ErrUnavailable, got %v", err) + } + doc, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + if mustData(t, doc).Enriched { + t.Error("expected document not to be marked as BiblioReads-enriched") + } + + source.mu.Lock() + source.err = nil + source.results = testResults + source.mu.Unlock() + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + if doc, _ = idx.DocumentByID(testID); mustData(t, doc).URL != wantDataSourceLink { + t.Errorf("expected document to be enriched on retry, got %q", mustData(t, doc).URL) + } +} + +func TestNewFileSchedulesEnrichment(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) + + if _, err := idx.NewFile("new.pdf", []byte("")); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(2 * time.Second) + for { + doc, err := idx.DocumentByID("new.pdf") + if err != nil { + t.Fatal(err) + } + if mustData(t, doc).Enriched { + if mustData(t, doc).URL != wantDataSourceLink { + t.Errorf("unexpected book URL %q", mustData(t, doc).URL) + } + assertNoDocumentAuthorURL(t, idx.documentsIdx) + return + } + if time.Now().After(deadline) { + t.Fatal("timed out waiting for background BiblioReads enrichment") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestEnrichWithoutSourceIsNoop(t *testing.T) { + idx := newTestIndexer(t, nil) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + if _, err := idx.NewFile("new.pdf", []byte("")); err != nil { + t.Fatal(err) + } + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + doc, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + if mustData(t, doc).Enriched { + t.Error("expected document not to be BiblioReads-enriched when the source is disabled") + } +} + +func TestEnrichSkipsSearchWhenDocumentHasLink(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + doc, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + updateData(t, &doc, func(data *DocumentData) { + data.URL = "/book/show/1-existing" + }) + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + + if len(source.Queries()) != 0 { + t.Errorf("expected no search for a document with a link, got %q", source.Queries()) + } + doc, err = idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + if !mustData(t, doc).Enriched || mustData(t, doc).URL != "/book/show/1-existing" { + t.Errorf("expected existing link kept and document marked enriched, got %+v", doc) + } +} + +func TestBook(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + doc, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + + got, err := idx.Book(context.Background(), doc) + if err != nil { + t.Fatal(err) + } + if got.BookURL != wantDataSourceLink || got.Rating != "3.82" { + t.Errorf("expected fresh book URL and search rating, got %+v", got) + } + stored, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + if !mustData(t, stored).Enriched || mustData(t, stored).URL != wantDataSourceLink { + t.Errorf("expected the link to be persisted, got %+v", stored) + } + + if got, _ := idx.Book(context.Background(), stored); got.BookURL != wantDataSourceLink || got.Rating != "3.82" { + t.Errorf("expected stored URL and rating, got %+v", got) + } + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{stored.ID})) + req.Fields = []string{"*"} + result, err := idx.documentsIdx.Search(req) + if err != nil || len(result.Hits) != 1 { + t.Fatalf("reading stored Goodreads fields: %v", err) + } + for _, field := range []string{"Rating", "rating"} { + if _, exists := result.Hits[0].Fields[field]; exists { + t.Errorf("rating must not be stored in document field %s", field) + } + } + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + if len(source.Queries()) != 1 { + t.Errorf("expected a single search, got %q", source.Queries()) + } +} + +func TestParseRating(t *testing.T) { + for _, tt := range []struct { + rating string + want float64 + }{ + {rating: "0.00", want: 0}, + {rating: "3.82", want: 3.82}, + {rating: "5.01", want: 0}, + {rating: "unavailable", want: 0}, + } { + t.Run(tt.rating, func(t *testing.T) { + if got := parseRating(tt.rating, testID); got != tt.want { + t.Errorf("parseBiblioReadsRating(%q) = %v, want %v", tt.rating, got, tt.want) + } + }) + } +} + +func TestBookReturnsErrorWhenUnavailable(t *testing.T) { + source := &fakeSource{err: ErrUnavailable} + idx := newTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + doc, _ := idx.DocumentByID(testID) + if _, err := idx.Book(context.Background(), doc); !errors.Is(err, ErrUnavailable) { + t.Fatalf("expected ErrUnavailable, got %v", err) + } + if doc, _ = idx.DocumentByID(testID); mustData(t, doc).Enriched { + t.Error("expected document to stay unenriched for a later retry") + } +} + +func TestTitleMatchScore(t *testing.T) { + for _, tt := range []struct { + name string + title string + candidate string + want int + }{ + {name: "normalized exact", title: "La promesa", candidate: "LA PROMESA!", want: 3}, + {name: "series suffix", title: "La promesa", candidate: "La promesa (Series, #1)", want: 3}, + {name: "document series suffix", title: "La promesa (Series, #1)", candidate: "La promesa", want: 3}, + {name: "subtitle", title: "La promesa", candidate: "La promesa: Una historia", want: 1}, + {name: "document subtitle", title: "La promesa: Una historia", candidate: "La promesa", want: 1}, + {name: "parenthetical", title: "La promesa", candidate: "La promesa (The Promise)", want: 1}, + {name: "different subtitles", title: "La promesa: Una historia", candidate: "La promesa: Otra historia"}, + {name: "standalone volume", title: "La promesa", candidate: "La promesa (#1)"}, + {name: "unrelated", title: "La promesa", candidate: "Otro libro"}, + {name: "empty candidate", title: "La promesa"}, + } { + t.Run(tt.name, func(t *testing.T) { + got := titleMatchScore(tt.candidate, normalizedTitle(tt.title), mainTitle(tt.title)) + if got != tt.want { + t.Fatalf("title match score = %d, want %d", got, tt.want) + } + }) + } +} + +func TestPreferBook(t *testing.T) { + for _, tt := range []struct { + name string + candidate, best SearchResult + score, bestScore int + want bool + }{ + {name: "first match", score: 1, want: true}, + {name: "lower ID before rating for equal strength", candidate: SearchResult{BookURL: "/book/show/1"}, best: SearchResult{BookURL: "/book/show/2", Rating: "4"}, score: 3, bestScore: 3, want: true}, + {name: "higher ID loses despite rating", candidate: SearchResult{BookURL: "/book/show/2", Rating: "4"}, best: SearchResult{BookURL: "/book/show/1"}, score: 3, bestScore: 3}, + {name: "more ratings before ID", candidate: SearchResult{BookURL: "/book/show/2", RatingCount: "100"}, best: SearchResult{BookURL: "/book/show/1", RatingCount: "3"}, score: 3, bestScore: 3, want: true}, + {name: "fewer ratings loses", candidate: SearchResult{RatingCount: "3"}, best: SearchResult{RatingCount: "100"}, score: 3, bestScore: 3}, + {name: "result rank is not book ID", candidate: SearchResult{ID: 2, BookURL: "/book/show/1"}, best: SearchResult{ID: 1, BookURL: "/book/show/2"}, score: 3, bestScore: 3, want: true}, + {name: "rating before strength for different strength", candidate: SearchResult{ID: 2, Rating: "4"}, best: SearchResult{ID: 1}, score: 1, bestScore: 3, want: true}, + {name: "stronger unrated loses to rated", best: SearchResult{Rating: "4"}, score: 3, bestScore: 1}, + {name: "strength with same rating status", score: 3, bestScore: 1, want: true}, + {name: "weaker match loses with same rating status", score: 1, bestScore: 3}, + {name: "unknown ID falls back to rating", candidate: SearchResult{Rating: "4"}, best: SearchResult{ID: 1}, score: 3, bestScore: 3, want: true}, + {name: "same ID falls back to rating", candidate: SearchResult{ID: 1, Rating: "4"}, best: SearchResult{ID: 1}, score: 3, bestScore: 3, want: true}, + {name: "complete tie retains first match", candidate: SearchResult{ID: 1, Rating: "4"}, best: SearchResult{ID: 1, Rating: "3"}, score: 3, bestScore: 3}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := preferBook(tt.candidate, tt.best, tt.score, tt.bestScore); got != tt.want { + t.Fatalf("candidate preference = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAuthorNameMatches(t *testing.T) { + tests := []struct { + name, candidate string + want bool + }{ + {"Paul Kennedy", "Paul M. Kennedy", true}, + {"Paul Kennedy", "Paul Michael Kennedy", true}, + {"Paul Michael Kennedy", "Paul Michael James Kennedy", true}, + {"J. R. R. Tolkien", "J.R.R. Tolkien", true}, + {"JRR Tolkien", "J.R.R. Tolkien", true}, + {"José García", "JOSE M. GARCIA", true}, + {"Paul M. Kennedy", "Paul A. Kennedy", false}, + {"Paul Michael James Kennedy", "Paul James Michael Kennedy", false}, + {"Paul Kennedy", "Pauline Kennedy", false}, + {"Paul Kennedy", "Paul Kennedys", false}, + {"Paul Kennedy", "John Kennedy", false}, + {"Kennedy", "Paul M. Kennedy", false}, + {"Paul", "Paul M. Kennedy", false}, + {"", "Paul Kennedy", false}, + {" . ", "", false}, + } + for _, tt := range tests { + t.Run(tt.name+"/"+tt.candidate, func(t *testing.T) { + if got := authorNameMatches(tt.name, tt.candidate); got != tt.want { + t.Errorf("match = %v, want %v", got, tt.want) + } + if got := authorNameMatches(tt.candidate, tt.name); got != tt.want { + t.Errorf("reverse match = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAuthorPrefersExactAndRejectsAmbiguousPartialMatches(t *testing.T) { + doc := index.Document{ + Metadata: metadata.Metadata{Authors: []string{"Paul Kennedy", "Paul M. Kennedy"}}, + AuthorsSlugs: []string{"paul-kennedy", "paul-m-kennedy"}, + } + if slug, name := matchedAuthor(doc, "Paul M. Kennedy"); slug != "paul-m-kennedy" || name != "Paul M. Kennedy" { + t.Fatalf("expected exact author, got %q, %q", slug, name) + } + doc.Authors = []string{"Paul M. Kennedy", "Paul A. Kennedy"} + if slug, name := matchedAuthor(doc, "Paul Kennedy"); slug != "" || name != "" { + t.Fatalf("ambiguous partial match must not select an author, got %q, %q", slug, name) + } +} + +func TestBookMatchesKennedyMiddleName(t *testing.T) { + const title = "Auge y caída de las grandes potencias" + const bookURL = "/book/show/27214" + const authorURL = "/author/show/8390124.Paul_M_Kennedy" + source := &fakeSource{results: []SearchResult{{ + Title: title, Author: "Paul M. Kennedy", BookURL: bookURL, AuthorURL: authorURL, Rating: "4.14", + }}} + idx := newTestIndexer(t, source) + doc := index.Document{ + ID: testID, AuthorsSlugs: []string{"paul-kennedy", "another-writer"}, + Metadata: metadata.Metadata{Title: title, Authors: []string{"Paul Kennedy", "Another Writer"}}, + } + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + got, err := idx.Book(context.Background(), doc) + if err != nil || got.BookURL != bookURL || got.Rating != "4.14" { + t.Fatalf("expected Kennedy's book and rating, got %+v, %v", got, err) + } + stored, err := idx.DocumentByID(doc.ID) + if err != nil || !mustData(t, stored).Enriched || mustData(t, stored).URL != bookURL || stored.DataSourceScoreValue != 4.14 { + t.Fatalf("expected persisted book match and rating, got %+v, %v", stored, err) + } + author, err := idx.Author("paul-kennedy", "") + if err != nil || author.DataSourceLinks["goodreads"] != authorURL || author.Name != "Paul Kennedy" { + t.Fatalf("expected author URL with original name preserved, got %+v, %v", author, err) + } + other, err := idx.Author("another-writer", "") + if err != nil || other.DataSourceLinks["goodreads"] != "" { + t.Fatalf("unmatched coauthor must not receive the URL, got %+v, %v", other, err) + } + if want := []string{title + " Paul Kennedy"}; !slices.Equal(source.Queries(), want) { + t.Fatalf("expected combined search to match without fallback, got %q", source.Queries()) + } +} + +func TestMatchingBook(t *testing.T) { + correct := SearchResult{ + Title: "El señor de los anillos", Author: "J.R.R. Tolkien", + BookURL: "/book/show/37910810", AuthorURL: "/author/show/656983.J_R_R_Tolkien", + } + parody := SearchResult{ + Title: "El sopor de los anillos", Author: "The Harvard Lampoon", + BookURL: "/book/show/7297403-el-sopor-de-los-anillos", + } + tests := []struct { + name string + title string + authors []string + results []SearchResult + want string + }{ + {name: "correct result after parody", results: []SearchResult{parody, correct}, want: correct.BookURL}, + {name: "parody rejected", results: []SearchResult{parody}}, + {name: "similar title with correct author rejected", results: []SearchResult{{Title: parody.Title, Author: correct.Author, BookURL: parody.BookURL}}}, + {name: "wrong author rejected", results: []SearchResult{{Title: correct.Title, Author: parody.Author, BookURL: parody.BookURL}}}, + {name: "normalization", title: " EL SEN\u0303OR DE LOS ANILLOS! ", results: []SearchResult{correct}, want: correct.BookURL}, + {name: "series suffix", results: []SearchResult{{Title: correct.Title + " (El señor de los anillos, #1)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "document series suffix", title: correct.Title + " (Middle Earth, #1)", results: []SearchResult{correct}, want: correct.BookURL}, + {name: "omnibus range", results: []SearchResult{{Title: correct.Title + " (#1-#3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "omnibus range with dash", results: []SearchResult{{Title: correct.Title + " (#1–3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "document omnibus range", title: correct.Title + " (#1-#3)", results: []SearchResult{correct}, want: correct.BookURL}, + {name: "standalone volume retained", results: []SearchResult{{Title: correct.Title + " (#1)", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "series suffix ranks equally", results: []SearchResult{{Title: correct.Title + " (Middle Earth, #1)", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: "/book/show/2"}, + {name: "lower Goodreads ID preferred over rating for equal titles", results: []SearchResult{ + {ID: 1, Title: correct.Title, Author: correct.Author, BookURL: "/book/show/1-unrated", Rating: "0.00"}, + {ID: 2, Title: correct.Title, Author: correct.Author, BookURL: "/book/show/2-rated", Rating: "4.05"}, + }, want: "/book/show/1-unrated"}, + {name: "unrated exact match retained when no rated match exists", results: []SearchResult{ + {Title: correct.Title, Author: correct.Author, BookURL: "/book/show/1-unrated", Rating: "0.00"}, + {Title: correct.Title, Author: correct.Author, BookURL: "/book/show/2"}, + }, want: "/book/show/1-unrated"}, + {name: "colon subtitle", results: []SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "period subtitle", results: []SearchResult{{Title: correct.Title + ". Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "dash subtitle", results: []SearchResult{{Title: correct.Title + " — Una historia", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "document subtitle", title: correct.Title + ": Una historia", results: []SearchResult{correct}, want: correct.BookURL}, + {name: "different subtitles rejected", title: correct.Title + ": Primera historia", results: []SearchResult{{Title: correct.Title + ": Otra historia", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "subtitle wrong author rejected", results: []SearchResult{{Title: correct.Title + ": Una historia", Author: parody.Author, BookURL: correct.BookURL}}}, + {name: "exact preferred over subtitle", results: []SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: "/book/show/2"}, correct}, want: correct.BookURL}, + {name: "series preferred over subtitle", results: []SearchResult{{Title: correct.Title + ": Una historia", Author: correct.Author, BookURL: "/book/show/2"}, {Title: correct.Title + " (#1-#3)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "arbitrary prefix rejected", results: []SearchResult{{Title: correct.Title + " y otras historias", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "volume retained", results: []SearchResult{{Title: correct.Title + " 2", Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "trailing parenthetical accepted as weaker match", results: []SearchResult{{Title: correct.Title + " (An edition)", Author: correct.Author, BookURL: correct.BookURL}}, want: correct.BookURL}, + {name: "missing result author", results: []SearchResult{{Title: correct.Title, BookURL: correct.BookURL}}}, + {name: "missing result title", results: []SearchResult{{Author: correct.Author, BookURL: correct.BookURL}}}, + {name: "missing URL", results: []SearchResult{{Title: correct.Title, Author: correct.Author}}}, + {name: "any document author", authors: []string{"Another Writer", "J. R. R. Tolkien"}, results: []SearchResult{correct}, want: correct.BookURL}, + {name: "no author", authors: []string{}, results: []SearchResult{correct}}, + {name: "no results"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + title, authors := tt.title, tt.authors + if title == "" { + title = correct.Title + } + if authors == nil { + authors = []string{"J. R. R. Tolkien"} + } + match, ok := matchingBook(index.Document{Metadata: metadata.Metadata{Title: title, Authors: authors}}, tt.results) + if match.BookURL != tt.want || ok != (tt.want != "") { + t.Fatalf("got %q, %v; want %q", match.BookURL, ok, tt.want) + } + }) + } +} + +func TestMatchingBookPrefersMoreRatingsWithSeriesSuffix(t *testing.T) { + document := index.Document{Metadata: metadata.Metadata{ + Title: "Los asesinatos de Manhattan", Authors: []string{"Douglas Preston", "Lincoln Child"}, + }} + popular := SearchResult{ + ID: 1, Title: "Los asesinatos de Manhattan (Pendergast, #3)", Author: "Douglas Preston", + BookURL: "/book/show/67005.Los_asesinatos_de_Manhattan", Rating: "4.26", RatingCount: "57,469", + } + obscure := SearchResult{ + ID: 3, Title: document.Title, Author: "Douglas Preston", + BookURL: "/book/show/152308668-los-asesinatos-de-manhattan", Rating: "3.67", RatingCount: "3", + } + for _, results := range [][]SearchResult{{popular, obscure}, {obscure, popular}} { + match, ok := matchingBook(document, results) + if !ok || match.BookURL != popular.BookURL { + t.Fatalf("expected popular series entry, got %+v, %v", match, ok) + } + } + popular.Author = "Another author" + match, ok := matchingBook(document, []SearchResult{popular, obscure}) + if !ok || match.BookURL != obscure.BookURL { + t.Fatalf("rating count must not bypass author matching, got %+v, %v", match, ok) + } +} + +func TestDataSourceScoreCount(t *testing.T) { + for _, tt := range []struct { + value string + want uint64 + }{ + {"", 0}, {"0", 0}, {"57469", 57469}, {" 57,469 ", 57469}, + {"unknown", 0}, {"-1", 0}, {"3.5", 0}, {"18446744073709551616", 0}, + } { + t.Run(tt.value, func(t *testing.T) { + if got := ratingCount(tt.value); got != tt.want { + t.Fatalf("rating count = %d, want %d", got, tt.want) + } + }) + } +} + +func TestMatchingBookPrefersLowerIDForEqualTitle(t *testing.T) { + const olderBookURL = "/book/show/1368314.El_Cercano_Oriente" + const newerBookURL = "/book/show/243175949-el-cercano-oriente" + document := index.Document{Metadata: metadata.Metadata{ + Title: "El cercano Oriente", Authors: []string{"Isaac Asimov"}, + }} + older := SearchResult{ + ID: 2, Title: "El Cercano Oriente", Author: "Isaac Asimov", + BookURL: olderBookURL, Rating: "4.10", + } + newer := SearchResult{ + ID: 1, Title: "El Cercano Oriente", Author: "Isaac Asimov", + BookURL: newerBookURL, Rating: "4.26", + } + + for _, results := range [][]SearchResult{{newer, older}, {older, newer}} { + match, ok := matchingBook(document, results) + if !ok || match.BookURL != olderBookURL { + t.Fatalf("expected lower Goodreads ID %q, got %+v, %v", olderBookURL, match, ok) + } + } +} + +func TestMatchingBookPrefersRatedParentheticalTitle(t *testing.T) { + doc := index.Document{Metadata: metadata.Metadata{Title: "La promesa", Authors: []string{"Pearl S. Buck"}}} + rated := SearchResult{ + Title: "La promesa (The Promise)", Author: "Pearl S. Buck", + BookURL: "/book/show/7377104-la-promesa", Rating: "4.04", + } + unrated := SearchResult{ + Title: "La promesa", Author: "Pearl S. Buck", + BookURL: "/book/show/210898619-la-promesa", Rating: "0.00", + } + for _, results := range [][]SearchResult{{rated, unrated}, {unrated, rated}} { + match, ok := matchingBook(doc, results) + if !ok || match.BookURL != rated.BookURL || match.Rating != rated.Rating { + t.Fatalf("expected rated translated-title entry, got %+v, %v", match, ok) + } + } + rated.Author = "Another author" + match, ok := matchingBook(doc, []SearchResult{rated, unrated}) + if !ok || match.BookURL != unrated.BookURL { + t.Fatalf("rated entry by another author must not win, got %+v, %v", match, ok) + } +} + +func TestBookMatchesDolnickSubtitle(t *testing.T) { + const title = "La escritura de los dioses" + const author = "Edward Dolnick" + const bookURL = "/book/show/208277727-la-escritura-de-los-dioses-descifrando-la-piedra-de-rosetta" + source := &fakeSource{results: []SearchResult{{ + Title: title + ". Descifrando la piedra de Rosetta", Author: author, + BookURL: bookURL, Rating: "4.19", + }}} + idx := newTestIndexer(t, source) + doc := index.Document{ + ID: testID, AuthorsSlugs: []string{"edward-dolnick"}, + Metadata: metadata.Metadata{Title: title, Authors: []string{author}}, + } + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + got, err := idx.Book(context.Background(), doc) + if err != nil || got.BookURL != bookURL || got.Rating != "4.19" { + t.Fatalf("expected Dolnick's book and score, got %+v, %v", got, err) + } + stored, err := idx.DocumentByID(doc.ID) + if err != nil || !mustData(t, stored).Enriched || mustData(t, stored).URL != bookURL { + t.Fatalf("expected subtitle match persisted, got %+v, %v", stored, err) + } + if want := []string{title + " " + author}; !slices.Equal(source.Queries(), want) { + t.Fatalf("expected combined title/author search to match, got %q", source.Queries()) + } +} + +func TestTitleFallback(t *testing.T) { + const title = "El señor de los anillos" + correct := SearchResult{Title: title, Author: "J.R.R. Tolkien", BookURL: "/book/show/37910810"} + parody := SearchResult{Title: "El sopor de los anillos", Author: "The Harvard Lampoon", BookURL: "/book/show/7297403"} + for _, fallbackUnavailable := range []bool{false, true} { + name := "matched" + if fallbackUnavailable { + name = "unavailable" + } + t.Run(name, func(t *testing.T) { + source := &fakeSource{byQuery: map[string][]SearchResult{ + title + " J. R. R. Tolkien": {parody}, + title: {parody, correct}, + }} + idx := newTestIndexer(t, source) + doc := index.Document{ID: testID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + if fallbackUnavailable { + source.queryErrors = map[string]error{title: ErrUnavailable} + } + got, err := idx.Book(context.Background(), doc) + if fallbackUnavailable { + if !errors.Is(err, ErrUnavailable) || got.BookURL != "" { + t.Fatalf("expected unavailable error, got %+v, %v", got, err) + } + stored, readErr := idx.DocumentByID(doc.ID) + if readErr != nil || mustData(t, stored).Enriched { + t.Fatalf("expected document left for retry, got %+v, %v", stored, readErr) + } + source.queryErrors = nil + got, err = idx.Book(context.Background(), stored) + } + if err != nil || got.BookURL != correct.BookURL { + t.Fatalf("got %+v, %v; want %q", got, err, correct.BookURL) + } + want := []string{title + " J. R. R. Tolkien", title} + if fallbackUnavailable { + want = append(want, want...) + } + if !slices.Equal(source.Queries(), want) { + t.Fatalf("queries = %q, want %q", source.Queries(), want) + } + }) + } +} + +func TestMatchesTolkienOmnibusSearchResults(t *testing.T) { + const title = "El señor de los anillos" + const wantURL = "/book/show/37910810-el-se-or-de-los-anillos" + // Titles and ordering from ' actual title-only search response. + results := []SearchResult{ + {Title: "La comunidad del anillo (El señor de los anillos, #1)", Author: "J.R.R. Tolkien", BookURL: "/book/show/222947.La_comunidad_del_anillo"}, + {Title: "Las dos torres (El señor de los anillos, #2)", Author: "J.R.R. Tolkien", BookURL: "/book/show/15280.Las_dos_torres"}, + {Title: "El retorno del rey (El señor de los anillos, #3)", Author: "J.R.R. Tolkien", BookURL: "/book/show/63358.El_retorno_del_rey"}, + {Title: "El señor de los anillos (#1-#3)", Author: "J.R.R. Tolkien", BookURL: wantURL, AuthorURL: "/author/show/656983.J_R_R_Tolkien"}, + {Title: "Apéndices (El Señor de los Anillos, #4)", Author: "J.R.R. Tolkien", BookURL: "/book/show/54694.Ap_ndices"}, + } + source := &fakeSource{byQuery: map[string][]SearchResult{title: results}} + idx := newTestIndexer(t, source) + doc := index.Document{ID: testID, AuthorsSlugs: []string{"j-r-r-tolkien"}, Metadata: metadata.Metadata{Title: title, Authors: []string{"J. R. R. Tolkien"}}} + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + got, err := idx.Book(context.Background(), doc) + if err != nil || got.BookURL != wantURL { + t.Fatalf("got %+v, %v; want %q", got, err, wantURL) + } + stored, err := idx.DocumentByID(doc.ID) + if err != nil { + t.Fatal(err) + } + if !mustData(t, stored).Enriched || mustData(t, stored).URL != wantURL { + t.Fatalf("expected persisted omnibus match, got %+v", stored) + } + author, err := idx.Author("j-r-r-tolkien", "") + if err != nil || author.DataSourceLinks["goodreads"] != results[3].AuthorURL { + t.Fatalf("expected author URL on author record, got %+v, %v", author, err) + } + if want := []string{title + " J. R. R. Tolkien", title}; !slices.Equal(source.Queries(), want) { + t.Fatalf("queries = %q, want %q", source.Queries(), want) + } +} + +func TestForceIndexingRefreshesMatch(t *testing.T) { + source := &fakeSource{results: testResults} + idx := newTestIndexer(t, source) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + doc, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + updateData(t, &doc, func(data *DocumentData) { + data.URL = "/book/show/1-wrong" + data.ResourceID = "wrong-work" + data.LegacyBookID = 1 + data.Enriched = true + }) + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + doc, err = idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + if mustData(t, doc).URL != wantDataSourceLink { + t.Fatalf("expected refreshed book link, got %q", mustData(t, doc).URL) + } + if mustData(t, doc).ResourceID != "" || mustData(t, doc).LegacyBookID != 0 { + t.Fatalf("expected old identifiers to be cleared, got %q and %d", mustData(t, doc).ResourceID, mustData(t, doc).LegacyBookID) + } +} + +func TestStoreDetails(t *testing.T) { + for _, scenario := range []string{"linked", "newly linked", "changed metadata", "changed match", "deleted", "partial identifiers"} { + t.Run(scenario, func(t *testing.T) { + idx := newTestIndexer(t, nil) + doc := withData(t, index.Document{ + ID: testID, + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + + TextRankWords: []string{"dragon"}, + }, DocumentData{URL: wantDataSourceLink, Enriched: true}) + + stale := doc + resourceID := "work-id" + switch scenario { + case "newly linked": + updateData(t, &stale, func(data *DocumentData) { + data.URL = "" + data.Enriched = false + }) + case "changed metadata": + doc.Title = "Another title" + case "changed match": + updateData(t, &doc, func(data *DocumentData) { + data.URL = "/book/show/2-other" + }) + case "partial identifiers": + resourceID = "" + } + if scenario != "deleted" { + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + } + + book := BookResponse{Rating: "4.12", ResourceID: resourceID, LegacyBookID: 12207444} + writes := idx.documentsIdx + writes.count.Store(0) + err := idx.StoreDetails(stale, wantDataSourceLink, book) + wantError := scenario != "linked" && scenario != "newly linked" && scenario != "partial identifiers" + if (err != nil) != wantError { + t.Fatalf("StoreDetails error = %v, want error %v", err, wantError) + } + if scenario == "deleted" { + if !errors.Is(err, index.ErrDocumentNotFound) { + t.Fatalf("expected index.ErrDocumentNotFound, got %v", err) + } + return + } + stored, err := idx.DocumentByID(doc.ID) + if err != nil { + t.Fatal(err) + } + if wantError { + if writes.count.Load() != 0 { + t.Fatal("stale details must not cause any index writes") + } + if mustData(t, stored).ResourceID != "" || mustData(t, stored).LegacyBookID != 0 || stored.DataSourceScoreValue != 0 { + t.Fatalf("unexpected details persisted: %+v", stored) + } + return + } + wantLegacyBookID := 12207444 + if scenario == "partial identifiers" { + wantLegacyBookID = 0 + } + if mustData(t, stored).ResourceID != resourceID || mustData(t, stored).LegacyBookID != wantLegacyBookID { + t.Fatalf("identifiers not persisted: %+v", stored) + } + if stored.DataSourceScoreValue != 4.12 { + t.Fatalf("rating not persisted with identifiers: %+v", stored) + } + if !mustData(t, stored).Enriched || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { + t.Fatalf("existing enrichment was lost: %+v", stored) + } + book.Rating = "4.120" + if err := idx.StoreDetails(stale, wantDataSourceLink, book); err != nil { + t.Fatalf("repeated store failed: %v", err) + } + if writes.count.Load() != 1 { + t.Fatalf("expected one combined write and no write for unchanged details, got %d", writes.count.Load()) + } + }) + } +} + +type countedIndex interface { + bleve.Index +} + +type writeCountingIndex struct { + countedIndex + count atomic.Int64 +} + +func (i *writeCountingIndex) Index(id string, data any) error { + i.count.Add(1) + return i.countedIndex.Index(id, data) +} + +func TestStoreDetailsPreservesMissingFields(t *testing.T) { + for _, tt := range []struct { + name string + book BookResponse + wantRating string + wantCount string + wantWork string + wantID int + }{ + {name: "rating only", book: BookResponse{Rating: "4.12"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "identifiers only", book: BookResponse{ResourceID: "new-work", LegacyBookID: 2}, wantRating: "3.82", wantWork: "new-work", wantID: 2}, + {name: "missing work", book: BookResponse{Rating: "4.12", LegacyBookID: 2}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "missing edition", book: BookResponse{Rating: "4.12", ResourceID: "new-work"}, wantRating: "4.12", wantWork: "old-work", wantID: 1}, + {name: "empty response", wantRating: "3.82", wantWork: "old-work", wantID: 1}, + {name: "count only", book: BookResponse{RatingCount: "650"}, wantRating: "3.82", wantCount: "650", wantWork: "old-work", wantID: 1}, + {name: "zero count", book: BookResponse{RatingCount: "0"}, wantRating: "3.82", wantCount: "0", wantWork: "old-work", wantID: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + idx := newTestIndexer(t, nil) + doc := withData(t, index.Document{ + ID: testID, + DataSourceScoreValue: 3.82, + }, DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 1}) + + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + if err := idx.StoreDetails(doc, mustData(t, doc).URL, tt.book); err != nil { + t.Fatal(err) + } + stored, err := idx.DocumentByID(doc.ID) + if err != nil { + t.Fatal(err) + } + if stored.DataSourceScoreValue != parseRating(tt.wantRating, doc.ID) || mustData(t, stored).ResourceID != tt.wantWork || mustData(t, stored).LegacyBookID != tt.wantID { + t.Fatalf("partial details changed fields that were not supplied: %+v", stored) + } + wantCount := tt.wantCount + if wantCount == "" { + wantCount = mustData(t, doc).RatingCount + } + if mustData(t, stored).RatingCount != wantCount { + t.Fatalf("expected rating count %q, got %q", wantCount, mustData(t, stored).RatingCount) + } + }) + } +} + +func TestUpdateDataSourceLink(t *testing.T) { + idx := newTestIndexer(t, nil) + doc := withData(t, index.Document{ + ID: testID, Slug: "la-reina-dragon", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Authors: []string{"Alice Borchardt"}}, + TextRankWords: []string{"dragon"}, + + DataSourceScoreValue: 3.82, + }, DocumentData{URL: wantDataSourceLink, RatingCount: "2,285", ResourceID: "old-work", LegacyBookID: 12207444, Enriched: true}) + + if err := index.IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + const newLink = "provider-specific-document-reference" + if err := idx.UpdateDataSourceLink(doc.ID, newLink); err != nil { + t.Fatal(err) + } + stored, err := idx.DocumentByID(doc.ID) + if err != nil { + t.Fatal(err) + } + if mustData(t, stored).URL != newLink || !mustData(t, stored).Enriched || + stored.DataSourceScoreValue != 0 || mustData(t, stored).RatingCount != "" || mustData(t, stored).ResourceID != "" || mustData(t, stored).LegacyBookID != 0 { + t.Fatalf("updating review link must clear cached score and provider lookup data: %+v", stored) + } + if stored.Title != doc.Title || stored.Slug != doc.Slug || !slices.Equal(stored.TextRankWords, doc.TextRankWords) { + t.Fatalf("review-link update changed unrelated document data: %+v", stored) + } + if err := idx.StoreDetails(doc, mustData(t, doc).URL, BookResponse{Rating: "4.5"}); err == nil { + t.Fatal("stale score write must not overwrite the updated link") + } +} + +func TestUpdateDataSourceLinkMissingDocument(t *testing.T) { + idx := newTestIndexer(t, nil) + if err := idx.UpdateDataSourceLink("missing", "provider-specific-document-reference"); !errors.Is(err, index.ErrDocumentNotFound) { + t.Fatalf("expected missing-document error, got %v", err) + } +} + +func TestDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { + idx := newTestIndexer(t, &fakeSource{results: testResults}) + if err := idx.AddLibrary(10, true, 1); err != nil { + t.Fatal(err) + } + stale, err := idx.DocumentByID(testID) + if err != nil { + t.Fatal(err) + } + current := stale + other := map[string]any{"id": "external-id", "nested": map[string]any{"count": float64(3)}} + if err := index.SetDataSourceData(¤t, "other", other); err != nil { + t.Fatal(err) + } + if _, exists := stale.DataSources["other"]; exists { + t.Fatal("updating a provider mutated another document's map") + } + if err := index.IndexDocument(idx.documentsIdx, current.ID, current); err != nil { + t.Fatal(err) + } + if err := idx.Enricher.Enrich(context.Background(), idx.BleveIndexer, stale); err != nil { + t.Fatal(err) + } + if err := idx.StoreDetails(stale, wantDataSourceLink, BookResponse{ + Rating: "4.5", RatingCount: "100", ResourceID: "work", LegacyBookID: 123, + }); err != nil { + t.Fatal(err) + } + if err := idx.UpdateDataSourceLink(current.ID, "/book/show/456"); err != nil { + t.Fatal(err) + } + stale.TextRankWords = []string{"updated"} + if err := idx.UpdateDocument(stale.ID, func(update *index.DocumentUpdate) (bool, error) { + update.TextRankWords = stale.TextRankWords + return true, nil + }); err != nil { + t.Fatal(err) + } + got, err := idx.DocumentByID(stale.ID) + if err != nil { + t.Fatal(err) + } + decoded, err := index.DataSourceData[map[string]any](got, "other") + if err != nil || !reflect.DeepEqual(decoded, other) { + t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) + } + if data := mustData(t, got); data != (DocumentData{URL: "/book/show/456", Enriched: true}) { + t.Fatalf("stale write restored old Goodreads state: %+v", data) + } +} + +func TestBookRejectsInvalidProviderData(t *testing.T) { + idx := newTestIndexer(t, nil) + document := index.Document{DataSources: map[string]json.RawMessage{Source: json.RawMessage(`{"url":42}`)}} + if _, err := idx.Book(context.Background(), document); err == nil { + t.Fatal("Goodreads accepted an incorrectly typed URL") + } +} + +type blockingSource struct { + started chan struct{} + release chan struct{} +} + +func (s *blockingSource) SearchBooks(ctx context.Context, _ string) (SearchBooksResponse, error) { + s.started <- struct{}{} + select { + case <-s.release: + return SearchBooksResponse{}, nil + case <-ctx.Done(): + return SearchBooksResponse{}, ctx.Err() + } +} + +func TestEnrichLimitsConcurrentLookupsAcrossCallers(t *testing.T) { + const callers = 2 * maxConcurrentLookups + source := &blockingSource{started: make(chan struct{}, callers), release: make(chan struct{})} + idx := newTestIndexer(t, source) + var callersDone sync.WaitGroup + defer func() { + close(source.release) + callersDone.Wait() + }() + // Separate callers model the batch and upload pools sharing one adapter. + for range callers { + callersDone.Go(func() { + _, _ = idx.enrich(context.Background(), idx.BleveIndexer, index.Document{Metadata: metadata.Metadata{Title: "Title"}}) + }) + } + for range maxConcurrentLookups { + select { + case <-source.started: + case <-time.After(time.Second): + t.Fatal("expected concurrent lookups") + } + } + select { + case <-source.started: + t.Fatal("lookups exceeded the adapter concurrency limit") + case <-time.After(30 * time.Millisecond): + } +} diff --git a/internal/datasource/biblioreads/cache.go b/internal/datasource/biblioreads/cache.go new file mode 100644 index 00000000..d4427df0 --- /dev/null +++ b/internal/datasource/biblioreads/cache.go @@ -0,0 +1,65 @@ +package biblioreads + +import ( + "sync" + "time" +) + +// DefaultCacheTTL is how long successful responses are cached by default. +const DefaultCacheTTL = 24 * time.Hour + +// cacheSweepInterval sets how often expired entries are purged, so entries that are never +// requested again don't stay in memory forever. +const cacheSweepInterval = 10 * time.Minute + +type cacheEntry struct { + body []byte + expiresAt time.Time +} + +// responseCache is an in-memory cache of raw response bodies with a fixed TTL. +type responseCache struct { + ttl time.Duration + mu sync.Mutex + entries map[string]cacheEntry + lastSweep time.Time +} + +func newResponseCache(ttl time.Duration) *responseCache { + return &responseCache{ + ttl: ttl, + entries: make(map[string]cacheEntry), + lastSweep: time.Now(), + } +} + +func (c *responseCache) get(key string) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + entry, ok := c.entries[key] + if !ok { + return nil, false + } + if !time.Now().Before(entry.expiresAt) { + delete(c.entries, key) + return nil, false + } + return entry.body, true +} + +func (c *responseCache) set(key string, body []byte) { + c.mu.Lock() + defer c.mu.Unlock() + + now := time.Now() + if now.Sub(c.lastSweep) >= cacheSweepInterval { + for k, entry := range c.entries { + if !now.Before(entry.expiresAt) { + delete(c.entries, k) + } + } + c.lastSweep = now + } + c.entries[key] = cacheEntry{body: body, expiresAt: now.Add(c.ttl)} +} diff --git a/internal/datasource/biblioreads/cache_test.go b/internal/datasource/biblioreads/cache_test.go new file mode 100644 index 00000000..a51c8baf --- /dev/null +++ b/internal/datasource/biblioreads/cache_test.go @@ -0,0 +1,213 @@ +package biblioreads_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +func TestIncompleteBookResponsesAreNotCached(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92"}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + for range 2 { + book, err := client.Book(context.Background(), "5898652") + if err != nil || book.Rating != "3.92" { + t.Fatalf("expected available score even without review identifiers, got %+v, %v", book, err) + } + } + if hits.Load() != 2 { + t.Fatal("incomplete book details must not be cached") + } +} + +func TestReviewsEmptyResponsesAreNotCached(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"totalCount":0,"reviews":[]}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + for range 2 { + response, err := client.Reviews(context.Background(), "work", 1, "", "") + if err != nil || len(response.Reviews) != 0 { + t.Fatalf("expected valid empty result, got %+v, %v", response, err) + } + } + if hits.Load() != 2 { + t.Fatal("empty results must not hide reviews behind the response cache") + } +} + +func TestReviewsLanguageCache(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + var payload struct { + LanguageCode string `json:"languageCode"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Errorf("decode request: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + if payload.LanguageCode != "es" && payload.LanguageCode != "en" { + t.Errorf("unexpected language %q", payload.LanguageCode) + w.WriteHeader(http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(biblioreads.ReviewsResponse{ + Version: "BiblioReads v4.4.0", Status: "Received", StatusCode: http.StatusOK, TotalCount: 1, + Reviews: []biblioreads.Review{{Text: payload.LanguageCode}}, + }) + })) + defer server.Close() + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + for _, language := range []string{"es", "en", "es", "en"} { + response, err := client.Reviews(context.Background(), "work", 1, "", language) + if err != nil { + t.Fatal(err) + } + if len(response.Reviews) != 1 || response.Reviews[0].Text != language { + t.Fatalf("expected %s reviews, got %+v", language, response.Reviews) + } + } + if hits.Load() != 2 { + t.Fatalf("expected one cached response per language, got %d requests", hits.Load()) + } +} + +func TestFailoverResponsesAreCached(t *testing.T) { + t.Run("book", func(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if hits.Add(1) == 1 { + w.WriteHeader(http.StatusGatewayTimeout) + _, _ = w.Write([]byte(`{"status":"Error - Book not found"}`)) + return + } + _, _ = w.Write([]byte(validBookResponse)) + }) + client := newFailoverClient(t, handler) + + for range 2 { + book, err := client.Book(context.Background(), "5898652") + if err != nil || book.Rating != "3.92" { + t.Fatalf("expected recovered book, got %+v, %v", book, err) + } + } + if hits.Load() != 2 { + t.Fatalf("expected recovered book to be cached after failover, got %d requests", hits.Load()) + } + }) + + t.Run("reviews", func(t *testing.T) { + var hits atomic.Int32 + fixture := readFixture(t, "reviews-scraper.json") + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if hits.Add(1) == 1 { + _, _ = w.Write([]byte(`{}`)) + return + } + _, _ = w.Write(fixture) + }) + client := newFailoverClient(t, handler) + + for range 2 { + response, err := client.Reviews(context.Background(), "work", 1, "", "") + if err != nil || len(response.Reviews) == 0 { + t.Fatalf("expected recovered reviews, got %+v, %v", response, err) + } + } + if hits.Load() != 2 { + t.Fatalf("expected recovered reviews to be cached after failover, got %d requests", hits.Load()) + } + }) +} + +func TestCache(t *testing.T) { + t.Run("successful responses are cached per request", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92","resourceID":"work-id","legacyBookID":5898652,"title":"La Reina Dragon","genres":["Fantasy"],"totalCount":1,"reviews":[{"id":"review-1"}]}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + + first, err := client.Book(context.Background(), "1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + first.Genres[0] = "modified" + + second, err := client.Book(context.Background(), "1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hits.Load() != 1 { + t.Errorf("expected cached response to be used, got %d requests", hits.Load()) + } + if second.Title != "La Reina Dragon" || second.Genres[0] != "Fantasy" { + t.Errorf("expected an unmodified copy of the cached response, got %+v", second) + } + + if _, err := client.Book(context.Background(), "2"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.SearchBooks(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "", ""); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "token", ""); err != nil { + t.Fatalf("unexpected error: %v", err) + } + for range 2 { + if _, err := client.Author(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if hits.Load() != 6 { + t.Errorf("expected different requests not to share cache entries, got %d requests", hits.Load()) + } + }) + + t.Run("entries expire after the TTL", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, validBookResponse, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(100*time.Millisecond)) + + for range 3 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if hits.Load() != 1 { + t.Fatalf("expected 1 request before expiry, got %d", hits.Load()) + } + + time.Sleep(150 * time.Millisecond) + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hits.Load() != 2 { + t.Errorf("expected expired entry to be refreshed, got %d requests", hits.Load()) + } + }) + + t.Run("caching can be disabled", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, validBookResponse, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithCacheTTL(0)) + + for range 3 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if hits.Load() != 3 { + t.Errorf("expected 3 requests with caching disabled, got %d", hits.Load()) + } + }) +} diff --git a/internal/datasource/biblioreads/client.go b/internal/datasource/biblioreads/client.go new file mode 100644 index 00000000..21c54b0c --- /dev/null +++ b/internal/datasource/biblioreads/client.go @@ -0,0 +1,402 @@ +// Package biblioreads implements a client and index enrichment adapter for the BiblioReads API +// (https://github.com/nesaku/biblioreads), an alternative Goodreads front-end. +package biblioreads + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math/rand/v2" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/svera/coreander/v5/internal/datasource/httpjson" + "golang.org/x/mod/semver" +) + +const ( + goodreadsBaseURL = "https://www.goodreads.com" + + searchBooksPath = "/api/search/books" + bookScraperPath = "/api/book-scraper" + authorInfoPath = "/api/author/info" + reviewsScraperPath = "/api/reviews-scraper" +) + +// ErrUnavailable is returned when no instance could serve a request (network errors, rate limits, +// non-JSON responses), as opposed to an instance answering with an error of its own. +var ErrUnavailable = errors.New("biblioreads: all instances failed") + +type Client struct { + instances []string + httpClient *http.Client + cache *responseCache + + mu sync.Mutex + cooldownUntil map[string]time.Time + nextRequest map[string]time.Time + requestInterval time.Duration +} + +type options struct { + instances []string + httpClient *http.Client + cacheTTL time.Duration + requestInterval time.Duration +} + +type Option func(*options) + +// WithRequestInterval sets the minimum spacing between request starts to each instance. +// Zero disables spacing. +func WithRequestInterval(interval time.Duration) Option { + return func(o *options) { + o.requestInterval = max(interval, 0) + } +} + +// WithInstances sets endpoints the caller hosts or has permission to automate. +// An empty list disables requests; there is no public-instance fallback. +func WithInstances(instances []string) Option { + return func(o *options) { + o.instances = instances + } +} + +// WithHTTPClient sets the HTTP client used for requests. A nil client keeps the default one, +// which has a 15 seconds timeout. +func WithHTTPClient(httpClient *http.Client) Option { + return func(o *options) { + if httpClient != nil { + o.httpClient = httpClient + } + } +} + +// WithCacheTTL sets how long successful responses are cached. A zero or negative ttl disables caching. +func WithCacheTTL(ttl time.Duration) Option { + return func(o *options) { + o.cacheTTL = ttl + } +} + +// NewClient returns a client that spreads requests across instances, preferring +// instances ready to accept a request and randomizing ties. Requests to each +// instance are spaced at least one second apart by default, including failover +// attempts. Unavailable instances are skipped until their Retry-After time, or +// httpjson.DefaultRetryAfter, has passed. Successful responses are cached in memory for +// DefaultCacheTTL unless changed with WithCacheTTL. No endpoints are configured +// by default; callers must supply them with WithInstances. +func NewClient(opts ...Option) *Client { + o := options{ + httpClient: &http.Client{Timeout: 15 * time.Second}, + cacheTTL: DefaultCacheTTL, + requestInterval: time.Second, + } + for _, opt := range opts { + opt(&o) + } + + normalized := make([]string, len(o.instances)) + for i, instance := range o.instances { + normalized[i] = strings.TrimRight(instance, "/") + } + + c := &Client{ + instances: normalized, + httpClient: o.httpClient, + cooldownUntil: make(map[string]time.Time), + nextRequest: make(map[string]time.Time), + requestInterval: o.requestInterval, + } + if o.cacheTTL > 0 { + c.cache = newResponseCache(o.cacheTTL) + } + return c +} + +type queryURLPayload struct { + QueryURL string `json:"queryURL"` +} + +type reviewsPayload struct { + ResourceID string `json:"resourceID"` + LegacyBookID int `json:"legacyBookID"` + NextPageToken string `json:"nextPageToken,omitempty"` + LanguageCode string `json:"languageCode,omitempty"` +} + +// SearchBooks searches Goodreads books matching query. +func (c *Client) SearchBooks(ctx context.Context, query string) (SearchBooksResponse, error) { + var res SearchBooksResponse + payload := queryURLPayload{ + QueryURL: goodreadsBaseURL + "/search?q=" + escapeQuery(query), + } + err := c.post(ctx, searchBooksPath, payload, &res) + return res, err +} + +// Book returns the details of the Goodreads book identified by id, which can be +// either a numeric ID ("12207444") or an ID with slug ("12207444-la-reina-dragon"). +func (c *Client) Book(ctx context.Context, id string) (BookResponse, error) { + var res BookResponse + payload := queryURLPayload{ + QueryURL: goodreadsBaseURL + "/book/show/" + url.PathEscape(id), + } + err := c.post(ctx, bookScraperPath, payload, &res) + return res, err +} + +// Author returns the details of the Goodreads author identified by id, which can be +// either a numeric ID ("151155") or an ID with slug ("151155.Alice_Borchardt"). +func (c *Client) Author(ctx context.Context, id string) (AuthorResponse, error) { + var res AuthorResponse + payload := queryURLPayload{ + QueryURL: goodreadsBaseURL + "/author/show/" + url.PathEscape(id), + } + err := c.post(ctx, authorInfoPath, payload, &res) + return res, err +} + +// Reviews returns a page of reviews for the work identified by resourceID and legacyBookID +// (both available in BookResponse). Pass an empty nextPageToken to get the first page. +// languageCode filters reviews by a two-letter ISO language code; empty means all languages. +func (c *Client) Reviews(ctx context.Context, resourceID string, legacyBookID int, nextPageToken, languageCode string) (ReviewsResponse, error) { + var res ReviewsResponse + payload := reviewsPayload{ + ResourceID: resourceID, + LegacyBookID: legacyBookID, + NextPageToken: nextPageToken, + LanguageCode: languageCode, + } + err := c.post(ctx, reviewsScraperPath, payload, &res) + return res, err +} + +// unavailableError means the instance can't serve requests right now and another one should be tried. +type unavailableError struct { + endpoint string + retryAfter time.Duration + reason error +} + +func (e *unavailableError) Error() string { + return fmt.Sprintf("%s: %v", e.endpoint, e.reason) +} + +func (e *unavailableError) Unwrap() error { + return e.reason +} + +func (c *Client) post(ctx context.Context, path string, payload, dest any) error { + if len(c.instances) == 0 { + return fmt.Errorf("%w: no BiblioReads endpoints configured", ErrUnavailable) + } + body, err := json.Marshal(payload) + if err != nil { + return err + } + + cacheKey := path + "\x00" + string(body) + if c.cache != nil { + if cached, ok := c.cache.get(cacheKey); ok { + return json.Unmarshal(cached, dest) + } + } + + var errs []error + var incompleteResponse []byte + tried := make(map[string]bool) + for len(tried) < len(c.instances) { + instance, err := c.acquireInstance(ctx, tried) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + errs = append(errs, err) + break + } + tried[instance] = true + endpoint := instance + path + respBody, err := c.postToInstance(ctx, endpoint, body) + if err == nil { + var complete bool + complete, err = decodeResponse(respBody, payload, dest) + if err != nil { + if path != bookScraperPath && path != reviewsScraperPath { + return fmt.Errorf("biblioreads: error decoding response from %s: %w", endpoint, err) + } + err = &unavailableError{endpoint: endpoint, retryAfter: httpjson.DefaultRetryAfter, reason: err} + } else if !complete { + // Keep a fallback, preferring book details that include a score. + book, isBook := dest.(*BookResponse) + if incompleteResponse == nil || !isBook || book.Rating != "" { + incompleteResponse = respBody + } + continue + } + } + if err == nil { + reviews, isReviews := dest.(*ReviewsResponse) + if c.cache != nil && (!isReviews || len(reviews.Reviews) > 0) { + c.cache.set(cacheKey, respBody) + } + return nil + } + var unavailable *unavailableError + if !errors.As(err, &unavailable) { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + c.setCooldown(instance, unavailable.retryAfter) + errs = append(errs, err) + } + if incompleteResponse != nil { + if err := ctx.Err(); err != nil { + return err + } + _, err := decodeResponse(incompleteResponse, payload, dest) + return err + } + return fmt.Errorf("%w: %w", ErrUnavailable, errors.Join(errs...)) +} + +// decodeResponse reports whether a valid response is complete enough to stop failover. +func decodeResponse(body []byte, payload, dest any) (bool, error) { + switch response := dest.(type) { + case *BookResponse: + *response = BookResponse{} + if err := json.Unmarshal(body, response); err != nil { + return false, err + } + if response.Status != "Received" || response.StatusCode != http.StatusOK { + return false, fmt.Errorf("invalid book response: status %q, code %d", response.Status, response.StatusCode) + } + return response.Rating != "" && response.ResourceID != "" && response.LegacyBookID > 0, nil + case *ReviewsResponse: + *response = ReviewsResponse{} + if err := json.Unmarshal(body, response); err != nil { + return false, err + } + request, ok := payload.(reviewsPayload) + if !ok { + return false, fmt.Errorf("invalid reviews request payload %T", payload) + } + if request.LanguageCode != "" && !supportsReviewLanguage(response.Version) { + return false, fmt.Errorf("review language filtering requires BiblioReads v4.4.0 or newer, got %q", response.Version) + } + if response.Status != "Received" || response.StatusCode != http.StatusOK || response.Reviews == nil || + (request.NextPageToken == "" && response.TotalCount > 0 && len(response.Reviews) == 0) { + return false, fmt.Errorf("invalid reviews response: status %q, code %d, total %d, reviews %d", + response.Status, response.StatusCode, response.TotalCount, len(response.Reviews)) + } + return request.NextPageToken != "" || len(response.Reviews) > 0, nil + default: + return true, json.Unmarshal(body, dest) + } +} + +func supportsReviewLanguage(version string) bool { + version = strings.TrimPrefix(version, "BiblioReads ") + return semver.IsValid(version) && semver.Compare(version, "v4.4.0") >= 0 +} + +// acquireInstance rechecks availability after waiting, so cancellation does not +// reserve a future slot and concurrent requests cannot take the same slot. +func (c *Client) acquireInstance(ctx context.Context, tried map[string]bool) (string, error) { + for { + if err := ctx.Err(); err != nil { + return "", err + } + c.mu.Lock() + start := rand.IntN(len(c.instances)) + instance := "" + now := time.Now() + for i := range c.instances { + candidate := c.instances[(start+i)%len(c.instances)] + if !tried[candidate] && !now.Before(c.cooldownUntil[candidate]) && + (instance == "" || c.nextRequest[candidate].Before(c.nextRequest[instance])) { + instance = candidate + } + } + if instance == "" { + c.mu.Unlock() + return "", fmt.Errorf("%w: no untried instance is outside its cooldown", ErrUnavailable) + } + wait := time.Until(c.nextRequest[instance]) + if wait <= 0 { + c.nextRequest[instance] = time.Now().Add(c.requestInterval) + c.mu.Unlock() + return instance, nil + } + c.mu.Unlock() + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return "", ctx.Err() + case <-timer.C: + } + } +} + +func (c *Client) setCooldown(instance string, d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.cooldownUntil[instance] = time.Now().Add(d) +} + +// postToInstance returns the body of a successful response. +func (c *Client) postToInstance(ctx context.Context, endpoint string, body []byte) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := httpjson.Do(c.httpClient, req) + if err != nil { + return nil, &unavailableError{endpoint: endpoint, retryAfter: httpjson.DefaultRetryAfter, reason: err} + } + + if resp.StatusCode == http.StatusTooManyRequests { + return nil, &unavailableError{ + endpoint: endpoint, + retryAfter: httpjson.ParseRetryAfter(resp.Header.Get("Retry-After")), + reason: fmt.Errorf("status code %d", resp.StatusCode), + } + } + if resp.StatusCode >= http.StatusInternalServerError { + return nil, &unavailableError{ + endpoint: endpoint, retryAfter: httpjson.DefaultRetryAfter, + reason: fmt.Errorf("status code %d: %s", resp.StatusCode, resp.Body), + } + } + + // BiblioReads always answers with JSON, even on errors (e.g. a 504 for a book that + // doesn't exist), so anything else comes from a broken instance or a proxy in front of it. + if !json.Valid(resp.Body) { + return nil, &unavailableError{ + endpoint: endpoint, + retryAfter: httpjson.DefaultRetryAfter, + reason: fmt.Errorf("status code %d with non-JSON response", resp.StatusCode), + } + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("biblioreads: request to %s failed with status code %d: %s", endpoint, resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + +// escapeQuery escapes a search string using %20 for spaces, as Goodreads does. +func escapeQuery(query string) string { + return strings.ReplaceAll(url.QueryEscape(query), "+", "%20") +} diff --git a/internal/datasource/biblioreads/client_test.go b/internal/datasource/biblioreads/client_test.go new file mode 100644 index 00000000..556afdf2 --- /dev/null +++ b/internal/datasource/biblioreads/client_test.go @@ -0,0 +1,405 @@ +package biblioreads_test + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +type capturedRequest struct { + method string + path string + contentType string + payload map[string]any +} + +const validBookResponse = `{"status":"Received","statusCode":200,"rating":"3.92","resourceID":"work-id","legacyBookID":5898652}` + +func newTestClient(opts ...biblioreads.Option) *biblioreads.Client { + return biblioreads.NewClient(append([]biblioreads.Option{biblioreads.WithRequestInterval(0)}, opts...)...) +} + +func newFailoverClient(t *testing.T, handler http.Handler) *biblioreads.Client { + t.Helper() + first := httptest.NewServer(handler) + t.Cleanup(first.Close) + second := httptest.NewServer(handler) + t.Cleanup(second.Close) + return newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL})) +} + +func readFixture(t *testing.T, name string) []byte { + t.Helper() + contents, err := os.ReadFile("fixtures/" + name) + if err != nil { + t.Fatalf("couldn't read fixture %s: %v", name, err) + } + return contents +} + +func captureRequest(t *testing.T, w http.ResponseWriter, r *http.Request, captured *capturedRequest) bool { + t.Helper() + captured.method = r.Method + captured.path = r.URL.Path + captured.contentType = r.Header.Get("Content-Type") + if err := json.NewDecoder(r.Body).Decode(&captured.payload); err != nil { + t.Errorf("couldn't decode request payload: %v", err) + w.WriteHeader(http.StatusBadRequest) + return false + } + return true +} + +func newMockServer(t *testing.T, status int, fixture string, captured *capturedRequest) *httptest.Server { + t.Helper() + contents := readFixture(t, fixture) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !captureRequest(t, w, r, captured) { + return + } + w.WriteHeader(status) + _, _ = w.Write(contents) + })) + t.Cleanup(server.Close) + return server +} + +func assertRequest(t *testing.T, captured capturedRequest, path string, payload map[string]any) { + t.Helper() + if captured.method != http.MethodPost { + t.Errorf("expected POST, got %s", captured.method) + } + if captured.path != path { + t.Errorf("expected path %s, got %s", path, captured.path) + } + if captured.contentType != "application/json" { + t.Errorf("expected application/json content type, got %s", captured.contentType) + } + if len(captured.payload) != len(payload) { + t.Errorf("expected payload %v, got %v", payload, captured.payload) + } + for key, value := range payload { + if captured.payload[key] != value { + t.Errorf("expected payload[%s] = %v, got %v", key, value, captured.payload[key]) + } + } +} + +func TestSearchBooks(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusOK, "search-books.json", &captured) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + res, err := client.SearchBooks(context.Background(), "La Reina Dragón Alice Borchardt") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertRequest(t, captured, "/api/search/books", map[string]any{ + "queryURL": "https://www.goodreads.com/search?q=La%20Reina%20Drag%C3%B3n%20Alice%20Borchardt", + }) + + if len(res.Result) != 1 { + t.Fatalf("expected 1 result, got %d", len(res.Result)) + } + got := res.Result[0] + if got.Title != "La Reina Dragon (La leyenda de Guinevere, #1)" || + got.BookURL != "/book/show/12207444-la-reina-dragon" || + got.Author != "Alice Borchardt" || + got.Rating != "3.82" || + got.RatingCount != "2285" || + got.ID != 1 { + t.Errorf("unexpected result: %+v", got) + } + if res.NumberOfResults != "1 results" || res.SearchType != "books" { + t.Errorf("unexpected response metadata: %+v", res) + } +} + +func TestBook(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusOK, "book-scraper.json", &captured) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + res, err := client.Book(context.Background(), "12207444-la-reina-dragon") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertRequest(t, captured, "/api/book-scraper", map[string]any{ + "queryURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + }) + + if res.LegacyBookID != 12207444 { + t.Errorf("expected legacy book ID 12207444, got %d", res.LegacyBookID) + } + if res.ResourceID != "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g" { + t.Errorf("unexpected resource ID %s", res.ResourceID) + } + if res.Title != "La Reina Dragon" || res.Series != "Tales of Guinevere" || res.SeriesPosition != "1" { + t.Errorf("unexpected book data: %+v", res) + } + if len(res.Author) != 1 || res.Author[0].Name != "Alice Borchardt" || res.Author[0].Role != "Author" { + t.Errorf("unexpected authors: %+v", res.Author) + } + if len(res.Genres) != 10 || res.Genres[0] != "Fantasy" { + t.Errorf("unexpected genres: %v", res.Genres) + } + if res.ISBN13 != "9788448033187" || res.Language != "Spanish; Castilian" { + t.Errorf("unexpected book data: %+v", res) + } + if res.ReviewBreakdown.Rating5 != "726" { + t.Errorf("unexpected review breakdown: %+v", res.ReviewBreakdown) + } +} + +func TestAuthor(t *testing.T) { + for _, tt := range []struct { + name string + id string + url string + }{ + {"numeric ID", "151155", "https://www.goodreads.com/author/show/151155"}, + {"ID with slug", "151155.Alice_Borchardt", "https://www.goodreads.com/author/show/151155.Alice_Borchardt"}, + {"escaped ID", "151155.Alice/Borchardt?#", "https://www.goodreads.com/author/show/151155.Alice%2FBorchardt%3F%23"}, + } { + t.Run(tt.name, func(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusOK, "author-info.json", &captured) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + res, err := client.Author(context.Background(), tt.id) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + assertRequest(t, captured, "/api/author/info", map[string]any{"queryURL": tt.url}) + + fixture := readFixture(t, "author-info.json") + encoded, err := json.Marshal(res) + if err != nil { + t.Fatalf("couldn't encode author response: %v", err) + } + var want, got map[string]any + if err := json.Unmarshal(fixture, &want); err != nil { + t.Fatalf("couldn't decode author fixture: %v", err) + } + if err := json.Unmarshal(encoded, &got); err != nil { + t.Fatalf("couldn't decode encoded author response: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("author response did not preserve all fixture fields:\ngot: %+v\nwant: %+v", got, want) + } + }) + } +} + +func TestAuthorErrors(t *testing.T) { + for _, tt := range []struct { + name string + status int + body string + unavailable bool + }{ + {"API error", http.StatusNotFound, `{"status":"Error - Author not found"}`, false}, + {"non-JSON response", http.StatusOK, "Bad Gateway", true}, + {"invalid field type", http.StatusOK, `{"books":"invalid"}`, false}, + } { + t.Run(tt.name, func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, tt.status, tt.body, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + + for range 2 { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + _, err := client.Author(ctx, "151155") + cancel() + if err == nil { + t.Fatal("expected error, got nil") + } + if errors.Is(err, biblioreads.ErrUnavailable) != tt.unavailable && !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("expected unavailable=%v, got %v", tt.unavailable, err) + } + } + wantHits := int32(2) + if tt.unavailable { + wantHits = 1 + } + if hits.Load() != wantHits { + t.Errorf("expected errors not to be cached, got %d requests", hits.Load()) + } + }) + } +} + +func TestReviewsInvalidResponses(t *testing.T) { + for _, body := range []string{ + `{}`, + `{"status":"Error - Invalid Query","statusCode":500}`, + `{"status":"Received","statusCode":200,"totalCount":90,"reviews":[]}`, + `{"status":"Received","statusCode":200,"totalCount":0}`, + `{"status":"Received","statusCode":200,"reviews":"invalid"}`, + } { + t.Run(body, func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, body, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + for range 2 { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + _, err := client.Reviews(ctx, "work", 1, "", "") + cancel() + if !errors.Is(err, biblioreads.ErrUnavailable) && !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected unavailable error, got %v", err) + } + } + if hits.Load() != 1 { + t.Fatal("invalid responses must trigger cooldown") + } + }) + } +} + +func TestReviewsLanguageVersionCompatibility(t *testing.T) { + for _, tt := range []struct { + version string + supported bool + }{ + {"", false}, + {"BiblioReads v4.3.1", false}, + {"BiblioReads v4.4.0-beta.1", false}, + {"unknown", false}, + {"BiblioReads v4.4.0", true}, + {"BiblioReads v4.5.0", true}, + } { + t.Run(tt.version, func(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + if err := json.NewEncoder(w).Encode(biblioreads.ReviewsResponse{ + Version: tt.version, Status: "Received", StatusCode: http.StatusOK, TotalCount: 1, + Reviews: []biblioreads.Review{{Text: "Review"}}, + }); err != nil { + t.Errorf("encode response: %v", err) + } + })) + defer server.Close() + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + for range 2 { + _, err := client.Reviews(context.Background(), "work", 1, "", "es") + if tt.supported && err != nil { + t.Fatalf("expected supported language filter, got %v", err) + } + if !tt.supported && !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected unsupported instance to fail, got %v", err) + } + } + wantHits := int32(1) + if hits.Load() != wantHits { + t.Fatalf("expected %d requests, got %d", wantHits, hits.Load()) + } + }) + } +} + +func TestReviews(t *testing.T) { + t.Run("fractional ratings", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"totalCount":4,"reviews":[{"rating":0.25},{"rating":3.5},{"rating":4.75},{"rating":5}]}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + response, err := client.Reviews(context.Background(), "work", 1, "", "") + if err != nil { + t.Fatal(err) + } + want := []float64{0.25, 3.5, 4.75, 5} + if len(response.Reviews) != len(want) { + t.Fatalf("expected %d reviews, got %d", len(want), len(response.Reviews)) + } + for i, rating := range want { + if response.Reviews[i].Rating != rating { + t.Errorf("review %d: expected rating %g, got %g", i, rating, response.Reviews[i].Rating) + } + } + }) + + t.Run("first page", func(t *testing.T) { + var captured capturedRequest + server := newMockServer(t, http.StatusOK, "reviews-scraper.json", &captured) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + res, err := client.Reviews(context.Background(), "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", 12207444, "", "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertRequest(t, captured, "/api/reviews-scraper", map[string]any{ + "resourceID": "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", + "legacyBookID": float64(12207444), + }) + + if res.TotalCount != 115 || res.NextPageToken != "NTgsMTcwNTAxNjc4NjQzNA" { + t.Errorf("unexpected response metadata: %+v", res) + } + if len(res.Reviews) == 0 { + t.Fatal("expected reviews, got none") + } + got := res.Reviews[0] + if got.Rating != 5 || got.Shelf != "read" || got.LikeCount != 18 || got.RecommendFor != nil { + t.Errorf("unexpected review: %+v", got) + } + if got.Reviewer.ID != 1259454 || got.Reviewer.Name != "Shawna Astley" || got.Reviewer.ReviewCount != 42 { + t.Errorf("unexpected reviewer: %+v", got.Reviewer) + } + }) + + t.Run("next page", func(t *testing.T) { + var captured capturedRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !captureRequest(t, w, r, &captured) { + return + } + _, _ = w.Write([]byte(`{"version":"BiblioReads v4.4.0","status":"Received","statusCode":200,"totalCount":1,"reviews":[{"text":"Una reseña"}]}`)) + })) + defer server.Close() + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + if _, err := client.Reviews(context.Background(), "kca://work/abc", 1, "token", "es"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertRequest(t, captured, "/api/reviews-scraper", map[string]any{ + "resourceID": "kca://work/abc", + "legacyBookID": float64(1), + "nextPageToken": "token", + "languageCode": "es", + }) + }) +} + +func TestErrorStatus(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL}), biblioreads.WithHTTPClient(server.Client()), biblioreads.WithCacheTTL(0)) + + if _, err := client.Book(context.Background(), "0"); err == nil { + t.Error("expected error on non-200 response, got nil") + } +} + +func newCountingServer(t *testing.T, status int, body string, hits *atomic.Int32) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} diff --git a/internal/datasource/biblioreads/document.go b/internal/datasource/biblioreads/document.go new file mode 100644 index 00000000..ceeee0d8 --- /dev/null +++ b/internal/datasource/biblioreads/document.go @@ -0,0 +1,16 @@ +package biblioreads + +// Source retains the persisted key for Goodreads data fetched through BiblioReads. +const Source = "goodreads" + +// Name is the user-facing name of the data source. +const Name = "Goodreads" + +// DocumentData is the persisted Goodreads state used by BiblioReads. +type DocumentData struct { + URL string `json:"url,omitempty"` + RatingCount string `json:"ratingCount,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + LegacyBookID int `json:"legacyBookId,omitempty"` + Enriched bool `json:"enriched"` +} diff --git a/internal/datasource/biblioreads/endpoints_test.go b/internal/datasource/biblioreads/endpoints_test.go new file mode 100644 index 00000000..09ba36ce --- /dev/null +++ b/internal/datasource/biblioreads/endpoints_test.go @@ -0,0 +1,47 @@ +package biblioreads_test + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +type unexpectedRequestTransport struct{ t *testing.T } + +func (transport unexpectedRequestTransport) RoundTrip(request *http.Request) (*http.Response, error) { + transport.t.Errorf("unexpected network request to %s", request.URL) + return nil, errors.New("network requests must require explicit endpoints") +} + +func TestClientWithoutEndpointsMakesNoRequests(t *testing.T) { + for _, tt := range []struct { + name string + opts []biblioreads.Option + }{ + {name: "default"}, + {name: "empty", opts: []biblioreads.Option{biblioreads.WithInstances(nil)}}, + {name: "explicitly cleared", opts: []biblioreads.Option{ + biblioreads.WithInstances([]string{"http://localhost:8080"}), + biblioreads.WithInstances(nil), + }}, + } { + t.Run(tt.name, func(t *testing.T) { + client := biblioreads.NewClient(append(tt.opts, biblioreads.WithHTTPClient( + &http.Client{Transport: unexpectedRequestTransport{t: t}}, + ))...) + ctx := context.Background() + _, searchErr := client.SearchBooks(ctx, "Book") + _, bookErr := client.Book(ctx, "123") + _, authorErr := client.Author(ctx, "123") + _, reviewsErr := client.Reviews(ctx, "work", 123, "", "en") + for _, err := range []error{searchErr, bookErr, authorErr, reviewsErr} { + if !errors.Is(err, biblioreads.ErrUnavailable) { + t.Errorf("expected explicit unavailable error without endpoints, got %v", err) + } + } + }) + } +} diff --git a/internal/datasource/biblioreads/failover_test.go b/internal/datasource/biblioreads/failover_test.go new file mode 100644 index 00000000..9c495308 --- /dev/null +++ b/internal/datasource/biblioreads/failover_test.go @@ -0,0 +1,390 @@ +package biblioreads_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +func TestBookResponseRecovery(t *testing.T) { + for _, tt := range []struct { + name string + status int + body string + }{ + {"JSON server error", http.StatusGatewayTimeout, `{"status":"Error - Book not found"}`}, + {"invalid success", http.StatusOK, `{}`}, + {"application error", http.StatusOK, `{"status":"Error","statusCode":503}`}, + {"missing score", http.StatusOK, `{"status":"Received","statusCode":200,"resourceID":"work-id","legacyBookID":5898652,"title":"Incomplete result"}`}, + {"missing review identifiers", http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92"}`}, + } { + t.Run(tt.name, func(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + w.WriteHeader(tt.status) + _, _ = w.Write([]byte(tt.body)) + return + } + _, _ = w.Write([]byte(validBookResponse)) + }) + client := newFailoverClient(t, handler) + book, err := client.Book(context.Background(), "5898652-la-cinta-roja") + if err != nil || book.Rating != "3.92" || book.ResourceID != "work-id" || book.LegacyBookID != 5898652 { + t.Fatalf("expected complete book after failover, got %+v, %v", book, err) + } + if book.Title != "" { + t.Fatalf("fields from previous attempts must not survive failover, got title %q", book.Title) + } + if hits.Load() != 2 { + t.Fatalf("expected both instances to be tried, got %d requests", hits.Load()) + } + }) + } +} + +func TestIncompleteBookFallbackPrefersScore(t *testing.T) { + scored := `{"status":"Received","statusCode":200,"rating":"3.92","title":"Scored result"}` + unscored := `{"status":"Received","statusCode":200,"resourceID":"work-id","legacyBookID":5898652,"title":"Unscored result"}` + for _, scoredFirst := range []bool{false, true} { + t.Run(fmt.Sprintf("scored first=%t", scoredFirst), func(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := unscored + if (hits.Add(1)%2 == 1) == scoredFirst { + body = scored + } + _, _ = w.Write([]byte(body)) + }) + client := newFailoverClient(t, handler) + for range 2 { + book, err := client.Book(context.Background(), "5898652") + if err != nil || book.Rating != "3.92" || book.Title != "Scored result" { + t.Fatalf("expected scored fallback, got %+v, %v", book, err) + } + if book.ResourceID != "" || book.LegacyBookID != 0 { + t.Fatalf("fields from separate incomplete responses must not be combined, got %+v", book) + } + } + if hits.Load() != 4 { + t.Fatalf("expected both instances tried without caching incomplete results, got %d requests", hits.Load()) + } + }) + } +} + +func TestReviewsResponseRecovery(t *testing.T) { + fixture := readFixture(t, "reviews-scraper.json") + for _, tt := range []struct { + name string + body string + }{ + {name: "invalid response", body: `{}`}, + {name: "empty response", body: `{"status":"Received","statusCode":200,"totalCount":0,"reviews":[]}`}, + } { + t.Run(tt.name, func(t *testing.T) { + var hits atomic.Int32 + client := newFailoverClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + _, _ = w.Write([]byte(tt.body)) + return + } + _, _ = w.Write(fixture) + })) + reviews, err := client.Reviews(context.Background(), "work", 1, "", "") + if err != nil || len(reviews.Reviews) == 0 { + t.Fatalf("expected reviews after failover, got %+v, %v", reviews, err) + } + if hits.Load() != 2 { + t.Fatalf("expected both instances to be tried, got %d requests", hits.Load()) + } + }) + } +} + +func TestReviewsEmptyFallbackSurvivesInvalidResponse(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := `{"version":"BiblioReads v4.4.0","status":"Received","statusCode":200,"totalCount":0,"reviews":[],"nextPageToken":"empty-page"}` + if hits.Add(1) > 1 { + body = `{"reviews":[{"text":"Unfiltered review"}],"nextPageToken":"invalid-page"}` + } + _, _ = w.Write([]byte(body)) + }) + client := newFailoverClient(t, handler) + response, err := client.Reviews(context.Background(), "work", 1, "", "es") + if err != nil || len(response.Reviews) != 0 || response.NextPageToken != "empty-page" { + t.Fatalf("expected original empty fallback without fields from the invalid response, got %+v, %v", response, err) + } + if hits.Load() != 2 { + t.Fatalf("expected both instances tried, got %d requests", hits.Load()) + } +} + +func TestReviewsLanguageFailsOverFromLegacyInstance(t *testing.T) { + var hits atomic.Int32 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response := biblioreads.ReviewsResponse{ + Status: "Received", StatusCode: http.StatusOK, TotalCount: 1, + Reviews: []biblioreads.Review{{Text: "An unfiltered English review"}}, + } + if hits.Add(1) > 1 { + response.Version = "BiblioReads v4.4.0" + response.Reviews[0].Text = "Una reseña en español" + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Errorf("encode response: %v", err) + } + }) + client := newFailoverClient(t, handler) + response, err := client.Reviews(context.Background(), "work", 1, "", "es") + if err != nil { + t.Fatal(err) + } + if len(response.Reviews) != 1 || response.Reviews[0].Text != "Una reseña en español" { + t.Fatalf("expected Spanish reviews rather than legacy unfiltered results, got %+v", response) + } + if hits.Load() != 2 { + t.Fatalf("expected both instances to be tried, got %d requests", hits.Load()) + } +} + +func TestFailover(t *testing.T) { + t.Run("unavailable instances are skipped", func(t *testing.T) { + var unavailableHits, rateLimitedHits, htmlHits, workingHits atomic.Int32 + unavailable := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &unavailableHits) + rateLimited := newCountingServer(t, http.StatusTooManyRequests, `{"status":"Too many requests"}`, &rateLimitedHits) + html := newCountingServer(t, http.StatusOK, "Parked domain", &htmlHits) + working := newCountingServer(t, http.StatusOK, `{"status":"Received","statusCode":200,"rating":"3.92","title":"La Reina Dragon"}`, &workingHits) + down := httptest.NewServer(http.NotFoundHandler()) + down.Close() + + instances := []string{down.URL, unavailable.URL, rateLimited.URL, html.URL, working.URL} + for range 20 { + client := newTestClient(biblioreads.WithInstances(instances), biblioreads.WithCacheTTL(0)) + res, err := client.Book(context.Background(), "12207444") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.Title != "La Reina Dragon" { + t.Fatalf("unexpected title %q", res.Title) + } + } + if workingHits.Load() != 20 { + t.Errorf("expected working instance to be hit 20 times, got %d", workingHits.Load()) + } + }) + + t.Run("requests start at different instances", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + first := newCountingServer(t, http.StatusOK, validBookResponse, &firstHits) + second := newCountingServer(t, http.StatusOK, validBookResponse, &secondHits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + for range 100 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected load to be spread, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) + + t.Run("JSON error responses are not retried", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) + second := newCountingServer(t, http.StatusNotFound, `{"status":"Error - Invalid Query"}`, &hits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + _, err := client.Book(context.Background(), "0") + if err == nil { + t.Fatal("expected error, got nil") + } + if errors.Is(err, biblioreads.ErrUnavailable) { + t.Errorf("expected a non-ErrUnavailable error, got %v", err) + } + if hits.Load() != 1 { + t.Errorf("expected a single request, got %d", hits.Load()) + } + }) + + t.Run("error when all instances fail", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) + second := newCountingServer(t, http.StatusServiceUnavailable, "", &hits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected ErrUnavailable, got %v", err) + } + if hits.Load() != 2 { + t.Errorf("expected both instances to be tried, got %d requests", hits.Load()) + } + }) + + t.Run("cancelled context stops failover", func(t *testing.T) { + var hits atomic.Int32 + first := newCountingServer(t, http.StatusOK, `{}`, &hits) + second := newCountingServer(t, http.StatusOK, `{}`, &hits) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(ctx, "1"); err == nil { + t.Fatal("expected error, got nil") + } + if hits.Load() != 0 { + t.Errorf("expected no requests, got %d", hits.Load()) + } + }) +} + +func TestCooldown(t *testing.T) { + t.Run("errors put the instance into cooldown", func(t *testing.T) { + var hits atomic.Int32 + server := newCountingServer(t, http.StatusGatewayTimeout, `{"status":"Error - Book not found"}`, &hits) + client := newTestClient(biblioreads.WithInstances([]string{server.URL})) + + for range 3 { + if _, err := client.Book(context.Background(), "0"); err == nil { + t.Fatal("expected error, got nil") + } + } + if hits.Load() != 1 { + t.Errorf("expected subsequent requests to respect cooldown, got %d requests", hits.Load()) + } + }) + + t.Run("failing instance is skipped on later requests", func(t *testing.T) { + var badHits, goodHits atomic.Int32 + bad := newCountingServer(t, http.StatusBadGateway, "Bad Gateway", &badHits) + good := newCountingServer(t, http.StatusOK, validBookResponse, &goodHits) + + client := newTestClient(biblioreads.WithInstances([]string{bad.URL, good.URL}), biblioreads.WithCacheTTL(0)) + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if badHits.Load() > 1 { + t.Errorf("expected failing instance to be hit at most once, got %d", badHits.Load()) + } + if goodHits.Load() != 20 { + t.Errorf("expected working instance to be hit 20 times, got %d", goodHits.Load()) + } + }) + + t.Run("instance is used again after Retry-After", func(t *testing.T) { + var limitedHits, otherHits atomic.Int32 + limited := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if limitedHits.Add(1) == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = w.Write([]byte(validBookResponse)) + })) + t.Cleanup(limited.Close) + other := newCountingServer(t, http.StatusOK, validBookResponse, &otherHits) + + client := newTestClient(biblioreads.WithInstances([]string{limited.URL, other.URL}), biblioreads.WithCacheTTL(0)) + for limitedHits.Load() == 0 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if limitedHits.Load() != 1 { + t.Fatalf("expected rate limited instance to be skipped during cooldown, got %d hits", limitedHits.Load()) + } + + time.Sleep(1100 * time.Millisecond) + for range 20 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if limitedHits.Load() == 1 { + t.Error("expected rate limited instance to be used again after its cooldown") + } + }) + + t.Run("respects cooldown when all instances are cooling down", func(t *testing.T) { + var hits atomic.Int32 + flaky := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = w.Write([]byte(validBookResponse)) + })) + t.Cleanup(flaky.Close) + + client := newTestClient(biblioreads.WithInstances([]string{flaky.URL}), biblioreads.WithCacheTTL(0)) + if _, err := client.Book(context.Background(), "1"); err == nil { + t.Fatal("expected error on first request, got nil") + } + if _, err := client.Book(context.Background(), "1"); !errors.Is(err, biblioreads.ErrUnavailable) { + t.Fatalf("expected unavailable during cooldown, got %v", err) + } + time.Sleep(1100 * time.Millisecond) + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("expected cooling down instance to be probed, got %v", err) + } + }) + + t.Run("JSON client error responses don't trigger cooldown", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + notFound := `{"status":"Error - Book not found"}` + first := newCountingServer(t, http.StatusNotFound, notFound, &firstHits) + second := newCountingServer(t, http.StatusNotFound, notFound, &secondHits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + for range 20 { + if _, err := client.Book(context.Background(), "0"); err == nil { + t.Fatal("expected error, got nil") + } + } + if firstHits.Load()+secondHits.Load() != 20 { + t.Errorf("expected one request per call, got %d", firstHits.Load()+secondHits.Load()) + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected no instance to be benched, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) + + t.Run("cancelled requests don't trigger cooldown", func(t *testing.T) { + var firstHits, secondHits atomic.Int32 + first := newCountingServer(t, http.StatusOK, validBookResponse, &firstHits) + second := newCountingServer(t, http.StatusOK, validBookResponse, &secondHits) + + client := newTestClient(biblioreads.WithInstances([]string{first.URL, second.URL}), biblioreads.WithCacheTTL(0)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for range 5 { + _, _ = client.Book(ctx, "1") + } + for range 40 { + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if firstHits.Load() == 0 || secondHits.Load() == 0 { + t.Errorf("expected no instance to be benched, got %d and %d hits", firstHits.Load(), secondHits.Load()) + } + }) +} diff --git a/internal/datasource/biblioreads/fixtures/author-info.json b/internal/datasource/biblioreads/fixtures/author-info.json new file mode 100644 index 00000000..388c0469 --- /dev/null +++ b/internal/datasource/biblioreads/fixtures/author-info.json @@ -0,0 +1,35 @@ +{ + "status": "Received", + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/author/show/151155.Alice_Borchardt", + "image": "https://images.gr-assets.com/authors/1237783268p5/151155.jpg", + "name": "Alice Borchardt", + "website": "", + "genre": ["Historical Fiction", "Science Fiction & Fantasy"], + "influences": [], + "birthDate": "\n October 06, 1939\n ", + "deathDate": "July 24, 2007", + "desc": "Author of The Silver Wolf.", + "books": [ + { + "id": 1, + "cover": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/books/1327131705i/258657._SX50_.jpg", + "title": "The Silver Wolf (Legends of the Wolf #1)", + "url": "/book/show/258657.The_Silver_Wolf", + "rating": " 3.92 avg rating From 4627 ratings", + "publishDate": "\n 3.92 avg rating — 4,627 ratings\n —\n published\n 1998\n —\n 2 editions\n " + } + ], + "series": [ + { + "id": 1, + "cover": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/books/1327131705l/258657._SX50_.jpg", + "title": "Legends of the Wolf", + "seriesURL": "/series/49775-legends-of-the-wolf", + "author": "Alice Borchardt", + "authorURL": "/author/show/151155.Alice_Borchardt", + "rating": " 3.91 avg rating — 8,472 ratings" + } + ], + "lastScraped": "2026-10-29T12:25:21.332Z" +} diff --git a/internal/datasource/biblioreads/fixtures/book-scraper.json b/internal/datasource/biblioreads/fixtures/book-scraper.json new file mode 100644 index 00000000..5b8abc6a --- /dev/null +++ b/internal/datasource/biblioreads/fixtures/book-scraper.json @@ -0,0 +1,60 @@ +{ + "status": "Received", + "statusCode": 200, + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + "legacyBookID": 12207444, + "bookID": "kca://book/amzn1.gr.book.v1.4FDu7eHal9NDfA3jGNhOXw", + "resourceID": "kca://work/amzn1.gr.work.v1.dbcQjzlbvXR8RBZBdd8J6g", + "cover": "https://m.media-amazon.com/images/S/compressed.photo.goodreads.com/books/1332599049i/12207444.jpg", + "series": "Tales of Guinevere", + "seriesURL": "/series/50072-tales-of-guinevere", + "seriesPosition": "1", + "workURL": "https://www.goodreads.com/book/show/12207444-la-reina-dragon", + "title": "La Reina Dragon", + "author": [ + { + "id": 1, + "name": "Alice Borchardt", + "role": "Author", + "url": "/author/show/151155.Alice_Borchardt" + } + ], + "rating": "3.82", + "ratingCount": "2,285", + "reviewsCount": "115", + "desc": "Arturo se dio la vuelta y vino resueltamente hacia nosotros. Estaba espléndido, y nunca olvidaré que en ese momento comencé a amarlo. Y ya nunca dejé de hacerlo. Nunca nadie me ha provocado tanta pena, dolor y alegría como él. Ni siquiera por mis hijos he sentido el amor que le profeso todavía ahora. Y creo que aunque hubiera sabido lo que nos deparaba el futuro (preocupaciones, tormentos, batallas y penas), aun así levaría entregado mi corazón tal como hice entonces.\r\n\r\nAlice Brondchardt entra en el reino de la leyenda artúrica y lo hace suyo. La reina Dragón es el primer volumen de la trilogía que recrea el mundo de Camelot con la reina Guinevere convertida en una guerrea sagaz y obstinada.\r\n\r\nNacida en un entorno en el que los enfrentamientos son constantes y las armas están siempre presentes en la vida de hombres y mujeres, Guinevere, hija de una poderosa reina pagana, es una amenaza para su gente y una preciada presa para el temible mago Merlín.\r\n\r\nEnviada a u lugar secreto, crece bajo la protección de un hombre-lobo y de un druida de agrio carácter. Pero ni siquiera en las remotas costas de Escocia, donde los dragones velan por ella, está a salvo del todopoderoso mago. Merlín conoce el destino de la bella joven y no se detendrá ante nada para evitar el cumplimiento de la profecía, puesto que si Guinevere y Arturo llegan a reinar, llevarán un paz al país que convertirá a Merlín, hambriento de poder, en un personaje secundario.", + "genres": [ + "Fantasy", + "Arthurian", + "Fiction", + "Historical Fiction", + "Dragons", + "Historical", + "Mythology", + "Magic", + "Science Fiction Fantasy", + "Retellings" + ], + "bookEdition": "452 pages, Hardcover", + "publishDate": "Wed Jan 01 2003", + "publisher": "Timun Mas", + "language": "Spanish; Castilian", + "asin": "8448033183", + "isbn": "8448033183", + "isbn13": "9788448033187", + "related": [], + "reviews": [], + "reviewBreakdown": { + "rating1": "72", + "rating2": "170", + "rating3": "586", + "rating4": "731", + "rating5": "726" + }, + "quotes": "4", + "quotesURL": "/work/quotes/2440721", + "questions": "1", + "questionsURL": "/book/258666/questions", + "lastScraped": "2026-10-28T18:34:27.525Z" +} diff --git a/internal/datasource/biblioreads/fixtures/reviews-scraper.json b/internal/datasource/biblioreads/fixtures/reviews-scraper.json new file mode 100644 index 00000000..1d7908c5 --- /dev/null +++ b/internal/datasource/biblioreads/fixtures/reviews-scraper.json @@ -0,0 +1,672 @@ +{ + "status": "Received", + "statusCode": 200, + "source": "https://github.com/nesaku/biblioreads", + "scrapeURL": "https://www.goodreads.com/book/show/12207444/reviews", + "totalCount": 115, + "nextPageToken": "NTgsMTcwNTAxNjc4NjQzNA", + "prevPageToken": "MTgwMSwxMjE0MTUxODU5MDAw", + "reviews": [ + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.gH7Sf4_G4iz72wvI_yVVyg", + "reviewer": { + "id": 1259454, + "name": "Shawna Astley", + "url": "/user/show/1259454-shawna-astley", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1214150161i/1259454._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 42, + "followerCount": 0 + }, + "rating": 5, + "text": "Women fighting men with swords - whats not to like? ", + "createdAt": "Sun Jun 22 2008", + "updatedAt": "Tue Jan 07 2025", + "spoiler": false, + "likeCount": 18, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/25118938" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.be94OXB9ezWIlFZvdGMKvA", + "reviewer": { + "id": 2478293, + "name": "Zeek", + "url": "/user/show/2478293-zeek", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1339154824i/2478293.jpg", + "isAuthor": false, + "reviewCount": 939, + "followerCount": 149 + }, + "rating": 3, + "text": "Although I liked this tale overall, there were pieces of it that bogged the story down for me.

The Dragon Queen is yet another entry into the tales of Arthur and The Knights of the Round Table. For those who love their Arthurian legends with Merlin as the big bad- then this book's for you. For those who don't-then Run away!

TDQ centers around Guinevere, the warrior queen of Arthur and in Borchardt's version she is indeed a warrior. Guinevere, has the beauty of previous Arthurian tales, but in TDQ she's not the sweet little misguided star-crossed lover of Lancelot that we usually see her portrayed to be. Courageous and bold she has power in her right hand where ferocious fire emanates (grand super power that!), and healing in her left. Moreover, arrows flyout her bow as if an extension of her arm, she communicates with dragons- as well with the dead- displays prophetic abilities, and audaciously defeats asundry of evil beasts that Merlin flings at her.

The story begins with Maeniel, Guinevere's soon to be foster father. Maeniel, otherwise knows as the Gray Watcher, is a shapeshifting man/wolf who has personally felt the destructive powers of Merlin. Deciding that being a wolf was much preferred to living with the humans, he escapes to the wilds and begins a family. It is during his time with the wolves that Guinevere falls into his care, and the Gray Watcher, along with the sorcerer Dugald- who Merlin chased into hiding long ago- take up the charge of raising her. The Gray Watcher's wolf mate nurses
Guinevere from the time they find her left on a cliff and later a woman joins them who will teach her the ways of her people- a people she never knew.

As a child, Guinevere helps her village to defeat a marauding group of seafarers- and it is here where we see her potential for greatness begin to come forth. Guinevere grows in stature, wisdom and power and, though still only a young teen through much of the story, she quickly finds herself thrust into peril. Merlin wants her dead, and though the author details many things in TDQ, I never quite got the gist of his motive for seeking to destroy her so passionately. The only thing I can figure is that she must be the one who either precipitates or actually carries out Merlin's eventual destruction.

Many other side characters show up to aid or engage her in battle. There are all kinds of fantasy creatures- including mythological gods- that she meets up with and, eventually, she meets her future in the boy of Arthur.

Arthur is just as strong as she- and closer to her age then what I've seen portrayed before. Set long before Camalot, he's young and only discovering who and what he's eventually going to be. Much of the middle of the book is from his POV, where he meets the Lady of the Lake. I was disappointed however, that at the end of TDQ, he and Guinevere do not meet up again. (I suppose that will come in later installments.)

The jumping around of POV's was rather confusing for me and the author lost me a bit until it began to pick up again toward the end. The book is full of detailed descriptions of the world Borchardt has created and some of it became overwhelming. To me her descriptions dragged the story down and lost me for most of the middle of the book. But, as I said before, by the end I was back with her.

This is definitely sequel bait- the story is far from over. I'm interested to see where the author- who was Anne Rice's sister, btw- will take it.", + "createdAt": "Thu May 20 2010", + "updatedAt": "Sun May 24 2026", + "spoiler": false, + "likeCount": 10, + "commentCount": 4, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/103546143" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.sC0IQDlAXY4LbkJ3LTJHvA", + "reviewer": { + "id": 80386185, + "name": "Rial", + "url": "/user/show/80386185-rial", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1545932910i/80386185._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 30, + "followerCount": 0 + }, + "rating": 1, + "text": "I did not like this book. However, it is my first 1 Star, so it's got that going for it.

Why did I read this? It was selected for my wife and roommate's book club, and I parasitically piggyback a lot of their books (gives me something to talk about...). It's Arthurian! It's from Guinivere's perspective! Magic! Celtic! Post 'Fall of the Western Empire'! These things and more!

No, it's terrible. Why did I keep reading it? It did take me on a journey: I experienced all the 'Stages of Grief' (some of them more than once.

Denial: Ha, it's so weird, but it'll get better! The chapters are long, but that's okay! Weird, the perspective shifted...?
Anger: I liked the other character; this one is dumb. She's a Mary Sue! WTF? That just worked out for her with zero effort! Okay, she needs to show some growth...
Bargaining: ...Okay, the parts with Arthur aren't bad, maybe?
Anger: Why are they having revenge sex in the forest?
Denial: Arthur is cool, yes, he's very cool... :|
Depression: I really shouldn't be reading this anymore; I think it's actively hurting me.
Acceptance: There is only book. Oh yeah, I guess your shoes can do that. Yep, you're definitely super fucking hot, and everyone wants to be or be with you. Sure why not? I don't have enough brain power to think it should be any different. After all, you're the best, Gwen!

But the real joy of this book is in not starting it. If you made that mistake and you're too masochistic to give up, then finishing it is an exultant experience

The contents are one thing, but they aren't even well served by the writing. Transitions are abrupt and jarring. I back-read multiple times to try and figure out where the hell I was, only to realize there wasn't any explanation; it just got there and it was fine (it wasn't fine).

This thing had such potential: Arthurian set in the 'proper' time frame, the Christianization of Britannia and the clashes with pagan beliefs that ensue, Celtic and Brythonic myth woven throughout. But it all reads so poorly in the end. The author is far too verbose and floral with her language. First person POV ruins any sense of actual accomplishment because we only see the character succeeding; no real growth or change is experienced. I find Divine Right to be problematic, and this book is very much into the romanticism of a destined ruler (both for Arthur and for Gwen).

Oh yeah, Merlin is evil and helps Arthur's mom torment Arthur, I guess? Like, if you hate the kid so much just kill him? Oh wait, 'destiny'...

Tedious tripe, glorified garbage, wretched rubbish. What a waste of time.", + "createdAt": "Mon Nov 05 2018", + "updatedAt": "Fri Jun 10 2022", + "spoiler": false, + "likeCount": 6, + "commentCount": 2, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2578561727" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.rwxYpuQEynxP2f5dSLFmPQ", + "reviewer": { + "id": 90733221, + "name": "Monique Mihalitsianos", + "url": "/user/show/90733221-monique-mihalitsianos", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1773156601i/90733221._UX200_CR0,29,200,200_.jpg", + "isAuthor": true, + "reviewCount": 88, + "followerCount": 9 + }, + "rating": 3, + "text": "This book was splendid in the story-telling sense. The story was epic and magical and engaging. Also, this is, by far, the best Guinevere I have encountered in modern fiction so far. She was strong but not brass, brave but not stupid, powerful yet humble, compassionate but not weak, feminine yet full of agency. Perhaps what I liked the most about her was that she was in communion with animals and gods alike and knew her place in the wheel of existence. She was wise like that, but never overbearingly so. I loved her. I wanted to *be* her.

Arthur I also liked very much, and I liked that this was a tortured Arthur who had suffered child abuse at the hands of her horrible queen mother Igrane (*the* worst evil queen ever) but was still good and noble and could not lie and never feared danger. I liked his relationship with his father, Uther, the High King, which was refreshingly healthy and loving. Merlin I hated. This is a book with a bad Merlin in it, which I don't like as a trope. I much prefer my Merlins to be good and Gandalf-like, but in this case it worked to have him be so evil and act as the villain. The black magic he wrought was one of the sickest and most horrifying ones I've ever read in a story.

Alice Borchardt seems to know her history well, so much so in fact that it's almost like if she lived in this period of time (Roman - Saxon) in a past life. I, for one, appreciated all of the extraneous detail relating to the customs and mores of the time because it made the world-building that much better.

The magic system in this book was refreshingly deep and believable. So many magic systems in recent fantasy books are light and superficial, as if they had been thought up in brainstorming sessions instead of being fully inspired. Not so with Borchardt's story; the magic here seemed inspired by the author's own psyche and thus, was multilayered and poliphacetic, changeable and malleable but also essentially true to its own core.

I also liked that an existencial and historical struggle was hinted at: the convergence between paganism and Christianity, if there can be a convergence at all. It was only hinted at here but I wouldn't mind if this theme in particular were elaborated more in the second book (The Raven Warrior) because it is of particular interest to me. On some level I think it was also of particular interest to Borchardt, as well. I am a huge fan of Anne Rice's (Alice Borchardt's sister) Vampire Chronicles, and Rice definitely waxes philosophical about her own struggles with different faith systems and beliefs in her own fiction, much more so than Borchardt, who chose to focus more on the story. But I wouldn't have minded if she had expanded on this in her fiction, just like her sister did. But then again, I am of a philosophical bent and like these kinds of tangents.

Why the three-star rating, then, if I liked the book so much? Well, the story was golden, but the writing was confusing at times, especially at the beginning and end. Sometimes it was almost stream-of-consciousness like, and I got lost and confused. At other times, when a character referred to somebody else as 'she', for example, I didn't know who they were talking about, because there were several 'she's' they could have been referring to. So some more specificity in this and other similar instances would have been appreciated. Sometimes, also, the writing got too dream-like, too airy fairy and dreamy...and perhaps more concrete writing would have served the book better overall. I also suspect this book could have had tighter editing. I know for a fact Anne Rice is not a fan of people editing her writing, and I don't know if Alice Borchardt felt the same as her sister does in this matter, but I for one think good editing is absolutely crucial in any book.

Putting this aside, though, I can say that I thoroughly enjoyed this book and am looking forward to reading the second and final installment. This was meant to be a trilogy, but unfortunately, the author died from illness before she could complete the third book, so two books is all we have. I will read her other books as well, though. She's a gem of a story-teller, at least in my estimation.", + "createdAt": "Wed Mar 18 2020", + "updatedAt": "Thu Nov 30 2023", + "spoiler": false, + "likeCount": 3, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3208179930" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.XvL2nHk7Bt4tULNXUE2yaA", + "reviewer": { + "id": 2207539, + "name": "Steen", + "url": "/user/show/2207539-steen", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1408254648i/2207539._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 473, + "followerCount": 4 + }, + "rating": 1, + "text": "This was a did not finish book for me.

So after reading about 185 pages of The Dragon Queen by Alice Borchardt I have decided to thrown it down and stop reading. I thought I would like it being about Quinevere and all but I hate it. It's not like the Arthur legends I've ever known. But that's not really even it though. I could handle a different take on the stories ( I still have yet to read any books about Arthur that are out there that are more well known) it's just the way she has written it. She jumps from one place to another and it's like when did that happen? Also sometimes when there are a few people talking 'she says...' your like who is talking? Which one is saying what. UGH. I hate not finishing a book but I really can't continue to read it anymore.

Having said that I really have no urge to read Bequiled which is also by her. I read the first in that series and while it was better written then The Dragon Queen I really don't love the characters in the first book to continue to read the second one anymore. Not after trying to get through The Dragon Queen and hating it. It's sad but yeah. I love Anne Rice's books and hoped that I would equally like her sister's books. But I guess they are just not for me.

I've read reviews on these two books and it sounds like her werewolf books are pretty good but I think I may wait a while to even attempt to try those ones.

So now its off to the library today to pick up some more to read.", + "createdAt": "Sat Apr 11 2009", + "updatedAt": "Wed Jan 23 2019", + "spoiler": false, + "likeCount": 3, + "commentCount": 3, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/52264098" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.rZ5icbmFfLHPKpEQOm2kNw", + "reviewer": { + "id": 41814516, + "name": "Alberto García", + "url": "/user/show/41814516-alberto-garc-a", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1508706260i/41814516._UX200_CR0,33,200,200_.jpg", + "isAuthor": true, + "reviewCount": 45, + "followerCount": 40 + }, + "rating": 4, + "text": "Voto de confianza... esta m¡¡bien escrito y la revisión de la leyenda artúrica es muy interesante. Al final, en algunos capítulos, se le va un poco la pinza con salidas algo paranoicas que me han dejado bastante descolocado... lo que es seguro es que no me lo esperaba... pero tampoco comprendo demasiado la razón para ello. Veremos cómo acaba y si se queda en una anécdota.", + "createdAt": "Fri Jun 21 2019", + "updatedAt": "Mon Sep 14 2026", + "spoiler": false, + "likeCount": 3, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2796875885" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.G3-a2b8-oopoAbpH6RmbsA", + "reviewer": { + "id": 73527602, + "name": "Rachel", + "url": "/user/show/73527602-rachel", + "avatar": "", + "isAuthor": false, + "reviewCount": 13, + "followerCount": 0 + }, + "rating": 3, + "text": "This was like reading someone's dream. The ideas were all loosely connected but the transitions were weird and random or completely non-existent. However, the characters and concepts were fun and interesting and I did like this book.", + "createdAt": "Fri Jan 04 2019", + "updatedAt": "Mon Nov 17 2025", + "spoiler": false, + "likeCount": 4, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2582029737" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.juoLRbi8R_urzbj6jgRAww", + "reviewer": { + "id": 101256289, + "name": "Malischa Nijenhuis", + "url": "/user/show/101256289-malischa-nijenhuis", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1773261867i/101256289._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 1222, + "followerCount": 32 + }, + "rating": 5, + "text": "Eigenlijks lees ik nu een boek van mijn man maar wauw wat is dit een gave uitgave en boek. Nu op naar boek 2", + "createdAt": "Sat Mar 21 2020", + "updatedAt": "Mon Mar 23 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3238259324" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.GLjLEKHIsU-RrmRdU73hGQ", + "reviewer": { + "id": 3504881, + "name": "Suzanne", + "url": "/user/show/3504881-suzanne", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1294105241i/3504881.jpg", + "isAuthor": false, + "reviewCount": 546, + "followerCount": 12 + }, + "rating": 2, + "text": "I'm very sorry to say, I could not finish this book. I REALLY wanted to be in love with this story. I even purchased the 2nd book from Thrift Books soon after I started this one. I figured they would become part of my Arthurian obsession, and I would need to own them. It started out with a great concept, and I liked the differences in the legend that I hadn't read before. I wanted to know more about the different characters, but it was so choppy, from one point of view to another back and forth, I couldn't keep track of which character I was reading about.

It just dragged on and on and on and on....it was trial after trial after trial. The characters would accomplish one win and suddenly face another trial. I felt like I was reading a bad rendition of The Odyssey. I REALLY wanted to love it. But reading became a chore, and I fell into a reading slump. The book sat in one spot for a couple weeks, and I felt it hanging over my head. I finally decided enough was enough. I wanted to be eager to read again. So I took out the bookmark, and this is going on my \"couldn't finish\" list.

I will donate the set to a Free Little Library somewhere, and I hope someone else will get some pleasure out of them.", + "createdAt": "Sun Oct 14 2018", + "updatedAt": "Fri Oct 30 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 2, + "recommendFor": null, + "shelf": "did-not-finish", + "reviewURL": "/review/show/2522667985" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.73ge1ENF_VjVNC1CSJb8iw", + "reviewer": { + "id": 67537582, + "name": "Kaylee", + "url": "/user/show/67537582-kaylee", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1627453497i/67537582._UX200_CR0,1,200,200_.jpg", + "isAuthor": false, + "reviewCount": 793, + "followerCount": 33 + }, + "rating": 1, + "text": "I do have to start by saying that this is the first Arthurian legend I can recall reading. I don't think that really affected my POV but...

Extremely hard to get through from the beginning. Kept hoping it would get better but nope. Not only was the switching between third and first POV annoying but it was made worse by the main character. Did not want her voice at all. It was also hard to tell what was going on/who was speaking at times (during the third POV of course). In the beginning I went back to the and re-read things but quickly learned there was no point. Sometimes things arrived randomly and others I realized I had just spaced out because of all the needless words.  You could take out half the words in this book and be all the better off for it...

Also if I had to read \"As I said...\" one more time...!

I can't remember the last time I didn't finish a book and if I hadn't had to read this because of my book club I don't think I would have finished. Says everything in a nutshell.

So much more but I'll just leave it at that. Extremely glad to be done with this book!", + "createdAt": "Fri Nov 09 2018", + "updatedAt": "Fri Oct 30 2020", + "spoiler": false, + "likeCount": 2, + "commentCount": 2, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2577596943" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.6kd8QVi2sBrNhUslyz1aUQ", + "reviewer": { + "id": 54738977, + "name": "TigerLily ", + "url": "/user/show/54738977-tigerlily", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1460164836i/54738977.jpg", + "isAuthor": false, + "reviewCount": 133, + "followerCount": 3 + }, + "rating": 5, + "text": "Really enjoyable, a page Turner that I didn't want to be finished. I am looking forward to reading the 2nd book.", + "createdAt": "Sun Jan 10 2021", + "updatedAt": "Sun Mar 21 2021", + "spoiler": false, + "likeCount": 2, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3732026445" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.njxZJZUHk2BVK8mcTJp2aQ", + "reviewer": { + "id": 40251914, + "name": "Elizabeth Sanders", + "url": "/user/show/40251914-elizabeth-sanders", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1530132382i/40251914.jpg", + "isAuthor": false, + "reviewCount": 38, + "followerCount": 0 + }, + "rating": 4, + "text": "It's an interesting retelling of Guinevere and Arthur. I loved every bit of it and can't wait to read the sequel.", + "createdAt": "Thu Jul 04 2019", + "updatedAt": "Thu Jul 11 2019", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/2871843590" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.HegnVxYZnLTsNGxJeVV2cA", + "reviewer": { + "id": 11384547, + "name": "Lyndsay MacDonald", + "url": "/user/show/11384547-lyndsay-macdonald", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1371265927i/11384547.jpg", + "isAuthor": false, + "reviewCount": 38, + "followerCount": 0 + }, + "rating": 2, + "text": "This book was weird. I think it might have been tolerable if the author didn't try to weave it into the Arthurian legend (ie. if she made up original characters), but as it is I just couldn't get into it. There were some parts/scenes that I did enjoy, and others that I didn't... but overall I found it very confusing, and often found myself asking \"What is happening right now?\" and \"Why?\"

The story line flip-flopped around a lot... the characters were not wholly believable (Guinevere did NOT seem like a 13 year old girl, by any stretch of the imagination).. and I found parts of the plot silly and unrelated.

I won't be reading any of the other books in this sequence.", + "createdAt": "Wed Jan 22 2014", + "updatedAt": "Fri Aug 12 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/831334767" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.oazwOZoGnu8yIQJ_48FVMw", + "reviewer": { + "id": 27373770, + "name": "Andrea Krivak", + "url": "/user/show/27373770-andrea-krivak", + "avatar": "", + "isAuthor": false, + "reviewCount": 26, + "followerCount": 1 + }, + "rating": 2, + "text": "I think you have to really like arthurian legends to read this book. I liked it in the beginning, it started telling the story of Geunivere as a young girl and the \"what if\" this was her upbringing. There were wizards, men who turned into wolves and of course, merlin and morgana. About halfway through she meets Arthur, and then the book divides into the telling of each of their lives and they each stray into their own mystic journeys. To me, it spent too much time on these mystic journeys and by the end, i was ready to be done with this whole story. Not interesting enough ending for me to read into the sequel.", + "createdAt": "Mon Mar 24 2014", + "updatedAt": "Fri Aug 12 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/890822661" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.IxTTRac3W8gbXoHFdfTu5g", + "reviewer": { + "id": 1009204, + "name": "Cindy", + "url": "/user/show/1009204-cindy", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1228465282i/1009204._UX200_CR0,28,200,200_.jpg", + "isAuthor": false, + "reviewCount": 856, + "followerCount": 105 + }, + "rating": 1, + "text": "Maybe some day I'll come back to this book, but I just don't think I can read this book. I find myself wondering if I'm not following something of if this book is just beyond my reading. The writing is in a way that makes it seems like, wow we want the read to think this author is just so descriptive and great. Only it becomes confusing and bored. If you aren't familiar with the story of Guinevere then there is no point in reading it because nothing is explained and you are just sitting there wondering what the heck is going on. ", + "createdAt": "Wed Aug 27 2008", + "updatedAt": "Thu Jan 25 2018", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/31374042" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1._BHlRCbTwy-DJRq5mCTQnw", + "reviewer": { + "id": 4804714, + "name": "Angela", + "url": "/user/show/4804714-angela", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1294946776i/4804714.jpg", + "isAuthor": false, + "reviewCount": 129, + "followerCount": 6 + }, + "rating": 3, + "text": "I enjoyed this book up until the last third and then it got way too much involved with Arthur. It didn't quite wrap up the way I had expected and I would suggest not reading it unless you want to continue on with the series. I may read the sequel at a later time but I need a break from the acid trip like sections with Arthur.", + "createdAt": "Thu Mar 28 2013", + "updatedAt": "Tue Aug 09 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/572964102" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.P4s-Ysv7_6bKQAupxDvCwA", + "reviewer": { + "id": 4018173, + "name": "Misty", + "url": "/user/show/4018173-misty", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1286250306i/4018173._UY200_CR33,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 307, + "followerCount": 0 + }, + "rating": 2, + "text": "I really wanted to like this book.

About halfway through, I got bored and tried to push through it, but ultimately gave up. There was so much description and not enough happening.

Also, considering this was titled the Dragon Queen, I expected more dragons. Their lack of presence was disappointing.", + "createdAt": "Tue Nov 26 2013", + "updatedAt": "Thu Jun 23 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "could-not-finish", + "reviewURL": "/review/show/774467120" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.2E0pSS7MyztIPHdoIbH4Nw", + "reviewer": { + "id": 120860, + "name": "Lassarina Aoibhell", + "url": "/user/show/120860-lassarina-aoibhell", + "avatar": "", + "isAuthor": false, + "reviewCount": 487, + "followerCount": 5 + }, + "rating": 1, + "text": "I am DNFing this book on page 84 and we're going to go on a wild ride about why.

NB: If you are a scholar of Arthuriana, history, religion, or....well, almost anything, you might want to put down your drink. I'm not responsible for the state of your monitor.

We begin with the cover copy: Daughter of a mighty pagan queen, Guinevere has become a threat to her people and a prize to the dreaded sorcerer Merlin. Sent into hiding, she grows up under the protection of a shapeshifting man-wolf and an ornery Druid. But even on the remote coast of Scotland, where dragons feed and watch over her, she is not safe. Merlin knows the young beauty's destiny--and he will stop at nothing to prevent what has been foretold...

I also need to strongly highlight one of the blurbs. Like, really strongly. \"In addition to being exceptionally well written, [The Dragon Queen] has enormous integrity: All of the stories--of Arthur's parentage, of Merlin, of the characters' daily struggles--are well grounded in the history, politics, and religions of the time.....This is a trilogy to watch.\" --San Jose Mercury News

I want you to look at that quote carefully, and then come on a ride with me.

I will acknowledge up front my own reading comprehension fail, because I had forgotten that the shapeshifter was actually in the blurb and was yelling about \"what the fuck.\". I'm still saying \"what the fuck,\" just about plenty of other things.

Let's start with: It takes until page 48 for Guinevere to show up in any real sense. 500 page book; the main character doesn't actually appear until 10% in. Don't like that. Instead we are following around Dugald, the druid, and Maeniel, the shapeshifter. He's allegedly from Gaul. We begin at Tintagel in the year 470 CE, where Merlin and some of his cronies murder Vortigen. Some kind of magical snake is involved. (Aside: absolutely no problem with Merlin as the villain, particularly in anything allegedly centering the women of this tale.) Maeniel the shifter teleports??? out of the carnage, finds himself a she-wolf who has recently given birth, and shacks up with her. They encounter Dugald and the toddler Guinevere, who is currently disguised as a boy. For some reason, although everyone complains about disliking the Romans, we re-enact the founding of Rome, with Maeniel's mate suckling little Guinevere. Why does the she-wolf agree to this? Who can say. Dugald, meanwhile, talks about the influence of St. Patrick. Yes, that one. \"We don't call ourselves [druids] any longer. Not since Patrick came to Ireland bearing Christ's message.\"

....ten pages ago we called Merlin a chief druid. Also. It's the year 470. We don't have exact dates of the saint's life, but we know it was roughly in that century. More importantly, it was the Romans who banned druidic practices, fucking 400 years earlier (Augustus had no chill.) So there is no way that Patrick's actions in Ireland are affecting the order in England. Remember that blurb I highlighted? HMMMM. ALSO WHY ARE YOU, A GUY IN CORNWALL, TALKING ABOUT PATRICK. why. what is even your deal.

Also also, in this variation of the story (?), apparently, Dugald had a philosophical disagreement with Merlin and \"lost,\" so he was expected by the rules of his order to commit suicide because he lost.

Me: I do not remember reading this in any of the books I consumed during my obsessive period about Celtic histories and cultures and archaeology. (Wiki and checking with knowledgeable friends suggests I am not alone in not remembering this.)

If nothing else, that seems like a giant fucking waste of a 20 year apprenticeship, man.

(At this point I also stopped to verify my memory that Dugald and Maeniel do not appear in other Arthurian sources. I seem to be correct. And like, yeah, we add characters to stories all the time, I'm just. why are you making me deal with men in a book about Guinevere!)

There's also a point where young Guinevere, Dugald, and Maeniel are involved in helping to fight off a ship of slavers, which conveniently results in there being a woman to join the household and teach Guinevere How To Be A Girl. I should specify that the captain of said slaver's ship was apparently named Cymry. Why is he literally named \"Welsh\"? fuck if I know.

There's also a particular kind of ambient misogyny that I find tiresome, and like, I know we're in the fifth century and the Romans were assholes, and also I am aware that Celtic cultures were not paragons of equality and virtue, but the available evidence both of Romans being bitchy about how ass-backward Celtic societies were (omg they let their women do things!!!!) and the remnants of brehon law allocating specific property to women after divorce and categorizing a type of marriage as \"she married down and gets to keep all her stuff\" [wildly paraphrased] I'm just like

what. what are you doing, ma'am

and once again I'm going to refer back to that blurb. About being well grounded in the history, politics, and religions of the time.

I do not recommend actually reading this unless you want to play a drinking game, in which case, may I suggest that the drink you use should perhaps be water?", + "createdAt": "Thu Apr 09 2026", + "updatedAt": "Thu Apr 09 2026", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "did-not-finish", + "reviewURL": "/review/show/1761732484" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.9X0QSJx7BxHSTHSctKMkwg", + "reviewer": { + "id": 11348002, + "name": "Laural", + "url": "/user/show/11348002-laural", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1349329030i/11348002._UX200_CR0,5,200,200_.jpg", + "isAuthor": false, + "reviewCount": 36, + "followerCount": 5 + }, + "rating": 2, + "text": "I got about halfway through this book. Someday I'll give it another chance, or maybe try to figure out what exactly the author did wrong. A sword weilding Gweneviere? It could have been great. I thought it would be after reading the description on the back of the gorgeous cover. ", + "createdAt": "Thu Oct 04 2012", + "updatedAt": "Thu Jun 23 2016", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/427412066" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.p0mSKsHft_BTv2vScLHOsQ", + "reviewer": { + "id": 1250008, + "name": "Giuila", + "url": "/user/show/1250008-giuila", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1244141968i/1250008._UY200_CR2,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 61, + "followerCount": 0 + }, + "rating": 0, + "text": "... t read", + "createdAt": "Thu Jul 17 2008", + "updatedAt": "Fri Mar 24 2017", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "to-read", + "reviewURL": "/review/show/27520982" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.QMO0-GkHiX0UEYcsOaO7lw", + "reviewer": { + "id": 1431136, + "name": "Matt", + "url": "/user/show/1431136-matt", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1218856549i/1431136._UY200_CR25,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 6, + "followerCount": 0 + }, + "rating": 5, + "text": "an overall awesome book", + "createdAt": "Sat Aug 16 2008", + "updatedAt": "Fri Mar 24 2017", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/30279062" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.qOdgrbVKU-XMQR5DSbHfDQ", + "reviewer": { + "id": 16979552, + "name": "Sarah Garner", + "url": "/user/show/16979552-sarah-garner", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1408284741i/16979552._UX200_CR0,53,200,200_.jpg", + "isAuthor": false, + "reviewCount": 84, + "followerCount": 18 + }, + "rating": 2, + "text": "I struggled with this book and I couldn't finish it, although it was a great idea, I didn't love this book.", + "createdAt": "Sat Sep 20 2014", + "updatedAt": "Thu Aug 20 2020", + "spoiler": false, + "likeCount": 1, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/1059961768" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.JjlmKD1eCF28HlpY4KmCGw", + "reviewer": { + "id": 34744007, + "name": "Michael D.", + "url": "/user/show/34744007-michael-d", + "avatar": "", + "isAuthor": false, + "reviewCount": 81, + "followerCount": 1 + }, + "rating": 2, + "text": "I had this book about 20 years ago, tried to read it, but I just couldn't get into it. I got another copy used and gave it another try. While not as bad as my first experience in the first 20 pages, I found in the ensuing pages that the narrative is quite disjointed, and timelines are fluid and unclear to me. Perhaps that was Borchardt's intent, but I was confused by it. Additionally, it seems that characters are magically transported to strange places without any transition that I could discern. This narrative was hard to follow.

The book begins in a scene in 470 in Tintigal (not Tintagil, the famous castle in Cornwall where Arthur was conceived and born in the traditional romance). Ostensibly, the main character is a wolfman (a wolf who changes into a man when he chooses to) named Maeniel, and he is a friend to the British king Vortigen, who is not the villain he is normally portrayed as in history and the romances. Merlin shows up with a contingent of Saxons, and a battle ensues in which Vortigen is slain and Maeniel is knocked unconscious by an unknown cudgel. In the next scene, he wakes up on a hillside far away, presumably the far north, near Hadrian's Wall, and he becomes the mate of a she-wolf, who bears him a son, Black Leg (who will become Lancelot in the second novel The Raven Warrior). Sometime later, we find that 70 years have passed since Vortigen's death. Oddly, there is a baby girl in the nearby village, Guenevere (Borchardt spells her name differently depending on who is addressing her), who is one of the major characters of this series, the eponymous destined Dragon Queen. When Guenevere is 15, we are then told that 100 years have passed since Vortigen died. Confusion abounds.

This story had occasional high points, with modern fantastical treatments of the transformation duels, but I really had a hard time following the flow of Borchardt's narrative. I am a pretty astute reader, so I find it difficult to believe this was my fault. Borchardt seemed to be playing coy with the unfolding of events, and that was annoying. She was the sister of the better-known Anne Rice, author of the Vampire Chronicles, so I tried hard to give this a chance.

Borchardt intended to make this a trilogy, but the passed away in 2007 before the third book in the trilogy could be published. I don't know if anyone picked up her notes with the intent to complete her work, and I can't imagine that there was a demand for it. I do intend to slog through The Raven Warrior, as it is my rule to read books through and evaluate them honestly. If they're bad, I point out why. The lack of comprehensible transitions in time and space are my primary complaint with this novel. I hope that The Raven Warrior is more cognizant of walking me through how characters get from Point A to Point B than this one was.
", + "createdAt": "Thu Nov 09 2023", + "updatedAt": "Thu Nov 09 2023", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5944805120" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v2.2025072400.a7efbaf3-dd61-4567-b4e1-d3f344fabb0c", + "reviewer": { + "id": 42506425, + "name": "Michelle M", + "url": "/user/show/42506425-michelle-m", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1715615833i/42506425._UX200_CR0,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 380, + "followerCount": 3 + }, + "rating": 5, + "text": "Honestly? 5/6 or 4.75/6
I really loved this book! It's hard to describe what exactly I loved about it though lol. It had a very ethereal quality to it... The magic in this world is just a fact of life, some peple have it, while others don't, and yet, it didn't feel fantastical. Everyone knows about magic and most peoples still believe in the gods and things even though \"The Church\" (christianity) is present on the British Isles.

Borchardt grounds the story in the time right after Rome leaves the Isles, though they are still very much an influence on the peoples. It was a very mystical vibe to the whole story, kind of like there is a shroud of fog that is the magic of the world. IDK, like I said, hard to describe lol.

I loved Guinevere, and I liked how different characters referred to her by various spellings/pronouciations thoughout the story, as if they were speaking different languages. It is a very different take on the Arthur/Guinevere story, and very much an action oriented one, with Guinevere being the focus of the action and not just a passive bystander.

I though the portrayal of the other major characters interesting, and they were given roles that they are not usual given in the story. Arthur is \"good\", Merlin and Ingrane are the \"evil\" ones, Guinevere is strong willed and knows what she wants and sets out to get it. She doesn't seem like the type of person who will betray Arthur with Lancelot, but we'll just have to find out! (I hope she doesn't). The portrayal of these characters (and other named characters) made me do some research to try to get a better picture of if they had ever been portayed like this before. My search was unsucessful lol, as they were very diffefrent from \"traditional\" stories, but I haven't really read many retellings, so I cant be sure if they are exclusive to this story.

My only compaint with this book is that I found out it was supposed to be a trilogy!!!! And YET there are only two books! T.T *cries* I know the author can't help that she passed away, but book 2 came out in 2003 and she passed in 2007! That seems like a decent amount of time to finish the third book (I'm being very ungrateful, I know I am, I just lament what never was). Also apparently Anne Rice of vampire fame is Borchardt's sister?!?!?! She doesn't have like an early draft of her sister's just laying around??? T.T

I will def be reading the next one! (this one took me so long because I paused to participate in a read a thon and then got distracted by other books lol and then I got busy with work so I didn't have as much time to read)", + "createdAt": "Thu Jul 24 2025", + "updatedAt": "Thu Jul 24 2025", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/7714101862" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.HXGbQg0020aQJjD9G6pW2A", + "reviewer": { + "id": 163241100, + "name": "Alex", + "url": "/user/show/163241100-alex", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1763011908i/163241100._UY200_CR11,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 98, + "followerCount": 0 + }, + "rating": 2, + "text": "I enjoyed Guinevere’s character early on in the book. It was interesting watching her being raised alongside her foster brother, Black Leg, by a wolf shifter (Maniel) and a she-wolf. I do appreciate her relationship with the She Wolf, who Guinevere calls Mother. The thing that really kept me going through this book was Guinevere’s interactions with the dragons of the land and her connection to the land. I was also intrigued by some of the same conversations about religion in this book since Guinevere is raised in nature and has regular interactions with a Druid and some interactions with Christiany.
All this being said I struggled with the writing. There are huge chunks of it that are confusing since it’s not always clear who is being talked about when he, she or they are mentioned considering there might be multiple people mentioned previously in the scene. Transitions between moments are also rough. I was questioning when things changed and what was being talked about. The long descriptions of things really didn’t help either and me zoning out as I read.
It was also a bit rough to read this because of how things were worded/certain sciences being really uncomfortable for seemingly no reason out of nowhere. The dynamics between Igraine and Merlin’s relationship being one them. Scenes with them were sexual with zero warning and it wasn’t just limited to them. It happened a lot with various characters. Things were also made pretty uncomfortable with minors even if it was a scene that wasn’t intended to uncomfortably sexual, it was made that way by directly saying things like they “were far too young for even a dream of passion.” With how confusing the writing was I couldn’t even tell if there was a reason for why these sort of scenes were written in detail outside of making the audience feel uncomfortable and actively dislike the characters who are at fault for these behaviors.", + "createdAt": "Sun May 24 2026", + "updatedAt": "Sun May 24 2026", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5900548383" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.VoNZbtbhrTyL1Fhg43jpGw", + "reviewer": { + "id": 42064438, + "name": "Briana", + "url": "/user/show/42064438-briana", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1715717411i/42064438._UY200_CR5,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 161, + "followerCount": 1 + }, + "rating": 2, + "text": "This book was not for me. I had high hopes, the summary sounded interesting. It took me much longer than usual to finish this book as I struggled to enjoy it and I will not be picking up the second installment. I almost DNF'd this book several times but decided to stick with it.

The first half of the book felt like it was the author's freewriting development of the book, which she never bothered to go back and actually edit into a cohesive and understandable story. It was like being is some kind of dream where everything was hazy and only loosely connected. Sometimes you ended up in an entirely different location or time without even realizing it. It was also almost impossible to understand who was being discussed as the author tends to avoid using names and simply use pronouns leaving the reader to try and figure out (usually without success) who the \"she\" is that is being discussed. It was also very interesting that the author (or editor) chose to spell Guinevere's name differently throughout the book.

The second half was a little better and is the only reason I didn't give this book a one star. I don't mind Merlin as the antagonist as some other reviewers do and I found his character interesting. However, I did not really like the relationship between Arthur and Guinevere. I'm not sure how I am supposed to be convinced that this was some fated love story when they met for all of maybe an hour total. It seemed more insta-lust than anything else and there was no relationship development at all after that.

Overall I spent my time either being very confused and a little bored or kind of confused and mildly interested.", + "createdAt": "Thu Aug 15 2024", + "updatedAt": "Thu Aug 15 2024", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/6728900786" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.IoZW54HX8Qad9w5IgbQB9A", + "reviewer": { + "id": 6424389, + "name": "Joy Pouros", + "url": "/user/show/6424389-joy-pouros", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1567889312i/6424389._UY200_CR47,0,200,200_.jpg", + "isAuthor": false, + "reviewCount": 125, + "followerCount": 2 + }, + "rating": 3, + "text": "2.5 stars.

I REALLY like the character of Guinevere, but there were other parts of the story I didn't enjoy.

The writing style can be very winding and even abstract in some ways. There are whole parts you can skim without missing much. There aren't a ton of people, but enough to get them confused, especially a couple with similar names.

I'm always very frustrated with stories that try to have strong female characters but still have an overly sexually charged world. There are so many wonderfully creative parts of this world and then we default back to several tired tropes. There's just no need.

Some chapters are 30 pages, some are 5. It doesn't appear based on POV or time, so why is this so inconsistent?

And finally... I understand this is the first book of the trilogy, but this book just felt like a long prologue. Other series the first book is still a worthwhile read and entertaining story. This is more of an origin story before you fully appreciate the main story. So there are very mixed feelings about continuing the series because while I love Guinevere, there's an awful lot to overlook. ", + "createdAt": "Tue Aug 30 2022", + "updatedAt": "Tue Aug 30 2022", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/4955253716" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.ahVbGwMWksFfbjRS9c6Vbg", + "reviewer": { + "id": 29405146, + "name": "Anita", + "url": "/user/show/29405146-anita", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1407269712i/29405146._CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 614, + "followerCount": 3 + }, + "rating": 1, + "text": "First of all, I love Arthurian legends. Second, I always finish books, constantly hopeful that the story will improve, but I must confess that I skimmed this book to the end. I just cannot read any more!
This story has no cohesion, no main narrator and skips from one location to another mid-chapter. Plus there are men suddenly shapeshifting into wolves, and back again; no reasons given; no context for the intrigue and killings taking place; and a young girl child, supposedly Guinevere of Arthurian legend, being suckled by a she-wolf after being kidnapped.
Imaginative description is great in a fantasy story; confusion is not. This book is one long puzzle - a maze with apparently no direction, interspersed with some flowery descriptions of sunsets and misty mornings; visions of ravens and dragons (fishing); and a cast of hundreds of unrelated and irrelevant characters. Reading this book feels akin to experiencing one of those dreams in which you are lost and desperately trying to find your way back home.
Need I add that I did not enjoy this book, and will not be reading the sequel?", + "createdAt": "Fri Oct 30 2020", + "updatedAt": "Fri Oct 30 2020", + "spoiler": true, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3618732305" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.zo7-0bYwOUS0LlPXyqeuXQ", + "reviewer": { + "id": 3198878, + "name": "Stevie", + "url": "/user/show/3198878-stevie", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1560478233i/3198878._UX200_CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 71, + "followerCount": 0 + }, + "rating": 2, + "text": "Rating this book was a challenge. It was an impulse grab. I suspected it would be wonderfully terrible or terribly wonderful and there was only one way to find out.

What I liked: Arthurian legend from Guin's point of view. Cool. That awkward time period in history when they were both Christian but still observed many pagan rites. It had all the names I knew, and some of the tropes I knew: Vortigen, the witch in the well, so on and so forth from various movies I've seen.

What I disliked: the writing. Some of the dialogue wasn't needed. The descriptions could be too much in places and entirely lacking in others. Who is speaking? What do they look like? They were a human in a tower and now they are a fish? When? How? Transitions don't exist. This makes it hard to follow.

It was a wild ride, and I hung on until the end because I was curious as to how it would turn out. Turns out there's a sequel...I don't know that I have that in me...", + "createdAt": "Sat Mar 21 2020", + "updatedAt": "Sat Mar 21 2020", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/3240571912" + }, + { + "id": "kca://review:goodreads/amzn1.gr.review:goodreads.v1.E7XotAu3DvsavlUw1u62eA", + "reviewer": { + "id": 107938758, + "name": "Danai", + "url": "/user/show/107938758-danai", + "avatar": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/users/1705239736i/107938758._UX200_CR0,33,200,200_.jpg", + "isAuthor": false, + "reviewCount": 133, + "followerCount": 0 + }, + "rating": 3, + "text": "I didn't love it, nor did I hate it.

Everything was magical, mystical, and mysterious; Borchardt painted an imaginative world with interesting and diverse characters. She evidently has a deep knowledge of the British Isles, its history, and its lore.

But the writing was, at times, completely chaotic and confusing. As if someone was just spilling their string of thoughts without sufficient context or details - or too many of the wrong details. More than once, I found myself confused by abrupt changes in the situation, only to learn - through narration - that a memory was being shared.

This book didn't capture me nor engross me by any means, but it did beckon me to come back and finish it. If I come across the next book in the series, I will read it simply out of curiosity - what happens next for our heroes and villains? But I won't go hunting for a copy of it anytime soon...", + "createdAt": "Thu Jan 11 2024", + "updatedAt": "Thu Jan 11 2024", + "spoiler": false, + "likeCount": 0, + "commentCount": 0, + "recommendFor": null, + "shelf": "read", + "reviewURL": "/review/show/5638508977" + } + ], + "lastScraped": "2026-10-28T18:38:19.808Z" +} \ No newline at end of file diff --git a/internal/datasource/biblioreads/fixtures/search-books.json b/internal/datasource/biblioreads/fixtures/search-books.json new file mode 100644 index 00000000..12001a0a --- /dev/null +++ b/internal/datasource/biblioreads/fixtures/search-books.json @@ -0,0 +1,20 @@ +{ + "status": "Received", + "source": "https://github.com/nesaku/biblioreads", + "apiURL": "https://www.goodreads.com/book/auto_complete?format=json&q=La%20Reina%20Drag%C3%B3n%20Alice%20Borchardt", + "searchType": "books", + "numberOfResults": "1 results", + "result": [ + { + "id": 1, + "cover": "https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/books/1332599049i/12207444._SX50_.jpg", + "title": "La Reina Dragon (La leyenda de Guinevere, #1)", + "bookURL": "/book/show/12207444-la-reina-dragon", + "author": "Alice Borchardt", + "authorURL": "/author/show/151155.Alice_Borchardt", + "rating": "3.82", + "ratingCount": "2285" + } + ], + "lastScraped": "2026-10-28T17:45:31.333Z" +} diff --git a/internal/datasource/biblioreads/matching.go b/internal/datasource/biblioreads/matching.go new file mode 100644 index 00000000..149c6a56 --- /dev/null +++ b/internal/datasource/biblioreads/matching.go @@ -0,0 +1,213 @@ +package biblioreads + +import ( + "log" + "math" + "regexp" + "strconv" + "strings" + "unicode" + + "github.com/svera/coreander/v5/internal/index" + "golang.org/x/text/unicode/norm" +) + +// Strip named series annotations and omnibus ranges, not standalone volume numbers. +var seriesSuffix = regexp.MustCompile(`\s*\((?:[^()]+,\s*#[0-9]+(?:\.[0-9]+)?|#[0-9]+\s*[-–]\s*#?[0-9]+)\)\s*$`) + +var subtitleSeparator = regexp.MustCompile(`:\s*|\.\s+|\s+[-–—]\s+`) + +var parentheticalTitle = regexp.MustCompile(`\s+\([\p{L}][^()]*\)\s*$`) + +var goodreadsBookPath = regexp.MustCompile(`^/book/show/([1-9][0-9]*)(?:[.-][A-Za-z0-9._~-]+)?/?$`) + +func normalizeName(value string) string { + var result strings.Builder + for _, r := range norm.NFD.String(strings.ToLower(value)) { + switch { + case unicode.Is(unicode.Mn, r): + continue + case unicode.IsLetter(r), unicode.IsNumber(r): + result.WriteRune(r) + default: + result.WriteByte(' ') + } + } + return strings.Join(strings.Fields(result.String()), " ") +} + +func authorNameMatches(name, candidate string) bool { + name = normalizeName(name) + candidate = normalizeName(candidate) + if name == "" || candidate == "" { + return false + } + if strings.ReplaceAll(name, " ", "") == strings.ReplaceAll(candidate, " ", "") { + return true + } + short, long := strings.Fields(name), strings.Fields(candidate) + if len(short) > len(long) { + short, long = long, short + } + if len(short) < 2 || short[0] != long[0] || short[len(short)-1] != long[len(long)-1] { + return false + } + // Allow omitted middle names, but not conflicting ones. + next := 1 + for _, part := range long[1 : len(long)-1] { + if next < len(short)-1 && part == short[next] { + next++ + } + } + return next == len(short)-1 +} + +func matchedAuthor(document index.Document, matchedName string) (string, string) { + nameKey := strings.ReplaceAll(normalizeName(matchedName), " ", "") + if nameKey == "" { + return "", "" + } + for i, name := range document.Authors { + if i < len(document.AuthorsSlugs) && strings.ReplaceAll(normalizeName(name), " ", "") == nameKey { + return document.AuthorsSlugs[i], name + } + } + var slug, authorName string + for i, name := range document.Authors { + if i < len(document.AuthorsSlugs) && authorNameMatches(name, matchedName) { + if slug != "" { + return "", "" + } + slug, authorName = document.AuthorsSlugs[i], name + } + } + return slug, authorName +} + +func normalizedTitle(value string) string { + return normalizeName(seriesSuffix.ReplaceAllString(value, "")) +} + +func mainTitle(value string) string { + value = seriesSuffix.ReplaceAllString(value, "") + if separator := subtitleSeparator.FindStringIndex(value); separator != nil { + value = value[:separator[0]] + } + return normalizeName(value) +} + +func titleMatchScore(candidate, title, documentMainTitle string) int { + candidateTitle := normalizedTitle(candidate) + if candidateTitle == "" { + return 0 + } + if candidateTitle == title { + return 3 + } + if mainTitle(candidate) == title || documentMainTitle == candidateTitle || + normalizedTitle(parentheticalTitle.ReplaceAllString(candidate, "")) == title { + return 1 + } + return 0 +} + +func preferBook(candidate, best SearchResult, score, bestScore int) bool { + if bestScore == 0 { + return true + } + if score == bestScore { + count, bestCount := ratingCount(candidate.RatingCount), ratingCount(best.RatingCount) + if count != bestCount { + return count > bestCount + } + id, bestID := goodreadsBookID(candidate.BookURL), goodreadsBookID(best.BookURL) + if id > 0 && bestID > 0 && id != bestID { + return id < bestID + } + } + hasRating, bestHasRating := hasRating(candidate.Rating), hasRating(best.Rating) + if hasRating != bestHasRating { + return hasRating + } + return score > bestScore +} + +func ratingCount(value string) uint64 { + if strings.TrimSpace(value) == "" { + return 0 + } + count, err := strconv.ParseUint(strings.ReplaceAll(strings.TrimSpace(value), ",", ""), 10, 64) + if err != nil { + log.Printf("Invalid BiblioReads rating count: %q", value) + return 0 + } + return count +} + +func goodreadsBookID(bookURL string) uint64 { + match := goodreadsBookPath.FindStringSubmatch(bookURL) + if match == nil { + return 0 + } + id, err := strconv.ParseUint(match[1], 10, 64) + if err != nil { + log.Printf("Invalid Goodreads book ID in URL: %q", bookURL) + return 0 + } + return id +} + +func matchingBook(document index.Document, results []SearchResult) (SearchResult, bool) { + title := normalizedTitle(document.Title) + if title == "" { + return SearchResult{}, false + } + mainTitle := mainTitle(document.Title) + var best SearchResult + bestScore := 0 + for _, result := range results { + if result.BookURL == "" { + continue + } + score := titleMatchScore(result.Title, title, mainTitle) + if score == 0 { + continue + } + for _, author := range document.Authors { + if authorNameMatches(author, result.Author) { + if preferBook(result, best, score, bestScore) { + best, bestScore = result, score + } + break + } + } + } + return best, bestScore > 0 +} + +func hasRating(rating string) bool { + value, err := strconv.ParseFloat(rating, 64) + return err == nil && value > 0 && value <= 5 && !math.IsNaN(value) && !math.IsInf(value, 0) +} + +// searchQuery returns the search string for document: its title +// followed by its first author, if any. +func searchQuery(document index.Document) string { + q := strings.TrimSpace(document.Title) + if len(document.Authors) > 0 { + q = strings.TrimSpace(q + " " + document.Authors[0]) + } + return q +} + +func parseRating(rating, documentID string) float64 { + if rating == "" { + return 0 + } + value, err := strconv.ParseFloat(rating, 64) + if err != nil || value < 0 || value > 5 || math.IsNaN(value) || math.IsInf(value, 0) { + log.Printf("Invalid BiblioReads rating for document %s: %q", documentID, rating) + return 0 + } + return value +} diff --git a/internal/datasource/biblioreads/model.go b/internal/datasource/biblioreads/model.go new file mode 100644 index 00000000..6f087099 --- /dev/null +++ b/internal/datasource/biblioreads/model.go @@ -0,0 +1,146 @@ +package biblioreads + +type SearchBooksResponse struct { + Status string `json:"status"` + Source string `json:"source"` + APIURL string `json:"apiURL"` + SearchType string `json:"searchType"` + NumberOfResults string `json:"numberOfResults"` + Result []SearchResult `json:"result"` + LastScraped string `json:"lastScraped"` +} + +type SearchResult struct { + ID int `json:"id"` + Cover string `json:"cover"` + Title string `json:"title"` + BookURL string `json:"bookURL"` + Author string `json:"author"` + AuthorURL string `json:"authorURL"` + Rating string `json:"rating"` + RatingCount string `json:"ratingCount"` +} + +type BookResponse struct { + Status string `json:"status"` + StatusCode int `json:"statusCode"` + Source string `json:"source"` + ScrapeURL string `json:"scrapeURL"` + LegacyBookID int `json:"legacyBookID"` + BookID string `json:"bookID"` + ResourceID string `json:"resourceID"` + Cover string `json:"cover"` + Series string `json:"series"` + SeriesURL string `json:"seriesURL"` + SeriesPosition string `json:"seriesPosition"` + WorkURL string `json:"workURL"` + Title string `json:"title"` + Author []BookAuthor `json:"author"` + Rating string `json:"rating"` + RatingCount string `json:"ratingCount"` + ReviewsCount string `json:"reviewsCount"` + Desc string `json:"desc"` + Genres []string `json:"genres"` + BookEdition string `json:"bookEdition"` + PublishDate string `json:"publishDate"` + Publisher string `json:"publisher"` + Language string `json:"language"` + ASIN string `json:"asin"` + ISBN string `json:"isbn"` + ISBN13 string `json:"isbn13"` + ReviewBreakdown ReviewBreakdown `json:"reviewBreakdown"` + Quotes string `json:"quotes"` + QuotesURL string `json:"quotesURL"` + Questions string `json:"questions"` + QuestionsURL string `json:"questionsURL"` + LastScraped string `json:"lastScraped"` +} + +type BookAuthor struct { + ID int `json:"id"` + Name string `json:"name"` + Role string `json:"role"` + URL string `json:"url"` +} + +type ReviewBreakdown struct { + Rating1 string `json:"rating1"` + Rating2 string `json:"rating2"` + Rating3 string `json:"rating3"` + Rating4 string `json:"rating4"` + Rating5 string `json:"rating5"` +} + +type AuthorResponse struct { + Status string `json:"status"` + Source string `json:"source"` + ScrapeURL string `json:"scrapeURL"` + Image string `json:"image"` + Name string `json:"name"` + Website string `json:"website"` + Genre []string `json:"genre"` + Influences []string `json:"influences"` + BirthDate string `json:"birthDate"` + DeathDate string `json:"deathDate"` + Desc string `json:"desc"` + Books []AuthorBook `json:"books"` + Series []AuthorSeries `json:"series"` + LastScraped string `json:"lastScraped"` +} + +type AuthorBook struct { + ID int `json:"id"` + Cover string `json:"cover"` + Title string `json:"title"` + URL string `json:"url"` + Rating string `json:"rating"` + PublishDate string `json:"publishDate"` +} + +type AuthorSeries struct { + ID int `json:"id"` + Cover string `json:"cover"` + Title string `json:"title"` + SeriesURL string `json:"seriesURL"` + Author string `json:"author"` + AuthorURL string `json:"authorURL"` + Rating string `json:"rating"` +} + +type ReviewsResponse struct { + Version string `json:"version"` + Status string `json:"status"` + StatusCode int `json:"statusCode"` + Source string `json:"source"` + ScrapeURL string `json:"scrapeURL"` + TotalCount int `json:"totalCount"` + NextPageToken string `json:"nextPageToken"` + PrevPageToken string `json:"prevPageToken"` + Reviews []Review `json:"reviews"` + LastScraped string `json:"lastScraped"` +} + +type Review struct { + ID string `json:"id"` + Reviewer Reviewer `json:"reviewer"` + Rating float64 `json:"rating"` + Text string `json:"text"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Spoiler bool `json:"spoiler"` + LikeCount int `json:"likeCount"` + CommentCount int `json:"commentCount"` + RecommendFor *string `json:"recommendFor"` + Shelf string `json:"shelf"` + ReviewURL string `json:"reviewURL"` +} + +type Reviewer struct { + ID int `json:"id"` + Name string `json:"name"` + URL string `json:"url"` + Avatar string `json:"avatar"` + IsAuthor bool `json:"isAuthor"` + ReviewCount int `json:"reviewCount"` + FollowerCount int `json:"followerCount"` +} diff --git a/internal/datasource/biblioreads/reviews.go b/internal/datasource/biblioreads/reviews.go new file mode 100644 index 00000000..16f79770 --- /dev/null +++ b/internal/datasource/biblioreads/reviews.go @@ -0,0 +1,224 @@ +package biblioreads + +import ( + "context" + "errors" + "fmt" + "log" + "net/url" + "strconv" + "strings" + "time" + + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/index" +) + +const goodreadsBookPrefix = "/book/show/" + +type ReviewClient interface { + Book(ctx context.Context, id string) (BookResponse, error) + Reviews(ctx context.Context, resourceID string, legacyBookID int, nextPageToken, languageCode string) (ReviewsResponse, error) +} + +type ReviewIndex interface { + Book(ctx context.Context, document index.Document) (SearchResult, error) + StoreDetails(document index.Document, bookURL string, book BookResponse) error + UpdateDataSourceLink(documentID, bookURL string) error +} + +type ReviewProvider struct { + client ReviewClient + index ReviewIndex +} + +// NewReviewProvider exposes BiblioReads data using the shared review models. +func NewReviewProvider(client ReviewClient, idx ReviewIndex) *ReviewProvider { + return &ReviewProvider{client: client, index: idx} +} + +func (b *ReviewProvider) Name() string { + return Name +} + +// AuthorLink returns the indexed author link without making a network request. +func (b *ReviewProvider) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { + return goodreadsDataSourceLink(author.DataSourceLinks["goodreads"], "/author/show/") +} + +// DocumentLink returns the indexed book link without making a network request. +func (b *ReviewProvider) DocumentLink(document index.Document) datasourcemodel.DataSourceLink { + data, err := index.DataSourceData[DocumentData](document, Source) + if err != nil { + log.Println(err) + return datasourcemodel.DataSourceLink{} + } + return goodreadsDataSourceLink(data.URL, "/book/show/") +} + +func goodreadsDataSourceLink(path, prefix string) datasourcemodel.DataSourceLink { + link := datasourcemodel.DataSourceLink{Source: Name, Icon: "/images/goodreads.svg", IconInvertible: true} + if id, ok := strings.CutPrefix(path, prefix); ok && id != "" { + link.URL = goodreadsBaseURL + path + } + return link +} + +// StoredScore builds a score from indexed data without making a network request. +func (b *ReviewProvider) StoredScore(document index.Document) (datasourcemodel.Score, error) { + data, err := index.DataSourceData[DocumentData](document, Source) + if err != nil { + return datasourcemodel.Score{}, err + } + return datasourcemodel.Score{ + Source: b.Name(), Value: fmt.Sprintf("%.2f", document.DataSourceScoreValue), + Count: data.RatingCount, URL: goodreadsDataSourceLink(data.URL, goodreadsBookPrefix).URL, + }, nil +} + +// CanEditDocumentLink reports that Goodreads book links can be corrected. +func (b *ReviewProvider) CanEditDocumentLink() bool { + return true +} + +// UpdateDocumentLink saves a corrected Goodreads book link and clears stale score details. +func (b *ReviewProvider) UpdateDocumentLink(document index.Document, link string) error { + path, err := normalizeGoodreadsBookURL(link) + if err != nil { + return fmt.Errorf("%w: invalid Goodreads book URL", datasourcemodel.ErrInvalidDocumentLink) + } + if err := b.index.UpdateDataSourceLink(document.ID, path); err != nil { + if errors.Is(err, index.ErrDocumentNotFound) { + return datasourcemodel.ErrDocumentNotFound + } + return err + } + return nil +} + +func normalizeGoodreadsBookURL(rawURL string) (string, error) { + bookURL, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || len(rawURL) > 2048 || bookURL.User != nil || + (bookURL.Scheme != "https" && bookURL.Scheme != "http") || + (bookURL.Host != "www.goodreads.com" && bookURL.Host != "goodreads.com") || + !goodreadsBookPath.MatchString(bookURL.Path) { + return "", fmt.Errorf("invalid Goodreads book URL") + } + return strings.TrimSuffix(bookURL.Path, "/"), nil +} + +func bookID(bookURL string) (string, error) { + id, ok := strings.CutPrefix(bookURL, goodreadsBookPrefix) + if !ok || id == "" { + return "", fmt.Errorf("invalid Goodreads book URL %q", bookURL) + } + return id, nil +} + +func (b *ReviewProvider) book(ctx context.Context, document index.Document) (BookResponse, error) { + data, err := index.DataSourceData[DocumentData](document, Source) + if err != nil { + return BookResponse{}, err + } + id, err := bookID(data.URL) + if err != nil { + return BookResponse{}, err + } + book, err := b.client.Book(ctx, id) + if err != nil { + return book, fmt.Errorf("retrieving Goodreads book %s: %w", id, err) + } + details := book + if book.Rating != "" && document.DataSourceScoreValue > 0 && isZeroRating(book.Rating) { + book.Rating = fmt.Sprintf("%.2f", document.DataSourceScoreValue) + details.Rating = "" + } + if err := b.index.StoreDetails(document, data.URL, details); err != nil { + log.Printf("Error saving Goodreads details for %s: %s", document.ID, err) + } + return book, nil +} + +func isZeroRating(rating string) bool { + value, err := strconv.ParseFloat(rating, 64) + return err == nil && value == 0 +} + +func (b *ReviewProvider) Score(ctx context.Context, document index.Document) (datasourcemodel.Score, error) { + score := datasourcemodel.Score{Source: Name} + data, err := index.DataSourceData[DocumentData](document, Source) + if err != nil { + return score, err + } + if data.URL == "" { + result, err := b.index.Book(ctx, document) + if err != nil { + return score, err + } + if result.BookURL == "" { + return score, nil + } + if _, err := bookID(result.BookURL); err != nil { + return score, err + } + data.URL = result.BookURL + if err := index.SetDataSourceData(&document, Source, data); err != nil { + return score, err + } + score.Value, score.Count, score.URL = result.Rating, result.RatingCount, b.DocumentLink(document).URL + return score, nil + } + book, err := b.book(ctx, document) + if err != nil { + return score, err + } + score.Value, score.Count, score.URL = book.Rating, book.RatingCount, b.DocumentLink(document).URL + return score, nil +} + +func (b *ReviewProvider) Reviews(ctx context.Context, document index.Document, language string) (datasourcemodel.ReviewPage, error) { + data, err := index.DataSourceData[DocumentData](document, Source) + if err != nil { + return datasourcemodel.ReviewPage{}, err + } + if data.URL == "" { + return datasourcemodel.ReviewPage{}, nil + } + book, err := b.book(ctx, document) + if err != nil { + return datasourcemodel.ReviewPage{}, err + } + if book.Rating == "" { + return datasourcemodel.ReviewPage{}, nil + } + resourceID, legacyBookID := data.ResourceID, data.LegacyBookID + if book.ResourceID != "" && book.LegacyBookID > 0 { + resourceID, legacyBookID = book.ResourceID, book.LegacyBookID + } + if resourceID == "" || legacyBookID <= 0 { + return datasourcemodel.ReviewPage{}, fmt.Errorf("missing Goodreads review identifiers for %s", document.ID) + } + response, err := b.client.Reviews(ctx, resourceID, legacyBookID, "", language) + if err != nil { + return datasourcemodel.ReviewPage{}, fmt.Errorf("retrieving Goodreads reviews for %s: %w", document.ID, err) + } + page := datasourcemodel.ReviewPage{Source: Name, URL: b.DocumentLink(document).URL, Available: true} + for _, review := range response.Reviews { + var createdAt time.Time + if review.CreatedAt != "" { + createdAt, err = time.Parse("Mon Jan 02 2006", review.CreatedAt) + if err != nil { + log.Printf("Invalid Goodreads review date for %s: %q: %s", review.ID, review.CreatedAt, err) + } + } + reviewURL := "" + if review.ReviewURL != "" { + reviewURL = goodreadsBaseURL + review.ReviewURL + } + page.Reviews = append(page.Reviews, datasourcemodel.Review{ + Reviewer: review.Reviewer.Name, Rating: review.Rating, Text: review.Text, + CreatedAt: createdAt, Spoiler: review.Spoiler, URL: reviewURL, + }) + } + return page, nil +} diff --git a/internal/datasource/biblioreads/reviews_test.go b/internal/datasource/biblioreads/reviews_test.go new file mode 100644 index 00000000..c53a758a --- /dev/null +++ b/internal/datasource/biblioreads/reviews_test.go @@ -0,0 +1,121 @@ +package biblioreads_test + +import ( + "context" + "errors" + "testing" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/index" +) + +func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + t.Fatal(err) + } + return document +} + +func TestBiblioReadsDataSourceLinks(t *testing.T) { + provider := biblioreads.NewReviewProvider(nil, nil) + author := provider.AuthorLink(index.Author{ + DataSourceLinks: map[string]string{"goodreads": "/author/show/123.Jane_Doe"}, + }, "en") + if author.URL != "https://www.goodreads.com/author/show/123.Jane_Doe" || + author.Source != "Goodreads" || author.Icon != "/images/goodreads.svg" || !author.IconInvertible { + t.Fatalf("unexpected Goodreads author link: %+v", author) + } + + document := provider.DocumentLink(withBiblioReadsData(t, index.Document{}, biblioreads.DocumentData{URL: "/book/show/456.A_Book"})) + if document.URL != "https://www.goodreads.com/book/show/456.A_Book" || + document.Source != "Goodreads" || document.Icon != "/images/goodreads.svg" || !document.IconInvertible { + t.Fatalf("unexpected Goodreads document link: %+v", document) + } + for _, links := range []map[string]string{ + nil, + {}, + {"other": "/author/show/123.Jane_Doe"}, + {"goodreads": "/book/show/456.A_Book"}, + } { + if link := provider.AuthorLink(index.Author{DataSourceLinks: links}, "en"); link.URL != "" { + t.Errorf("unexpected Goodreads link for %v: %+v", links, link) + } + } +} + +type reviewLinkIndex struct { + link string + err error +} + +func (i *reviewLinkIndex) Book(context.Context, index.Document) (biblioreads.SearchResult, error) { + return biblioreads.SearchResult{}, nil +} + +func (i *reviewLinkIndex) StoreDetails(index.Document, string, biblioreads.BookResponse) error { + return nil +} + +func (i *reviewLinkIndex) UpdateDataSourceLink(_ string, link string) error { + i.link = link + return i.err +} + +func TestBiblioReadsUpdateDocumentLink(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + {name: "numeric ID", url: "https://www.goodreads.com/book/show/1368314", want: "/book/show/1368314"}, + {name: "dotted slug", url: "https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente", want: "/book/show/1368314.El_Cercano_Oriente"}, + {name: "hyphenated slug", url: "https://www.goodreads.com/book/show/243175949-el-cercano-oriente", want: "/book/show/243175949-el-cercano-oriente"}, + {name: "normalize shared URL", url: " http://goodreads.com/book/show/1368314/?from_search=true#reviews ", want: "/book/show/1368314"}, + {name: "empty"}, + {name: "relative", url: "/book/show/1368314"}, + {name: "other domain", url: "https://example.com/book/show/1368314"}, + {name: "deceptive domain", url: "https://www.goodreads.com.example.com/book/show/1368314"}, + {name: "credentials", url: "https://reader@www.goodreads.com/book/show/1368314"}, + {name: "port", url: "https://www.goodreads.com:4000/book/show/1368314"}, + {name: "wrong scheme", url: "ftp://www.goodreads.com/book/show/1368314"}, + {name: "author", url: "https://www.goodreads.com/author/show/16667.Isaac_Asimov"}, + {name: "non-numeric ID", url: "https://www.goodreads.com/book/show/not-a-book"}, + {name: "zero ID", url: "https://www.goodreads.com/book/show/0"}, + {name: "extra path", url: "https://www.goodreads.com/book/show/1368314/other"}, + {name: "encoded path separator", url: "https://www.goodreads.com/book/show/1368314%2Fother"}, + {name: "malformed URL", url: "https://%zz/book/show/1368314"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + idx := &reviewLinkIndex{} + provider := biblioreads.NewReviewProvider(nil, idx) + err := provider.UpdateDocumentLink(index.Document{ID: "doc-id"}, tt.url) + if tt.want == "" { + if !errors.Is(err, datasourcemodel.ErrInvalidDocumentLink) { + t.Fatalf("expected invalid-link error, got %v", err) + } + if idx.link != "" { + t.Fatalf("invalid link was passed to index: %q", idx.link) + } + return + } + if err != nil { + t.Fatal(err) + } + if idx.link != tt.want { + t.Fatalf("normalized link = %q, want %q", idx.link, tt.want) + } + }) + } +} + +func TestBiblioReadsUpdateDocumentLinkMissingDocument(t *testing.T) { + idx := &reviewLinkIndex{err: index.ErrDocumentNotFound} + provider := biblioreads.NewReviewProvider(nil, idx) + err := provider.UpdateDocumentLink(index.Document{ID: "missing"}, "https://www.goodreads.com/book/show/1368314") + if !errors.Is(err, datasourcemodel.ErrDocumentNotFound) { + t.Fatalf("expected missing-document error, got %v", err) + } +} diff --git a/internal/datasource/biblioreads/throttle_test.go b/internal/datasource/biblioreads/throttle_test.go new file mode 100644 index 00000000..d54bbb7b --- /dev/null +++ b/internal/datasource/biblioreads/throttle_test.go @@ -0,0 +1,80 @@ +package biblioreads_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sort" + "sync" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/biblioreads" +) + +func TestRequestsAreThrottledPerInstance(t *testing.T) { + const interval = 200 * time.Millisecond + var mu sync.Mutex + arrivals := make([][]time.Time, 2) + var instances []string + for i := range arrivals { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + arrivals[i] = append(arrivals[i], time.Now()) + mu.Unlock() + _, _ = w.Write([]byte(validBookResponse)) + })) + t.Cleanup(server.Close) + instances = append(instances, server.URL) + } + client := biblioreads.NewClient(biblioreads.WithInstances(instances), + biblioreads.WithCacheTTL(0), biblioreads.WithRequestInterval(interval)) + var wg sync.WaitGroup + for range 6 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Errorf("Book: %v", err) + } + }() + } + wg.Wait() + for i, times := range arrivals { + if len(times) != 3 { + t.Fatalf("instance %d: expected three requests, got %d", i, len(times)) + } + sort.Slice(times, func(i, j int) bool { return times[i].Before(times[j]) }) + for j := 1; j < len(times); j++ { + if gap := times[j].Sub(times[j-1]); gap < interval-20*time.Millisecond { + t.Errorf("instance %d: request gap %s is below %s", i, gap, interval) + } + } + } + gap := arrivals[0][0].Sub(arrivals[1][0]) + if gap.Abs() >= interval/2 { + t.Errorf("independent instances should start concurrently, gap %s", gap) + } +} + +func TestThrottleWaitHonorsCancellation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(validBookResponse)) + })) + defer server.Close() + client := biblioreads.NewClient(biblioreads.WithInstances([]string{server.URL}), + biblioreads.WithCacheTTL(0), biblioreads.WithRequestInterval(time.Second)) + if _, err := client.Book(context.Background(), "1"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := client.Book(ctx, "2"); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected deadline exceeded during throttle wait, got %v", err) + } + if time.Since(start) >= 500*time.Millisecond { + t.Fatal("cancellation did not interrupt the throttle wait") + } +} diff --git a/internal/datasource/httpjson/httpjson.go b/internal/datasource/httpjson/httpjson.go new file mode 100644 index 00000000..100c931d --- /dev/null +++ b/internal/datasource/httpjson/httpjson.go @@ -0,0 +1,72 @@ +// Package httpjson contains helpers shared by the datasource clients that consume JSON HTTP APIs. +package httpjson + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" +) + +const ( + UserAgent = "Mozilla/5.0 (compatible; coreander/1.0; +https://github.com/svera/coreander)" + + // DefaultRetryAfter is used when a Retry-After header is missing or invalid. + DefaultRetryAfter = 60 * time.Second +) + +type Response struct { + StatusCode int + Header http.Header + Body []byte +} + +// Do sends req with the default headers and reads the whole response body. +func Do(client *http.Client, req *http.Request) (Response, error) { + req.Header.Set("User-Agent", UserAgent) + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return Response{}, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return Response{}, err + } + return Response{StatusCode: resp.StatusCode, Header: resp.Header, Body: body}, nil +} + +// Decode unmarshals the response body into dest, returning an error if the status code is not 200. +func (r Response) Decode(dest any) error { + if r.StatusCode != http.StatusOK { + return fmt.Errorf("request failed with status code %d: %s", r.StatusCode, r.Body) + } + return json.Unmarshal(r.Body, dest) +} + +// ParseRetryAfter returns the wait time specified by a Retry-After header value, which can be +// either a number of seconds or an HTTP date. DefaultRetryAfter is returned if it can't be used. +func ParseRetryAfter(header string) time.Duration { + if header == "" { + return DefaultRetryAfter + } + if seconds, err := strconv.Atoi(header); err == nil { + if seconds <= 0 { + return DefaultRetryAfter + } + return time.Duration(seconds) * time.Second + } + if retryTime, err := http.ParseTime(header); err == nil { + wait := time.Until(retryTime) + if wait <= 0 { + return DefaultRetryAfter + } + return wait + } + return DefaultRetryAfter +} diff --git a/internal/datasource/httpjson/httpjson_test.go b/internal/datasource/httpjson/httpjson_test.go new file mode 100644 index 00000000..d44c33ed --- /dev/null +++ b/internal/datasource/httpjson/httpjson_test.go @@ -0,0 +1,77 @@ +package httpjson_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/datasource/httpjson" +) + +func TestParseRetryAfter(t *testing.T) { + t.Run("seconds", func(t *testing.T) { + if got := httpjson.ParseRetryAfter("30"); got != 30*time.Second { + t.Fatalf("expected 30s, got %s", got) + } + }) + + t.Run("http date", func(t *testing.T) { + retryTime := time.Now().Add(45 * time.Second).UTC() + got := httpjson.ParseRetryAfter(retryTime.Format(http.TimeFormat)) + if got < 44*time.Second || got > 46*time.Second { + t.Fatalf("expected about 45s, got %s", got) + } + }) + + t.Run("missing header uses default", func(t *testing.T) { + if got := httpjson.ParseRetryAfter(""); got != httpjson.DefaultRetryAfter { + t.Fatalf("expected default %s, got %s", httpjson.DefaultRetryAfter, got) + } + }) + + t.Run("invalid header uses default", func(t *testing.T) { + for _, header := range []string{"-5", "0", "soon"} { + if got := httpjson.ParseRetryAfter(header); got != httpjson.DefaultRetryAfter { + t.Fatalf("expected default %s for %q, got %s", httpjson.DefaultRetryAfter, header, got) + } + } + }) +} + +func TestDo(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("User-Agent") != httpjson.UserAgent { + t.Errorf("unexpected User-Agent %q", r.Header.Get("User-Agent")) + } + if r.Header.Get("Accept") != "application/json" { + t.Errorf("unexpected Accept %q", r.Header.Get("Accept")) + } + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTeapot) + _, _ = w.Write([]byte(`{"a":1}`)) + })) + defer server.Close() + + req, _ := http.NewRequest(http.MethodGet, server.URL, nil) + resp, err := httpjson.Do(server.Client(), req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusTeapot || resp.Header.Get("Retry-After") != "5" || string(resp.Body) != `{"a":1}` { + t.Errorf("unexpected response %+v", resp) + } +} + +func TestDecode(t *testing.T) { + var dest struct{ A int } + if err := (httpjson.Response{StatusCode: http.StatusOK, Body: []byte(`{"a":1}`)}).Decode(&dest); err != nil || dest.A != 1 { + t.Errorf("expected A=1 without error, got %d, %v", dest.A, err) + } + if err := (httpjson.Response{StatusCode: http.StatusNotFound, Body: []byte(`{}`)}).Decode(&dest); err == nil { + t.Error("expected error on non-200 status") + } + if err := (httpjson.Response{StatusCode: http.StatusOK, Body: []byte(``)}).Decode(&dest); err == nil { + t.Error("expected error on invalid JSON") + } +} diff --git a/internal/datasource/model/author.go b/internal/datasource/model/author.go index 599f00ce..82356915 100644 --- a/internal/datasource/model/author.go +++ b/internal/datasource/model/author.go @@ -14,8 +14,10 @@ type Author interface { DateOfBirth() precisiondate.PrecisionDate DateOfDeath() precisiondate.PrecisionDate Image() string - Website() string - WikipediaLink(language string) string + // Links returns the current links owned by this datasource, including language-specific keys. + Links() map[string]string + // OwnsLink identifies entries replaced by a metadata refresh, even if absent from Links. + OwnsLink(key string) bool SourceID() string RetrievedOn() time.Time Pseudonyms() []string diff --git a/internal/datasource/model/data_source_link.go b/internal/datasource/model/data_source_link.go new file mode 100644 index 00000000..fb665585 --- /dev/null +++ b/internal/datasource/model/data_source_link.go @@ -0,0 +1,10 @@ +package model + +type DataSourceLink struct { + Source string + URL string + Icon string + IconClass string + IconInvertible bool + Label string +} diff --git a/internal/datasource/model/review.go b/internal/datasource/model/review.go new file mode 100644 index 00000000..08aa8be7 --- /dev/null +++ b/internal/datasource/model/review.go @@ -0,0 +1,53 @@ +package model + +import ( + "errors" + "strconv" + "strings" + "time" +) + +var ( + ErrInvalidDocumentLink = errors.New("invalid document link") + ErrDocumentNotFound = errors.New("document not found") +) + +// Score has an empty URL for an unmatched document and an empty Value for an unavailable score. +type Score struct { + Source string + Value string + Count string + URL string +} + +// HasScore reports whether the value is scored or has valuations despite a zero value. +func (s Score) HasScore() bool { + if s.Value == "" { + return false + } + value, err := strconv.ParseFloat(s.Value, 64) + if err != nil || value != 0 { + return true + } + count, err := strconv.ParseFloat(strings.ReplaceAll(strings.TrimSpace(s.Count), ",", ""), 64) + return err == nil && count > 0 +} + +type Review struct { + Reviewer string + // Rating is on a five-point scale and may be fractional; zero means unrated. + Rating float64 + // Text may contain untrusted HTML, which must be sanitized before rendering. + Text string + CreatedAt time.Time + Spoiler bool + URL string +} + +// ReviewPage distinguishes an unavailable review section from an available, empty list. +type ReviewPage struct { + Source string + URL string + Available bool + Reviews []Review +} diff --git a/internal/datasource/wikidata/author.go b/internal/datasource/wikidata/author.go index 21f3b73a..70780204 100644 --- a/internal/datasource/wikidata/author.go +++ b/internal/datasource/wikidata/author.go @@ -1,6 +1,7 @@ package wikidata import ( + "strings" "time" "github.com/svera/coreander/v5/internal/precisiondate" @@ -90,8 +91,21 @@ func (a Author) DateOfDeath() precisiondate.PrecisionDate { return a.dateOfDeath } -func (a Author) Website() string { - return a.website +func (a Author) Links() map[string]string { + links := make(map[string]string, len(a.wikipediaLink)+1) + if a.website != "" { + links["website"] = a.website + } + for language, url := range a.wikipediaLink { + if url != "" { + links["wikipedia:"+language] = url + } + } + return links +} + +func (a Author) OwnsLink(key string) bool { + return key == "website" || strings.HasPrefix(key, "wikipedia:") } func (a Author) Image() string { @@ -102,10 +116,6 @@ func (a Author) InstanceOf() float64 { return a.instanceOf } -func (a Author) WikipediaLink(language string) string { - return a.wikipediaLink[language] -} - func (a Author) Gender() float64 { return a.gender } diff --git a/internal/datasource/wikidata/httpclient.go b/internal/datasource/wikidata/httpclient.go index 18d90a0f..f2e0b5f6 100644 --- a/internal/datasource/wikidata/httpclient.go +++ b/internal/datasource/wikidata/httpclient.go @@ -1,17 +1,13 @@ package wikidata import ( - "encoding/json" - "fmt" - "io" "log" "net/http" - "strconv" "sync" "time" -) -const defaultRetryAfter = 60 * time.Second + "github.com/svera/coreander/v5/internal/datasource/httpjson" +) var apiHTTPClient = newRateLimitedHTTPClient() @@ -35,33 +31,18 @@ func (c *rateLimitedHTTPClient) getJSON(url string, dest any) error { if err != nil { return err } - for key, value := range wikidataAPIHeaders() { - req.Header.Set(key, value) - } - resp, err := c.client.Do(req) + resp, err := httpjson.Do(c.client, req) if err != nil { return err } if resp.StatusCode == http.StatusTooManyRequests { - _ = resp.Body.Close() c.waitUntil(resp.Header.Get("Retry-After")) continue } - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - return fmt.Errorf("request failed with status code %d: %s", resp.StatusCode, body) - } - - body, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if err != nil { - return err - } - return json.Unmarshal(body, dest) + return resp.Decode(dest) } } @@ -71,7 +52,7 @@ func (c *rateLimitedHTTPClient) waitUntil(retryAfterHeader string) { c.mu.Lock() var rateLimitWait time.Duration if retryAfterHeader != "" { - rateLimitWait = parseRetryAfter(retryAfterHeader) + rateLimitWait = httpjson.ParseRetryAfter(retryAfterHeader) resumeAt := time.Now().Add(rateLimitWait) if resumeAt.After(c.resumeAt) { c.resumeAt = resumeAt @@ -86,29 +67,3 @@ func (c *rateLimitedHTTPClient) waitUntil(retryAfterHeader string) { time.Sleep(wait) } } - -func parseRetryAfter(header string) time.Duration { - if header == "" { - return defaultRetryAfter - } - if seconds, err := strconv.Atoi(header); err == nil { - if seconds <= 0 { - return defaultRetryAfter - } - return time.Duration(seconds) * time.Second - } - if retryTime, err := http.ParseTime(header); err == nil { - wait := time.Until(retryTime) - if wait <= 0 { - return defaultRetryAfter - } - return wait - } - return defaultRetryAfter -} - -func wikidataAPIHeaders() map[string]string { - return map[string]string{ - "User-Agent": "Mozilla/5.0 (compatible; coreander/1.0; +https://github.com/svera/coreander)", - } -} diff --git a/internal/datasource/wikidata/httpclient_test.go b/internal/datasource/wikidata/httpclient_test.go index 61fcfa30..116e07fb 100644 --- a/internal/datasource/wikidata/httpclient_test.go +++ b/internal/datasource/wikidata/httpclient_test.go @@ -8,28 +8,6 @@ import ( "time" ) -func TestParseRetryAfter(t *testing.T) { - t.Run("seconds", func(t *testing.T) { - if got := parseRetryAfter("30"); got != 30*time.Second { - t.Fatalf("expected 30s, got %s", got) - } - }) - - t.Run("http date", func(t *testing.T) { - retryTime := time.Now().Add(45 * time.Second).UTC() - got := parseRetryAfter(retryTime.Format(http.TimeFormat)) - if got < 44*time.Second || got > 46*time.Second { - t.Fatalf("expected about 45s, got %s", got) - } - }) - - t.Run("missing header uses default", func(t *testing.T) { - if got := parseRetryAfter(""); got != defaultRetryAfter { - t.Fatalf("expected default %s, got %s", defaultRetryAfter, got) - } - }) -} - func TestRateLimitedHTTPClientWaitsForRetryAfter(t *testing.T) { var requests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/datasource/wikidata/service.go b/internal/datasource/wikidata/service.go index 44cab53c..727f3942 100644 --- a/internal/datasource/wikidata/service.go +++ b/internal/datasource/wikidata/service.go @@ -14,6 +14,7 @@ import ( gowikidata "github.com/Navid2zp/go-wikidata" "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/precisiondate" ) @@ -42,6 +43,23 @@ type WikidataSource struct { wikidata wikidata } +func (WikidataSource) OwnsLink(key string) bool { + return (Author{}).OwnsLink(key) +} + +func (WikidataSource) AuthorLink(author index.Author, language string) model.DataSourceLink { + url := author.DataSourceLinks["wikipedia:"+language] + if url == "" { + return model.DataSourceLink{} + } + return model.DataSourceLink{ + Source: "Wikipedia", + URL: url, + IconClass: "bi-wikipedia", + Label: "Go to Wikipedia entry", + } +} + func NewWikidataSource(w wikidata) WikidataSource { return WikidataSource{w} } diff --git a/internal/datasource/wikidata/service_test.go b/internal/datasource/wikidata/service_test.go index 795712a1..aa00113e 100644 --- a/internal/datasource/wikidata/service_test.go +++ b/internal/datasource/wikidata/service_test.go @@ -6,9 +6,65 @@ import ( gowikidata "github.com/Navid2zp/go-wikidata" "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/precisiondate" ) +func TestAuthorLinks(t *testing.T) { + author := Author{ + website: "https://author.example", + wikipediaLink: map[string]string{ + "en": "https://en.wikipedia.org/wiki/Example", + "es": "https://es.wikipedia.org/wiki/Ejemplo", + "fr": "", + }, + } + want := map[string]string{ + "website": "https://author.example", + "wikipedia:en": "https://en.wikipedia.org/wiki/Example", + "wikipedia:es": "https://es.wikipedia.org/wiki/Ejemplo", + } + if got := author.Links(); !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected author links: %+v", got) + } + links := author.Links() + links["website"] = "https://changed.example" + links["wikipedia:en"] = "https://changed.example/en" + if got := author.Links(); !reflect.DeepEqual(got, want) { + t.Fatalf("caller mutated author links: %+v", got) + } + for _, key := range []string{"website", "wikipedia:en", "wikipedia:de"} { + if !author.OwnsLink(key) { + t.Errorf("Wikidata should own %q, including missing language links", key) + } + } + for _, key := range []string{"goodreads", "catalog", "wikipedia", "website:other"} { + if author.OwnsLink(key) { + t.Errorf("Wikidata should not own %q", key) + } + } + if links := (Author{}).Links(); len(links) != 0 { + t.Fatalf("expected no links for an empty author: %+v", links) + } +} + +func TestWikipediaDataSourceLink(t *testing.T) { + source := NewWikidataSource(Gowikidata{}) + link := source.AuthorLink(index.Author{ + DataSourceLinks: map[string]string{ + "wikipedia:en": "https://en.wikipedia.org/wiki/Example", + "wikipedia:es": "https://es.wikipedia.org/wiki/Ejemplo", + }, + }, "es") + if link.Source != "Wikipedia" || link.URL != "https://es.wikipedia.org/wiki/Ejemplo" || + link.IconClass != "bi-wikipedia" || link.Label != "Go to Wikipedia entry" { + t.Fatalf("unexpected Wikipedia data-source link: %+v", link) + } + if missing := source.AuthorLink(index.Author{}, "en"); missing.URL != "" { + t.Fatalf("expected no link for missing Wikipedia URL, got %+v", missing) + } +} + func TestAuthor(t *testing.T) { mockServer := NewMockServer(t, "fixtures") defer mockServer.Close() diff --git a/internal/index/author.go b/internal/index/author.go index 757b52c4..98fcdf5c 100644 --- a/internal/index/author.go +++ b/internal/index/author.go @@ -1,6 +1,7 @@ package index import ( + "maps" "strings" "time" @@ -15,16 +16,31 @@ type Author struct { BirthName string DataSourceID string RetrievedOn time.Time - WikipediaLink map[string]string InstanceOf float64 Description map[string]string DateOfBirth precisiondate.PrecisionDate DateOfDeath precisiondate.PrecisionDate - Website string + DataSourceLinks map[string]string DataSourceImage string Gender float64 Pseudonyms []string DocumentCount uint64 + // ownsRefreshedLink marks links replaced by a metadata refresh or explicit reset. + ownsRefreshedLink func(string) bool +} + +func (a Author) ClearMetadata(ownsLink func(string) bool) Author { + links := maps.Clone(a.DataSourceLinks) + for source := range links { + if ownsLink(source) { + delete(links, source) + } + } + return Author{ + Slug: a.Slug, Name: a.Name, DocumentCount: a.DocumentCount, + RetrievedOn: time.Now().UTC(), DataSourceLinks: links, + ownsRefreshedLink: ownsLink, + } } func (a Author) YearOfBirthAbs() int { diff --git a/internal/index/author_search_test.go b/internal/index/author_search_test.go index 6a9879ad..5304cd74 100644 --- a/internal/index/author_search_test.go +++ b/internal/index/author_search_test.go @@ -258,7 +258,7 @@ func TestSearchAuthors(t *testing.T) { }, } for _, doc := range docs { - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } } diff --git a/internal/index/authors_enrich_test.go b/internal/index/authors_enrich_test.go index 52199f68..fce084b0 100644 --- a/internal/index/authors_enrich_test.go +++ b/internal/index/authors_enrich_test.go @@ -2,6 +2,8 @@ package index_test import ( "errors" + "maps" + "strings" "sync" "testing" "time" @@ -76,7 +78,9 @@ func (m *mockAuthorDataSource) RetrieveAuthors(candidates map[string][]string, _ } type stubAuthor struct { - sourceID string + sourceID string + links map[string]string + linkPrefix string } func (a stubAuthor) BirthName() string { return "" } @@ -86,11 +90,131 @@ func (a stubAuthor) Gender() float64 { return 0 } func (a stubAuthor) DateOfBirth() precisiondate.PrecisionDate { return precisiondate.PrecisionDate{} } func (a stubAuthor) DateOfDeath() precisiondate.PrecisionDate { return precisiondate.PrecisionDate{} } func (a stubAuthor) Image() string { return "" } -func (a stubAuthor) Website() string { return "" } -func (a stubAuthor) WikipediaLink(string) string { return "" } -func (a stubAuthor) SourceID() string { return a.sourceID } -func (a stubAuthor) RetrievedOn() time.Time { return time.Now().UTC() } -func (a stubAuthor) Pseudonyms() []string { return nil } +func (a stubAuthor) Links() map[string]string { return maps.Clone(a.links) } +func (a stubAuthor) OwnsLink(key string) bool { + return a.linkPrefix != "" && strings.HasPrefix(key, a.linkPrefix) +} +func (a stubAuthor) SourceID() string { return a.sourceID } +func (a stubAuthor) RetrievedOn() time.Time { return time.Now().UTC() } +func (a stubAuthor) Pseudonyms() []string { return nil } + +func TestAuthorMetadataRefreshUpdatesLinks(t *testing.T) { + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + defer authors.Close() + idx := index.NewBleve(nil, authors, nil, "", nil, index.Config{}) + author := index.Author{ + Slug: "writer", Name: "Writer", + DataSourceLinks: map[string]string{ + "example:website": "https://old.example", + "example:en": "https://en.example/Old", + "example:es": "https://es.example/Old", + "goodreads": "/author/show/1.Writer", + "catalog": "https://catalog.example/writer", + }, + } + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + for _, source := range []stubAuthor{ + {sourceID: "Q1", linkPrefix: "example:", links: map[string]string{ + "example:website": "https://new.example", "example:en": "https://en.example/New", + }}, + {sourceID: "Q1", linkPrefix: "example:"}, + } { + index.CombineWithDataSource(&author, source, []string{"en", "es"}) + current, err := idx.Author("writer", "en") + if err != nil { + t.Fatal(err) + } + current.DataSourceLinks["another-provider"] = "https://another.example/writer" + if err := idx.IndexAuthor(current); err != nil { + t.Fatal(err) + } + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + got, err := idx.Author("writer", "en") + if err != nil { + t.Fatal(err) + } + if got.DataSourceLinks["example:website"] != source.links["example:website"] || + got.DataSourceLinks["example:en"] != source.links["example:en"] || + got.DataSourceLinks["example:es"] != "" { + t.Fatalf("metadata links were not refreshed: %+v", got.DataSourceLinks) + } + if got.DataSourceLinks["goodreads"] != "/author/show/1.Writer" || + got.DataSourceLinks["catalog"] != "https://catalog.example/writer" || + got.DataSourceLinks["another-provider"] != "https://another.example/writer" { + t.Fatalf("metadata refresh lost independent provider links: %+v", got.DataSourceLinks) + } + if source.links != nil { + author.DataSourceLinks["example:website"] = "https://caller.example" + if source.links["example:website"] != "https://new.example" { + t.Fatal("combining metadata retained a mutable datasource map") + } + } + } +} + +func TestAuthorMetadataResetPreservesConcurrentProviderLinks(t *testing.T) { + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + defer authors.Close() + idx := index.NewBleve(nil, authors, nil, "", nil, index.Config{}) + author := index.Author{ + Slug: "writer", Name: "Writer", DataSourceID: "Q1", DocumentCount: 3, + DataSourceLinks: map[string]string{ + "example:website": "https://old.example", "example:en": "https://en.example/Old", + "catalog": "https://catalog.example/writer", + }, + } + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + reset := author.ClearMetadata((stubAuthor{linkPrefix: "example:"}).OwnsLink) + if author.DataSourceLinks["example:website"] == "" { + t.Fatal("reset must not mutate the caller's links") + } + current, err := idx.Author("writer", "en") + if err != nil { + t.Fatal(err) + } + current.DataSourceLinks["goodreads"] = "/author/show/1.Writer" + if err := idx.IndexAuthor(current); err != nil { + t.Fatal(err) + } + if err := idx.IndexAuthor(reset); err != nil { + t.Fatal(err) + } + stored, err := idx.Author("writer", "en") + if err != nil { + t.Fatal(err) + } + if stored.DataSourceLinks["example:website"] != "" || stored.DataSourceLinks["example:en"] != "" || + stored.DataSourceID != "" || stored.RetrievedOn.IsZero() { + t.Fatalf("reset must remove metadata association and owned links: %+v", stored) + } + if stored.DataSourceLinks["catalog"] != "https://catalog.example/writer" || + stored.DataSourceLinks["goodreads"] != "/author/show/1.Writer" || stored.DocumentCount != 3 { + t.Fatalf("reset must retain independent and concurrent author data: %+v", stored) + } + stale := index.Author{Slug: "writer", Name: "Writer"} + if err := idx.IndexAuthor(stale); err != nil { + t.Fatal(err) + } + stored, err = idx.Author("writer", "en") + if err != nil { + t.Fatal(err) + } + if len(stored.DataSourceLinks) != 2 { + t.Fatalf("ordinary stale writes must preserve independent links: %+v", stored.DataSourceLinks) + } +} func TestAuthorsWithoutInfo(t *testing.T) { authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) diff --git a/internal/index/bleve.go b/internal/index/bleve.go index da4cc666..93e3d513 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -4,6 +4,7 @@ import ( "errors" "log" "path/filepath" + "slices" "strconv" "strings" "sync" @@ -30,11 +31,11 @@ import ( // DocumentVersion identifies the mapping used for indexing documents. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. -const DocumentVersion = "v13" +const DocumentVersion = "v14" // AuthorVersion identifies the mapping used for indexing authors. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. -const AuthorVersion = "2" +const AuthorVersion = "3" // Metadata fields var ( @@ -182,6 +183,7 @@ type Config struct { // to already be resolved (e.g. via ResolveMetadataWorkers); <= 0 falls // back to 1. TextRankEnrichWorkers int + DocumentEnrichers []DocumentEnricher } type BleveIndexer struct { @@ -215,6 +217,15 @@ type BleveIndexer struct { // textRankEnrichJobs feeds documents queued by indexFile to a fixed-size // pool of long-lived worker goroutines; see runTextRankEnrichWorker. textRankEnrichJobs chan Document + documentEnrichJobs chan string + documentEnrichers []DocumentEnricher + // documentEnrichProgress measures each provider's current run rate, by enricher index. + documentEnrichProgress []progressTracker + providerProgressMu sync.Mutex + providerProgressUntil time.Time + providerProgress []providerProgressCount + providerProgressVersion atomic.Uint64 + cachedProviderProgressVersion uint64 // fileLocks holds a *sync.Mutex per document ID, serializing concurrent // indexFile calls for the same file (upload vs. file watcher reacting to // the same write). @@ -261,6 +272,16 @@ func (p *progressTracker) recordN(n int) { p.processed.Add(uint64(n)) } +// remainingTime estimates how long the given remaining entries take at the current run's rate. +func (p *progressTracker) remainingTime(remaining uint64) time.Duration { + start, processed := p.startNanos.Load(), p.processed.Load() + if start == 0 || processed == 0 || remaining == 0 { + return 0 + } + elapsed := float64(time.Now().UnixNano() - start) + return time.Duration(elapsed * float64(remaining) / float64(processed)) +} + // NewBleve creates a new BleveIndexer instance using the passed parameters func NewBleve(documentsIndex bleve.Index, authorsIndex bleve.Index, fs afero.Fs, libraryPath string, read map[string]metadata.Reader, cfg Config) *BleveIndexer { maxSimilarityCandidates := cfg.MaxSimilarityCandidates @@ -319,12 +340,21 @@ func NewBleve(documentsIndex bleve.Index, authorsIndex bleve.Index, fs afero.Fs, minCommonTextRankAbsoluteCount: minCommonTextRankAbsoluteCount, pruneChangeTriggerRatio: pruneChangeTriggerRatio, textRankEnrichJobs: make(chan Document, textRankEnrichWorkers), + documentEnrichers: slices.Clone(cfg.DocumentEnrichers), + documentEnrichProgress: make([]progressTracker, len(cfg.DocumentEnrichers)), } for range textRankEnrichWorkers { go b.runTextRankEnrichWorker() } + if len(b.documentEnrichers) > 0 { + b.documentEnrichJobs = make(chan string, documentEnrichWorkers) + for range documentEnrichWorkers { + go b.runDocumentEnrichWorker() + } + } + return b } @@ -376,6 +406,17 @@ func CreateDocumentsMapping() mapping.IndexMapping { keywordFieldMapping := bleve.NewKeywordFieldMapping() + // Stored only, so the field can be read back but never matches a search. + // IncludeInAll=false is redundant with Index=false today, but keeps the + // field out of the composite _all field explicitly. + storedOnlyFieldMapping := bleve.NewKeywordFieldMapping() + storedOnlyFieldMapping.Index = false + storedOnlyFieldMapping.IncludeInAll = false + storedOnlyFieldMapping.DocValues = false + + enrichedSourcesFieldMapping := bleve.NewKeywordFieldMapping() + enrichedSourcesFieldMapping.IncludeInAll = false + simpleTextFieldMapping := bleve.NewTextFieldMapping() simpleTextFieldMapping.Analyzer = defaultAnalyzer @@ -434,6 +475,9 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.TypeMapping[lang].AddFieldMappingsAt("Illustrations", numericFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.TypeMapping[lang].AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSources", storedOnlyFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("EnrichedDataSources", enrichedSourcesFieldMapping) + indexMapping.TypeMapping[lang].AddFieldMappingsAt("DataSourceScoreValue", numericFieldMapping) } indexMapping.DefaultMapping.DefaultAnalyzer = defaultAnalyzer @@ -458,6 +502,9 @@ func CreateDocumentsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("Illustrations", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("AddedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("TextRankEnriched", booleanFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("DataSources", storedOnlyFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("EnrichedDataSources", enrichedSourcesFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceScoreValue", numericFieldMapping) return indexMapping } @@ -483,6 +530,10 @@ func CreateAuthorsMapping() mapping.IndexMapping { keywordFieldMapping := bleve.NewKeywordFieldMapping() keywordFieldMappingNotIndexable := bleve.NewKeywordFieldMapping() keywordFieldMappingNotIndexable.Index = false + storedOnlyFieldMapping := bleve.NewKeywordFieldMapping() + storedOnlyFieldMapping.Index = false + storedOnlyFieldMapping.IncludeInAll = false + storedOnlyFieldMapping.DocValues = false simpleTextFieldMapping := bleve.NewTextFieldMapping() simpleTextFieldMapping.Analyzer = defaultAnalyzer @@ -497,7 +548,7 @@ func CreateAuthorsMapping() mapping.IndexMapping { indexMapping.DefaultMapping.AddFieldMappingsAt("RetrievedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceID", keywordFieldMappingNotIndexable) indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceImage", keywordFieldMappingNotIndexable) - indexMapping.DefaultMapping.AddFieldMappingsAt("Website", keywordFieldMappingNotIndexable) + indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceLinks", storedOnlyFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DateOfBirth.Date", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DateOfBirth.Precision", numericFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DateOfDeath.Date", numericFieldMapping) diff --git a/internal/index/bleve_author_migrate.go b/internal/index/bleve_author_migrate.go new file mode 100644 index 00000000..4fdc6900 --- /dev/null +++ b/internal/index/bleve_author_migrate.go @@ -0,0 +1,136 @@ +package index + +import ( + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/blevesearch/bleve/v2" +) + +// OpenAuthorsIndex upgrades version 2 without discarding stored author metadata. +// The previous index is kept only until the migrated index opens successfully. +func OpenAuthorsIndex(path string, batchSize int) (bleve.Index, error) { + authors, err := bleve.Open(path) + if errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { + backup := path + ".v2-backup" + if _, statErr := os.Stat(backup); statErr == nil { + // Recover an interrupted swap before treating the index as missing. + if err := os.Rename(backup, path); err != nil { + return nil, fmt.Errorf("restoring authors index: %w", err) + } + authors, err = bleve.Open(path) + } else if !os.IsNotExist(statErr) { + return nil, fmt.Errorf("checking authors index backup: %w", statErr) + } + } + if err != nil { + return nil, err + } + version, err := authors.GetInternal(internalVersion) + if err != nil { + return nil, errors.Join(err, authors.Close()) + } + if string(version) == "2" { + upgraded, err := migrateAuthorsIndex(authors, path, batchSize) + if err != nil { + return nil, errors.Join(err, authors.Close()) + } + authors = upgraded + log.Printf("Migrated authors index to version %s", AuthorVersion) + } else if string(version) != AuthorVersion { + return authors, nil + } + if err := os.RemoveAll(path + ".v2-backup"); err != nil { + return nil, errors.Join(fmt.Errorf("removing previous authors index: %w", err), authors.Close()) + } + return authors, nil +} + +func migrateAuthorsIndex(authors bleve.Index, path string, batchSize int) (bleve.Index, error) { + if batchSize <= 0 { + return nil, fmt.Errorf("authors migration batch size must be positive") + } + backup := path + ".v2-backup" + if _, err := os.Stat(backup); err == nil { + return nil, fmt.Errorf("authors migration backup already exists: %s", backup) + } else if !os.IsNotExist(err) { + return nil, err + } + stagingDir, err := os.MkdirTemp(filepath.Dir(path), ".authors-migration-") + if err != nil { + return nil, err + } + defer func() { + if err := os.RemoveAll(stagingDir); err != nil { + log.Printf("Error removing authors migration staging directory: %s", err) + } + }() + stagingPath := filepath.Join(stagingDir, "index") + staged, err := bleve.New(stagingPath, CreateAuthorsMapping()) + if err != nil { + return nil, err + } + if err := copyAuthors(authors, staged, batchSize); err != nil { + return nil, errors.Join(err, staged.Close()) + } + if err := staged.Close(); err != nil { + return nil, err + } + if err := authors.Close(); err != nil { + return nil, err + } + if err := os.Rename(path, backup); err != nil { + return nil, err + } + if err := os.Rename(stagingPath, path); err != nil { + return nil, errors.Join(err, os.Rename(backup, path)) + } + return bleve.Open(path) +} + +func copyAuthors(source, destination bleve.Index, batchSize int) error { + for from := 0; ; from += batchSize { + req := bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), batchSize, from, false) + req.Fields = []string{"*"} + req.SortBy([]string{"_id"}) + result, err := source.Search(req) + if err != nil { + return err + } + if len(result.Hits) == 0 { + break + } + batch := destination.NewBatch() + for _, hit := range result.Hits { + author := hydrateAuthor(hit) + for field, value := range hit.Fields { + source := "" + switch { + case field == "Website": + source = "website" + case strings.HasPrefix(field, "WikipediaLink."): + source = "wikipedia:" + strings.TrimPrefix(field, "WikipediaLink.") + } + if link, ok := value.(string); source != "" && ok && link != "" { + if author.DataSourceLinks == nil { + author.DataSourceLinks = make(map[string]string) + } + if author.DataSourceLinks[source] == "" { + author.DataSourceLinks[source] = link + } + } + } + if err := indexAuthor(batch, hit.ID, author); err != nil { + return err + } + } + if err := destination.Batch(batch); err != nil { + return err + } + } + return destination.SetInternal(internalVersion, []byte(AuthorVersion)) +} diff --git a/internal/index/bleve_author_migrate_test.go b/internal/index/bleve_author_migrate_test.go new file mode 100644 index 00000000..4d03ef04 --- /dev/null +++ b/internal/index/bleve_author_migrate_test.go @@ -0,0 +1,261 @@ +package index + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/rickb777/date/v2" + "github.com/svera/coreander/v5/internal/precisiondate" +) + +func legacyAuthorsIndex(t *testing.T, path string) bleve.Index { + t.Helper() + m := CreateAuthorsMapping().(*mapping.IndexMappingImpl) + delete(m.DefaultMapping.Properties, "DataSourceLinks") + idx, err := bleve.New(path, m) + if err != nil { + t.Fatal(err) + } + if err := idx.SetInternal(internalVersion, []byte("2")); err != nil { + t.Fatal(err) + } + return idx +} + +func TestOpenAuthorsIndexMigratesStoredMetadata(t *testing.T) { + path := filepath.Join(t.TempDir(), "authors_index") + legacy := legacyAuthorsIndex(t, path) + retrieved := time.Date(2026, 10, 28, 12, 0, 0, 0, time.UTC) + want := Author{ + Slug: "alice-borchardt", Name: "Alice Borchardt", BirthName: "Alice O'Brien", + DataSourceID: "Q1", RetrievedOn: retrieved, + DataSourceLinks: map[string]string{ + "wikipedia:en": "https://en.wikipedia.org/wiki/Alice_Borchardt", + "wikipedia:es": "https://es.wikipedia.org/wiki/Alice_Borchardt", + "website": "https://example.org", + }, + Description: map[string]string{"en": "Writer", "es": "Escritora"}, InstanceOf: 5, + DateOfBirth: precisiondate.PrecisionDate{Date: date.New(1939, 10, 6), Precision: precisiondate.PrecisionDay}, + DateOfDeath: precisiondate.PrecisionDate{Date: date.New(2007, 7, 24), Precision: precisiondate.PrecisionDay}, + DataSourceImage: "image.jpg", Gender: 6581072, + Pseudonyms: []string{"Alice", "A. Borchardt"}, DocumentCount: 7, + } + for _, author := range []Author{want, {Slug: "other", Name: "Other", RetrievedOn: retrieved}, {Slug: "third", Name: "Third", RetrievedOn: retrieved}} { + if err := legacy.Index(author.Slug, struct { + Author + Website string + WikipediaLink map[string]string + }{author, author.DataSourceLinks["website"], map[string]string{ + "en": author.DataSourceLinks["wikipedia:en"], "es": author.DataSourceLinks["wikipedia:es"], + }}); err != nil { + t.Fatal(err) + } + } + if err := legacy.Close(); err != nil { + t.Fatal(err) + } + + upgraded, err := OpenAuthorsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + documents, err := bleve.NewMemOnly(CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + idx := NewBleve(documents, upgraded, nil, "", nil, Config{}) + got, err := idx.Author(want.Slug, "en") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("migration changed author metadata:\ngot: %#v\nwant: %#v", got, want) + } + if count, err := idx.TotalAuthors(); err != nil || count != 3 { + t.Fatalf("expected all 3 authors migrated, got %d, %v", count, err) + } + if pending, err := idx.AuthorsWithoutInfo(); err != nil || len(pending) != 0 { + t.Fatalf("migration must not trigger new Wikidata lookups: %+v, %v", pending, err) + } + if version, err := upgraded.GetInternal(internalVersion); err != nil || string(version) != AuthorVersion { + t.Fatalf("unexpected migrated version %q, %v", version, err) + } + if _, err := os.Stat(path + ".v2-backup"); !os.IsNotExist(err) { + t.Fatalf("expected original index removed after successful migration, got %v", err) + } + + got.DataSourceLinks["goodreads"] = "https://example.org/authors/writer" + if err := idx.IndexAuthor(got); err != nil { + t.Fatal(err) + } + if err := idx.Close(); err != nil { + t.Fatal(err) + } + reopened, err := OpenAuthorsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{want.Slug})) + req.Fields = []string{"*"} + result, err := reopened.Search(req) + if err != nil || len(result.Hits) != 1 { + t.Fatalf("reading reopened author: %v", err) + } + want.DataSourceLinks["goodreads"] = "https://example.org/authors/writer" + if got := hydrateAuthor(result.Hits[0]); !reflect.DeepEqual(got, want) { + t.Fatalf("migrated mapping or URL did not survive reopening: %#v", got) + } +} + +func TestOpenAuthorsIndexRecoversInterruptedSwap(t *testing.T) { + path := filepath.Join(t.TempDir(), "authors_index") + legacy := legacyAuthorsIndex(t, path+".v2-backup") + if err := legacy.Index("saved", Author{Slug: "saved", Name: "Saved Author", DataSourceID: "Q1"}); err != nil { + t.Fatal(err) + } + if err := legacy.Close(); err != nil { + t.Fatal(err) + } + upgraded, err := OpenAuthorsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + defer upgraded.Close() + if count, err := upgraded.DocCount(); err != nil || count != 1 { + t.Fatalf("expected saved author recovered, got %d, %v", count, err) + } +} + +func TestOpenAuthorsIndexRemovesPreviouslyRetainedBackup(t *testing.T) { + path := filepath.Join(t.TempDir(), "authors_index") + backup := legacyAuthorsIndex(t, path+".v2-backup") + if err := backup.Close(); err != nil { + t.Fatal(err) + } + current := CreateAuthorsIndex(path) + if err := current.Index("saved", Author{Slug: "saved", Name: "Saved Author", DataSourceID: "Q1"}); err != nil { + t.Fatal(err) + } + if err := current.Close(); err != nil { + t.Fatal(err) + } + reopened, err := OpenAuthorsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if count, err := reopened.DocCount(); err != nil || count != 1 { + t.Fatalf("current index changed while removing backup: %d, %v", count, err) + } + if _, err := os.Stat(path + ".v2-backup"); !os.IsNotExist(err) { + t.Fatalf("expected previously retained backup removed, got %v", err) + } +} + +func TestOpenAuthorsIndexMigrationFailurePreservesOriginal(t *testing.T) { + path := filepath.Join(t.TempDir(), "authors_index") + legacy := legacyAuthorsIndex(t, path) + if err := legacy.Index("saved", Author{Slug: "saved", Name: "Saved Author", DataSourceID: "Q1"}); err != nil { + t.Fatal(err) + } + + if err := legacy.Close(); err != nil { + t.Fatal(err) + } + if _, err := OpenAuthorsIndex(path, 0); err == nil { + t.Fatal("expected invalid batch size to fail migration") + } + original, err := bleve.Open(path) + if err != nil { + t.Fatal(err) + } + defer original.Close() + if version, err := original.GetInternal(internalVersion); err != nil || string(version) != "2" { + t.Fatalf("original version changed after migration failure: %q, %v", version, err) + } + if count, err := original.DocCount(); err != nil || count != 1 { + t.Fatalf("original metadata lost after migration failure: %d, %v", count, err) + } +} + +func TestOpenAuthorsIndexMigratesLegacyAuthorLinks(t *testing.T) { + path := filepath.Join(t.TempDir(), "authors_index") + m := CreateAuthorsMapping().(*mapping.IndexMappingImpl) + delete(m.DefaultMapping.Properties, "DataSourceLinks") + legacyURL := bleve.NewKeywordFieldMapping() + legacyURL.Index = false + legacyURL.IncludeInAll = false + legacyURL.DocValues = false + m.DefaultMapping.AddFieldMappingsAt("Website", legacyURL) + legacy, err := bleve.New(path, m) + if err != nil { + t.Fatal(err) + } + fields := map[string]any{ + "Slug": "alice", "Name": "Alice", + "DataSourceID": "Q1", "DocumentCount": 7, + "Website": "https://example.org", + "WikipediaLink": map[string]string{ + "en": "https://en.wikipedia.org/wiki/Alice", + "es": "https://es.wikipedia.org/wiki/Alice", + }, + } + if err := legacy.Index("alice", fields); err != nil { + t.Fatal(err) + } + if err := legacy.SetInternal(internalVersion, []byte("2")); err != nil { + t.Fatal(err) + } + if err := legacy.Close(); err != nil { + t.Fatal(err) + } + upgraded, err := OpenAuthorsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + idx := NewBleve(nil, upgraded, nil, "", nil, Config{}) + author, err := idx.Author("alice", "") + if err != nil || author.DataSourceID != "Q1" || author.DocumentCount != 7 || + author.DataSourceLinks["website"] != "https://example.org" || + author.DataSourceLinks["wikipedia:en"] != "https://en.wikipedia.org/wiki/Alice" || + author.DataSourceLinks["wikipedia:es"] != "https://es.wikipedia.org/wiki/Alice" { + t.Fatalf("migration lost legacy links or metadata: %+v, %v", author, err) + } + author.DataSourceLinks["other"] = "https://example.org/alice" + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + if err := upgraded.Close(); err != nil { + t.Fatal(err) + } + reopened, err := OpenAuthorsIndex(path, 1) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{"alice"})) + req.Fields = []string{"*"} + result, err := reopened.Search(req) + if err != nil || len(result.Hits) != 1 { + t.Fatalf("reading reopened author: %v", err) + } + got := hydrateAuthor(result.Hits[0]) + if !reflect.DeepEqual(got, author) { + t.Fatalf("links or metadata changed after reopening:\ngot: %#v\nwant: %#v", got, author) + } + for field := range result.Hits[0].Fields { + if field == "Website" || strings.HasPrefix(field, "WikipediaLink.") { + t.Fatalf("migrated author still stores legacy field %s", field) + } + } + if _, err := os.Stat(path + ".v2-backup"); !os.IsNotExist(err) { + t.Fatalf("migration backup was not removed: %v", err) + } +} diff --git a/internal/index/bleve_author_read.go b/internal/index/bleve_author_read.go index 43932d9a..6f049541 100644 --- a/internal/index/bleve_author_read.go +++ b/internal/index/bleve_author_read.go @@ -1,6 +1,8 @@ package index import ( + "encoding/json" + "log" "strings" "time" @@ -39,9 +41,6 @@ func (b *BleveIndexer) Author(slug, lang string) (Author, error) { author := hydrateAuthor(searchResult.Hits[0]) // Override language-specific fields if requested language is available - if value, ok := searchResult.Hits[0].Fields["WikipediaLink."+lang].(string); ok { - author.WikipediaLink[lang] = value - } if value, ok := searchResult.Hits[0].Fields["Description."+lang].(string); ok { author.Description[lang] = value } @@ -116,13 +115,6 @@ func hydrateAuthorFromFields(fields map[string]any, docID string) Author { } } - website := "" - if val, ok := fields["Website"]; ok && val != nil { - if str, ok := val.(string); ok { - website = str - } - } - dataSourceImage := "" if val, ok := fields["DataSourceImage"]; ok && val != nil { if str, ok := val.(string); ok { @@ -157,26 +149,22 @@ func hydrateAuthorFromFields(fields map[string]any, docID string) Author { Slug: slug, DataSourceID: dataSourceID, RetrievedOn: retrievedOn, - WikipediaLink: make(map[string]string), InstanceOf: instanceOf, Description: make(map[string]string), DateOfBirth: dateOfBirth, DateOfDeath: dateOfDeath, - Website: website, DataSourceImage: dataSourceImage, Gender: gender, Pseudonyms: slicer(fields["Pseudonyms"]), DocumentCount: documentCount, } - - // Extract Wikipedia links and descriptions for all languages - for key, value := range fields { - if strings.HasPrefix(key, "WikipediaLink.") { - lang := strings.TrimPrefix(key, "WikipediaLink.") - if str, ok := value.(string); ok { - author.WikipediaLink[lang] = str - } + if links, ok := fields["DataSourceLinks"].(string); ok { + if err := json.Unmarshal([]byte(links), &author.DataSourceLinks); err != nil { + log.Printf("Error reading data source links for author %s: %s", docID, err) } + } + // Extract descriptions for all languages. + for key, value := range fields { if strings.HasPrefix(key, "Description.") { lang := strings.TrimPrefix(key, "Description.") if str, ok := value.(string); ok { diff --git a/internal/index/bleve_author_write.go b/internal/index/bleve_author_write.go index 2b3dc909..4f25ae9c 100644 --- a/internal/index/bleve_author_write.go +++ b/internal/index/bleve_author_write.go @@ -1,13 +1,29 @@ package index import ( + "encoding/json" + "fmt" "log" + "maps" "time" "github.com/blevesearch/bleve/v2" datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" ) +// indexAuthor stores links as one stored-only field so new sources need no mapping changes. +func indexAuthor(destination interface{ Index(string, any) error }, id string, author Author) error { + links, err := json.Marshal(author.DataSourceLinks) + if err != nil { + return err + } + author.DataSourceLinks = nil + return destination.Index(id, struct { + Author + DataSourceLinks string + }{author, string(links)}) +} + func (b *BleveIndexer) incrementAuthorCounts(names, slugs []string) error { for i, name := range names { if i < len(slugs) { @@ -25,7 +41,9 @@ func (b *BleveIndexer) incrementAuthorCount(name, authorSlug string) error { if name == "" || authorSlug == "" { return nil } - existing, err := b.Author(authorSlug, "") + b.authorsMu.Lock() + defer b.authorsMu.Unlock() + existing, err := b.authorBySlugLocked(authorSlug) if err != nil { return err } @@ -34,9 +52,7 @@ func (b *BleveIndexer) incrementAuthorCount(name, authorSlug string) error { } else { existing.DocumentCount++ } - b.authorsMu.Lock() - defer b.authorsMu.Unlock() - return b.authorsIdx.Index(authorSlug, existing) + return indexAuthor(b.authorsIdx, authorSlug, existing) } // RebuildAuthorsFromDocuments recalculates DocumentCount for every author from the documents @@ -51,10 +67,10 @@ func (b *BleveIndexer) RebuildAuthorsFromDocuments(batchSize int) error { } // Fetch all existing authors so their enriched metadata is preserved. - b.authorsMu.RLock() + b.authorsMu.Lock() + defer b.authorsMu.Unlock() authorDocCount, err := b.authorsIdx.DocCount() if err != nil { - b.authorsMu.RUnlock() return err } existingAuthors := make(map[string]Author, authorDocCount) @@ -63,7 +79,6 @@ func (b *BleveIndexer) RebuildAuthorsFromDocuments(batchSize int) error { req.Fields = []string{"*"} result, err := b.authorsIdx.Search(req) if err != nil { - b.authorsMu.RUnlock() return err } for _, hit := range result.Hits { @@ -71,7 +86,6 @@ func (b *BleveIndexer) RebuildAuthorsFromDocuments(batchSize int) error { existingAuthors[a.Slug] = a } } - b.authorsMu.RUnlock() batch := b.authorsIdx.NewBatch() for authorSlug, count := range counts { @@ -80,13 +94,11 @@ func (b *BleveIndexer) RebuildAuthorsFromDocuments(batchSize int) error { author = Author{Name: names[authorSlug], Slug: authorSlug} } author.DocumentCount = count - if err := batch.Index(authorSlug, author); err != nil { + if err := indexAuthor(batch, authorSlug, author); err != nil { return err } if batch.Size() >= batchSize { - b.authorsMu.Lock() err := b.authorsIdx.Batch(batch) - b.authorsMu.Unlock() if err != nil { return err } @@ -94,8 +106,6 @@ func (b *BleveIndexer) RebuildAuthorsFromDocuments(batchSize int) error { } } if batch.Size() > 0 { - b.authorsMu.Lock() - defer b.authorsMu.Unlock() return b.authorsIdx.Batch(batch) } return nil @@ -105,6 +115,9 @@ func (b *BleveIndexer) RebuildAuthorsFromDocuments(batchSize int) error { // and one representative name per author slug. Pages through the index batchSize documents // at a time, requesting only the four contributor fields it needs. func (b *BleveIndexer) countDocumentsPerAuthor(batchSize int) (counts map[string]uint64, names map[string]string, err error) { + if batchSize <= 0 { + return nil, nil, fmt.Errorf("author rebuild batch size must be positive") + } b.documentsMu.RLock() docCount, err := b.documentsIdx.DocCount() b.documentsMu.RUnlock() @@ -121,6 +134,7 @@ func (b *BleveIndexer) countDocumentsPerAuthor(batchSize int) (counts map[string for from := 0; uint64(from) < docCount; from += batchSize { req := bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), batchSize, from, false) req.Fields = []string{"Authors", "AuthorsSlugs", "Illustrators", "IllustratorsSlugs"} + req.SortBy([]string{"_id"}) b.documentsMu.RLock() result, err := b.documentsIdx.Search(req) b.documentsMu.RUnlock() @@ -154,12 +168,39 @@ func accumulateContributors(counts map[string]uint64, names map[string]string, s func (b *BleveIndexer) IndexAuthor(author Author) error { b.authorsMu.Lock() defer b.authorsMu.Unlock() - if err := b.authorsIdx.Index(author.Slug, author); err != nil { + current, err := b.authorBySlugLocked(author.Slug) + if err != nil { + return err + } + // Link enrichment runs independently of metadata enrichment and UI edits. + links := maps.Clone(author.DataSourceLinks) + if len(current.DataSourceLinks) > 0 { + if links == nil { + links = make(map[string]string) + } + for source, link := range current.DataSourceLinks { + if author.ownsRefreshedLink == nil || !author.ownsRefreshedLink(source) { + links[source] = link + } + } + } + author.DataSourceLinks = links + if err := indexAuthor(b.authorsIdx, author.Slug, author); err != nil { return err } return nil } +func (b *BleveIndexer) authorBySlugLocked(slug string) (Author, error) { + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{slug})) + req.Fields = []string{"*"} + result, err := b.authorsIdx.Search(req) + if err != nil || len(result.Hits) == 0 { + return Author{}, err + } + return hydrateAuthor(result.Hits[0]), nil +} + func (b *BleveIndexer) beginAuthorEnrichment(total int) { b.authorEnrichProgress.begin(total) } @@ -190,12 +231,25 @@ func CombineWithDataSource(author *Author, authorDataSource datasourcemodel.Auth author.DataSourceID = authorDataSource.SourceID() author.BirthName = authorDataSource.BirthName() author.RetrievedOn = authorDataSource.RetrievedOn() - author.WikipediaLink = make(map[string]string) + author.DataSourceLinks = maps.Clone(author.DataSourceLinks) + author.ownsRefreshedLink = authorDataSource.OwnsLink + if author.DataSourceLinks == nil { + author.DataSourceLinks = make(map[string]string) + } + for source := range author.DataSourceLinks { + if authorDataSource.OwnsLink(source) { + delete(author.DataSourceLinks, source) + } + } author.InstanceOf = authorDataSource.InstanceOf() author.Description = make(map[string]string) author.DateOfBirth = authorDataSource.DateOfBirth() author.DateOfDeath = authorDataSource.DateOfDeath() - author.Website = authorDataSource.Website() + for key, link := range authorDataSource.Links() { + if link != "" { + author.DataSourceLinks[key] = link + } + } author.DataSourceImage = authorDataSource.Image() author.Gender = authorDataSource.Gender() author.Pseudonyms = make([]string, 0, len(authorDataSource.Pseudonyms())) @@ -207,7 +261,6 @@ func CombineWithDataSource(author *Author, authorDataSource datasourcemodel.Auth } for _, lang := range supportedLanguages { - author.WikipediaLink[lang] = authorDataSource.WikipediaLink(lang) author.Description[lang] = authorDataSource.Description(lang) } } diff --git a/internal/index/bleve_document_data_sources.go b/internal/index/bleve_document_data_sources.go new file mode 100644 index 00000000..4b7438a2 --- /dev/null +++ b/internal/index/bleve_document_data_sources.go @@ -0,0 +1,134 @@ +package index + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2" +) + +// AuthorLink is a provider-owned link discovered while enriching a document. +type AuthorLink struct { + Slug string + Name string + Source string + URL string +} + +// DocumentUpdate groups document changes and discovered author links. +type DocumentUpdate struct { + Document + AuthorLinks []AuthorLink +} + +// DocumentByID retrieves a document using its index ID rather than its public slug. +func (b *BleveIndexer) DocumentByID(id string) (Document, error) { + return b.documentByIndexID(id) +} + +// UpdateDocument applies changes to the latest stored document under the write lock. +// The callback returns false when no write is needed. It must not call other index methods. +func (b *BleveIndexer) UpdateDocument(id string, update func(*DocumentUpdate) (bool, error)) error { + b.documentsMu.Lock() + defer b.documentsMu.Unlock() + document, found, err := b.documentByIndexIDLocked(id) + if err != nil { + return fmt.Errorf("reading document %s for update: %w", id, err) + } + if !found { + return ErrDocumentNotFound + } + pending := DocumentUpdate{Document: document} + changed, err := update(&pending) + if err != nil || !changed { + return err + } + for _, link := range pending.AuthorLinks { + if err := b.storeAuthorLinkLocked(link); err != nil { + return err + } + } + if err := IndexDocument(b.documentsIdx, id, pending.Document); err != nil { + return err + } + b.providerProgressVersion.Add(1) + return nil +} + +// storeAuthorLinkLocked requires documentsMu to be held. +func (b *BleveIndexer) storeAuthorLinkLocked(link AuthorLink) error { + if link.Slug == "" || link.Source == "" || link.URL == "" { + return fmt.Errorf("author link must have a slug, source and URL") + } + b.authorsMu.Lock() + defer b.authorsMu.Unlock() + author, err := b.authorBySlugLocked(link.Slug) + if err != nil { + return err + } + if author.Slug == "" { + author = Author{Slug: link.Slug, Name: link.Name} + for _, field := range []string{"AuthorsSlugs", "IllustratorsSlugs"} { + q := bleve.NewTermQuery(link.Slug) + q.SetField(field) + result, err := b.documentsIdx.Search(bleve.NewSearchRequestOptions(q, 0, 0, false)) + if err != nil { + return fmt.Errorf("counting documents for author %s: %w", link.Slug, err) + } + author.DocumentCount += result.Total + } + } + if author.DataSourceLinks[link.Source] != "" { + return nil + } + if author.DataSourceLinks == nil { + author.DataSourceLinks = make(map[string]string) + } + author.DataSourceLinks[link.Source] = link.URL + return indexAuthor(b.authorsIdx, link.Slug, author) +} + +func (b *BleveIndexer) documentByIndexIDLocked(id string) (Document, bool, error) { + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) + req.Fields = []string{"*"} + result, err := b.documentsIdx.Search(req) + if err != nil || len(result.Hits) == 0 { + return Document{}, false, err + } + document := hydrateDocument(result.Hits[0]) + document.DataSources, err = decodeDataSources(result.Hits[0].Fields, id) + return document, true, err +} + +// preserveDataSourcesLocked protects provider updates and removals from stale writers. +// It requires documentsMu to be held for writing. +func (b *BleveIndexer) preserveDataSourcesLocked(documents []Document) error { + if len(documents) == 0 { + return nil + } + ids := make([]string, len(documents)) + for i, document := range documents { + ids[i] = document.ID + } + req := bleve.NewSearchRequestOptions(bleve.NewDocIDQuery(ids), len(ids), 0, false) + req.Fields = []string{"DataSources", "DataSourceScoreValue"} + result, err := b.documentsIdx.Search(req) + if err != nil { + return fmt.Errorf("reading stored data source fields: %w", err) + } + stored := make(map[string]Document, len(result.Hits)) + for _, hit := range result.Hits { + ratingValue, _ := hit.Fields["DataSourceScoreValue"].(float64) + sources, err := decodeDataSources(hit.Fields, hit.ID) + if err != nil { + return err + } + stored[hit.ID] = Document{DataSources: sources, DataSourceScoreValue: ratingValue} + } + for i := range documents { + if current, ok := stored[documents[i].ID]; ok { + documents[i].DataSources = current.DataSources + documents[i].DataSourceScoreValue = current.DataSourceScoreValue + } + } + return nil +} diff --git a/internal/index/bleve_document_enrich.go b/internal/index/bleve_document_enrich.go new file mode 100644 index 00000000..df19a292 --- /dev/null +++ b/internal/index/bleve_document_enrich.go @@ -0,0 +1,185 @@ +package index + +import ( + "context" + "errors" + "fmt" + "log" + "sync" + + "github.com/blevesearch/bleve/v2" +) + +// documentEnrichWorkers sets lookup parallelism for each pool. Enrichers must enforce +// their own source request limits because batch and upload pools run concurrently. +const documentEnrichWorkers = 4 + +// DocumentEnricher owns a provider's lookup policy and persisted payload. +type DocumentEnricher interface { + // Source is the persisted storage key; Name is shown to users. + Source() string + Name() string + NeedsEnrichment(Document) (bool, error) + // Enrich persists an enriched=true payload for completed lookups, including no matches. + // Transient failures must leave the document pending. Enrich may be called concurrently + // by multiple pools, so implementations must limit their own requests to the source. + Enrich(context.Context, *BleveIndexer, Document) error +} + +func (b *BleveIndexer) documentsNeedingProvider(source string, size int, excluded map[string]struct{}) ([]Document, uint64, error) { + enriched := bleve.NewTermQuery(source) + enriched.SetField("EnrichedDataSources") + q := bleve.NewBooleanQuery() + q.AddMust(bleve.NewMatchAllQuery()) + q.AddMustNot(enriched) + if len(excluded) > 0 { + ids := make([]string, 0, len(excluded)) + for id := range excluded { + ids = append(ids, id) + } + q.AddMustNot(bleve.NewDocIDQuery(ids)) + } + req := bleve.NewSearchRequestOptions(q, size, 0, false) + req.Fields = []string{"*"} + b.documentsMu.RLock() + result, err := b.documentsIdx.Search(req) + b.documentsMu.RUnlock() + if err != nil { + return nil, 0, err + } + documents := make([]Document, 0, len(result.Hits)) + for _, hit := range result.Hits { + documents = append(documents, hydrateDocument(hit)) + } + return documents, result.Total, nil +} + +// EnrichDocuments resumes pending lookups for every configured provider. +// A provider failure does not prevent independent providers from running. +func (b *BleveIndexer) EnrichDocuments(batchSize int) error { + if batchSize <= 0 { + return fmt.Errorf("document enrichment batch size must be positive") + } + var failures []error + for i, provider := range b.documentEnrichers { + if err := b.enrichProvider(provider, b.documentEnrichTracker(i), batchSize); err != nil { + failures = append(failures, fmt.Errorf("enriching documents with %s: %w", provider.Source(), err)) + } + } + return errors.Join(failures...) +} + +func (b *BleveIndexer) documentEnrichTracker(i int) *progressTracker { + if i < len(b.documentEnrichProgress) { + return &b.documentEnrichProgress[i] + } + return &progressTracker{} +} + +func (b *BleveIndexer) enrichProvider(provider DocumentEnricher, tracker *progressTracker, batchSize int) error { + _, total, err := b.documentsNeedingProvider(provider.Source(), 0, nil) + if err != nil || total == 0 { + return err + } + tracker.begin(int(total)) + defer tracker.end() + log.Printf("Enriching %d documents with %s data", total, provider.Source()) + var failures []error + failedDocuments := make(map[string]struct{}) + for iteration := 0; iteration < int(total)+10; iteration++ { + documents, _, err := b.documentsNeedingProvider(provider.Source(), batchSize, failedDocuments) + if err != nil { + failures = append(failures, err) + break + } + if len(documents) == 0 { + return errors.Join(failures...) + } + for id, err := range b.enrichProviderBatch(provider, tracker, documents) { + failedDocuments[id] = struct{}{} + failures = append(failures, err) + } + } + if len(failures) > 0 { + return errors.Join(failures...) + } + return fmt.Errorf("document enrichment did not finish; some documents remain pending") +} + +func (b *BleveIndexer) enrichProviderBatch(provider DocumentEnricher, tracker *progressTracker, documents []Document) map[string]error { + jobs := make(chan Document, len(documents)) + for _, document := range documents { + jobs <- document + } + close(jobs) + var workers sync.WaitGroup + var failureMu sync.Mutex + failures := make(map[string]error) + for range min(documentEnrichWorkers, len(documents)) { + workers.Go(func() { + for document := range jobs { + needed, err := provider.NeedsEnrichment(document) + if err == nil && needed { + err = provider.Enrich(context.Background(), b, document) + } + if err != nil { + failureMu.Lock() + failures[document.ID] = fmt.Errorf("enriching document %s: %w", document.ID, err) + failureMu.Unlock() + } + tracker.record() + } + }) + } + workers.Wait() + return failures +} + +func (b *BleveIndexer) runDocumentEnrichWorker() { + for id := range b.documentEnrichJobs { + for _, provider := range b.documentEnrichers { + document, err := b.documentByIndexID(id) + if err != nil { + log.Printf("Error reading document %s for %s enrichment: %s", id, provider.Source(), err) + continue + } + if document.ID == "" { + break + } + needed, err := provider.NeedsEnrichment(document) + if err == nil && needed { + err = provider.Enrich(context.Background(), b, document) + } + if err != nil { + log.Printf("Error enriching document %s with %s: %s", id, provider.Source(), err) + } + } + } +} + +func (b *BleveIndexer) scheduleDocumentEnrichment(id string) { + if len(b.documentEnrichers) > 0 { + go func() { b.documentEnrichJobs <- id }() + } +} + +// StoreDocumentData replaces only one provider's payload under the document write lock. +func (b *BleveIndexer) StoreDocumentData(documentID, source string, data any) error { + b.documentsMu.Lock() + defer b.documentsMu.Unlock() + document, found, err := b.documentByIndexIDLocked(documentID) + if err != nil { + return err + } + if !found { + return ErrDocumentNotFound + } + if err := SetDataSourceData(&document, source, data); err != nil { + return err + } + if err := IndexDocument(b.documentsIdx, documentID, document); err != nil { + return err + } + b.providerProgressVersion.Add(1) + return nil +} diff --git a/internal/index/bleve_document_enrich_test.go b/internal/index/bleve_document_enrich_test.go new file mode 100644 index 00000000..69fcc838 --- /dev/null +++ b/internal/index/bleve_document_enrich_test.go @@ -0,0 +1,318 @@ +package index + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/svera/coreander/v5/internal/metadata" +) + +type testDocumentEnricher struct { + source string + calls atomic.Int32 + failure error + failureID string +} + +type blockingDocumentEnricher struct { + started chan struct{} + release chan struct{} +} + +func (*blockingDocumentEnricher) Source() string { return "blocking" } + +func (*blockingDocumentEnricher) Name() string { return "Blocking" } + +func (*blockingDocumentEnricher) NeedsEnrichment(Document) (bool, error) { + return true, nil +} + +func (p *blockingDocumentEnricher) Enrich(ctx context.Context, _ *BleveIndexer, _ Document) error { + p.started <- struct{}{} + select { + case <-p.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func TestDocumentEnrichmentUsesBoundedConcurrency(t *testing.T) { + const documentCount = 2 * documentEnrichWorkers + provider := &blockingDocumentEnricher{ + started: make(chan struct{}, documentCount), + release: make(chan struct{}), + } + idx, _ := newTestTextRankIndexer(t, Config{}) + documents := make([]Document, documentCount) + done := make(chan map[string]error, 1) + go func() { done <- idx.enrichProviderBatch(provider, &progressTracker{}, documents) }() + defer func() { + close(provider.release) + if failures := <-done; len(failures) != 0 { + t.Errorf("enrichment failed: %v", failures) + } + }() + for range documentEnrichWorkers { + select { + case <-provider.started: + case <-time.After(time.Second): + t.Fatal("expected concurrent provider lookups") + } + } + select { + case <-provider.started: + t.Fatal("enrichment exceeded its concurrency bound") + case <-time.After(30 * time.Millisecond): + } +} + +func TestBackgroundProgressTracksPersistedProviderLookups(t *testing.T) { + first := &testDocumentEnricher{source: "first", failure: errors.New("unavailable")} + second := &testDocumentEnricher{source: "second"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{first, second}}) + t.Cleanup(func() { _ = idx.Close() }) + for _, id := range []string{"matched", "unmatched", "pending"} { + if err := IndexDocument(idx.documentsIdx, id, Document{ID: id}); err != nil { + t.Fatal(err) + } + } + // A completed lookup without a score is still processed. + for _, id := range []string{"matched", "unmatched"} { + if err := idx.StoreDocumentData(id, "first", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + } + idx.indexProgress.begin(3) + idx.indexProgress.record() + idx.textRankEnrichProgress.begin(3) + idx.textRankEnrichProgress.record() + idx.authorEnrichProgress.begin(3) + idx.authorEnrichProgress.record() + idx.pruneProgress.begin(3) + idx.pruneProgress.record() + progress, err := idx.BackgroundProgress() + if err != nil { + t.Fatal(err) + } + if len(progress) != 6 || progress[0].Kind != ProgressDocuments || progress[1].Kind != ProgressAuthors || + progress[2].Kind != ProgressTextRank || progress[3].Kind != ProgressPruning || + progress[4].Name != "first" || progress[4].Percentage != 66 || + progress[5].Name != "second" || progress[5].Percentage != 0 { + t.Fatalf("incorrect simultaneous progress: %+v", progress) + } + if progress[4].RemainingTime != 0 { + t.Fatalf("estimated time without an active lookup run: %v", progress[4].RemainingTime) + } + // One lookup in this run took a minute, so the remaining one should take about a minute. + run := &idx.documentEnrichProgress[0] + run.begin(1) + run.startNanos.Store(time.Now().Add(-time.Minute).UnixNano()) + run.record() + progress, err = idx.BackgroundProgress() + if err != nil || progress[4].RemainingTime < time.Minute || progress[4].RemainingTime > time.Minute+time.Second { + t.Fatalf("incorrect review lookup estimate: %+v, %v", progress, err) + } + run.end() + idx.indexProgress.end() + idx.textRankEnrichProgress.end() + idx.authorEnrichProgress.end() + idx.pruneProgress.end() + if err := idx.EnrichDocuments(1); err == nil { + t.Fatal("expected unavailable provider to fail") + } + progress, err = idx.BackgroundProgress() + if err != nil || len(progress) != 1 || progress[0].Name != "first" || progress[0].Percentage != 66 { + t.Fatalf("failed lookups must remain visible; successful providers must disappear: %+v, %v", progress, err) + } + // Progress is reconstructed from stored data, not a running worker counter. + resumed := &BleveIndexer{documentsIdx: idx.documentsIdx, documentEnrichers: []DocumentEnricher{first}} + progress, err = resumed.BackgroundProgress() + if err != nil || len(progress) != 1 || progress[0].Percentage != 66 { + t.Fatalf("persisted progress lost on restart: %+v, %v", progress, err) + } + if err := idx.StoreDocumentData("pending", "first", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + progress, err = idx.BackgroundProgress() + if err != nil || len(progress) != 0 { + t.Fatalf("completed lookups still reported as pending: %+v, %v", progress, err) + } +} + +func TestBackgroundProgressNeverRoundsPendingLookupsToComplete(t *testing.T) { + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{&testDocumentEnricher{source: "test"}}}) + t.Cleanup(func() { _ = idx.Close() }) + for i := range 201 { + document := Document{ID: fmt.Sprintf("book-%d", i)} + if i < 200 { + if err := SetDataSourceData(&document, "test", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + } + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + } + progress, err := idx.BackgroundProgress() + if err != nil || len(progress) != 1 || progress[0].Percentage != 99 { + t.Fatalf("unfinished lookups must not display 100%%: %+v, %v", progress, err) + } +} + +func TestBackgroundProgressWithoutPendingProviders(t *testing.T) { + for _, config := range []Config{{}, {DocumentEnrichers: []DocumentEnricher{&testDocumentEnricher{source: "test"}}}} { + idx, _ := newTestTextRankIndexer(t, config) + t.Cleanup(func() { _ = idx.Close() }) + progress, err := idx.BackgroundProgress() + if err != nil || len(progress) != 0 { + t.Fatalf("empty or disabled provider should have no progress: %+v, %v", progress, err) + } + } +} + +func (p *testDocumentEnricher) Source() string { return p.source } + +func (p *testDocumentEnricher) Name() string { return p.source } + +func (p *testDocumentEnricher) NeedsEnrichment(document Document) (bool, error) { + data, err := DataSourceData[struct{ Enriched bool }](document, p.source) + return !data.Enriched, err +} + +func (p *testDocumentEnricher) Enrich(_ context.Context, idx *BleveIndexer, document Document) error { + p.calls.Add(1) + if p.failure != nil && (p.failureID == "" || p.failureID == document.ID) { + return p.failure + } + return idx.StoreDocumentData(document.ID, p.source, map[string]any{ + "enriched": true, "reference": p.source + "-reference", + }) +} + +func TestDocumentEnrichmentContinuesAfterDocumentFailure(t *testing.T) { + unavailable := errors.New("document unavailable") + provider := &testDocumentEnricher{source: "first", failure: unavailable, failureID: "book-2"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{provider}}) + t.Cleanup(func() { _ = idx.Close() }) + for _, id := range []string{"book-1", "book-2", "book-3"} { + if err := IndexDocument(idx.documentsIdx, id, Document{ID: id}); err != nil { + t.Fatal(err) + } + } + + if err := idx.EnrichDocuments(1); !errors.Is(err, unavailable) { + t.Fatalf("expected the individual lookup error, got %v", err) + } + if provider.calls.Load() != 3 { + t.Fatalf("expected all three documents to be attempted, got %d", provider.calls.Load()) + } + for _, id := range []string{"book-1", "book-3"} { + document, err := idx.documentByIndexID(id) + if err != nil { + t.Fatal(err) + } + data, err := DataSourceData[map[string]any](document, "first") + if err != nil || data["enriched"] != true { + t.Errorf("%s was not enriched despite another document failing: %+v, %v", id, data, err) + } + } + pending, _, err := idx.documentsNeedingProvider("first", 10, nil) + if err != nil { + t.Fatal(err) + } + if len(pending) != 1 || pending[0].ID != "book-2" { + t.Fatalf("failed document should remain pending for a later retry: %+v", pending) + } +} + +func TestDocumentEnrichmentSupportsIndependentProviders(t *testing.T) { + first := &testDocumentEnricher{source: "first"} + second := &testDocumentEnricher{source: "second"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{first, second}}) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + for range 2 { + if err := idx.EnrichDocuments(1); err != nil { + t.Fatal(err) + } + } + stored, err := idx.documentByIndexID(document.ID) + if err != nil { + t.Fatal(err) + } + for _, provider := range []*testDocumentEnricher{first, second} { + if provider.calls.Load() != 1 { + t.Fatalf("%s enriched an already processed document", provider.source) + } + data, err := DataSourceData[map[string]any](stored, provider.source) + if err != nil || data["reference"] != provider.source+"-reference" { + t.Fatalf("provider payload lost: %+v, %v", data, err) + } + } +} + +func TestDocumentEnrichmentContinuesAfterProviderFailure(t *testing.T) { + unavailable := errors.New("provider unavailable") + first := &testDocumentEnricher{source: "first", failure: unavailable} + second := &testDocumentEnricher{source: "second"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{first, second}}) + if err := IndexDocument(idx.documentsIdx, "book", Document{ID: "book", Metadata: metadata.Metadata{Title: "Book"}}); err != nil { + t.Fatal(err) + } + if err := idx.EnrichDocuments(1); !errors.Is(err, unavailable) { + t.Fatalf("expected provider error, got %v", err) + } + if second.calls.Load() != 1 { + t.Fatal("one provider failure prevented another provider from running") + } + first.failure = nil + if err := idx.EnrichDocuments(1); err != nil { + t.Fatal(err) + } + if first.calls.Load() != 2 || second.calls.Load() != 1 { + t.Fatal("retry did not resume only the failed provider") + } +} + +func TestDocumentEnrichmentQueueUsesConfiguredProviders(t *testing.T) { + first := &testDocumentEnricher{source: "first"} + second := &testDocumentEnricher{source: "second"} + idx, _ := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{first, second}}) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + idx.scheduleDocumentEnrichment(document.ID) + deadline := time.Now().Add(2 * time.Second) + for { + stored, err := idx.documentByIndexID(document.ID) + if err != nil { + t.Fatal(err) + } + a, err := first.NeedsEnrichment(stored) + if err != nil { + t.Fatal(err) + } + b, err := second.NeedsEnrichment(stored) + if err != nil { + t.Fatal(err) + } + if !a && !b { + break + } + if time.Now().After(deadline) { + t.Fatal("queued document was not enriched by both providers") + } + time.Sleep(time.Millisecond) + } + if err := idx.EnrichDocuments(0); err == nil { + t.Fatal("accepted invalid batch size") + } +} diff --git a/internal/index/bleve_document_migrate.go b/internal/index/bleve_document_migrate.go new file mode 100644 index 00000000..113ab63a --- /dev/null +++ b/internal/index/bleve_document_migrate.go @@ -0,0 +1,121 @@ +package index + +import ( + "errors" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/blevesearch/bleve/v2" +) + +func RecoverDocumentsIndex(path string) error { + if _, err := os.Stat(path); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + backup := path + ".v13-backup" + if _, err := os.Stat(backup); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + return os.Rename(backup, path) +} + +func MigrateDocumentsIndexV13(documents bleve.Index, path string, batchSize int) (bleve.Index, error) { + if batchSize <= 0 { + return nil, fmt.Errorf("documents migration batch size must be positive") + } + backup := path + ".v13-backup" + if _, err := os.Stat(backup); err == nil { + return nil, fmt.Errorf("documents migration backup already exists: %s", backup) + } else if !os.IsNotExist(err) { + return nil, err + } + stagingDir, err := os.MkdirTemp(filepath.Dir(path), ".documents-migration-") + if err != nil { + return nil, err + } + defer func() { + if err := os.RemoveAll(stagingDir); err != nil { + log.Printf("Error removing documents migration staging directory: %s", err) + } + }() + + stagingPath := filepath.Join(stagingDir, "index") + staged, err := bleve.New(stagingPath, CreateDocumentsMapping()) + if err != nil { + return nil, err + } + if err := copyDocumentsV13(documents, staged, batchSize); err != nil { + return nil, errors.Join(err, staged.Close()) + } + if err := staged.Close(); err != nil { + return nil, err + } + if err := documents.Close(); err != nil { + return nil, err + } + if err := os.Rename(path, backup); err != nil { + return nil, err + } + if err := os.Rename(stagingPath, path); err != nil { + return nil, errors.Join(err, os.Rename(backup, path)) + } + migrated, err := bleve.Open(path) + if err != nil { + return nil, errors.Join(err, os.Rename(backup, path)) + } + if err := os.RemoveAll(backup); err != nil { + return nil, errors.Join(fmt.Errorf("removing previous documents index: %w", err), migrated.Close()) + } + return migrated, nil +} + +func copyDocumentsV13(source, destination bleve.Index, batchSize int) error { + for from := 0; ; from += batchSize { + request := bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), batchSize, from, false) + request.Fields = []string{"*"} + request.SortBy([]string{"_id"}) + result, err := source.Search(request) + if err != nil { + return err + } + if len(result.Hits) == 0 { + break + } + batch := destination.NewBatch() + for _, hit := range result.Hits { + document := hydrateDocument(hit) + if err := IndexDocument(batch, hit.ID, document); err != nil { + return err + } + } + if err := destination.Batch(batch); err != nil { + return err + } + } + + for _, key := range [][]byte{ + internalLanguages, + internalIllustratedMinSize, + internalMinOccurrenceRatio, + internalMaxTextRankWords, + internalDocCountAtLastPrune, + } { + value, err := source.GetInternal(key) + if err != nil { + return err + } + if value != nil { + if err := destination.SetInternal(key, value); err != nil { + return err + } + } + } + return destination.SetInternal(internalVersion, []byte(DocumentVersion)) +} diff --git a/internal/index/bleve_document_read.go b/internal/index/bleve_document_read.go index 48ee2b13..32adbfd5 100644 --- a/internal/index/bleve_document_read.go +++ b/internal/index/bleve_document_read.go @@ -3,10 +3,12 @@ package index import ( "cmp" "crypto/sha256" + "encoding/json" "errors" "fmt" "html/template" "image" + "log" "math" "net/url" "os" @@ -40,6 +42,27 @@ const titleBoost = 3.0 // document detail page's "related documents" widget. var DefaultDocumentSortBy = []string{"-_score", "Series", "SeriesIndex"} +// reviewScoreSort treats a stored zero as missing without changing the index. +type reviewScoreSort struct { + search.SortField +} + +func (s *reviewScoreSort) Value(hit *search.DocumentMatch) string { + value := s.SortField.Value(hit) + if s.SortField.DecodeValue(value) == "0" { + if s.Desc { + return search.LowTerm + } + return search.HighTerm + } + return value +} + +func (s *reviewScoreSort) Copy() search.SearchSort { + copied := *s + return &copied +} + // Search look for documents which match the passed keywords and filters. // Returns a maximum documents, offset by func (b *BleveIndexer) Search(searchFields SearchFields, page, resultsPerPage int) (result.CappedPaginatedResult[[]Document], error) { @@ -226,6 +249,13 @@ func (b *BleveIndexer) addFilters(searchFields SearchFields, filtersQuery *query bq.SetBoost(0) filtersQuery.AddQuery(bq) } + if max := searchFields.DataSourceScoreUpperBound(); searchFields.DataSourceScoreFrom != 0 || max != 5 { + min := searchFields.DataSourceScoreFrom + ratingQuery := newInclusiveNumericRangeQuery(&min, &max) + ratingQuery.SetField("DataSourceScoreValue") + ratingQuery.SetBoost(0) + filtersQuery.AddQuery(ratingQuery) + } if searchFields.IllustratedOnly && b.illustratedMinAmount > 0 { minIllustrations := float64(b.illustratedMinAmount) q := bleve.NewNumericRangeQuery(&minIllustrations, nil) @@ -388,6 +418,16 @@ func (b *BleveIndexer) searchAndHydrate(query query.Query, size, offset int, sor // with an empty sort order, which falls back to arbitrary (index) order instead of relevance. if len(sortBy) > 0 { searchOptions.SortBy(sortBy) + for i, key := range sortBy { + if strings.TrimPrefix(key, "-") == "DataSourceScoreValue" { + searchOptions.Sort[i] = &reviewScoreSort{SortField: search.SortField{ + Field: "DataSourceScoreValue", + Desc: strings.HasPrefix(key, "-"), + Type: search.SortFieldAsNumber, + Missing: search.SortFieldMissingLast, + }} + } + } } searchOptions.Fields = []string{"*"} b.documentsMu.RLock() @@ -577,6 +617,14 @@ func compareDocumentsBySortKey(a, b Document, key string) int { c = cmp.Compare(a.Publication.Date, b.Publication.Date) case "Words": c = cmp.Compare(a.Words, b.Words) + case "DataSourceScoreValue": + if (a.DataSourceScoreValue == 0) != (b.DataSourceScoreValue == 0) { + if a.DataSourceScoreValue == 0 { + return 1 + } + return -1 + } + c = cmp.Compare(a.DataSourceScoreValue, b.DataSourceScoreValue) case "Series": c = cmp.Compare(a.Series, b.Series) case "SeriesIndex": @@ -1023,6 +1071,8 @@ func hydrateDocument(match *search.DocumentMatch) Document { textRankEnriched = match.Fields["TextRankEnriched"].(bool) } + reviewScoreValue, _ := match.Fields["DataSourceScoreValue"].(float64) + doc := Document{ ID: match.ID, Metadata: metadata.Metadata{ @@ -1040,20 +1090,40 @@ func hydrateDocument(match *search.DocumentMatch) Document { Illustrations: illustrations, Format: match.Fields["Format"].(string), }, - Slug: match.Fields["Slug"].(string), - AuthorsSlugs: slicer(match.Fields["AuthorsSlugs"]), - IllustratorsSlugs: illustratorsSlugs, - SeriesSlug: match.Fields["SeriesSlug"].(string), - SubjectsSlugs: slicer(match.Fields["SubjectsSlugs"]), - AddedOn: addedOn, - TextRankPhrases: textRankPhrases, - TextRankWords: textRankWords, - TextRankEnriched: textRankEnriched, + Slug: match.Fields["Slug"].(string), + AuthorsSlugs: slicer(match.Fields["AuthorsSlugs"]), + IllustratorsSlugs: illustratorsSlugs, + SeriesSlug: match.Fields["SeriesSlug"].(string), + SubjectsSlugs: slicer(match.Fields["SubjectsSlugs"]), + AddedOn: addedOn, + TextRankPhrases: textRankPhrases, + TextRankWords: textRankWords, + TextRankEnriched: textRankEnriched, + DataSources: hydrateDataSources(match.Fields, match.ID), + DataSourceScoreValue: reviewScoreValue, } return doc } +func hydrateDataSources(fields map[string]any, documentID string) map[string]json.RawMessage { + sources, err := decodeDataSources(fields, documentID) + if err != nil { + log.Println(err) + } + return sources +} + +func decodeDataSources(fields map[string]any, documentID string) (map[string]json.RawMessage, error) { + var sources map[string]json.RawMessage + if raw, ok := fields["DataSources"].(string); ok { + if err := json.Unmarshal([]byte(raw), &sources); err != nil { + return nil, fmt.Errorf("reading data sources for document %s: %w", documentID, err) + } + } + return sources, nil +} + func slicer(val any) []string { var ( terms []any diff --git a/internal/index/bleve_document_write.go b/internal/index/bleve_document_write.go index 110a892a..8e29a4b1 100644 --- a/internal/index/bleve_document_write.go +++ b/internal/index/bleve_document_write.go @@ -56,12 +56,85 @@ func (b *BleveIndexer) progressFrom(kind ProgressKind, startNanos int64, process return progress } +// BackgroundProgress includes concurrent tasks and persisted pending provider lookups. +func (b *BleveIndexer) BackgroundProgress() ([]Progress, error) { + var progress []Progress + for _, task := range []struct { + kind ProgressKind + tracker *progressTracker + }{ + {ProgressDocuments, &b.indexProgress}, + {ProgressAuthors, &b.authorEnrichProgress}, + {ProgressTextRank, &b.textRankEnrichProgress}, + {ProgressPruning, &b.pruneProgress}, + } { + if start := task.tracker.startNanos.Load(); start != 0 { + progress = append(progress, b.progressFrom(task.kind, start, task.tracker.processed.Load(), task.tracker.total.Load())) + } + } + if len(b.documentEnrichers) == 0 { + return progress, nil + } + providers, err := b.providerProgressSnapshot() + if err != nil { + return nil, err + } + for i, counts := range providers { + if counts.enriched < counts.total { + progress = append(progress, Progress{ + Kind: ProgressDataSources, InProgress: true, Name: b.documentEnrichers[i].Name(), + // Do not round pending work up to 100%. + Percentage: math.Floor(100 * float64(counts.enriched) / float64(counts.total)), + RemainingTime: b.documentEnrichTracker(i).remainingTime(counts.total - counts.enriched), + }) + } + } + return progress, nil +} + +type providerProgressCount struct { + total, enriched uint64 +} + +// Cache index queries across page renders and polling clients; timing estimates +// remain live. Provider writes and metadata reindexing invalidate the snapshot. +func (b *BleveIndexer) providerProgressSnapshot() ([]providerProgressCount, error) { + b.providerProgressMu.Lock() + defer b.providerProgressMu.Unlock() + version := b.providerProgressVersion.Load() + if b.cachedProviderProgressVersion == version && time.Now().Before(b.providerProgressUntil) { + return slices.Clone(b.providerProgress), nil + } + b.documentsMu.RLock() + defer b.documentsMu.RUnlock() + total, err := b.documentsIdx.DocCount() + if err != nil { + return nil, fmt.Errorf("counting documents for enrichment progress: %w", err) + } + var counts []providerProgressCount + for _, provider := range b.documentEnrichers { + query := bleve.NewTermQuery(provider.Source()) + query.SetField("EnrichedDataSources") + result, err := b.documentsIdx.Search(bleve.NewSearchRequestOptions(query, 0, 0, false)) + if err != nil { + return nil, fmt.Errorf("reading %s enrichment progress: %w", provider.Source(), err) + } + counts = append(counts, providerProgressCount{total: total, enriched: result.Total}) + } + b.providerProgress = slices.Clone(counts) + b.cachedProviderProgressVersion = version + b.providerProgressUntil = time.Now().Add(5 * time.Second) + return counts, nil +} + // BeginIndexing marks startup indexing active before the HTTP server can accept mutations. func (b *BleveIndexer) BeginIndexing() { + b.providerProgressVersion.Add(1) b.indexProgress.begin(0) } func (b *BleveIndexer) endIndexing() { + b.providerProgressVersion.Add(1) b.indexProgress.end() } @@ -117,6 +190,7 @@ type indexedFileState struct { } func (b *BleveIndexer) indexFileLocked(file string) (string, error) { + defer b.providerProgressVersion.Add(1) ext := strings.ToLower(filepath.Ext(file)) if _, ok := b.reader[ext]; !ok { return "", fmt.Errorf("file extension %s not supported", ext) @@ -145,8 +219,13 @@ func (b *BleveIndexer) indexFileLocked(file string) (string, error) { } document.Metadata = job.meta document.TextRankEnriched = !b.supportsTextRank(file) + documents := []Document{document} b.documentsMu.Lock() - err = b.documentsIdx.Index(id, document) + err = b.preserveDataSourcesLocked(documents) + if err == nil { + document = documents[0] + err = IndexDocument(b.documentsIdx, id, document) + } b.documentsMu.Unlock() if err != nil { return "", fmt.Errorf("error updating indexed file %s: %w", file, err) @@ -164,7 +243,7 @@ func (b *BleveIndexer) indexFileLocked(file string) (string, error) { document.TextRankEnriched = !b.supportsTextRank(file) b.documentsMu.Lock() - err = b.documentsIdx.Index(document.ID, document) + err = IndexDocument(b.documentsIdx, document.ID, document) b.documentsMu.Unlock() if err != nil { return "", fmt.Errorf("error indexing file %s: %s", file, err) @@ -181,6 +260,7 @@ func (b *BleveIndexer) indexFileLocked(file string) (string, error) { if !document.TextRankEnriched { b.scheduleTextRankEnrichment(document) } + b.scheduleDocumentEnrichment(document.ID) b.maybePruneForLibraryChange() @@ -217,6 +297,7 @@ func (b *BleveIndexer) removeFile(file string) error { // DeleteDocument removes the document identified by slug from the index and deletes its file from the filesystem. func (b *BleveIndexer) DeleteDocument(slug string) error { + defer b.providerProgressVersion.Add(1) if b.indexProgress.startNanos.Load() != 0 { return ErrLibraryIndexing } @@ -239,6 +320,7 @@ func (b *BleveIndexer) DeleteDocument(slug string) error { } func (b *BleveIndexer) deleteDocumentFromIndex(document Document) error { + defer b.providerProgressVersion.Add(1) b.documentsMu.Lock() err := b.documentsIdx.Delete(document.ID) b.documentsMu.Unlock() @@ -259,7 +341,7 @@ func (b *BleveIndexer) deleteDocumentFromIndex(document Document) error { err = b.authorsIdx.Delete(authorSlug) } else { author.DocumentCount-- - err = b.authorsIdx.Index(authorSlug, author) + err = indexAuthor(b.authorsIdx, authorSlug, author) } b.authorsMu.Unlock() if err != nil { @@ -328,7 +410,7 @@ func (b *BleveIndexer) AddLibrary(batchSize int, forceIndexing bool, metadataWor // consider them. document.TextRankEnriched = !b.supportsTextRank(job.path) - if err = batch.Index(document.ID, document); err != nil { + if err = IndexDocument(batch, document.ID, document); err != nil { log.Printf("Error indexing file %s: %s\n", job.path, err) continue } diff --git a/internal/index/bleve_file_hash_test.go b/internal/index/bleve_file_hash_test.go index b812776a..32441fab 100644 --- a/internal/index/bleve_file_hash_test.go +++ b/internal/index/bleve_file_hash_test.go @@ -146,7 +146,7 @@ func TestIndexFileUpdatesHashWithoutMetadataChanges(t *testing.T) { t.Fatal(err) } doc.TextRankWords = []string{"enriched"} - if err := idx.documentsIdx.Index(doc.ID, doc); err != nil { + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { t.Fatal(err) } for _, content := range []string{"original content", "modified content", "modified content"} { diff --git a/internal/index/bleve_provider_preservation_test.go b/internal/index/bleve_provider_preservation_test.go new file mode 100644 index 00000000..6b0105ca --- /dev/null +++ b/internal/index/bleve_provider_preservation_test.go @@ -0,0 +1,130 @@ +package index + +import ( + "context" + "reflect" + "slices" + "testing" + + "github.com/svera/coreander/v5/internal/metadata" +) + +func TestTextRankWritesPreserveProviderRemovals(t *testing.T) { + for _, keepIndependent := range []bool{false, true} { + name := "all providers reset" + if keepIndependent { + name = "one provider reset" + } + t.Run(name, func(t *testing.T) { + idx, documents := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + document := withProviderData(t, Document{ + ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}, + DataSourceScoreValue: 3.82, + }, testProviderData{URL: "/book/show/123", Enriched: true}) + if err := SetDataSourceData(&document, "other", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + if err := IndexDocument(documents, document.ID, document); err != nil { + t.Fatal(err) + } + stale, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + if err := idx.UpdateDocument("book", func(update *DocumentUpdate) (bool, error) { + if keepIndependent { + delete(update.DataSources, "dummy") + } else { + update.DataSources = nil + } + update.DataSourceScoreValue = 0 + return true, nil + }); err != nil { + t.Fatal(err) + } + latest, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + + idx.enrichTextRankAndReindex(stale) + got, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.DataSources, latest.DataSources) || got.DataSourceScoreValue != 0 { + t.Fatalf("stale TextRank write restored removed provider data: %+v", got) + } + if !got.TextRankEnriched { + t.Fatal("TextRank result must still be persisted") + } + pending, total, err := idx.documentsNeedingProvider("dummy", 10, nil) + if err != nil || total != 1 || len(pending) != 1 || pending[0].ID != "book" { + t.Fatalf("reset provider lookup must remain pending: %+v, %d, %v", pending, total, err) + } + }) + } +} + +// TestTextRankWritesPreserveProviderFields guards against TextRank +// enrichment, which writes back documents read before ranking them, +// discarding provider data persisted in the meantime, and vice versa. +func TestTextRankWritesPreserveProviderFields(t *testing.T) { + source := &testDocumentEnricher{source: "dummy"} + idx, documents := newTestTextRankIndexer(t, Config{DocumentEnrichers: []DocumentEnricher{source}}) + t.Cleanup(func() { _ = idx.Close() }) + if err := IndexDocument(documents, "book", Document{ID: "book", Metadata: metadata.Metadata{Title: "Book"}}); err != nil { + t.Fatal(err) + } + stale, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + stale.TextRankEnriched = false + want := testProviderData{ + URL: "provider-reference", Enriched: true, RatingCount: "2,285", + ResourceID: "work-id", LegacyBookID: 12207444, + } + if err := idx.UpdateDocument(stale.ID, func(update *DocumentUpdate) (bool, error) { + update.DataSourceScoreValue = 3.82 + return true, SetDataSourceData(&update.Document, source.Source(), want) + }); err != nil { + t.Fatal(err) + } + stale.TextRankWords = []string{"dragon"} + idx.enrichTextRankAndReindex(stale) + + doc, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + if data := mustProviderData(t, doc); data != want || doc.DataSourceScoreValue != 3.82 { + t.Errorf("expected provider fields and identifiers to survive a TextRank write, got %+v", doc) + } + if !doc.TextRankEnriched { + t.Error("expected TextRank write to be persisted") + } + + // A Provider write must keep TextRank fields written after the document was read. + stale, _ = idx.documentByIndexID("book") + if err := SetDataSourceData(&stale, source.Source(), testProviderData{Enriched: false}); err != nil { + t.Fatal(err) + } + doc.TextRankWords = []string{"guinevere"} + idx.documentsMu.Lock() + if err := IndexDocument(idx.documentsIdx, doc.ID, doc); err != nil { + t.Fatal(err) + } + idx.documentsMu.Unlock() + if err := source.Enrich(context.Background(), idx, stale); err != nil { + t.Fatal(err) + } + if doc, _ = idx.documentByIndexID("book"); !slices.Equal(doc.TextRankWords, []string{"guinevere"}) { + t.Errorf("expected TextRank words to survive a Provider write, got %q", doc.TextRankWords) + } +} diff --git a/internal/index/bleve_textrank_write.go b/internal/index/bleve_textrank_write.go index ad9335b1..8600c504 100644 --- a/internal/index/bleve_textrank_write.go +++ b/internal/index/bleve_textrank_write.go @@ -85,8 +85,12 @@ func (b *BleveIndexer) scheduleTextRankEnrichment(document Document) { // waiting on TextRank. func (b *BleveIndexer) enrichTextRankAndReindex(document Document) { enriched := b.rankDocument(document) + documents := []Document{enriched} b.documentsMu.Lock() - err := b.documentsIdx.Index(enriched.ID, enriched) + err := b.preserveDataSourcesLocked(documents) + if err == nil { + err = IndexDocument(b.documentsIdx, enriched.ID, documents[0]) + } b.documentsMu.Unlock() if err != nil { log.Printf("Error indexing TextRank-enriched document %s: %s\n", enriched.ID, err) @@ -263,13 +267,18 @@ func (b *BleveIndexer) EnrichTextRankKeywords(batchSize, workers int) error { enriched := b.rankDocuments(chunk, workers) + b.documentsMu.Lock() + if err := b.preserveDataSourcesLocked(enriched); err != nil { + b.documentsMu.Unlock() + b.endTextRankEnrichment() + return err + } batch := b.documentsIdx.NewBatch() for _, document := range enriched { - if err := batch.Index(document.ID, document); err != nil { + if err := IndexDocument(batch, document.ID, document); err != nil { log.Printf("Error indexing enriched document %s: %s\n", document.ID, err) } } - b.documentsMu.Lock() err = b.documentsIdx.Batch(batch) b.documentsMu.Unlock() if err != nil { @@ -529,7 +538,7 @@ func (b *BleveIndexer) rewriteCommonTextRankEntries(docCount uint64, batchSize i return pruned, err } - batch := b.documentsIdx.NewBatch() + var documents []Document for _, hit := range docSearchResult.Hits { document := hydrateDocument(hit) p, ok := toPrune[document.ID] @@ -538,20 +547,28 @@ func (b *BleveIndexer) rewriteCommonTextRankEntries(docCount uint64, batchSize i } document.TextRankPhrases = p.phrases document.TextRankWords = p.words - if err := batch.Index(document.ID, document); err != nil { + documents = append(documents, document) + } + + b.documentsMu.Lock() + if err := b.preserveDataSourcesLocked(documents); err != nil { + b.documentsMu.Unlock() + return pruned, err + } + batch := b.documentsIdx.NewBatch() + for _, document := range documents { + if err := IndexDocument(batch, document.ID, document); err != nil { log.Printf("Error indexing document %s while pruning common TextRank entries: %s\n", document.ID, err) continue } pruned++ } - if batch.Size() > 0 { - b.documentsMu.Lock() err = b.documentsIdx.Batch(batch) - b.documentsMu.Unlock() - if err != nil { - return pruned, err - } + } + b.documentsMu.Unlock() + if err != nil { + return pruned, err } b.recordPruningProgress(len(result.Hits)) } diff --git a/internal/index/bleve_textrank_write_test.go b/internal/index/bleve_textrank_write_test.go index 36eea4cf..d869c730 100644 --- a/internal/index/bleve_textrank_write_test.go +++ b/internal/index/bleve_textrank_write_test.go @@ -36,7 +36,7 @@ func TestCommonTextRankEntriesNeedPrune(t *testing.T) { }) for i := 0; i < 4; i++ { - if err := documentsIndexMem.Index(strconv.Itoa(i), Document{ID: strconv.Itoa(i)}); err != nil { + if err := IndexDocument(documentsIndexMem, strconv.Itoa(i), Document{ID: strconv.Itoa(i)}); err != nil { t.Fatal(err) } } @@ -60,7 +60,7 @@ func TestCommonTextRankEntriesNeedPrune(t *testing.T) { // PruneChangeTriggerRatio is 0.5 of the 4 documents recorded above (i.e. // 2), so 3 more documents is enough to cross it. for i := 0; i < 3; i++ { - if err := documentsIndexMem.Index("extra"+strconv.Itoa(i), Document{ID: "extra" + strconv.Itoa(i)}); err != nil { + if err := IndexDocument(documentsIndexMem, "extra"+strconv.Itoa(i), Document{ID: "extra" + strconv.Itoa(i)}); err != nil { t.Fatal(err) } } @@ -83,7 +83,7 @@ func TestPruningReportsProgress(t *testing.T) { TextRankPhrases: []string{"phrase " + id}, TextRankWords: []string{"word" + id}, } - if err := documentsIndexMem.Index(id, document); err != nil { + if err := IndexDocument(documentsIndexMem, id, document); err != nil { t.Fatal(err) } } @@ -140,7 +140,7 @@ func TestPruneCommonTextRankEntriesRewritesCommonEntries(t *testing.T) { TextRankPhrases: []string{"common phrase", "commonword distinctive" + id, "unique phrase " + id}, TextRankWords: []string{"commonword", "unique" + id}, } - if err := documentsIndexMem.Index(id, document); err != nil { + if err := IndexDocument(documentsIndexMem, id, document); err != nil { t.Fatal(err) } } diff --git a/internal/index/data_source_fields_test.go b/internal/index/data_source_fields_test.go new file mode 100644 index 00000000..07dd7083 --- /dev/null +++ b/internal/index/data_source_fields_test.go @@ -0,0 +1,322 @@ +package index_test + +import ( + "maps" + "slices" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" +) + +type storedProviderData struct { + URL string `json:"url"` + ScoreCount string `json:"scoreCount"` + ReferenceID string `json:"referenceId"` + RecordID int `json:"recordId"` +} + +func mustStoredProviderData(t *testing.T, document index.Document) storedProviderData { + t.Helper() + data, err := index.DataSourceData[storedProviderData](document, "dummy") + if err != nil { + t.Fatal(err) + } + return data +} + +func withStoredProviderData(t *testing.T, document index.Document, data storedProviderData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, "dummy", data); err != nil { + t.Fatal(err) + } + return document +} + +func TestDataSourceScoreSorting(t *testing.T) { + docs := []index.Document{ + {ID: "high", Slug: "high", DataSourceScoreValue: 4.75}, + {ID: "unrated", Slug: "unrated"}, + {ID: "low", Slug: "low", DataSourceScoreValue: 3.25}, + } + for i := range docs { + docs[i].Metadata = metadata.Metadata{Title: docs[i].Slug, Authors: []string{"Writer"}, Format: "EPUB", Series: "Series"} + docs[i].AuthorsSlugs = []string{"writer"} + docs[i].SeriesSlug = "series" + docs[i].TextRankPhrases = []string{"atomic bomb", "los alamos", "manhattan project"} + } + reference := index.Document{ + ID: "reference", Slug: "reference", + Metadata: metadata.Metadata{Title: "Reference", Authors: []string{"Other"}, Format: "EPUB"}, + AuthorsSlugs: []string{"other"}, + TextRankPhrases: docs[0].TextRankPhrases, + } + idx := newTextRankTestIndex(t, append(docs, reference)) + t.Cleanup(func() { _ = idx.Close() }) + + for _, scope := range []string{"search", "author", "series", "similar"} { + for _, direction := range []string{"asc", "desc"} { + t.Run(scope+"/"+direction, func(t *testing.T) { + key := "DataSourceScoreValue" + want := []string{"low", "high"} + if direction == "desc" { + key = "-" + key + slices.Reverse(want) + } + want = append(want, "unrated") + fields := index.SearchFields{SortBy: []string{key}} + switch scope { + case "search": + fields.Keywords = "writer" + case "author": + fields.AuthorSlug = "writer" + case "series": + fields.Keywords = "series" + case "similar": + fields.SimilarTo = "reference" + } + var got []string + for page := 1; page <= 2; page++ { + switch scope { + case "author": + res, err := idx.SearchByAuthor(fields, page, 2) + if err != nil { + t.Fatal(err) + } + got = append(got, slugsOf(res.Hits())...) + case "series": + res, err := idx.SearchBySeries(fields, page, 2) + if err != nil { + t.Fatal(err) + } + got = append(got, slugsOf(res.Hits())...) + default: + res, err := idx.Search(fields, page, 2) + if err != nil { + t.Fatal(err) + } + got = append(got, slugsOf(res.Hits())...) + } + } + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + } + } +} + +func TestDataSourceFieldsAreStoredButNotSearchable(t *testing.T) { + documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, index.Config{}) + t.Cleanup(func() { _ = idx.Close() }) + + const ( + bookURL = "https://example.org/books/12207444-zyxwvbook" + authorURL = "https://example.org/authors/151155.Zyxwv_Author" + referenceID = "opaque-zyxwvreference" + recordID = 12207444 + ) + for _, lang := range []string{"es", ""} { + doc := withStoredProviderData(t, index.Document{ + ID: "doc-" + lang, + Slug: "doc-" + lang, + Metadata: metadata.Metadata{ + Title: "Example Book", + Authors: []string{"Example Writer"}, + Language: lang, + Format: "EPUB", + }, + AuthorsSlugs: []string{"example-writer"}, + + DataSourceScoreValue: 4.25, + }, storedProviderData{URL: bookURL, ScoreCount: "2285", ReferenceID: referenceID, RecordID: recordID}) + + if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { + t.Fatal(err) + } + + got, err := idx.Document(doc.Slug) + if err != nil { + t.Fatal(err) + } + if mustStoredProviderData(t, got).URL != bookURL { + t.Errorf("language %q: expected stored book URL, got %q", lang, mustStoredProviderData(t, got).URL) + } + if got.DataSourceScoreValue != 4.25 { + t.Errorf("language %q: expected stored rating, got %v", lang, got.DataSourceScoreValue) + } + if mustStoredProviderData(t, got).ScoreCount != "2285" { + t.Errorf("language %q: expected stored rating count, got %q", lang, mustStoredProviderData(t, got).ScoreCount) + } + if mustStoredProviderData(t, got).ReferenceID != referenceID || mustStoredProviderData(t, got).RecordID != recordID { + t.Errorf("language %q: expected stored identifiers, got %q and %d", lang, mustStoredProviderData(t, got).ReferenceID, mustStoredProviderData(t, got).RecordID) + } + } + if err := index.IndexDocument(documentsIndexMem, "unscored", index.Document{ + ID: "unscored", + Slug: "unscored", + Metadata: metadata.Metadata{ + Title: "Unscored document", + Format: "EPUB", + }, + }); err != nil { + t.Fatal(err) + } + + for _, keywords := range []string{"zyxwvbook", "Zyxwv_Author", "12207444", "151155", "zyxwvreference", referenceID, "4.25", "2285"} { + res, err := idx.Search(index.SearchFields{Keywords: keywords}, 1, 10) + if err != nil { + t.Fatal(err) + } + if res.TotalHits() != 0 { + t.Errorf("expected no results searching keywords %q, got %d", keywords, res.TotalHits()) + } + } + filtered, err := idx.Search(index.SearchFields{DataSourceScoreFrom: 4.2, DataSourceScoreTo: new(4.3)}, 1, 10) + if err != nil || filtered.TotalHits() != 2 { + t.Errorf("expected both documents in the data-source score range, got %d results and error %v", filtered.TotalHits(), err) + } + filtered, err = idx.Search(index.SearchFields{DataSourceScoreFrom: 4.3}, 1, 10) + if err != nil || filtered.TotalHits() != 0 { + t.Errorf("expected no documents above the data-source score, got %d results and error %v", filtered.TotalHits(), err) + } + filtered, err = idx.Search(index.SearchFields{DataSourceScoreTo: new(0.0)}, 1, 10) + if err != nil || filtered.TotalHits() != 1 || filtered.Hits()[0].DataSourceScoreValue != 0 { + t.Errorf("expected only the unscored document, got %d results and error %v", filtered.TotalHits(), err) + } + + queries := map[string]bleve.SearchRequest{ + "provider data field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources", bookURL)), + "provider URL field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources.dummy.url", bookURL)), + "author URL field term": *bleve.NewSearchRequest(fieldTermQuery("DataSourceLinks.dummy", authorURL)), + "resource ID field term": *bleve.NewSearchRequest(fieldTermQuery("DataSources.dummy.referenceId", referenceID)), + "legacy ID numeric field": *bleve.NewSearchRequest(providerRecordIDQuery(recordID)), + "_all match": *bleve.NewSearchRequest(bleve.NewMatchQuery("zyxwvbook")), + "_all term": *bleve.NewSearchRequest(bleve.NewTermQuery(bookURL)), + "query string field": *bleve.NewSearchRequest(bleve.NewQueryStringQuery(`DataSources:"` + bookURL + `"`)), + } + for name, req := range queries { + res, err := documentsIndexMem.Search(&req) + if err != nil { + t.Fatal(err) + } + if res.Total != 0 { + t.Errorf("%s: expected no hits, got %d", name, res.Total) + } + } +} + +func TestDataSourceScoreSortingMissingFieldsAndTies(t *testing.T) { + documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + idx := index.NewBleve(documents, authors, nil, "", nil, index.Config{}) + t.Cleanup(func() { _ = idx.Close() }) + for _, doc := range []map[string]any{ + {"Slug": "unrated-zero", "DataSourceScoreValue": 0.0, "SeriesIndex": 2.0}, + {"Slug": "high", "DataSourceScoreValue": 4.75, "SeriesIndex": 1.0}, + {"Slug": "unrated-missing", "SeriesIndex": 1.0}, + {"Slug": "low-second", "DataSourceScoreValue": 3.25, "SeriesIndex": 2.0}, + {"Slug": "low-first", "DataSourceScoreValue": 3.25, "SeriesIndex": 1.0}, + } { + for _, field := range []string{"Title", "Description", "Series", "SeriesSlug", "Format"} { + doc[field] = "" + } + doc["Words"], doc["Pages"] = 0.0, 0.0 + if err := documents.Index(doc["Slug"].(string), doc); err != nil { + t.Fatal(err) + } + } + for _, direction := range []string{"asc", "desc"} { + t.Run(direction, func(t *testing.T) { + key := "DataSourceScoreValue" + want := []string{"low-first", "low-second", "high", "unrated-missing", "unrated-zero"} + if direction == "desc" { + key = "-" + key + want = []string{"high", "low-first", "low-second", "unrated-missing", "unrated-zero"} + } + var got []string + for page := 1; page <= 3; page++ { + res, err := idx.Search(index.SearchFields{SortBy: []string{key, "SeriesIndex"}}, page, 2) + if err != nil { + t.Fatal(err) + } + if res.TotalHits() != len(want) { + t.Errorf("got %d hits, want %d", res.TotalHits(), len(want)) + } + got = append(got, slugsOf(res.Hits())...) + } + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + } +} + +func providerRecordIDQuery(id int) *query.NumericRangeQuery { + value := float64(id) + q := bleve.NewNumericRangeQuery(&value, &value) + q.SetField("DataSources.dummy.recordId") + return q +} + +func TestAuthorDataSourceLinksAreStoredButNotSearchable(t *testing.T) { + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + defer authors.Close() + idx := index.NewBleve(nil, authors, nil, "", nil, index.Config{}) + const authorURL = "https://example.org/authors/151155.Zyxwv_Author" + links := map[string]string{ + "dummy": authorURL, + "other": "https://example.org/Zyxwv_Author", + } + if err := idx.IndexAuthor(index.Author{Slug: "alice", Name: "Example Writer", DataSourceLinks: links}); err != nil { + t.Fatal(err) + } + author, err := idx.Author("alice", "") + if err != nil { + t.Fatal(err) + } + if !maps.Equal(author.DataSourceLinks, links) { + t.Fatalf("expected stored author links, got %+v", author.DataSourceLinks) + } + for _, q := range []query.Query{ + fieldTermQuery("DataSourceLinks", authorURL), + fieldTermQuery("DataSourceLinks.dummy", authorURL), + fieldTermQuery("DataSourceLinks.other", links["other"]), + bleve.NewTermQuery(authorURL), + bleve.NewMatchQuery("Zyxwv_Author"), + bleve.NewQueryStringQuery(`DataSourceLinks:"` + authorURL + `"`), + } { + result, err := authors.Search(bleve.NewSearchRequest(q)) + if err != nil { + t.Fatal(err) + } + if result.Total != 0 { + t.Errorf("author URL must not affect search results, got %d hits", result.Total) + } + } +} + +func fieldTermQuery(field, term string) *query.TermQuery { + q := bleve.NewTermQuery(term) + q.SetField(field) + return q +} diff --git a/internal/index/document.go b/internal/index/document.go index 07dfe299..24928eb5 100644 --- a/internal/index/document.go +++ b/internal/index/document.go @@ -1,12 +1,18 @@ package index import ( + "encoding/json" + "fmt" + "maps" "time" "github.com/rickb777/date/v2" "github.com/svera/coreander/v5/internal/metadata" ) +// DataSourceScoreValueSortField is the index sort key for the configured data-source score. +const DataSourceScoreValueSortField = "DataSourceScoreValue" + type SearchFields struct { Keywords string // AuthorSlug, when set, scopes the search to documents by (or illustrated @@ -20,16 +26,26 @@ type SearchFields struct { // author and its own series - deliberately not its subjects, see // similarToQuery), ranked by score and pruned to those scoring close // enough to the best match (see runSimilarityQuery). - SimilarTo string - PubDateFrom date.Date - PubDateTo date.Date - EstReadTimeFrom float64 - EstReadTimeTo float64 - WordsPerMinute float64 - PagesFrom float64 - PagesTo float64 - IllustratedOnly bool - SortBy []string + SimilarTo string + PubDateFrom date.Date + PubDateTo date.Date + EstReadTimeFrom float64 + EstReadTimeTo float64 + WordsPerMinute float64 + PagesFrom float64 + PagesTo float64 + DataSourceScoreFrom float64 + DataSourceScoreTo *float64 + IllustratedOnly bool + SortBy []string +} + +// DataSourceScoreUpperBound defaults an unspecified upper bound to the maximum score. +func (s SearchFields) DataSourceScoreUpperBound() float64 { + if s.DataSourceScoreTo == nil { + return 5 + } + return *s.DataSourceScoreTo } type Document struct { @@ -76,6 +92,74 @@ type Document struct { // in the background. See documentsNeedingTextRank, which drives that pass // off this field so it survives (and resumes correctly across) restarts. TextRankEnriched bool + // DataSources holds opaque provider-owned JSON objects, stored but not indexed. + DataSources map[string]json.RawMessage `json:"-"` + // DataSourceScoreValue is the configured data source's score, stored and indexed for filtering and sorting. + DataSourceScoreValue float64 +} + +// DataSourceData decodes a provider's payload. Missing providers have zero-valued data. +func DataSourceData[T any](document Document, source string) (T, error) { + var data T + if raw, ok := document.DataSources[source]; ok { + if err := json.Unmarshal(raw, &data); err != nil { + return data, fmt.Errorf("decoding %s data for document %s: %w", source, document.ID, err) + } + } + return data, nil +} + +// SetDataSourceData replaces one provider's payload without mutating shared document maps. +func SetDataSourceData(document *Document, source string, data any) error { + raw, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("encoding %s data for document %s: %w", source, document.ID, err) + } + var object map[string]json.RawMessage + if err := json.Unmarshal(raw, &object); err != nil || object == nil { + return fmt.Errorf("data for provider %s must be a JSON object", source) + } + document.DataSources = maps.Clone(document.DataSources) + if document.DataSources == nil { + document.DataSources = make(map[string]json.RawMessage) + } + document.DataSources[source] = raw + return nil +} + +// IndexDocument stores provider JSON without exposing its contents to Bleve's dynamic mapping. +func IndexDocument(destination interface{ Index(string, any) error }, id string, document Document) error { + for source, raw := range document.DataSources { + var object map[string]json.RawMessage + if err := json.Unmarshal(raw, &object); err != nil || object == nil { + return fmt.Errorf("invalid JSON object for provider %s in document %s", source, id) + } + } + raw, err := json.Marshal(document.DataSources) + if err != nil { + return fmt.Errorf("encoding data sources for document %s: %w", id, err) + } + var enriched []string + for source, payload := range document.DataSources { + var state struct { + Enriched bool `json:"enriched"` + } + if err := json.Unmarshal(payload, &state); err != nil { + return fmt.Errorf("reading enrichment state for provider %s in document %s: %w", source, id, err) + } + if state.Enriched { + enriched = append(enriched, source) + } + } + return destination.Index(id, struct { + Document + DataSources string + EnrichedDataSources []string + }{document, string(raw), enriched}) +} + +func (d Document) DataSourceScore() float64 { + return d.DataSourceScoreValue } func (d Document) SupportsAnnotations() bool { diff --git a/internal/index/document_data_sources_test.go b/internal/index/document_data_sources_test.go new file mode 100644 index 00000000..8e0f7b67 --- /dev/null +++ b/internal/index/document_data_sources_test.go @@ -0,0 +1,332 @@ +package index + +import ( + "encoding/json" + "errors" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/svera/coreander/v5/internal/metadata" +) + +func TestDocumentDataSourcesRoundTrip(t *testing.T) { + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + for _, language := range []string{"es", ""} { + document := withProviderData(t, Document{ + ID: "book-" + language, Slug: "book-" + language, + Metadata: metadata.Metadata{Title: "A book", Language: language, Format: "EPUB"}, + DataSourceScoreValue: 4.25, + }, testProviderData{URL: "/book/show/123", Enriched: true}) + other := map[string]any{ + "id": "opaqueprovideridentifier", "nested": map[string]any{"items": []any{"privatecatalogtoken", float64(42)}}, + } + if err := SetDataSourceData(&document, "other", other); err != nil { + t.Fatal(err) + } + if err := IndexDocument(idx.documentsIdx, document.ID, document); err != nil { + t.Fatal(err) + } + got, err := idx.documentByIndexID(document.ID) + if err != nil { + t.Fatal(err) + } + decoded, err := DataSourceData[map[string]any](got, "other") + if err != nil || !reflect.DeepEqual(decoded, other) { + t.Fatalf("provider payload changed: %+v, %v", decoded, err) + } + if mustProviderData(t, got) != mustProviderData(t, document) || got.DataSourceScoreValue != 4.25 { + t.Fatalf("provider payload or score changed: %+v", got) + } + for _, term := range []string{"opaqueprovideridentifier", "privatecatalogtoken", "dummy", "other"} { + request := bleve.NewSearchRequest(bleve.NewMatchQuery(term)) + result, err := idx.documentsIdx.Search(request) + if err != nil || result.Total != 0 { + t.Fatalf("opaque data affected keyword search for %q: %+v, %v", term, result, err) + } + } + } + if _, total, err := idx.documentsNeedingProvider("dummy", 10, nil); err != nil || total != 0 { + t.Fatalf("enriched payloads were queued again: %d, %v", total, err) + } + pending := Document{ID: "other-only", Slug: "other-only", Metadata: metadata.Metadata{Title: "Pending book"}} + if err := SetDataSourceData(&pending, "other", map[string]any{"enriched": true}); err != nil { + t.Fatal(err) + } + if err := IndexDocument(idx.documentsIdx, pending.ID, pending); err != nil { + t.Fatal(err) + } + if documents, total, err := idx.documentsNeedingProvider("dummy", 10, nil); err != nil || total != 1 || documents[0].ID != pending.ID { + t.Fatalf("another provider marked provider as enriched: %+v, %d, %v", documents, total, err) + } +} + +func TestDocumentDataSourcesRejectInvalidData(t *testing.T) { + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}} + for _, invalid := range []any{nil, "not an object", []int{1}, make(chan int)} { + if err := SetDataSourceData(&document, "invalid", invalid); err == nil { + t.Fatalf("accepted non-object provider payload: %#v", invalid) + } + if document.DataSources != nil { + t.Fatal("failed update mutated document data") + } + } + for _, raw := range []string{`{"url":`, `null`, `[]`} { + document.DataSources = map[string]json.RawMessage{"dummy": json.RawMessage(raw)} + if err := IndexDocument(idx.documentsIdx, document.ID, document); err == nil { + t.Fatalf("indexed invalid payload: %s", raw) + } + } +} + +func TestDocumentDataSourcesSurviveProviderAndStaleWrites(t *testing.T) { + idx, _ := newTestTextRankIndexer(t, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + if err := IndexDocument(idx.documentsIdx, "book", Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Book"}}); err != nil { + t.Fatal(err) + } + stale, err := idx.documentByIndexID("book") + if err != nil { + t.Fatal(err) + } + current := stale + other := map[string]any{"id": "external-id", "nested": map[string]any{"count": float64(3)}} + if err := SetDataSourceData(¤t, "other", other); err != nil { + t.Fatal(err) + } + if _, exists := stale.DataSources["other"]; exists { + t.Fatal("updating a provider mutated another document's map") + } + if err := IndexDocument(idx.documentsIdx, current.ID, current); err != nil { + t.Fatal(err) + } + if err := idx.StoreDocumentData(stale.ID, "dummy", testProviderData{URL: "/book/show/123", Enriched: true}); err != nil { + t.Fatal(err) + } + if err := idx.UpdateDocument(stale.ID, func(update *DocumentUpdate) (bool, error) { + update.DataSourceScoreValue = 4.5 + return true, SetDataSourceData(&update.Document, "dummy", testProviderData{URL: "/book/show/123", Enriched: true, RatingCount: "100", ResourceID: "work", LegacyBookID: 123}) + }); err != nil { + t.Fatal(err) + } + if err := idx.StoreDocumentData(stale.ID, "dummy", testProviderData{URL: "/book/show/456", Enriched: true}); err != nil { + t.Fatal(err) + } + stale.TextRankWords = []string{"updated"} + documents := []Document{stale} + idx.documentsMu.Lock() + err = idx.preserveDataSourcesLocked(documents) + if err == nil { + err = IndexDocument(idx.documentsIdx, stale.ID, documents[0]) + } + idx.documentsMu.Unlock() + if err != nil { + t.Fatal(err) + } + got, err := idx.documentByIndexID(stale.ID) + if err != nil { + t.Fatal(err) + } + decoded, err := DataSourceData[map[string]any](got, "other") + if err != nil || !reflect.DeepEqual(decoded, other) { + t.Fatalf("another provider's payload was lost: %+v, %v", decoded, err) + } + if data := mustProviderData(t, got); data != (testProviderData{URL: "/book/show/456", Enriched: true}) { + t.Fatalf("stale write restored old provider state: %+v", data) + } +} + +type testProviderData struct { + URL string `json:"url"` + Enriched bool `json:"enriched"` + RatingCount string `json:"ratingCount,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + LegacyBookID int `json:"legacyBookId,omitempty"` +} + +func mustProviderData(t *testing.T, document Document) testProviderData { + t.Helper() + data, err := DataSourceData[testProviderData](document, "dummy") + if err != nil { + t.Fatal(err) + } + return data +} + +func withProviderData(t *testing.T, document Document, data testProviderData) Document { + t.Helper() + if err := SetDataSourceData(&document, "dummy", data); err != nil { + t.Fatal(err) + } + return document +} + +type documentUpdateBaseIndex interface { + bleve.Index +} + +type documentUpdateCountingIndex struct { + documentUpdateBaseIndex + writes int +} + +func (i *documentUpdateCountingIndex) Index(id string, data any) error { + i.writes++ + return i.documentUpdateBaseIndex.Index(id, data) +} + +func newDocumentUpdateTestIndexer(t *testing.T) (*BleveIndexer, *documentUpdateCountingIndex, *documentUpdateCountingIndex) { + t.Helper() + documents, err := bleve.NewMemOnly(CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authors, err := bleve.NewMemOnly(CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + documentWrites := &documentUpdateCountingIndex{documentUpdateBaseIndex: documents} + authorWrites := &documentUpdateCountingIndex{documentUpdateBaseIndex: authors} + idx := NewBleve(documentWrites, authorWrites, nil, "", nil, Config{}) + t.Cleanup(func() { _ = idx.Close() }) + return idx, documentWrites, authorWrites +} + +func TestUpdateDocumentSkipsUnchangedAndFailedUpdates(t *testing.T) { + failure := errors.New("provider failed") + for _, scenario := range []struct { + name string + changed bool + err error + }{ + {name: "unchanged"}, + {name: "failed", changed: true, err: failure}, + } { + t.Run(scenario.name, func(t *testing.T) { + idx, documents, authors := newDocumentUpdateTestIndexer(t) + document := Document{ID: "book", Slug: "book", Metadata: metadata.Metadata{Title: "Original"}} + if err := IndexDocument(documents, document.ID, document); err != nil { + t.Fatal(err) + } + before, err := idx.DocumentByID(document.ID) + if err != nil { + t.Fatal(err) + } + documents.writes = 0 + called := false + err = idx.UpdateDocument(document.ID, func(update *DocumentUpdate) (bool, error) { + called = true + update.Title = "Changed" + update.AuthorLinks = []AuthorLink{{Slug: "writer", Name: "Writer", Source: "dummy", URL: "reference"}} + if err := SetDataSourceData(&update.Document, "dummy", testProviderData{Enriched: true}); err != nil { + t.Fatal(err) + } + return scenario.changed, scenario.err + }) + if !called || !errors.Is(err, scenario.err) { + t.Fatalf("callback called = %v, error = %v, want %v", called, err, scenario.err) + } + if documents.writes != 0 || authors.writes != 0 { + t.Fatalf("skipped update wrote documents/authors: %d/%d", documents.writes, authors.writes) + } + after, err := idx.DocumentByID(document.ID) + if err != nil || !reflect.DeepEqual(after, before) { + t.Fatalf("skipped update changed document: %+v, %v", after, err) + } + }) + } +} + +func TestUpdateDocumentMissingDocumentDoesNotCallCallback(t *testing.T) { + idx, documents, authors := newDocumentUpdateTestIndexer(t) + called := false + err := idx.UpdateDocument("missing", func(*DocumentUpdate) (bool, error) { + called = true + return true, nil + }) + if !errors.Is(err, ErrDocumentNotFound) || called || documents.writes != 0 || authors.writes != 0 { + t.Fatalf("missing-document update: error %v, callback %v, writes %d/%d", err, called, documents.writes, authors.writes) + } +} + +func TestUpdateDocumentRejectsMalformedProviderData(t *testing.T) { + for _, raw := range []string{`{"url":`, `null`, `[]`} { + t.Run(raw, func(t *testing.T) { + idx, documents, _ := newDocumentUpdateTestIndexer(t) + if err := IndexDocument(documents, "book", Document{ID: "book", Metadata: metadata.Metadata{Title: "Original"}}); err != nil { + t.Fatal(err) + } + documents.writes = 0 + err := idx.UpdateDocument("book", func(update *DocumentUpdate) (bool, error) { + update.Title = "Invalid update" + update.DataSources = map[string]json.RawMessage{"dummy": json.RawMessage(raw)} + return true, nil + }) + if err == nil || documents.writes != 0 { + t.Fatalf("malformed data accepted: error %v, writes %d", err, documents.writes) + } + stored, err := idx.DocumentByID("book") + if err != nil || stored.Title != "Original" || len(stored.DataSources) != 0 { + t.Fatalf("malformed update changed document: %+v, %v", stored, err) + } + }) + } +} + +func TestUpdateDocumentStoresAuthorLinksFirstWins(t *testing.T) { + for _, existing := range []bool{false, true} { + name := "new author" + if existing { + name = "existing author" + } + t.Run(name, func(t *testing.T) { + idx, documents, _ := newDocumentUpdateTestIndexer(t) + for _, document := range []Document{ + {ID: "book", AuthorsSlugs: []string{"writer"}}, + {ID: "second", AuthorsSlugs: []string{"writer"}}, + {ID: "illustrated", IllustratorsSlugs: []string{"writer"}}, + {ID: "unrelated", AuthorsSlugs: []string{"other"}}, + } { + if err := IndexDocument(documents, document.ID, document); err != nil { + t.Fatal(err) + } + } + if existing { + if err := idx.IndexAuthor(Author{ + Name: "Existing Writer", Slug: "writer", DocumentCount: 42, DataSourceID: "external-id", + DataSourceLinks: map[string]string{"other": "other-reference"}, + }); err != nil { + t.Fatal(err) + } + } + for _, url := range []string{"first-reference", "second-reference"} { + if err := idx.UpdateDocument("book", func(update *DocumentUpdate) (bool, error) { + update.Title = url + update.AuthorLinks = []AuthorLink{{Slug: "writer", Name: "Writer", Source: "dummy", URL: url}} + return true, nil + }); err != nil { + t.Fatal(err) + } + } + author, err := idx.Author("writer", "") + if err != nil || author.DataSourceLinks["dummy"] != "first-reference" { + t.Fatalf("first author link lost: %+v, %v", author, err) + } + if existing { + if author.Name != "Existing Writer" || author.DocumentCount != 42 || + author.DataSourceID != "external-id" || author.DataSourceLinks["other"] != "other-reference" { + t.Fatalf("existing author metadata changed: %+v", author) + } + } else if author.Name != "Writer" || author.DocumentCount != 3 { + t.Fatalf("new author name or author/illustrator count incorrect: %+v", author) + } + document, err := idx.DocumentByID("book") + if err != nil || document.Title != "second-reference" { + t.Fatalf("author link prevented document update: %+v, %v", document, err) + } + }) + } +} diff --git a/internal/index/progress.go b/internal/index/progress.go index 87bd467a..72948b28 100644 --- a/internal/index/progress.go +++ b/internal/index/progress.go @@ -5,10 +5,11 @@ import "time" type ProgressKind string const ( - ProgressDocuments ProgressKind = "documents" - ProgressAuthors ProgressKind = "authors" - ProgressTextRank ProgressKind = "textrank" - ProgressPruning ProgressKind = "pruning" + ProgressDocuments ProgressKind = "documents" + ProgressAuthors ProgressKind = "authors" + ProgressTextRank ProgressKind = "textrank" + ProgressPruning ProgressKind = "pruning" + ProgressDataSources ProgressKind = "data-sources" ) type Progress struct { @@ -16,4 +17,5 @@ type Progress struct { InProgress bool RemainingTime time.Duration Percentage float64 + Name string } diff --git a/internal/index/provider_progress_test.go b/internal/index/provider_progress_test.go new file mode 100644 index 00000000..4b8c01ce --- /dev/null +++ b/internal/index/provider_progress_test.go @@ -0,0 +1,71 @@ +package index + +import ( + "testing" + "time" + + "github.com/blevesearch/bleve/v2" +) + +type progressCountingIndex struct { + documentUpdateBaseIndex + searches int + counts int +} + +func (i *progressCountingIndex) Search(request *bleve.SearchRequest) (*bleve.SearchResult, error) { + i.searches++ + return i.documentUpdateBaseIndex.Search(request) +} + +func (i *progressCountingIndex) DocCount() (uint64, error) { + i.counts++ + return i.documentUpdateBaseIndex.DocCount() +} + +func TestProviderProgressCachesCounts(t *testing.T) { + idx, documents := newTestTextRankIndexer(t, Config{ + DocumentEnrichers: []DocumentEnricher{&testDocumentEnricher{source: "dummy"}}, + }) + t.Cleanup(func() { _ = idx.Close() }) + counting := &progressCountingIndex{documentUpdateBaseIndex: documents} + idx.documentsIdx = counting + if err := IndexDocument(documents, "book", Document{ID: "book"}); err != nil { + t.Fatal(err) + } + documentReads := 0 + check := func(wantTasks, wantQueries int) { + t.Helper() + tasks, err := idx.BackgroundProgress() + if err != nil { + t.Fatal(err) + } + if len(tasks) != wantTasks || counting.counts != wantQueries || counting.searches != wantQueries+documentReads { + t.Fatalf("tasks=%+v counts=%d searches=%d; want %d tasks and %d queries", + tasks, counting.counts, counting.searches, wantTasks, wantQueries) + } + } + check(1, 1) + check(1, 1) + idx.providerProgressMu.Lock() + idx.providerProgressUntil = time.Now().Add(-time.Second) + idx.providerProgressMu.Unlock() + check(1, 2) + + if err := idx.StoreDocumentData("book", "dummy", testProviderData{Enriched: true}); err != nil { + t.Fatal(err) + } + // StoreDocumentData itself reads the document once. + documentReads++ + check(0, 3) + check(0, 3) + if err := idx.UpdateDocument("book", func(update *DocumentUpdate) (bool, error) { + update.DataSources = nil + return true, nil + }); err != nil { + t.Fatal(err) + } + documentReads++ + check(1, 4) + check(1, 4) +} diff --git a/internal/index/rebuild_authors_test.go b/internal/index/rebuild_authors_test.go index bfb4b4fc..b9cf9971 100644 --- a/internal/index/rebuild_authors_test.go +++ b/internal/index/rebuild_authors_test.go @@ -35,7 +35,7 @@ func TestRebuildAuthorsFromDocuments(t *testing.T) { }, } for _, doc := range docs { - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } } @@ -63,6 +63,21 @@ func TestRebuildAuthorsFromDocuments(t *testing.T) { if author.DocumentCount != 1 { t.Fatalf("expected DocumentCount=1 for George Orwell, got %d", author.DocumentCount) } + author.DataSourceLinks = map[string]string{ + "goodreads": "/author/show/3706.George_Orwell", + "other": "https://example.org/authors/orwell", + } + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + if err := idx.RebuildAuthorsFromDocuments(10); err != nil { + t.Fatal(err) + } + author, err = idx.Author("george-orwell", "en") + if err != nil || author.DataSourceLinks["goodreads"] != "/author/show/3706.George_Orwell" || + author.DataSourceLinks["other"] != "https://example.org/authors/orwell" { + t.Fatalf("expected existing author URL preserved during rebuild, got %+v, %v", author, err) + } illustrator, err := idx.Author("jane-austen", "en") if err != nil { diff --git a/internal/index/search_filter_scoring_test.go b/internal/index/search_filter_scoring_test.go index 816d0989..0873f3a9 100644 --- a/internal/index/search_filter_scoring_test.go +++ b/internal/index/search_filter_scoring_test.go @@ -68,7 +68,7 @@ func TestAddFiltersDoesNotAffectScoring(t *testing.T) { } idx := NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, Config{}) for _, doc := range docs { - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + if err := IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } } diff --git a/internal/index/textrank_related_test.go b/internal/index/textrank_related_test.go index 4751abdd..2fd7ca1d 100644 --- a/internal/index/textrank_related_test.go +++ b/internal/index/textrank_related_test.go @@ -35,7 +35,7 @@ func newTextRankTestIndexWithConfig(t *testing.T, docs []index.Document, cfg ind idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, cfg) for _, doc := range docs { - if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documentsIndexMem, doc.ID, doc); err != nil { t.Fatal(err) } } diff --git a/internal/webserver/author_summary_test.go b/internal/webserver/author_summary_test.go index 12baff0d..70978fea 100644 --- a/internal/webserver/author_summary_test.go +++ b/internal/webserver/author_summary_test.go @@ -6,12 +6,17 @@ import ( "net/http" "net/url" "os" + "strings" "testing" + "time" gowikidata "github.com/Navid2zp/go-wikidata" "github.com/PuerkitoBio/goquery" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/datasource/wikidata" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/infrastructure" ) @@ -41,7 +46,7 @@ func TestClearAuthorRemovesImage(t *testing.T) { appFS := loadDirInMemoryFs("testdata/library") cfg := defaultTestConfig() cfg.CacheDir = "/cache" - app := bootstrapApp(db, &infrastructure.NoEmail{}, appFS, cfg) + app, idx := bootstrapAppWithIndexer(db, &infrastructure.NoEmail{}, appFS, cfg) cookie, err := login(app, "admin@example.com", "admin", t) if err != nil { @@ -54,6 +59,20 @@ func TestClearAuthorRemovesImage(t *testing.T) { response.Body.Close() mustReturnStatus(response, http.StatusOK, t) + author, err := idx.Author("john-doe", "en") + if err != nil { + t.Fatal(err) + } + author.DataSourceLinks = map[string]string{ + "website": "https://old.example", "wikipedia:en": "https://en.wikipedia.org/wiki/Old", + "wikipedia:es": "https://es.wikipedia.org/wiki/Old", + "goodreads": "/author/show/1.Writer", "catalog": "https://catalog.example/writer", + } + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + documentCount := author.DocumentCount + imagePath := cfg.CacheDir + "/authors/john-doe.webp" if tc.cached { if err := appFS.MkdirAll(cfg.CacheDir+"/authors", 0755); err != nil { @@ -78,7 +97,7 @@ func TestClearAuthorRemovesImage(t *testing.T) { if err != nil { t.Fatal(err) } - imageURL, _ := doc.Find("img").Attr("src") + imageURL, _ := doc.Find(`img[src^="/authors/"]`).Attr("src") if imageURL != "/authors/john-doe_.webp" { t.Fatalf("expected image URL with cleared ID and no cached version, got %q", imageURL) } @@ -110,13 +129,195 @@ func TestClearAuthorRemovesImage(t *testing.T) { if err != nil { t.Fatal(err) } - if imageURL, _ := doc.Find("img").Attr("src"); imageURL != "/authors/john-doe_.webp" { + if imageURL, _ := doc.Find(`img[src^="/authors/"]`).Attr("src"); imageURL != "/authors/john-doe_.webp" { t.Fatalf("expected cleared image URL to persist, got %q", imageURL) } + stored, err := idx.Author("john-doe", "en") + if err != nil { + t.Fatal(err) + } + if stored.DataSourceID != "" || stored.DataSourceImage != "" || + stored.DataSourceLinks["website"] != "" || stored.DataSourceLinks["wikipedia:en"] != "" || + stored.DataSourceLinks["wikipedia:es"] != "" { + t.Fatalf("metadata reset restored stale metadata: %+v", stored) + } + if stored.DataSourceLinks["goodreads"] != "/author/show/1.Writer" || + stored.DataSourceLinks["catalog"] != "https://catalog.example/writer" || + stored.DocumentCount != documentCount { + t.Fatalf("metadata reset lost independent author data: %+v", stored) + } }) } } +type translatedDataSourceLinks struct{} + +func (translatedDataSourceLinks) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { + if author.DataSourceID == "" { + return datasourcemodel.DataSourceLink{} + } + return datasourcemodel.DataSourceLink{ + Source: "Example catalog", URL: "https://example.org/authors/" + author.Slug, + IconClass: "bi-bookmark", + } +} + +func TestAuthorSummaryDataSourceLinkTranslations(t *testing.T) { + app, idx := newDataSourceProvidersTestApp(t, nil, nil, translatedDataSourceLinks{}) + const ( + authorURL = "https://example.org/authors/" + wikipediaURL = "https://en.wikipedia.org/wiki/Alice_Borchardt" + ) + locales := []struct { + lang string + label string + wikipediaLabel string + }{ + {"en", "Go to Example catalog author page", "Go to Wikipedia entry"}, + {"es", "Ir a la página del autor en Example catalog", "Ir al artículo en Wikipedia"}, + {"de", "Zur Autorenseite auf Example catalog", "Zur Wikipedia-Seite"}, + {"fr", "Voir la page de l'auteur sur Example catalog", "Aller à l'entrée Wikipedia"}, + {"ru", "Перейти на страницу автора в Example catalog", "Перейти к статье в Википедии"}, + } + for _, tc := range []struct { + name string + dataSourceID string + hasDataSource bool + wikipedia bool + }{ + {name: "with-wikipedia", dataSourceID: "Q1", hasDataSource: true, wikipedia: true}, + {name: "without-wikipedia", dataSourceID: "Q1", hasDataSource: true}, + {name: "without-data-source-id"}, + } { + author := index.Author{ + Slug: tc.name, Name: "Alice Borchardt", + DataSourceID: tc.dataSourceID, RetrievedOn: time.Now().UTC(), + DataSourceLinks: map[string]string{"website": "https://author.example/" + tc.name}, + } + if tc.wikipedia { + for _, locale := range locales { + author.DataSourceLinks["wikipedia:"+locale.lang] = strings.Replace(wikipediaURL, "en.wikipedia.org", locale.lang+".wikipedia.org", 1) + } + } + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + for _, style := range []struct { + name string + query string + }{ + {"summary", ""}, + {"document-detail", "?style=clear"}, + } { + for _, locale := range locales { + t.Run(tc.name+"/"+style.name+"/"+locale.lang, func(t *testing.T) { + req := mustGetRequest(t, "/authors/"+tc.name+"/summary"+style.query) + req.Header.Set("Accept-Language", locale.lang) + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + wikipediaLink := page.Find(`a[href="https://` + locale.lang + `.wikipedia.org/wiki/Alice_Borchardt"]`) + websiteLink := page.Find(`a[href="https://author.example/` + tc.name + `"]`) + if websiteLink.Length() != 1 || websiteLink.PrevFiltered("i.bi-globe-europe-africa").Length() != 1 { + t.Fatal("expected the stored website link and globe icon") + } + if tc.wikipedia && (wikipediaLink.Length() != 1 || strings.TrimSpace(wikipediaLink.Text()) != locale.wikipediaLabel) { + t.Fatalf("expected localized Wikipedia data-source link %q, got %d links with text %q", locale.wikipediaLabel, wikipediaLink.Length(), wikipediaLink.Text()) + } + if tc.wikipedia && wikipediaLink.PrevFiltered("i.bi-wikipedia").Length() != 1 { + t.Fatal("expected the Wikipedia provider's Bootstrap icon") + } + if !tc.wikipedia && page.Find(`a[href$="wikipedia.org/wiki/Alice_Borchardt"]`).Length() != 0 { + t.Fatal("expected no Wikipedia link when there is no link for the current language") + } + link := page.Find(`a[href="` + authorURL + tc.name + `"]`) + if !tc.hasDataSource { + if link.Length() != 0 { + t.Fatal("expected no data-source link without a provider URL") + } + return + } + if link.Length() != 1 || strings.TrimSpace(link.Text()) != locale.label { + t.Fatalf("expected one localized data-source link %q, got %d links with text %q", locale.label, link.Length(), link.Text()) + } + if link.PrevFiltered("i.bi-bookmark").Length() != 1 { + t.Error("expected the provider's Bootstrap icon before the data-source link") + } + if tc.wikipedia { + wantWikipediaURL := strings.Replace(wikipediaURL, "en.wikipedia.org", locale.lang+".wikipedia.org", 1) + if href, _ := link.Parent().Prev().Find("a").Attr("href"); href != wantWikipediaURL { + t.Fatalf("expected data-source link immediately below Wikipedia, previous link is %q", href) + } + } + }) + } + } + } +} + +type alternativeDataSourceLinks struct{} + +func (alternativeDataSourceLinks) AuthorLink(author index.Author, _ string) datasourcemodel.DataSourceLink { + return datasourcemodel.DataSourceLink{Source: "Another source", URL: "https://example.org/authors/" + author.Slug} +} + +func TestAlternativeDataSourceLinks(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "4", ResourceID: "work", LegacyBookID: 1, + }} + app, idx := newDataSourceProvidersTestApp(t, source, nil, alternativeDataSourceLinks{}) + for _, slug := range []string{"jane-doe", "john-doe"} { + if err := idx.IndexAuthor(index.Author{ + Slug: slug, Name: slug, RetrievedOn: time.Now().UTC(), + }); err != nil { + t.Fatal(err) + } + } + for _, path := range []string{ + "/authors/jane-doe", "/authors/jane-doe/summary", + "/authors/jane-doe/summary?style=clear", "/documents/unlinked", + } { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) + if err != nil { + t.Fatal(err) + } + link := page.Find(`a[href="https://example.org/authors/jane-doe"]`) + if link.Length() != 1 || link.Text() != "Go to Another source author page" { + t.Fatalf("%s: expected injected author data-source link, got %q", path, link.Text()) + } + if page.Find(`img[src^="/images/goodreads.svg"]`).Length() != 0 { + t.Fatalf("%s: unexpected Goodreads branding", path) + } + } + for path, wantLinks := range map[string]int{"/documents/linked/score": 2, "/documents/linked/reviews": 1} { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) + if err != nil { + t.Fatal(err) + } + if page.Find(`a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`).Length() != wantLinks { + t.Fatalf("%s: BiblioReads must retain its Goodreads document URL regardless of the author link provider", path) + } + } +} + +func TestDataSourceLinksWithoutReviewProvider(t *testing.T) { + app, idx := newDataSourceProvidersTestApp(t, nil, nil, alternativeDataSourceLinks{}) + if err := idx.IndexAuthor(index.Author{Slug: "jane-doe", Name: "Jane Doe", RetrievedOn: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + body := getBody(t, app, "/authors/jane-doe/summary") + if !strings.Contains(body, "https://example.org/authors/jane-doe") { + t.Fatal("links must not require a review provider") + } +} + func TestAuthorSummary(t *testing.T) { db := infrastructure.Connect(":memory:", 250) smtpMock := &infrastructure.SMTPMock{} diff --git a/internal/webserver/background_progress_test.go b/internal/webserver/background_progress_test.go new file mode 100644 index 00000000..00190ae4 --- /dev/null +++ b/internal/webserver/background_progress_test.go @@ -0,0 +1,220 @@ +package webserver_test + +import ( + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/gofiber/fiber/v3" + "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/datasource/wikidata" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/webserver" + "github.com/svera/coreander/v5/internal/webserver/infrastructure" + "github.com/svera/coreander/v5/internal/webserver/model" +) + +type progressCountingInfo struct { + webserver.IndexInfo + calls atomic.Int64 +} + +func (i *progressCountingInfo) BackgroundProgress() ([]index.Progress, error) { + i.calls.Add(1) + return i.IndexInfo.BackgroundProgress() +} + +func TestBackgroundProgressOnlyComputedWhenRendered(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + cfg := defaultTestConfig() + cfg.ServerStaticCacheTTL = 60 + sender := &infrastructure.NoEmail{} + appFS := afero.NewMemMapFs() + unusedApp, idx := bootstrapAppWithIndexer(db, sender, appFS, cfg) + t.Cleanup(func() { _ = unusedApp.Shutdown() }) + t.Cleanup(func() { _ = idx.Close() }) + source := wikidata.NewWikidataSource(wikidata.Gowikidata{}) + provider := biblioreads.NewReviewProvider(nil, nil) + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, source, nil, source, provider) + counting := &progressCountingInfo{IndexInfo: idx} + app := webserver.New(cfg, controllers, sender, counting, &model.UserRepository{DB: db}) + t.Cleanup(func() { _ = app.Shutdown() }) + app.Get("/cached-progress-page", func(c fiber.Ctx) error { + c.Set("Cache-Time", "60") + return c.Render("auth/login", fiber.Map{}, "layout") + }) + + for _, path := range []string{ + "/js/feedback.js", "/js/feedback.js", "/css/bootstrap.min.css", + "/images/male.webp", "/reader-service-worker.js", "/documents/missing/score", + } { + response, err := app.Test(mustGetRequest(t, path)) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("%s: unexpected status %d", path, response.StatusCode) + } + if counting.calls.Load() != 0 { + t.Fatalf("%s: progress must not be loaded for assets or unrelated fragments", path) + } + } + for _, path := range []string{"/", "/background-progress", "/sessions/new"} { + before := counting.calls.Load() + response, err := app.Test(mustGetRequest(t, path)) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusOK || counting.calls.Load() != before+1 { + t.Fatalf("%s: expected exactly one progress render", path) + } + } + before := counting.calls.Load() + for range 2 { + response, err := app.Test(mustGetRequest(t, "/cached-progress-page")) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("cached page returned status %d", response.StatusCode) + } + } + if counting.calls.Load() != before+1 { + t.Fatal("cached page responses must not recompute progress") + } + cfg.RequireAuth = true + privateApp := webserver.New(cfg, controllers, sender, counting, &model.UserRepository{DB: db}) + t.Cleanup(func() { _ = privateApp.Shutdown() }) + before = counting.calls.Load() + response, err := privateApp.Test(mustGetRequest(t, "/sessions/new")) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusOK || counting.calls.Load() != before { + t.Fatal("anonymous sign-in pages must not compute inaccessible progress") + } +} + +func TestBackgroundProgressPollingAccess(t *testing.T) { + for _, requireAuth := range []bool{false, true} { + name := "public library" + if requireAuth { + name = "private library" + } + t.Run(name, func(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + cfg := defaultTestConfig() + cfg.RequireAuth = requireAuth + app, idx := bootstrapAppWithIndexer(db, &infrastructure.SMTPMock{}, afero.NewMemMapFs(), cfg) + t.Cleanup(func() { _ = idx.Close() }) + t.Cleanup(func() { _ = app.Shutdown() }) + idx.BeginIndexing() + + if err := db.Model(&model.User{}).Where("email = ?", "admin@example.com").Updates(map[string]interface{}{ + "recovery_uuid": "progress-reset", + "recovery_valid_until": time.Now().UTC().Add(time.Hour), + }).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.Invitation{ + Email: "invite@example.com", + UUID: "progress-invite", + ValidUntil: time.Now().UTC().Add(time.Hour), + }).Error; err != nil { + t.Fatal(err) + } + + check := func(path string, cookie *http.Cookie, status int, polling bool) { + t.Helper() + response, err := getRequest(cookie, app, path, t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != status { + t.Fatalf("%s: expected status %d, got %d", path, status, response.StatusCode) + } + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + banner := page.Find("#background-progress") + if !polling { + if page.Find(`[hx-get^="/background-progress"]`).Length() != 0 { + t.Errorf("%s: inaccessible progress endpoint must not be polled", path) + } + return + } + if banner.AttrOr("hx-trigger", "") != "every 5s" || + banner.AttrOr("hx-get", "") != "/background-progress?l=en" || + banner.AttrOr("hx-swap", "") != "outerHTML" || + banner.AttrOr("data-error-message", "") == "" { + t.Errorf("%s: expected live progress polling", path) + } + if banner.Find(`[data-progress-kind="documents"]`).Length() != 1 { + t.Errorf("%s: expected indexing progress", path) + } + + } + + anonymous := &http.Cookie{} + for _, path := range []string{ + "/sessions/new", "/recover", "/reset-password?id=progress-reset", "/invite?id=progress-invite", + } { + check(path, anonymous, http.StatusOK, !requireAuth) + } + status := http.StatusOK + if requireAuth { + status = http.StatusForbidden + } + for _, path := range []string{"/", "/background-progress"} { + check(path, anonymous, status, !requireAuth) + check(path, &http.Cookie{Name: "session", Value: "invalid"}, status, !requireAuth) + } + + cookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + check("/", cookie, http.StatusOK, true) + check("/background-progress", cookie, http.StatusOK, true) + }) + } +} + +func TestBackgroundProgressErrorHandlerUsesFeedbackScript(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + app := bootstrapApp(db, &infrastructure.NoEmail{}, afero.NewMemMapFs(), webserver.Config{}) + response, err := app.Test(mustGetRequest(t, "/")) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if page.Find(`script[src^="/js/feedback.js"]`).Length() != 1 { + t.Fatal("expected feedback script to be loaded") + } + if page.Find(`script[src^="/js/background-progress.js"]`).Length() != 0 { + t.Fatal("expected no standalone background progress script") + } +} diff --git a/internal/webserver/controller.go b/internal/webserver/controller.go index 0eeb4802..3b4363d9 100644 --- a/internal/webserver/controller.go +++ b/internal/webserver/controller.go @@ -30,7 +30,7 @@ type Controllers struct { Series *series.Controller } -func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender Sender, appFs afero.Fs, dataSource author.DataSource) Controllers { +func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender Sender, appFs afero.Fs, authorDataSource author.DataSource, reviewProvider document.ReviewProvider, dataSourceLinkProviders ...author.DataSourceLinkProvider) Controllers { usersRepository := &model.UserRepository{DB: db} invitationsRepository := &model.InvitationRepository{DB: db} highlightsRepository := &model.HighlightRepository{DB: db, Idx: idx, IllustratedMinAmount: cfg.IllustratedMinAmount} @@ -80,20 +80,23 @@ func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender S } authorsCfg := author.Config{ - WordsPerMinute: cfg.WordsPerMinute, - CacheDir: cfg.CacheDir, - CacheMaxSize: cfg.CacheMaxSize, - AuthorImageMaxWidth: cfg.AuthorImageMaxWidth, - ClientImageCacheTTL: cfg.ClientDynamicImageCacheTTL, - ServerImageCacheTTL: cfg.ServerDynamicImageCacheTTL, + WordsPerMinute: cfg.WordsPerMinute, + CacheDir: cfg.CacheDir, + CacheMaxSize: cfg.CacheMaxSize, + AuthorImageMaxWidth: cfg.AuthorImageMaxWidth, + ClientImageCacheTTL: cfg.ClientDynamicImageCacheTTL, + ServerImageCacheTTL: cfg.ServerDynamicImageCacheTTL, + ReviewSourceConfigured: cfg.ReviewSourceConfigured, } seriesCfg := series.Config{ - WordsPerMinute: cfg.WordsPerMinute, + WordsPerMinute: cfg.WordsPerMinute, + ReviewSourceConfigured: cfg.ReviewSourceConfigured, } searchCfg := search.Config{ - WordsPerMinute: cfg.WordsPerMinute, + WordsPerMinute: cfg.WordsPerMinute, + ReviewSourceConfigured: cfg.ReviewSourceConfigured, } homeCfg := home.Config{ @@ -102,7 +105,7 @@ func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender S LatestDocsLimit: 6, } - authorsController := author.NewController(highlightsRepository, readingRepository, sender, idx, authorsCfg, dataSource, appFs, imagesFS) + authorsController := author.NewController(highlightsRepository, readingRepository, sender, idx, authorsCfg, authorDataSource, appFs, imagesFS, dataSourceLinkProviders...) go authorsController.MigrateJPEGsToWebP() return Controllers{ @@ -110,8 +113,8 @@ func SetupControllers(cfg Config, db *gorm.DB, idx *index.BleveIndexer, sender S Auth: auth.NewController(usersRepository, sender, authCfg, translator), Users: user.NewController(usersRepository, invitationsRepository, usersCfg, sender, translator), Completed: completed.NewController(readingRepository, idx), - Highlights: highlight.NewController(highlightsRepository, readingRepository, usersRepository, sender, cfg.WordsPerMinute, idx), - Documents: document.NewController(highlightsRepository, usersRepository, readingRepository, annotationsRepository, sender, idx, appFs, documentsCfg, translator), + Highlights: highlight.NewController(highlightsRepository, readingRepository, usersRepository, sender, cfg.WordsPerMinute, cfg.ReviewSourceConfigured, idx), + Documents: document.NewController(highlightsRepository, usersRepository, readingRepository, annotationsRepository, sender, idx, appFs, documentsCfg, translator, reviewProvider, dataSourceLinkProviders...), Home: home.NewController(highlightsRepository, readingRepository, sender, idx, homeCfg), Authors: authorsController, Search: search.NewController(highlightsRepository, readingRepository, sender, idx, searchCfg), diff --git a/internal/webserver/controller/author/controller.go b/internal/webserver/controller/author/controller.go index e004b5dc..d44a0eac 100644 --- a/internal/webserver/controller/author/controller.go +++ b/internal/webserver/controller/author/controller.go @@ -4,6 +4,7 @@ import ( "io/fs" "github.com/spf13/afero" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/result" "github.com/svera/coreander/v5/internal/webserver/model" @@ -13,6 +14,10 @@ type Sender interface { From() string } +type DataSourceLinkProvider interface { + AuthorLink(author index.Author, language string) datasourcemodel.DataSourceLink +} + // IdxReader defines a set of author reading operations over an index type IdxReader interface { SearchAuthors(searchFields index.AuthorSearchFields, page, resultsPerPage int) (result.Paginated[[]index.Author], error) @@ -31,34 +36,55 @@ type readingRepository interface { } type Config struct { - WordsPerMinute float64 - CacheDir string - CacheMaxSize int - AuthorImageMaxWidth int - ClientImageCacheTTL int - ServerImageCacheTTL int + WordsPerMinute float64 + CacheDir string + CacheMaxSize int + AuthorImageMaxWidth int + ClientImageCacheTTL int + ServerImageCacheTTL int + ReviewSourceConfigured bool } type Controller struct { - hlRepository highlightsRepository - readingRepository readingRepository - idx IdxReader - sender Sender - config Config - dataSource DataSource - appFs afero.Fs - embeddedImagesFS fs.FS + hlRepository highlightsRepository + readingRepository readingRepository + idx IdxReader + sender Sender + config Config + dataSource DataSource + appFs afero.Fs + embeddedImagesFS fs.FS + dataSourceLinkProviders []DataSourceLinkProvider } -func NewController(hlRepository highlightsRepository, readingRepository readingRepository, sender Sender, idx IdxReader, cfg Config, dataSource DataSource, appFs afero.Fs, embeddedImagesFS fs.FS) *Controller { +func NewController(hlRepository highlightsRepository, readingRepository readingRepository, sender Sender, idx IdxReader, cfg Config, dataSource DataSource, appFs afero.Fs, embeddedImagesFS fs.FS, dataSourceLinkProviders ...DataSourceLinkProvider) *Controller { return &Controller{ - hlRepository: hlRepository, - readingRepository: readingRepository, - idx: idx, - sender: sender, - config: cfg, - dataSource: dataSource, - appFs: appFs, - embeddedImagesFS: embeddedImagesFS, + hlRepository: hlRepository, + readingRepository: readingRepository, + idx: idx, + sender: sender, + config: cfg, + dataSource: dataSource, + appFs: appFs, + embeddedImagesFS: embeddedImagesFS, + dataSourceLinkProviders: dataSourceLinkProviders, + } +} + +func (a *Controller) authorDataSourceLinks(author index.Author, language string) []datasourcemodel.DataSourceLink { + return DataSourceLinks(author, language, a.dataSourceLinkProviders...) +} + +func DataSourceLinks(author index.Author, language string, providers ...DataSourceLinkProvider) []datasourcemodel.DataSourceLink { + var links []datasourcemodel.DataSourceLink + for _, provider := range providers { + if provider == nil { + continue + } + link := provider.AuthorLink(author, language) + if link.URL != "" { + links = append(links, link) + } } + return links } diff --git a/internal/webserver/controller/author/datasource.go b/internal/webserver/controller/author/datasource.go index f9652a32..5339a743 100644 --- a/internal/webserver/controller/author/datasource.go +++ b/internal/webserver/controller/author/datasource.go @@ -3,6 +3,7 @@ package author import "github.com/svera/coreander/v5/internal/datasource/model" type DataSource interface { + OwnsLink(key string) bool SearchAuthor(name string, languages []string) (model.Author, error) RetrieveAuthor(IDs []string, languages []string) (model.Author, error) } diff --git a/internal/webserver/controller/author/documents.go b/internal/webserver/controller/author/documents.go index 0092fdd4..d24e0115 100644 --- a/internal/webserver/controller/author/documents.go +++ b/internal/webserver/controller/author/documents.go @@ -34,7 +34,8 @@ func (a *Controller) Documents(c fiber.Ctx) error { page = 1 } - author, err := a.idx.Author(authorSlug, c.Locals("Lang").(string)) + lang := c.Locals("Lang").(string) + author, err := a.idx.Author(authorSlug, lang) if err != nil { log.Println(err) } @@ -64,27 +65,20 @@ func (a *Controller) Documents(c fiber.Ctx) error { } templateVars := fiber.Map{ - "Author": author, - "ImageVersion": a.getImageVersion(author.Slug), - "DocumentSearchFields": searchFields, - "DocumentsOnly": true, - "Results": searchResults, - "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), - "Title": author.Name, - "EmailFrom": a.sender.From(), - "WordsPerMinute": a.config.WordsPerMinute, - "URL": view.URL(c), - "SortURL": view.BaseURLWithout(c, "sort-by", "page"), - "SortBy": c.Query("sort-by"), - "AdditionalSortOptions": []struct { - Key string - Value string - }{ - {"pub-date-older-first", "older"}, - {"pub-date-newer-first", "newer"}, - {"est-read-time-shorter-first", "shorter"}, - {"est-read-time-longer-first", "longer"}, - }, + "Author": author, + "DataSourceLinks": a.authorDataSourceLinks(author, lang), + "ImageVersion": a.getImageVersion(author.Slug), + "DocumentSearchFields": searchFields, + "DocumentsOnly": true, + "Results": searchResults, + "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), + "Title": author.Name, + "EmailFrom": a.sender.From(), + "WordsPerMinute": a.config.WordsPerMinute, + "URL": view.URL(c), + "SortURL": view.BaseURLWithout(c, "sort-by", "page"), + "SortBy": c.Query("sort-by"), + "AdditionalSortOptions": authorDocumentSortOptions(a.config.ReviewSourceConfigured), } if c.Get("hx-request") == "true" { @@ -101,3 +95,28 @@ func (a *Controller) Documents(c fiber.Ctx) error { } return nil } + +func authorDocumentSortOptions(reviewSourceConfigured bool) []struct { + Key string + Value string +} { + options := []struct { + Key string + Value string + }{ + {"pub-date-older-first", "older"}, + {"pub-date-newer-first", "newer"}, + {"est-read-time-shorter-first", "shorter"}, + {"est-read-time-longer-first", "longer"}, + } + if reviewSourceConfigured { + options = append(options, []struct { + Key string + Value string + }{ + {"review-score-asc", "review-score-asc"}, + {"review-score-desc", "review-score-desc"}, + }...) + } + return options +} diff --git a/internal/webserver/controller/author/summary.go b/internal/webserver/controller/author/summary.go index b18dc474..ccec6635 100644 --- a/internal/webserver/controller/author/summary.go +++ b/internal/webserver/controller/author/summary.go @@ -47,8 +47,9 @@ func (a *Controller) Summary(c fiber.Ctx) error { if !author.RetrievedOn.IsZero() { templateVars := fiber.Map{ - "Author": author, - "ImageVersion": imageVersion, + "Author": author, + "DataSourceLinks": a.authorDataSourceLinks(author, lang), + "ImageVersion": imageVersion, } if err = c.Render(template, templateVars); err != nil { @@ -75,8 +76,9 @@ func (a *Controller) Summary(c fiber.Ctx) error { } templateVars := fiber.Map{ - "Author": author, - "ImageVersion": imageVersion, + "Author": author, + "DataSourceLinks": a.authorDataSourceLinks(author, lang), + "ImageVersion": imageVersion, } if err = c.Render(template, templateVars); err != nil { diff --git a/internal/webserver/controller/author/update.go b/internal/webserver/controller/author/update.go index f1c6131a..25fe54ad 100644 --- a/internal/webserver/controller/author/update.go +++ b/internal/webserver/controller/author/update.go @@ -4,7 +4,6 @@ import ( "fmt" "log" "os" - "time" "github.com/gofiber/fiber/v3" "github.com/svera/coreander/v5/internal/index" @@ -37,7 +36,7 @@ func (a *Controller) Update(c fiber.Ctx) error { } if sourceID == "" { - author = clear(author) + author = author.ClearMetadata(a.dataSource.OwnsLink) } else { authorDataSource, err := a.dataSource.RetrieveAuthor([]string{sourceID}, supportedLanguages) if err != nil { @@ -59,13 +58,15 @@ func (a *Controller) Update(c fiber.Ctx) error { if err := a.idx.IndexAuthor(author); err != nil { log.Println(err) + return fiber.ErrInternalServerError } templateVars := fiber.Map{ - "Author": author, - "ImageVersion": a.getImageVersion(author.Slug), - "Lang": lang, - "Session": session, + "Author": author, + "DataSourceLinks": a.authorDataSourceLinks(author, lang), + "ImageVersion": a.getImageVersion(author.Slug), + "Lang": lang, + "Session": session, } if err = c.Render(template, templateVars); err != nil { @@ -74,11 +75,3 @@ func (a *Controller) Update(c fiber.Ctx) error { } return nil } - -func clear(author index.Author) index.Author { - cleanAuthor := index.Author{} - cleanAuthor.Slug = author.Slug - cleanAuthor.Name = author.Name - cleanAuthor.RetrievedOn = time.Now().UTC() - return cleanAuthor -} diff --git a/internal/webserver/controller/document/controller.go b/internal/webserver/controller/document/controller.go index d3a77edb..e28e2667 100644 --- a/internal/webserver/controller/document/controller.go +++ b/internal/webserver/controller/document/controller.go @@ -1,18 +1,32 @@ package document import ( + "context" "image" "time" "github.com/spf13/afero" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/i18n" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/result" + authorcontroller "github.com/svera/coreander/v5/internal/webserver/controller/author" "github.com/svera/coreander/v5/internal/webserver/model" ) const relatedDocuments = 4 +type ReviewProvider interface { + Name() string + Score(ctx context.Context, document index.Document) (datasourcemodel.Score, error) + // Reviews returns reviews in the current UI language (a two-letter ISO code). + Reviews(ctx context.Context, document index.Document, language string) (datasourcemodel.ReviewPage, error) + // CanEditDocumentLink reports whether this provider supports correcting its document link. + CanEditDocumentLink() bool + // UpdateDocumentLink saves an administrator's correction to the provider-specific link. + UpdateDocumentLink(document index.Document, link string) error +} + type Sender interface { SendBCC(addresses []string, subject, body string) error SendDocument(address, subject string, file []byte, fileName string) error @@ -88,9 +102,12 @@ type Controller struct { config Config appFs afero.Fs translator i18n.Translator + reviewProvider ReviewProvider + linkProviders []authorcontroller.DataSourceLinkProvider } -func NewController(hlRepository highlightsRepository, usersRepository usersRepository, readingRepository readingRepository, annotationsRepository annotationsRepository, sender Sender, idx IdxReaderWriter, appFs afero.Fs, cfg Config, translator i18n.Translator) *Controller { +// NewController creates a document controller. A nil reviewProvider disables scores and reviews. +func NewController(hlRepository highlightsRepository, usersRepository usersRepository, readingRepository readingRepository, annotationsRepository annotationsRepository, sender Sender, idx IdxReaderWriter, appFs afero.Fs, cfg Config, translator i18n.Translator, reviewProvider ReviewProvider, linkProviders ...authorcontroller.DataSourceLinkProvider) *Controller { return &Controller{ hlRepository: hlRepository, usersRepository: usersRepository, @@ -101,5 +118,34 @@ func NewController(hlRepository highlightsRepository, usersRepository usersRepos config: cfg, appFs: appFs, translator: translator, + reviewProvider: reviewProvider, + linkProviders: linkProviders, + } +} + +// DataSourceName returns the source name used for review-score labels. +func (d *Controller) DataSourceName() string { + if d.reviewProvider == nil { + return "" + } + return d.reviewProvider.Name() +} + +// StoredScore reads cached scores without making requests to a review service. +func (d *Controller) StoredScore(document index.Document) (datasourcemodel.Score, error) { + type storedScoreProvider interface { + StoredScore(index.Document) (datasourcemodel.Score, error) + } + if d.reviewProvider != nil { + if provider, ok := d.reviewProvider.(storedScoreProvider); ok { + return provider.StoredScore(document) + } + return datasourcemodel.Score{}, nil + } + for _, linkProvider := range d.linkProviders { + if provider, ok := linkProvider.(storedScoreProvider); ok { + return provider.StoredScore(document) + } } + return datasourcemodel.Score{}, nil } diff --git a/internal/webserver/controller/document/detail.go b/internal/webserver/controller/document/detail.go index 29a49a6e..86f4998f 100644 --- a/internal/webserver/controller/document/detail.go +++ b/internal/webserver/controller/document/detail.go @@ -6,7 +6,9 @@ import ( "github.com/gofiber/fiber/v3" "github.com/gosimple/slug" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" + authorcontroller "github.com/svera/coreander/v5/internal/webserver/controller/author" "github.com/svera/coreander/v5/internal/webserver/controller/fsutil" "github.com/svera/coreander/v5/internal/webserver/model" ) @@ -15,8 +17,9 @@ import ( // render eagerly (i.e. without the async /summary htmx round-trip), so it can // only fall back to that round-trip when the author hasn't been enriched yet. type authorSummary struct { - Author index.Author - ImageVersion string + Author index.Author + ImageVersion string + DataSourceLinks []datasourcemodel.DataSourceLink } func (d *Controller) Detail(c fiber.Ctx) error { @@ -71,9 +74,20 @@ func (d *Controller) Detail(c fiber.Ctx) error { "AuthorSummaries": authorSummaries, "IllustratorSummaries": illustratorSummaries, "Debug": c.RequestCtx().QueryArgs().Has("debug"), + "ReviewsEnabled": d.reviewProvider != nil, + "DocumentLink": d.dataSourceDocumentLink(document), }, "layout") } +func (d *Controller) dataSourceDocumentLink(document index.Document) datasourcemodel.DataSourceLink { + if provider, ok := d.reviewProvider.(interface { + DocumentLink(index.Document) datasourcemodel.DataSourceLink + }); ok { + return provider.DocumentLink(document) + } + return datasourcemodel.DataSourceLink{} +} + // authorAndIllustratorSummaries resolves the (up to 2) authors and (up to 2) // illustrators shown on a document's detail page, so their summary partials can // render eagerly when the author is already enriched, skipping the async @@ -117,7 +131,8 @@ func (d *Controller) authorSummaryFor(authorSlug, lang string) authorSummary { log.Println(err) } - return authorSummary{Author: author, ImageVersion: fsutil.AuthorImageVersion(d.appFs, d.config.CacheDir, author.Slug)} + links := authorcontroller.DataSourceLinks(author, lang, d.linkProviders...) + return authorSummary{Author: author, ImageVersion: fsutil.AuthorImageVersion(d.appFs, d.config.CacheDir, author.Slug), DataSourceLinks: links} } func (d *Controller) sameSeries(slug string, sessionID int) ([]model.AugmentedDocument, int, error) { diff --git a/internal/webserver/controller/document/reviews.go b/internal/webserver/controller/document/reviews.go new file mode 100644 index 00000000..53ad8459 --- /dev/null +++ b/internal/webserver/controller/document/reviews.go @@ -0,0 +1,83 @@ +package document + +import ( + "context" + "html/template" + "log" + "math" + "time" + + "github.com/gofiber/fiber/v3" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/metadata" + "golang.org/x/exp/slices" +) + +type documentReview struct { + datasourcemodel.Review + HTML template.HTML + CreatedOn string + StarFills [5]int +} + +// Reviews loads the first page of user reviews when the detail page's bottom is visible. +func (d *Controller) Reviews(c fiber.Ctx) error { + if d.reviewProvider == nil { + return c.SendString("") + } + + document, err := d.idx.Document(c.Params("slug")) + if err != nil { + log.Println(err) + return fiber.ErrInternalServerError + } + if document.Slug == "" { + return fiber.ErrNotFound + } + ctx, cancel := context.WithTimeout(c.Context(), reviewsTimeout) + defer cancel() + lang := c.Locals("Lang").(string) + reviewLanguage := c.Query("language") + supportedLanguages := c.Locals("SupportedLanguages").([]string) + if reviewLanguage == "" { + reviewLanguage = lang + } + if !slices.Contains(supportedLanguages, reviewLanguage) { + return fiber.ErrBadRequest + } + page, err := d.reviewProvider.Reviews(ctx, document, reviewLanguage) + if err != nil { + log.Printf("Error retrieving reviews for %s: %s", document.ID, err) + return fiber.ErrInternalServerError + } + if !page.Available { + link := d.dataSourceDocumentLink(document) + if link.URL == "" { + return c.SendString("") + } + return c.Render("partials/document-reviews", fiber.Map{ + "Slug": document.Slug, "ReviewLanguage": reviewLanguage, + "Source": link.Source, "BookURL": link.URL, "Unavailable": true, + }) + } + renderedReviews := make([]documentReview, 0, len(page.Reviews)) + for _, review := range page.Reviews { + var createdOn string + if !review.CreatedAt.IsZero() { + createdOn = review.CreatedAt.Format(time.DateOnly) + } + rendered := documentReview{ + Review: review, + HTML: template.HTML(metadata.SanitizeDescription(review.Text)), + CreatedOn: createdOn, + } + for point := range rendered.StarFills { + rendered.StarFills[point] = int(math.Round(min(1, max(0, review.Rating-float64(point))) * 100)) + } + renderedReviews = append(renderedReviews, rendered) + } + return c.Render("partials/document-reviews", fiber.Map{ + "Slug": document.Slug, "ReviewLanguage": reviewLanguage, + "Reviews": renderedReviews, "Source": page.Source, "BookURL": page.URL, + }) +} diff --git a/internal/webserver/controller/document/score.go b/internal/webserver/controller/document/score.go new file mode 100644 index 00000000..893ae4ae --- /dev/null +++ b/internal/webserver/controller/document/score.go @@ -0,0 +1,50 @@ +package document + +import ( + "context" + "log" + "time" + + "github.com/gofiber/fiber/v3" + "github.com/svera/coreander/v5/internal/webserver/model" +) + +const reviewsTimeout = 30 * time.Second + +// Score loads a document's score without blocking the detail page. +func (d *Controller) Score(c fiber.Ctx) error { + if d.reviewProvider == nil { + return c.SendString("") + } + document, err := d.idx.Document(c.Params("slug")) + if err != nil { + log.Println(err) + return c.SendString("") + } + if document.Slug == "" { + return c.SendString("") + } + ctx, cancel := context.WithTimeout(c.Context(), reviewsTimeout) + defer cancel() + score, err := d.reviewProvider.Score(ctx, document) + session, _ := c.Locals("Session").(model.Session) + canEditDocumentLink := session.Role == model.RoleAdmin && d.reviewProvider.CanEditDocumentLink() + if err != nil { + log.Printf("Error retrieving score for %s: %s", document.ID, err) + if !canEditDocumentLink { + return c.SendString("") + } + score.Value, score.Count = "", "" + score.URL = d.dataSourceDocumentLink(document).URL + } + return c.Render("partials/document-score", fiber.Map{ + "Source": score.Source, + "Rating": score.Value, + "RatingCount": score.Count, + "Scored": score.HasScore(), + "URL": score.URL, + "DocumentSlug": document.Slug, + "CanEditDocumentLink": canEditDocumentLink, + "ScoreUnavailable": err != nil, + }) +} diff --git a/internal/webserver/controller/document/update_review_link.go b/internal/webserver/controller/document/update_review_link.go new file mode 100644 index 00000000..8785f1bd --- /dev/null +++ b/internal/webserver/controller/document/update_review_link.go @@ -0,0 +1,38 @@ +package document + +import ( + "errors" + "log" + + "github.com/gofiber/fiber/v3" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" +) + +func (d *Controller) UpdateReviewLink(c fiber.Ctx) error { + if d.reviewProvider == nil || !d.reviewProvider.CanEditDocumentLink() { + return fiber.ErrNotFound + } + document, err := d.idx.Document(c.Params("slug")) + if err != nil { + log.Printf("Error reading document for link correction: %s", err) + return fiber.ErrInternalServerError + } + if document.Slug == "" { + return fiber.ErrNotFound + } + link := c.FormValue("bookURL") + if err := d.reviewProvider.UpdateDocumentLink(document, link); err != nil { + if errors.Is(err, datasourcemodel.ErrInvalidDocumentLink) { + return c.Status(fiber.StatusUnprocessableEntity).Render("partials/document-review-link-form", fiber.Map{ + "Slug": document.Slug, "BookURL": link, "InvalidURL": true, "Source": d.reviewProvider.Name(), + }) + } + if errors.Is(err, datasourcemodel.ErrDocumentNotFound) { + return fiber.ErrNotFound + } + log.Printf("Error correcting document link for %s: %s", document.ID, err) + return fiber.ErrInternalServerError + } + c.Set("HX-Refresh", "true") + return c.SendStatus(fiber.StatusNoContent) +} diff --git a/internal/webserver/controller/highlight/controller.go b/internal/webserver/controller/highlight/controller.go index caa800c5..4dfab1a4 100644 --- a/internal/webserver/controller/highlight/controller.go +++ b/internal/webserver/controller/highlight/controller.go @@ -37,21 +37,23 @@ type Sender interface { } type Controller struct { - hlRepository highlightsRepository - readingRepository readingRepository - usrRepository usersRepository - idx IdxReaderWriter - sender Sender - wordsPerMinute float64 + hlRepository highlightsRepository + readingRepository readingRepository + usrRepository usersRepository + idx IdxReaderWriter + sender Sender + wordsPerMinute float64 + reviewSourceConfigured bool } -func NewController(hlRepository highlightsRepository, readingRepository readingRepository, usrRepository usersRepository, sender Sender, wordsPerMinute float64, idx IdxReaderWriter) *Controller { +func NewController(hlRepository highlightsRepository, readingRepository readingRepository, usrRepository usersRepository, sender Sender, wordsPerMinute float64, reviewSourceConfigured bool, idx IdxReaderWriter) *Controller { return &Controller{ - hlRepository: hlRepository, - readingRepository: readingRepository, - usrRepository: usrRepository, - idx: idx, - sender: sender, - wordsPerMinute: wordsPerMinute, + hlRepository: hlRepository, + readingRepository: readingRepository, + usrRepository: usrRepository, + idx: idx, + sender: sender, + wordsPerMinute: wordsPerMinute, + reviewSourceConfigured: reviewSourceConfigured, } } diff --git a/internal/webserver/controller/highlight/list.go b/internal/webserver/controller/highlight/list.go index 4c6e687c..f7731ddd 100644 --- a/internal/webserver/controller/highlight/list.go +++ b/internal/webserver/controller/highlight/list.go @@ -79,16 +79,17 @@ func (h *Controller) List(c fiber.Ctx) error { } templateVars := fiber.Map{ - "Results": paginatedResults, - "Paginator": view.Pagination(model.MaxPagesNavigator, paginatedResults, c.Queries()), - "Title": "Highlights", - "EmailFrom": h.sender.From(), - "WordsPerMinute": h.wordsPerMinute, - "SortURL": view.BaseURLWithout(c, "sort-by", "page"), - "SortBy": c.Query("sort-by"), - "HighlightsFilter": filter, - "SearchFields": searchFields, - "HighlightsTotalAll": totalAll, + "Results": paginatedResults, + "Paginator": view.Pagination(model.MaxPagesNavigator, paginatedResults, c.Queries()), + "Title": "Highlights", + "ReviewSourceConfigured": h.reviewSourceConfigured, + "EmailFrom": h.sender.From(), + "WordsPerMinute": h.wordsPerMinute, + "SortURL": view.BaseURLWithout(c, "sort-by", "page"), + "SortBy": c.Query("sort-by"), + "HighlightsFilter": filter, + "SearchFields": searchFields, + "HighlightsTotalAll": totalAll, "AdditionalSortOptions": []struct { Key string Value string diff --git a/internal/webserver/controller/search/controller.go b/internal/webserver/controller/search/controller.go index 40bb005c..792f04c1 100644 --- a/internal/webserver/controller/search/controller.go +++ b/internal/webserver/controller/search/controller.go @@ -35,7 +35,8 @@ type readingRepository interface { } type Config struct { - WordsPerMinute float64 + WordsPerMinute float64 + ReviewSourceConfigured bool } type Controller struct { diff --git a/internal/webserver/controller/search/parse.go b/internal/webserver/controller/search/parse.go index 6ea798f2..1f7eb1bb 100644 --- a/internal/webserver/controller/search/parse.go +++ b/internal/webserver/controller/search/parse.go @@ -12,20 +12,27 @@ import ( // ParseDocumentSearchQuery reads document search/filter query params (search, // language, subjects, similar, sort-by, pub-date-from/to, est-read-time-from/to, -// pages-from/to, illustrated-only) into an index.SearchFields. +// pages-from/to, review score, illustrated-only) into an index.SearchFields. func ParseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.SearchFields, error) { searchFields := index.SearchFields{ - Keywords: c.Query("search"), - Language: c.Query("language"), - Subjects: c.Query("subjects"), - SimilarTo: similarToSlug(c), - SortBy: parseDocumentSortBy(c), - EstReadTimeFrom: fiber.Query[float64](c, "est-read-time-from", 0), - EstReadTimeTo: fiber.Query[float64](c, "est-read-time-to", 0), - WordsPerMinute: wordsPerMinute, - PagesFrom: fiber.Query[float64](c, "pages-from", 0), - PagesTo: fiber.Query[float64](c, "pages-to", 0), - IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", + Keywords: c.Query("search"), + Language: c.Query("language"), + Subjects: c.Query("subjects"), + SimilarTo: similarToSlug(c), + SortBy: parseDocumentSortBy(c), + EstReadTimeFrom: fiber.Query[float64](c, "est-read-time-from", 0), + EstReadTimeTo: fiber.Query[float64](c, "est-read-time-to", 0), + WordsPerMinute: wordsPerMinute, + PagesFrom: fiber.Query[float64](c, "pages-from", 0), + PagesTo: fiber.Query[float64](c, "pages-to", 0), + DataSourceScoreFrom: fiber.Query[float64](c, "review-score-from", 0), + DataSourceScoreTo: new(fiber.Query[float64](c, "review-score-to", 5)), + IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", + } + // Preserve existing links using the former no-score query value. + if c.Query("review-score-from") == "none" || c.Query("review-score-to") == "none" { + searchFields.DataSourceScoreFrom = 0 + searchFields.DataSourceScoreTo = new(0.0) } if c.Query("pub-date-from") != "" { @@ -56,6 +63,10 @@ func ParseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.Search searchFields.PagesFrom, searchFields.PagesTo = searchFields.PagesTo, searchFields.PagesFrom } + if searchFields.DataSourceScoreFrom > *searchFields.DataSourceScoreTo { + searchFields.DataSourceScoreFrom, *searchFields.DataSourceScoreTo = *searchFields.DataSourceScoreTo, searchFields.DataSourceScoreFrom + } + return searchFields, nil } @@ -146,6 +157,10 @@ func parseDocumentSortBy(c fiber.Ctx) []string { return []string{"Words"} case "est-read-time-longer-first": return []string{"-Words"} + case "review-score-asc": + return []string{index.DataSourceScoreValueSortField} + case "review-score-desc": + return []string{"-" + index.DataSourceScoreValueSortField} } } return index.DefaultDocumentSortBy diff --git a/internal/webserver/controller/search/search.go b/internal/webserver/controller/search/search.go index f06735c7..f618edc1 100644 --- a/internal/webserver/controller/search/search.go +++ b/internal/webserver/controller/search/search.go @@ -87,7 +87,7 @@ func (s *Controller) renderDocumentSearch(c fiber.Ctx, session model.Session, pa templateVars["Paginator"] = view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()) templateVars["Title"] = "Search results" templateVars["WordsPerMinute"] = wordsPerMinute - templateVars["AdditionalSortOptions"] = documentSortOptions() + templateVars["AdditionalSortOptions"] = documentSortOptions(s.config.ReviewSourceConfigured) templateVars["DocumentsTotalHits"] = docCount templateVars["AuthorsTotalHits"] = authorCount templateVars["SimilarCandidatesCapped"] = documentResults.Candidates.Capped() @@ -151,17 +151,18 @@ func (s *Controller) renderAuthorSearch(c fiber.Ctx, session model.Session, page func (s *Controller) baseTemplateVars(c fiber.Ctx, searchType string, similarToDocument index.Document) fiber.Map { return fiber.Map{ - "SearchType": searchType, - "SearchPage": true, - "EmailFrom": s.sender.From(), - "URL": view.URL(c), - "SortURL": view.BaseURLWithout(c, "sort-by", "page"), - "SortBy": c.Query("sort-by"), - "AuthorSearchFields": index.AuthorSearchFields{}, - "DocumentSearchFields": index.SearchFields{}, - "SimilarToDocument": similarToDocument, - "SimilarToActive": similarToDocument.Slug != "", - "DocumentsOnly": similarToDocument.Slug != "", + "SearchType": searchType, + "SearchPage": true, + "ReviewSourceConfigured": s.config.ReviewSourceConfigured, + "EmailFrom": s.sender.From(), + "URL": view.URL(c), + "SortURL": view.BaseURLWithout(c, "sort-by", "page"), + "SortBy": c.Query("sort-by"), + "AuthorSearchFields": index.AuthorSearchFields{}, + "DocumentSearchFields": index.SearchFields{}, + "SimilarToDocument": similarToDocument, + "SimilarToActive": similarToDocument.Slug != "", + "DocumentsOnly": similarToDocument.Slug != "", } } @@ -209,11 +210,11 @@ func (s *Controller) tabCounts(docFields index.SearchFields, authorFields index. return docResults, authorCount, nil } -func documentSortOptions() []struct { +func documentSortOptions(reviewSourceConfigured bool) []struct { Key string Value string } { - return []struct { + options := []struct { Key string Value string }{ @@ -223,6 +224,16 @@ func documentSortOptions() []struct { {"est-read-time-shorter-first", "shorter"}, {"est-read-time-longer-first", "longer"}, } + if reviewSourceConfigured { + options = append(options, []struct { + Key string + Value string + }{ + {"review-score-asc", "review-score-asc"}, + {"review-score-desc", "review-score-desc"}, + }...) + } + return options } func authorSortOptions() []struct { diff --git a/internal/webserver/controller/series/controller.go b/internal/webserver/controller/series/controller.go index 6c42d561..cbbd0449 100644 --- a/internal/webserver/controller/series/controller.go +++ b/internal/webserver/controller/series/controller.go @@ -26,7 +26,8 @@ type readingRepository interface { } type Config struct { - WordsPerMinute float64 + WordsPerMinute float64 + ReviewSourceConfigured bool } type Controller struct { diff --git a/internal/webserver/controller/series/documents.go b/internal/webserver/controller/series/documents.go index 840cddbe..61e06f8a 100644 --- a/internal/webserver/controller/series/documents.go +++ b/internal/webserver/controller/series/documents.go @@ -72,28 +72,18 @@ func (a *Controller) Documents(c fiber.Ctx) error { } templateVars := fiber.Map{ - "SeriesTitle": title, - "DocumentSearchFields": searchFields, - "DocumentsOnly": true, - "Results": searchResults, - "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), - "Title": title, - "EmailFrom": a.sender.From(), - "WordsPerMinute": a.config.WordsPerMinute, - "URL": view.URL(c), - "SortURL": view.BaseURLWithout(c, "sort-by", "page"), - "SortBy": c.Query("sort-by"), - "AdditionalSortOptions": []struct { - Key string - Value string - }{ - {"number", "series number"}, - {"number-desc", "series number (descending)"}, - {"pub-date-older-first", "older"}, - {"pub-date-newer-first", "newer"}, - {"est-read-time-shorter-first", "shorter"}, - {"est-read-time-longer-first", "longer"}, - }, + "SeriesTitle": title, + "DocumentSearchFields": searchFields, + "DocumentsOnly": true, + "Results": searchResults, + "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), + "Title": title, + "EmailFrom": a.sender.From(), + "WordsPerMinute": a.config.WordsPerMinute, + "URL": view.URL(c), + "SortURL": view.BaseURLWithout(c, "sort-by", "page"), + "SortBy": c.Query("sort-by"), + "AdditionalSortOptions": seriesDocumentSortOptions(a.config.ReviewSourceConfigured), } if c.Get("hx-request") == "true" { @@ -111,6 +101,33 @@ func (a *Controller) Documents(c fiber.Ctx) error { return nil } +func seriesDocumentSortOptions(reviewSourceConfigured bool) []struct { + Key string + Value string +} { + options := []struct { + Key string + Value string + }{ + {"number", "series number"}, + {"number-desc", "series number (descending)"}, + {"pub-date-older-first", "older"}, + {"pub-date-newer-first", "newer"}, + {"est-read-time-shorter-first", "shorter"}, + {"est-read-time-longer-first", "longer"}, + } + if reviewSourceConfigured { + options = append(options, []struct { + Key string + Value string + }{ + {"review-score-asc", "review-score-asc"}, + {"review-score-desc", "review-score-desc"}, + }...) + } + return options +} + func (d *Controller) parseSortBy(c fiber.Ctx) []string { if c.Query("sort-by") != "" { switch c.Query("sort-by") { @@ -124,6 +141,10 @@ func (d *Controller) parseSortBy(c fiber.Ctx) []string { return []string{"Words"} case "est-read-time-longer-first": return []string{"-Words"} + case "review-score-asc": + return []string{index.DataSourceScoreValueSortField, "SeriesIndex"} + case "review-score-desc": + return []string{"-" + index.DataSourceScoreValueSortField, "SeriesIndex"} } } return []string{"SeriesIndex"} diff --git a/internal/webserver/document_biblioreads_test.go b/internal/webserver/document_biblioreads_test.go new file mode 100644 index 00000000..0993d67d --- /dev/null +++ b/internal/webserver/document_biblioreads_test.go @@ -0,0 +1,709 @@ +package webserver_test + +import ( + "context" + "errors" + "io" + "net/http" + "net/url" + "strings" + "sync" + "testing" + + "github.com/PuerkitoBio/goquery" + "github.com/blevesearch/bleve/v2" + "github.com/gofiber/fiber/v3" + "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/datasource/wikidata" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" + "github.com/svera/coreander/v5/internal/webserver" + authorcontroller "github.com/svera/coreander/v5/internal/webserver/controller/author" + "github.com/svera/coreander/v5/internal/webserver/controller/document" + "github.com/svera/coreander/v5/internal/webserver/infrastructure" + "github.com/svera/coreander/v5/internal/webserver/model" +) + +type fakeBookSource struct { + mu sync.Mutex + ids []string + queries []string + book biblioreads.BookResponse + results []biblioreads.SearchResult + err error + searchErr error + reviews biblioreads.ReviewsResponse + reviewErr error + reviewCalls []reviewCall +} + +func mustBiblioReadsData(t *testing.T, document index.Document) biblioreads.DocumentData { + t.Helper() + data, err := index.DataSourceData[biblioreads.DocumentData](document, biblioreads.Source) + if err != nil { + t.Fatal(err) + } + return data +} + +func withBiblioReadsData(t *testing.T, document index.Document, data biblioreads.DocumentData) index.Document { + t.Helper() + if err := index.SetDataSourceData(&document, biblioreads.Source, data); err != nil { + t.Fatal(err) + } + return document +} + +type reviewCall struct { + resourceID string + legacyBookID int + nextPageToken string + languageCode string +} + +func (f *fakeBookSource) Reviews(_ context.Context, resourceID string, legacyBookID int, nextPageToken, languageCode string) (biblioreads.ReviewsResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.reviewCalls = append(f.reviewCalls, reviewCall{resourceID, legacyBookID, nextPageToken, languageCode}) + return f.reviews, f.reviewErr +} + +func (f *fakeBookSource) SearchBooks(_ context.Context, query string) (biblioreads.SearchBooksResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.queries = append(f.queries, query) + return biblioreads.SearchBooksResponse{Result: f.results}, f.searchErr +} + +func (f *fakeBookSource) Book(_ context.Context, id string) (biblioreads.BookResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.ids = append(f.ids, id) + return f.book, f.err +} + +func newBiblioReadsTestApp(t *testing.T, source *fakeBookSource) (*fiber.App, *index.BleveIndexer) { + t.Helper() + return newDataSourceProvidersTestApp(t, source, nil, nil) +} + +func TestReviewEnrichmentProgressBanner(t *testing.T) { + source := &fakeBookSource{searchErr: biblioreads.ErrUnavailable} + app, idx := newBiblioReadsTestApp(t, source) + idx.BeginIndexing() + page := getPage(t, app, "/documents/linked") + banner := page.Find("#background-progress") + if banner.Find(`[data-progress-kind="documents"]`).Length() != 1 || + banner.Find(`[data-progress-kind="data-sources"]`).Length() != 1 { + t.Fatal("indexing and review progress must share a banner") + } + if banner.AttrOr("hx-trigger", "") != "every 5s" || + !strings.Contains(banner.Text(), "Score filters may omit documents") { + t.Fatal("expected live polling and score-filter warning") + } + checkPending := func() { + t.Helper() + page := getPage(t, app, "/background-progress?l=en") + reviews := page.Find(`[data-progress-kind="data-sources"]`) + if !strings.Contains(reviews.Text(), "(Goodreads)") || + reviews.Find(`[role="progressbar"]`).AttrOr("aria-valuenow", "") != "66" { + t.Fatalf("incorrect persisted lookup progress: %s", reviews.Text()) + } + } + checkPending() + spanish := getPage(t, app, "/background-progress?l=es") + if !strings.Contains(spanish.Find("#background-progress").Text(), "La información de valoraciones está incompleta") { + t.Fatal("live progress must retain the selected language") + } + if err := idx.EnrichDocuments(1); err == nil { + t.Fatal("expected unavailable source to fail") + } + checkPending() + source.searchErr = nil + if err := idx.EnrichDocuments(1); err != nil { + t.Fatal(err) + } + page = getPage(t, app, "/background-progress?l=en") + if page.Find(`[data-progress-kind="data-sources"]`).Length() != 0 || + page.Find(`[data-progress-kind="documents"]`).Length() != 1 { + t.Fatal("completion must remove only the review bar, including no-match lookups") + } + response, err := app.Test(mustGetRequest(t, "/background-progress")) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.Header.Get("Cache-Control") != "no-store" { + t.Fatal("live progress responses must not be cached") + } +} + +func newDataSourceProvidersTestApp(t *testing.T, source *fakeBookSource, provider document.ReviewProvider, dataSourceLinkProvider authorcontroller.DataSourceLinkProvider) (*fiber.App, *index.BleveIndexer) { + t.Helper() + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + + documents, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authors, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + appFS := afero.NewMemMapFs() + idxCfg := index.Config{} + var adapter *biblioreads.Enricher + if source != nil { + adapter = biblioreads.NewEnricher(source) + } else { + adapter = biblioreads.NewEnricher(nil) + } + if source != nil { + idxCfg.DocumentEnrichers = []index.DocumentEnricher{adapter} + } + idx := index.NewBleve(documents, authors, appFS, testLibraryDir, nil, idxCfg) + adapter.SetIndex(idx) + if source != nil { + provider = biblioreads.NewReviewProvider(source, adapter) + } + biblioReadsLinks := biblioreads.NewReviewProvider(nil, adapter) + t.Cleanup(func() { _ = idx.Close() }) + + for _, doc := range []index.Document{withBiblioReadsData(t, index.Document{ + ID: "linked.epub", Slug: "linked", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, + + DataSourceScoreValue: 3.82, + }, biblioreads.DocumentData{URL: "/book/show/12207444-la-reina-dragon", Enriched: true}), { + ID: "unlinked.epub", Slug: "unlinked", + Metadata: metadata.Metadata{Title: "Unlinked", Authors: []string{"Jane Doe", "John Doe"}, Format: "EPUB"}, + AuthorsSlugs: []string{"jane-doe", "john-doe"}, + }, withBiblioReadsData(t, index.Document{ + ID: "unmatched.epub", Slug: "unmatched", + Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, + }, biblioreads.DocumentData{Enriched: true}), + } { + if err := index.IndexDocument(documents, doc.ID, doc); err != nil { + t.Fatal(err) + } + } + + cfg := defaultTestConfig() + cfg.ReviewSourceConfigured = provider != nil + sender := &infrastructure.NoEmail{} + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, wikidata.NewWikidataSource(wikidata.Gowikidata{}), provider, wikidata.NewWikidataSource(wikidata.Gowikidata{}), biblioReadsLinks, dataSourceLinkProvider) + app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) + t.Cleanup(func() { _ = app.Shutdown() }) + return app, idx +} + +func TestDocumentDetailBiblioReadsRating(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "3.82", RatingCount: "2,285", + ResourceID: "work-id", LegacyBookID: 12207444, + }} + app, idx := newBiblioReadsTestApp(t, source) + + placeholder := documentScorePlaceholders(t, getBody(t, app, "/documents/linked")) + if placeholder.Length() != 1 { + t.Fatalf("expected one BiblioReads placeholder, got %d", placeholder.Length()) + } + if url, _ := placeholder.Attr("hx-get"); url != "/documents/linked/score" { + t.Errorf("unexpected placeholder URL %q", url) + } + if placeholder.ParentsFiltered(`[id="document-metadata-linked"]`).Length() != 1 { + t.Error("expected the placeholder inside the document metadata list") + } + + page := getPage(t, app, "/documents/linked/score") + link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if page.Find("li i.bi-star-fill").Length() != 1 { + t.Error("expected the rating row to retain its star icon") + } + if strings.TrimSpace(link.Text()) != "3.82" { + t.Errorf("expected rating link with 3.82, got %q", link.Text()) + } + if !strings.Contains(page.Find("li").Text(), "2,285") { + t.Errorf("expected rating count in %q", page.Find("li").Text()) + } + if len(source.ids) != 1 || source.ids[0] != "12207444-la-reina-dragon" { + t.Errorf("expected Book to be called with the stored id, got %q", source.ids) + } + if len(source.queries) != 0 { + t.Errorf("expected no search for a linked document, got %q", source.queries) + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if mustBiblioReadsData(t, stored).ResourceID != source.book.ResourceID || mustBiblioReadsData(t, stored).LegacyBookID != source.book.LegacyBookID { + t.Fatalf("expected book identifiers to be stored, got %+v", stored) + } +} + +func TestBiblioReadsRatingCountEscaping(t *testing.T) { + for _, count := range []string{ + "2,285", + ``, + `<img src=x> & "quoted" 'count'`, + } { + t.Run(count, func(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "3.82", RatingCount: count, + }} + app, idx := newBiblioReadsTestApp(t, source) + for _, language := range []struct { + code string + label string + }{ + {code: "en", label: "ratings"}, + {code: "de", label: "Bewertungen"}, + } { + for _, path := range []string{ + "/documents/linked/score?l=" + language.code, + "/search?type=documents&search=La&l=" + language.code, + } { + page := getPage(t, app, path) + row := page.Find("i.bi-star-fill").Closest("li") + if row.Length() != 1 { + t.Fatalf("%s: expected one rating row, got %d", path, row.Length()) + } + if row.Find("small").Text() != "("+count+" "+language.label+")" { + t.Errorf("%s: rating count must remain literal text, got %q", path, row.Find("small").Text()) + } + if row.Find("img, script, [onerror]").Length() != 0 { + t.Errorf("%s: provider rating count must not create HTML elements", path) + } + } + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if got := mustBiblioReadsData(t, stored).RatingCount; got != count { + t.Fatalf("escaping must not change the stored count: got %q", got) + } + }) + } +} + +func TestDocumentUpdateBiblioReads(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "3.82", ResourceID: "old-work", LegacyBookID: 12207444, + }} + app, idx := newBiblioReadsTestApp(t, source) + getBody(t, app, "/documents/linked/score") + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + response, err := putRequest(url.Values{ + "bookURL": {"https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente?from_search=true"}, + }, adminCookie, app, "/documents/linked/review-link", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusNoContent, t) + if response.Header.Get("HX-Refresh") != "true" { + t.Fatal("successful correction must reload the detail page") + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if mustBiblioReadsData(t, stored).URL != "/book/show/1368314.El_Cercano_Oriente" || + stored.DataSourceScoreValue != 0 || mustBiblioReadsData(t, stored).ResourceID != "" || mustBiblioReadsData(t, stored).LegacyBookID != 0 || + !mustBiblioReadsData(t, stored).Enriched { + t.Fatalf("expected correction persisted without stale rating/review identifiers: %+v", stored) + } + source.book = biblioreads.BookResponse{Rating: "4.2", ResourceID: "new-work", LegacyBookID: 1368314} + body := getBody(t, app, "/documents/linked/score") + if !strings.Contains(body, `href="https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente"`) || + source.ids[len(source.ids)-1] != "1368314.El_Cercano_Oriente" { + t.Fatal("subsequent scores must use the corrected book") + } + getBody(t, app, "/documents/linked/reviews") + if len(source.reviewCalls) != 1 || source.reviewCalls[0].resourceID != "new-work" || + source.reviewCalls[0].legacyBookID != 1368314 { + t.Fatalf("subsequent reviews must use the corrected book: %+v", source.reviewCalls) + } +} + +func TestDocumentUpdateBiblioReadsValidation(t *testing.T) { + app, idx := newBiblioReadsTestApp(t, &fakeBookSource{}) + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"", "https://example.org/book/show/123", "https://www.goodreads.com/author/show/16667"} { + response, err := putRequest(url.Values{"bookURL": {value}}, adminCookie, app, "/documents/linked/review-link", t) + if err != nil { + t.Fatal(err) + } + mustReturnStatus(response, http.StatusUnprocessableEntity, t) + page, err := goquery.NewDocumentFromReader(response.Body) + response.Body.Close() + if err != nil { + t.Fatal(err) + } + if page.Find(`form#review-link-form-linked input[aria-invalid="true"]`).Length() != 1 || + page.Find(`[role="alert"]`).Text() != "Enter a valid Goodreads book URL." { + t.Fatal("invalid URLs must render inline validation feedback") + } + if got, _ := page.Find(`input[name="bookURL"]`).Attr("value"); got != value { + t.Fatalf("invalid input should be retained for correction, got %q", got) + } + stored, err := idx.Document("linked") + if err != nil || mustBiblioReadsData(t, stored).URL != "/book/show/12207444-la-reina-dragon" || stored.DataSourceScoreValue != 3.82 { + t.Fatalf("invalid input must not change the stored match: %+v, %v", stored, err) + } + } + response, err := putRequest(url.Values{ + "bookURL": {"https://www.goodreads.com/book/show/1368314"}, + }, adminCookie, app, "/documents/missing/review-link", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusNotFound, t) +} + +func TestDocumentDetailUpdatesStoredBiblioReadsRating(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "4.12", ResourceID: "work-id", LegacyBookID: 12207444, + }} + app, idx := newBiblioReadsTestApp(t, source) + + page := getPage(t, app, "/documents/linked/score") + link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if got := strings.TrimSpace(link.Text()); got != "4.12" { + t.Fatalf("expected latest Goodreads rating 4.12, got %q", got) + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if stored.DataSourceScoreValue != 4.12 { + t.Fatalf("expected latest rating and searchable value to be stored, got %+v", stored) + } + results, err := idx.Search(index.SearchFields{DataSourceScoreFrom: 4.1, DataSourceScoreTo: new(4.2)}, 1, 10) + if err != nil || results.TotalHits() != 1 { + t.Fatalf("expected document to match updated rating range, got %d results and error %v", results.TotalHits(), err) + } +} + +func TestDocumentDetailKeepsStoredBiblioReadsRatingWhenScraperReturnsZero(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ + Rating: "0.00", RatingCount: "650", ResourceID: "work-id", LegacyBookID: 12207444, + }} + app, idx := newBiblioReadsTestApp(t, source) + + page := getPage(t, app, "/documents/linked/score") + link := page.Find(`li a[href="https://www.goodreads.com/book/show/12207444-la-reina-dragon"]`) + if got := strings.TrimSpace(link.Text()); got != "3.82" { + t.Fatalf("expected known Goodreads rating to be retained, got %q", got) + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if stored.DataSourceScoreValue != 3.82 { + t.Fatalf("zero scraper rating must not replace indexed Goodreads score, got %+v", stored) + } +} + +func TestDocumentDetailBiblioReadsSearchesUnlinkedDocuments(t *testing.T) { + source := &fakeBookSource{ + book: biblioreads.BookResponse{Rating: "4.20", RatingCount: "10", ResourceID: "work-id", LegacyBookID: 1}, + results: []biblioreads.SearchResult{{Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", AuthorURL: "/author/show/1.Jane_Doe", Rating: "4.10"}, {BookURL: "/book/show/2-other", Rating: "5.00"}}, + } + app, idx := newBiblioReadsTestApp(t, source) + + if got := documentScorePlaceholders(t, getBody(t, app, "/documents/unlinked")).Length(); got != 1 { + t.Fatalf("expected a placeholder for a document not looked up yet, got %d", got) + } + + page := getPage(t, app, "/documents/unlinked/score") + if got := strings.TrimSpace(page.Find(`li a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.10" { + t.Errorf("expected rating link with 4.10, got %q", got) + } + if len(source.queries) != 1 || source.queries[0] != "Unlinked Jane Doe" { + t.Errorf("expected a single search with title and first author, got %q", source.queries) + } + if len(source.ids) != 0 { + t.Errorf("expected search rating without a book-scraper call, got %q", source.ids) + } + if page.Find("small").Length() != 0 { + t.Error("search response should not display a rating count") + } + + stored, err := idx.Document("unlinked") + if err != nil { + t.Fatal(err) + } + if mustBiblioReadsData(t, stored).URL != "/book/show/1-unlinked" || !mustBiblioReadsData(t, stored).Enriched { + t.Errorf("expected the found link to be stored, got %+v", stored) + } + author, err := idx.Author("jane-doe", "") + if err != nil || author.DataSourceLinks["goodreads"] != "/author/show/1.Jane_Doe" || author.DocumentCount != 1 { + t.Fatalf("expected missing author entry created with URL and document count, got %+v, %v", author, err) + } + if mustBiblioReadsData(t, stored).ResourceID != "" || mustBiblioReadsData(t, stored).LegacyBookID != 0 { + t.Fatalf("search-only lookup must not populate scraper identifiers, got %+v", stored) + } + + if err := idx.EnrichDocuments(10); err != nil { + t.Fatal(err) + } + page = getPage(t, app, "/documents/unlinked/score") + if got := strings.TrimSpace(page.Find(`li a[href="https://www.goodreads.com/book/show/1-unlinked"]`).Text()); got != "4.20" { + t.Errorf("expected later request to use book-scraper rating, got %q", got) + } + if len(source.queries) != 1 { + t.Errorf("expected the stored link to be reused without searching again, got %q", source.queries) + } + if len(source.ids) != 1 || source.ids[0] != "1-unlinked" { + t.Errorf("expected later request to scrape the stored book URL, got %q", source.ids) + } +} + +func TestDocumentDetailBiblioReadsSkipsUnmatchedDocuments(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{Rating: "3.82"}} + app, _ := newBiblioReadsTestApp(t, source) + if got := documentScorePlaceholders(t, getBody(t, app, "/documents/unmatched")).Length(); got != 1 { + t.Errorf("expected an async score placeholder, got %d", got) + } + for _, path := range []string{"/documents/unmatched/score"} { + page := getPage(t, app, path) + score := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { + return item.Find("i.bi-star-fill").Length() == 1 + }) + if strings.TrimSpace(score.Text()) != "Not scored" || score.Find("a").Length() != 0 { + t.Errorf("%s: expected unlinked no-score row, got %q", path, score.Text()) + } + } + if len(source.queries) != 0 || len(source.ids) != 0 { + t.Errorf("expected no calls, got queries %q and ids %q", source.queries, source.ids) + } +} + +func TestDocumentDetailBiblioReadsIdentifiersWithoutRating(t *testing.T) { + source := &fakeBookSource{book: biblioreads.BookResponse{ResourceID: "work-id", LegacyBookID: 12207444}} + app, idx := newBiblioReadsTestApp(t, source) + page := getPage(t, app, "/documents/linked/score") + if strings.TrimSpace(page.Find("li").Text()) != "Not scored" || page.Find("h2").Text() != "Goodreads user reviews" { + t.Fatal("expected no score and a reviews heading for a matched book without a rating") + } + stored, err := idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if mustBiblioReadsData(t, stored).ResourceID != "work-id" || mustBiblioReadsData(t, stored).LegacyBookID != 12207444 { + t.Fatalf("expected identifiers even without a rating, got %+v", stored) + } + + for _, book := range []biblioreads.BookResponse{ + {}, + {ResourceID: "partial-work"}, + {LegacyBookID: 2}, + } { + source.book = book + getBody(t, app, "/documents/linked/score") + stored, err = idx.Document("linked") + if err != nil { + t.Fatal(err) + } + if mustBiblioReadsData(t, stored).ResourceID != "work-id" || mustBiblioReadsData(t, stored).LegacyBookID != 12207444 { + t.Fatalf("incomplete response overwrote stored identifiers: %+v", stored) + } + } +} + +func TestDocumentDetailBiblioReadsSearchDoesNotScrape(t *testing.T) { + for _, tt := range []struct { + name string + results []biblioreads.SearchResult + err error + queries int + enriched bool + noScore bool + }{ + { + name: "matching book without rating", + results: []biblioreads.SearchResult{{Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked"}}, + queries: 1, enriched: true, + }, + {name: "no match", queries: 2, enriched: true, noScore: true}, + {name: "unavailable search", err: biblioreads.ErrUnavailable, queries: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + source := &fakeBookSource{ + results: tt.results, searchErr: tt.err, + book: biblioreads.BookResponse{Rating: "5.00"}, + } + app, idx := newBiblioReadsTestApp(t, source) + body := strings.TrimSpace(getBody(t, app, "/documents/unlinked/score")) + if tt.noScore { + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(page.Find("li").Text()) != "Not scored" || page.Find("a").Length() != 0 { + t.Errorf("expected no-score row without a link, got %q", body) + } + } else if tt.enriched && len(tt.results) > 0 { + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if page.Find("#document-reviews-unlinked h2").Text() != "Goodreads user reviews" || + page.Find("#document-reviews-unlinked[hx-get]").Length() != 0 { + t.Fatal("a match without a rating must show the heading without fetching reviews") + } + } else if body != "" { + t.Errorf("expected no rating from search, got %q", body) + } + if len(source.ids) != 0 || len(source.queries) != tt.queries { + t.Errorf("expected %d searches and no book-scraper calls, got queries %q and ids %q", tt.queries, source.queries, source.ids) + } + stored, err := idx.Document("unlinked") + if err != nil || mustBiblioReadsData(t, stored).Enriched != tt.enriched { + t.Fatalf("unexpected stored lookup status: %+v, %v", stored, err) + } + }) + } +} + +func TestDocumentDetailBiblioReadsReviews(t *testing.T) { + source := &fakeBookSource{ + book: biblioreads.BookResponse{Rating: "4.20", ResourceID: "work-id", LegacyBookID: 12207444}, + reviews: biblioreads.ReviewsResponse{Reviews: []biblioreads.Review{ + {Reviewer: biblioreads.Reviewer{Name: ""}, Rating: 4, + Text: `

Great
book

`, + CreatedAt: "Fri Oct 30 2026", ReviewURL: "/review/show/123"}, + {Reviewer: biblioreads.Reviewer{Name: "Spoiler reviewer"}, Text: "The ending", Spoiler: true}, + }}, + } + app, idx := newBiblioReadsTestApp(t, source) + page := getPage(t, app, "/documents/linked/reviews") + if len(source.reviewCalls) != 1 || source.reviewCalls[0] != (reviewCall{"work-id", 12207444, "", "en"}) { + t.Fatalf("unexpected review calls: %+v", source.reviewCalls) + } + if page.Find("article").Length() != 2 || page.Find(`span[role="img"][aria-label="4/5"]`).Length() != 1 { + t.Fatal("expected reviews and their ratings") + } + rating := page.Find(`span[role="img"][aria-label="4/5"]`) + if rating.Find("i.bi-star-fill").Length() != 4 || rating.Find("i.bi-star").Length() != 1 || rating.Find(`i[aria-hidden="true"]`).Length() != 5 { + t.Fatal("expected four solid stars and one hollow star") + } + page.Find("article").Each(func(i int, article *goquery.Selection) { + content := article.Find(".review-content.clamp") + button := article.Find(`button.review-toggle[aria-expanded="false"]`) + id, _ := content.Attr("id") + target, _ := button.Attr("aria-controls") + if content.Length() != 1 || button.Length() != 1 || id == "" || target != id || strings.TrimSpace(button.Text()) != "Show more" { + t.Errorf("review %d must start collapsed with an accessible expansion button", i) + } + }) + if page.Find("details .review-content.clamp").Length() != 1 { + t.Fatal("spoiler content must also start with a collapsed preview") + } + if page.Find("script, img, [onerror]").Length() != 0 || page.Find("article p br").Length() != 1 { + t.Fatal("expected sanitized review HTML retaining paragraph breaks") + } + if page.Find("article strong").First().Text() != "" { + t.Fatal("reviewer names must be escaped") + } + if page.Find("details:not([open])").Length() != 1 || !strings.Contains(page.Find("details").Text(), "The ending") { + t.Fatal("spoiler review must be collapsed") + } + if page.Find(`a[href="https://www.goodreads.com/review/show/123"]`).Length() != 1 { + t.Fatal("expected a link to the original review") + } + if page.Find(`a time.locale[datetime="2026-10-30"]`).Text() != "2026-10-30" { + t.Fatal("expected review dates in ISO format for client-side localization") + } + stored, err := idx.Document("linked") + if err != nil || mustBiblioReadsData(t, stored).ResourceID != "work-id" || mustBiblioReadsData(t, stored).LegacyBookID != 12207444 { + t.Fatalf("expected review identifiers to be stored: %+v, %v", stored, err) + } +} + +func TestDocumentDetailBiblioReadsReviewsAvailability(t *testing.T) { + for _, tt := range []struct { + name string + source *fakeBookSource + slug string + status int + calls int + text string + language string + }{ + {name: "disabled", slug: "linked", status: http.StatusOK}, + {name: "unmatched", source: &fakeBookSource{}, slug: "unmatched", status: http.StatusOK}, + {name: "no score", source: &fakeBookSource{book: biblioreads.BookResponse{ResourceID: "work", LegacyBookID: 1}}, slug: "linked", status: http.StatusOK, text: "Goodreads user reviews"}, + {name: "book unavailable", source: &fakeBookSource{err: errors.New("unavailable")}, slug: "linked", status: http.StatusInternalServerError}, + {name: "missing identifiers", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "4"}}, slug: "linked", status: http.StatusInternalServerError}, + {name: "reviews unavailable", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}, reviewErr: errors.New("unavailable")}, slug: "linked", status: http.StatusInternalServerError, calls: 1}, + {name: "empty reviews", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}}, slug: "linked", status: http.StatusOK, calls: 1, text: "No user reviews available in the current language"}, + {name: "empty Spanish reviews", source: &fakeBookSource{book: biblioreads.BookResponse{Rating: "4", ResourceID: "work", LegacyBookID: 1}}, slug: "linked", status: http.StatusOK, calls: 1, language: "es", text: "No hay reseñas de usuarios disponibles en el idioma actual"}, + } { + t.Run(tt.name, func(t *testing.T) { + app, _ := newBiblioReadsTestApp(t, tt.source) + req := mustGetRequest(t, "/documents/"+tt.slug+"/reviews") + req.Header.Set("Accept-Language", "en") + if tt.language != "" { + req.Header.Set("Accept-Language", tt.language) + } + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + + defer response.Body.Close() + mustReturnStatus(response, tt.status, t) + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if tt.status == http.StatusOK && tt.text == "" && strings.TrimSpace(string(body)) != "" { + t.Fatalf("expected no reviews content, got %q", body) + } + if tt.text != "" && !strings.Contains(string(body), tt.text) { + t.Fatalf("expected %q in %q", tt.text, body) + } + if tt.source != nil && len(tt.source.reviewCalls) != tt.calls { + t.Fatalf("expected %d review requests, got %+v", tt.calls, tt.source.reviewCalls) + } + }) + } +} + +func TestDocumentDetailBiblioReadsReviewsAfterSearch(t *testing.T) { + source := &fakeBookSource{ + results: []biblioreads.SearchResult{{Title: "Unlinked", Author: "Jane Doe", BookURL: "/book/show/1-unlinked", Rating: "4"}}, + book: biblioreads.BookResponse{Rating: "4", ResourceID: "work-id", LegacyBookID: 1}, + } + app, _ := newBiblioReadsTestApp(t, source) + getBody(t, app, "/documents/unlinked/score") + if len(source.ids) != 0 || len(source.reviewCalls) != 0 { + t.Fatal("search rating must not scrape the book or load reviews") + } + getBody(t, app, "/documents/unlinked/reviews") + if len(source.ids) != 1 || source.ids[0] != "1-unlinked" || len(source.reviewCalls) != 1 { + t.Fatal("reviews must resolve identifiers for a search-only match") + } + + source.book = biblioreads.BookResponse{Rating: "4"} + getBody(t, app, "/documents/unlinked/reviews") + if len(source.reviewCalls) != 2 || source.reviewCalls[1] != (reviewCall{"work-id", 1, "", "en"}) { + t.Fatalf("expected stored identifiers to survive incomplete book responses, got %+v", source.reviewCalls) + } +} diff --git a/internal/webserver/document_detail_test.go b/internal/webserver/document_detail_test.go index 373f8c97..bb02c6b4 100644 --- a/internal/webserver/document_detail_test.go +++ b/internal/webserver/document_detail_test.go @@ -183,7 +183,7 @@ func TestRelatedDocumentHeadersShowFullTotals(t *testing.T) { Metadata: metadata.Metadata{Title: "Source", Series: "Collection", Format: "EPUB"}, TextRankPhrases: []string{"atomic bomb", "los alamos"}, } - if err := documents.Index(source.ID, source); err != nil { + if err := index.IndexDocument(documents, source.ID, source); err != nil { t.Fatal(err) } for i := 0; i < 6; i++ { @@ -198,7 +198,7 @@ func TestRelatedDocumentHeadersShowFullTotals(t *testing.T) { } else { doc.TextRankPhrases = source.TextRankPhrases } - if err := documents.Index(doc.ID, doc); err != nil { + if err := index.IndexDocument(documents, doc.ID, doc); err != nil { t.Fatal(err) } } @@ -206,7 +206,7 @@ func TestRelatedDocumentHeadersShowFullTotals(t *testing.T) { cfg := defaultTestConfig() sender := &infrastructure.NoEmail{} - controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, nil) + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, nil, nil, nil) app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) t.Cleanup(func() { _ = app.Shutdown() }) diff --git a/internal/webserver/document_reviews_test.go b/internal/webserver/document_reviews_test.go new file mode 100644 index 00000000..f26f840c --- /dev/null +++ b/internal/webserver/document_reviews_test.go @@ -0,0 +1,597 @@ +package webserver_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/gofiber/fiber/v3" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/webserver" + "github.com/svera/coreander/v5/internal/webserver/controller/document" +) + +func TestDocumentReviewsCanSelectSupportedLanguage(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true}, + } + app, _ := newReviewProviderTestApp(t, provider) + req := mustGetRequest(t, "/documents/linked/reviews?language=fr") + req.Header.Set("Accept-Language", "es") + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if len(provider.reviewLanguages) != 1 || provider.reviewLanguages[0] != "fr" { + t.Fatalf("expected reviews in French, got %+v", provider.reviewLanguages) + } + if page.Find("h2").Text() != "Reseñas de usuarios de Another service" { + t.Fatalf("expected page language to remain Spanish while selecting French reviews, got %q", page.Find("h2").Text()) + } + selectBox := page.Find(`#review-language-linked`) + if selectBox.Length() != 1 || selectBox.Find(`option[value="fr"][selected]`).Length() != 1 { + html, _ := selectBox.Html() + t.Fatalf("expected the review-language selector to retain the selected language: %q", html) + } + if indicator, _ := selectBox.Attr("hx-indicator"); indicator != "#document-reviews-linked" || + page.Find(`#document-reviews-linked .document-reviews-loading > div`).Length() != 2 { + t.Fatal("expected a review skeleton indicator for language changes") + } + for _, language := range webserver.SupportedLanguages() { + if selectBox.Find(`option[value="`+language+`"]`).Length() != 1 { + t.Errorf("expected supported language %q in the review selector", language) + } + } +} + +func TestDocumentReviewsRejectUnsupportedLanguage(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true}, + } + app, _ := newReviewProviderTestApp(t, provider) + response, err := app.Test(mustGetRequest(t, "/documents/linked/reviews?language=zz")) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusBadRequest, t) + if len(provider.reviewLanguages) != 0 { + t.Fatalf("unsupported language must not be sent to the provider, got %+v", provider.reviewLanguages) + } +} + +type fakeReviewProvider struct { + score datasourcemodel.Score + page datasourcemodel.ReviewPage + scoreErr error + reviewsErr error + canEditDocumentLink bool + scoreSlugs []string + reviewSlugs []string + reviewLanguages []string + documentURL string +} + +func (f *fakeReviewProvider) CanEditDocumentLink() bool { + return f.canEditDocumentLink +} + +func (f *fakeReviewProvider) Name() string { + return f.score.Source +} + +func (f *fakeReviewProvider) DocumentLink(index.Document) datasourcemodel.DataSourceLink { + return datasourcemodel.DataSourceLink{Source: f.Name(), URL: f.documentURL} +} + +func (f *fakeReviewProvider) UpdateDocumentLink(index.Document, string) error { + return nil +} + +func (f *fakeReviewProvider) Score(_ context.Context, document index.Document) (datasourcemodel.Score, error) { + f.scoreSlugs = append(f.scoreSlugs, document.Slug) + return f.score, f.scoreErr +} + +func (f *fakeReviewProvider) StoredScore(index.Document) (datasourcemodel.Score, error) { + return f.score, f.scoreErr +} + +func (f *fakeReviewProvider) Reviews(_ context.Context, document index.Document, language string) (datasourcemodel.ReviewPage, error) { + f.reviewSlugs = append(f.reviewSlugs, document.Slug) + f.reviewLanguages = append(f.reviewLanguages, language) + return f.page, f.reviewsErr +} + +func TestDocumentReviewsUILanguage(t *testing.T) { + for _, tt := range []struct { + name string + query string + cookie string + header string + want string + }{ + {name: "unsupported header falls back to English", header: "zz", want: "en"}, + {name: "header", header: "fr", want: "fr"}, + {name: "cookie overrides header", cookie: "es", header: "en", want: "es"}, + {name: "query overrides cookie", query: "?l=de", cookie: "es", header: "en", want: "de"}, + } { + t.Run(tt.name, func(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true}, + } + app, _ := newReviewProviderTestApp(t, provider) + req := mustGetRequest(t, "/documents/linked/reviews"+tt.query) + if tt.header != "" { + req.Header.Set("Accept-Language", tt.header) + } + if tt.cookie != "" { + req.AddCookie(&http.Cookie{Name: "locale", Value: tt.cookie}) + } + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + if len(provider.reviewLanguages) != 1 || provider.reviewLanguages[0] != tt.want { + t.Fatalf("expected review language %q, got %+v", tt.want, provider.reviewLanguages) + } + }) + } +} + +func TestDocumentAlternativeReviewProvider(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service", Value: "4.5", Count: "123", URL: "https://example.org/books/book-1"}, + page: datasourcemodel.ReviewPage{ + Source: "Another service", URL: "https://example.org/books/book-1", Available: true, + Reviews: []datasourcemodel.Review{{ + Reviewer: "Reader", Rating: 5, Text: "

Excellent

", + CreatedAt: time.Date(2026, 10, 30, 0, 0, 0, 0, time.UTC), URL: "https://example.org/reviews/review-1", + }}, + }, + } + app, _ := newReviewProviderTestApp(t, provider) + searchPage := getPage(t, app, "/search?type=documents&search=La") + if !strings.Contains(searchPage.Find(`fieldset legend`).Text(), "Another service score") || + searchPage.Find(`#sort-by option[value="review-score-asc"]`).Text() != "Another service score (ascending)" { + t.Fatal("search filters and sort options must use the active review provider's name") + } + if searchPage.Find(`#list a[href="https://example.org/books/book-1"]`).Length() != 1 || + searchPage.Find(`#list a[href^="https://www.goodreads.com/book/"]`).Length() != 0 { + t.Fatal("cached search-result scores must use the configured provider") + } + // This document has a cached Goodreads non-match; it must still use the chosen provider. + detail := getBody(t, app, "/documents/unmatched") + if documentScorePlaceholders(t, detail).Length() != 1 || len(provider.scoreSlugs) != 0 || len(provider.reviewSlugs) != 0 { + t.Fatal("detail must defer lookups to the selected provider") + } + for _, path := range []string{"/documents/unmatched/score"} { + body := getBody(t, app, path) + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + link := page.Find(`li a[href="https://example.org/books/book-1"]`) + if link.Text() != "4.5" || !strings.Contains(body, "Another service rating") || strings.Contains(body, "goodreads.com") { + t.Fatalf("score must use the selected provider's labels, value and URL: %s", body) + } + } + body := getBody(t, app, "/documents/unmatched/reviews") + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if page.Find("h2").Text() != "Another service user reviews" || + page.Find(`a[href="https://example.org/reviews/review-1"] time[datetime="2026-10-30"]`).Length() != 1 || + page.Find("article strong").Text() != "Reader" || + page.Find("i.bi-star-fill").Length() != 5 || page.Find("script").Length() != 0 { + t.Fatalf("reviews must render normalized provider data safely: %s", body) + } + if len(provider.scoreSlugs) != 1 || len(provider.reviewSlugs) != 1 || provider.reviewSlugs[0] != "unmatched" { + t.Fatal("expected calls through the provider interface") + } + if len(provider.reviewLanguages) != 1 || provider.reviewLanguages[0] != "en" { + t.Fatalf("expected the current UI language to reach alternative providers, got %q", provider.reviewLanguages) + } +} + +func TestDocumentScoreMarksZeroWithoutValuationsAsNotScored(t *testing.T) { + for _, tt := range []struct { + name string + count string + wantScore string + wantScored bool + }{ + {name: "zero valuations", count: "0", wantScore: "Not scored"}, + {name: "valuations exist", count: "650", wantScore: "0.00", wantScored: true}, + } { + t.Run(tt.name, func(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{ + Source: "Another service", Value: "0.00", Count: tt.count, + URL: "https://example.org/books/book-1", + }, + } + app, _ := newReviewProviderTestApp(t, provider) + page := getPage(t, app, "/documents/unmatched/score") + row := page.Find("li") + if tt.wantScored { + if got := strings.TrimSpace(row.Find("a").Text()); got != tt.wantScore { + t.Fatalf("expected score %q, got %q", tt.wantScore, got) + } + return + } + if got := strings.TrimSpace(row.Text()); got != tt.wantScore || row.Find("a").Length() != 0 { + t.Fatalf("expected unlinked %q label, got %q", tt.wantScore, got) + } + }) + } +} + +func newReviewProviderTestApp(t *testing.T, provider document.ReviewProvider) (*fiber.App, *index.BleveIndexer) { + t.Helper() + return newDataSourceProvidersTestApp(t, nil, provider, nil) +} + +func getBody(t *testing.T, app *fiber.App, path string) string { + t.Helper() + req := mustGetRequest(t, path) + req.Header.Set("Accept-Language", "en") + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("%s: status = %d, want %d", path, response.StatusCode, http.StatusOK) + } + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func getPage(t *testing.T, app *fiber.App, path string) *goquery.Document { + t.Helper() + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, path))) + if err != nil { + t.Fatal(err) + } + return page +} + +func documentScorePlaceholders(t *testing.T, html string) *goquery.Selection { + t.Helper() + page, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + if err != nil { + t.Fatal(err) + } + return page.Find(`li[hx-get$="/score"]`) +} + +func TestDocumentReviewFractionalStars(t *testing.T) { + for _, tt := range []struct { + rating float64 + full int + empty int + width int + }{ + {rating: 0}, + {rating: 0.25, empty: 4, width: 25}, + {rating: 3.5, full: 3, empty: 1, width: 50}, + {rating: 4.25, full: 4, width: 25}, + {rating: 4.75, full: 4, width: 75}, + {rating: 4, full: 4, empty: 1}, + {rating: 5, full: 5}, + } { + t.Run(fmt.Sprint(tt.rating), func(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true, Reviews: []datasourcemodel.Review{ + {Reviewer: "Reader", Rating: tt.rating}, + }}, + } + app, _ := newReviewProviderTestApp(t, provider) + page := getPage(t, app, "/documents/linked/reviews") + stars := page.Find(`article span[role="img"]`) + if tt.rating == 0 { + if stars.Length() != 0 { + t.Fatal("unrated reviews must not display stars") + } + if page.Find("article header span.text-muted").Text() != "Not scored" { + t.Fatal("unrated reviews must display Not scored") + } + return + } + if page.Find("article header span.text-muted").Length() != 0 { + t.Fatal("scored reviews must not display the unrated label") + } + if label, _ := stars.Attr("aria-label"); label != fmt.Sprintf("%g/5", tt.rating) { + t.Fatalf("expected exact accessible rating, got %q", label) + } + if stars.ChildrenFiltered("i.bi-star-fill").Length() != tt.full || + stars.ChildrenFiltered("i.bi-star").Length() != tt.empty || + stars.Children().Length() != 5 { + t.Fatalf("unexpected full/empty star counts: %s", stars.Text()) + } + partial := stars.Find(".review-star-fill") + if tt.width == 0 { + if partial.Length() != 0 { + t.Fatal("whole ratings must not include a partial star") + } + } else if style, _ := partial.Attr("style"); style != fmt.Sprintf("width: %d%%", tt.width) || + stars.Find(`.review-star[aria-hidden="true"]`).Length() != 1 { + t.Fatalf("expected a %d%% decorative star fill, got %q", tt.width, style) + } + }) + } +} + +func TestDocumentReviewNotScoredTranslations(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service"}, + page: datasourcemodel.ReviewPage{Source: "Another service", Available: true, Reviews: []datasourcemodel.Review{ + {Reviewer: "Reader", Text: "A review without a rating"}, + }}, + } + app, _ := newReviewProviderTestApp(t, provider) + for _, tt := range []struct { + lang string + text string + }{ + {"en", "Not scored"}, + {"de", "Nicht bewertet"}, + {"es", "Sin puntuar"}, + {"fr", "Non noté"}, + {"ru", "Не оценено"}, + } { + t.Run(tt.lang, func(t *testing.T) { + page := getPage(t, app, "/documents/linked/reviews?l="+tt.lang) + if got := page.Find("article header span.text-muted").Text(); got != tt.text { + t.Fatalf("expected %q, got %q", tt.text, got) + } + }) + } +} + +func TestDocumentScoreNoScoreTranslations(t *testing.T) { + provider := &fakeReviewProvider{score: datasourcemodel.Score{Source: "Another service"}} + app, _ := newReviewProviderTestApp(t, provider) + for _, locale := range []struct { + lang string + text string + }{ + {"en", "Not scored"}, + {"es", "Sin puntuar"}, + {"de", "Nicht bewertet"}, + {"fr", "Non noté"}, + {"ru", "Не оценено"}, + } { + for _, path := range []string{"/documents/unmatched/score"} { + t.Run(locale.lang+path, func(t *testing.T) { + req := mustGetRequest(t, path) + req.Header.Set("Accept-Language", locale.lang) + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + score := page.Find("li").FilterFunction(func(_ int, item *goquery.Selection) bool { + return item.Find("i.bi-star-fill").Length() == 1 + }) + if strings.TrimSpace(score.Text()) != locale.text || score.Find("a").Length() != 0 { + t.Fatalf("expected localized no-score row %q, got %q", locale.text, score.Text()) + } + }) + } + } + if len(provider.reviewSlugs) != 0 { + t.Fatal("score requests must not fetch reviews") + } +} + +func TestDocumentReviewsLazyPlaceholder(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service", Value: "3.82", URL: "https://example.org/books/book-1"}, + documentURL: "https://example.org/books/book-1", + } + app, _ := newReviewProviderTestApp(t, provider) + page := getPage(t, app, "/documents/linked") + target := page.Find("#document-reviews-linked") + if target.Length() != 1 || target.Is("[hx-get]") { + t.Fatal("expected an inactive reviews target before the rating loads") + } + if target.Find("h2").Text() != "Another service user reviews" { + t.Fatal("expected the reviews heading before the score loads") + } + if !target.Is(".document-detail-content > :last-child") { + t.Fatal("reviews must appear at the bottom of the document content") + } + page = getPage(t, app, "/documents/linked/score") + placeholder := page.Find(`#document-reviews-linked[hx-get="/documents/linked/reviews"][hx-trigger="intersect once"][hx-swap-oob="outerHTML"]`) + if placeholder.Length() != 1 { + t.Fatal("expected successful rating to enable reviews on intersection only") + } + if placeholder.Find("h2").Text() != "Another service user reviews" || placeholder.Find(`[aria-hidden="true"] h2`).Length() != 0 { + t.Fatal("the reviews heading must remain visible and accessible while reviews load") + } + skeletons := placeholder.Find(".border-top") + if skeletons.Length() != 2 || placeholder.Find(`div[aria-hidden="true"]`).Length() != 1 { + t.Fatal("expected two decorative review-shaped skeletons") + } + skeletons.Each(func(i int, skeleton *goquery.Selection) { + if skeleton.Find(".fs-5 .placeholder").Length() != 1 || + skeleton.Find("i.bi-star").Length() != 5 || + skeleton.Find(".placeholder.ms-auto").Length() != 1 || + skeleton.ChildrenFiltered("div").Length() != 6 { + t.Errorf("skeleton %d must include a name, five stars, a date and five text lines", i) + } + }) + if len(provider.reviewSlugs) != 0 { + t.Fatal("page and rating requests must not fetch reviews") + } +} + +func TestDocumentReviewsHeader(t *testing.T) { + for _, tt := range []struct { + name string + provider *fakeReviewProvider + slug string + want int + }{ + {name: "matched", provider: &fakeReviewProvider{documentURL: "https://example.org/books/book-1"}, slug: "linked", want: 1}, + {name: "zero score", provider: &fakeReviewProvider{score: datasourcemodel.Score{Value: "0.00"}, documentURL: "https://example.org/books/book-1"}, slug: "linked", want: 1}, + {name: "unavailable", provider: &fakeReviewProvider{scoreErr: errors.New("unavailable"), documentURL: "https://example.org/books/book-1"}, slug: "linked", want: 1}, + {name: "disabled", slug: "linked"}, + {name: "unmatched", provider: &fakeReviewProvider{}, slug: "unmatched"}, + {name: "not looked up", provider: &fakeReviewProvider{}, slug: "unlinked"}, + } { + t.Run(tt.name, func(t *testing.T) { + var provider document.ReviewProvider + if tt.provider != nil { + tt.provider.score.Source = "Another service" + provider = tt.provider + } + app, _ := newReviewProviderTestApp(t, provider) + page := getPage(t, app, "/documents/"+tt.slug) + header := page.Find(`#document-reviews-` + tt.slug + ` h2 a`) + if header.Length() != tt.want { + t.Fatalf("expected %d reviews headings, got %d", tt.want, header.Length()) + } + if tt.want == 1 { + url, _ := header.Attr("href") + if header.Text() != "Another service user reviews" || url != "https://example.org/books/book-1" { + t.Fatal("expected a linked provider reviews heading") + } + } + if tt.provider != nil && (len(tt.provider.scoreSlugs) != 0 || len(tt.provider.reviewSlugs) != 0) { + t.Fatal("rendering the heading must not make any API calls") + } + }) + } +} + +func TestDocumentReviewsDisabled(t *testing.T) { + app, _ := newReviewProviderTestApp(t, nil) + if getPage(t, app, "/background-progress").Find(".alert").Length() != 0 { + t.Fatal("disabled providers must not show incomplete review progress") + } + for _, slug := range []string{"linked", "unlinked", "unmatched"} { + body := getBody(t, app, "/documents/"+slug) + if got := documentScorePlaceholders(t, body).Length(); got != 0 { + t.Errorf("%s: expected no placeholder when reviews are disabled, got %d", slug, got) + } + if body := strings.TrimSpace(getBody(t, app, "/documents/"+slug+"/score")); body != "" { + t.Errorf("%s: expected empty response, got %q", slug, body) + } + if strings.Contains(body, "bi-star-fill") { + t.Errorf("%s: expected no score row when integration is disabled", slug) + } + if strings.Contains(body, `id="document-reviews-`) { + t.Errorf("%s: expected no reviews target when integration is disabled", slug) + } + } +} + +func TestDocumentScoreRendersNothingWhenUnavailable(t *testing.T) { + provider := &fakeReviewProvider{scoreErr: errors.New("unavailable")} + app, _ := newReviewProviderTestApp(t, provider) + if body := strings.TrimSpace(getBody(t, app, "/documents/linked/score")); body != "" { + t.Errorf("expected empty response, got %q", body) + } +} + +func TestDocumentReviewLinkEditor(t *testing.T) { + provider := &fakeReviewProvider{ + score: datasourcemodel.Score{Source: "Another service", Value: "3.82", URL: "https://example.org/books/book-1"}, + documentURL: "https://example.org/books/book-1", + canEditDocumentLink: true, + } + app, _ := newReviewProviderTestApp(t, provider) + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + cookie *http.Cookie + want int + }{ + {name: "anonymous", cookie: &http.Cookie{}}, + {name: "administrator", cookie: adminCookie, want: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + response, err := getRequest(tt.cookie, app, "/documents/linked/score", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, http.StatusOK, t) + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + control := page.Find(`li a[data-bs-toggle="collapse"][href="#edit-review-link-linked"]`) + if control.Length() != tt.want { + t.Fatalf("expected %d edit links, got %d", tt.want, control.Length()) + } + if tt.want == 0 { + if page.Find(`input[name="bookURL"]`).Length() != 0 { + t.Fatal("anonymous readers must not receive the editor") + } + return + } + if control.Text() != "" || + control.Find(`i.bi-pencil[alt="Edit Another service link"]`).Length() != 1 || + control.Filter(`[title="Edit Another service link"][aria-label="Edit Another service link"]`).Length() != 1 || + page.Find(`#edit-review-link-linked.collapse:not(.show)`).Length() != 1 || + page.Find(`form[hx-put="/documents/linked/review-link"][hx-target="this"]`).Length() != 1 || + page.Find(`label[for="document-book-url-linked"]`).Length() != 1 { + t.Fatal("expected accessible, initially collapsed Another service editor beside the score") + } + if value, _ := page.Find(`input[name="bookURL"]`).Attr("value"); value != "https://example.org/books/book-1" { + t.Fatalf("editor must be prefilled with the current link, got %q", value) + } + }) + } + + provider.scoreErr = errors.New("unavailable") + response, err := getRequest(adminCookie, app, "/documents/linked/score", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + page, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(page.Find("li").Text(), "Unable to load Another service score") || + page.Find(`input[name="bookURL"]`).Length() != 1 { + t.Fatal("an unavailable score must not prevent administrators from correcting the link") + } +} diff --git a/internal/webserver/embedded/css/display.css b/internal/webserver/embedded/css/display.css index 8c16977f..7de2cd82 100644 --- a/internal/webserver/embedded/css/display.css +++ b/internal/webserver/embedded/css/display.css @@ -347,6 +347,36 @@ main .form-control:focus { overflow: hidden; } +.review-content.clamp { + -webkit-line-clamp: 5; + line-clamp: 5; +} + +.document-reviews-loading { + display: none; +} + +.document-reviews.htmx-request .document-reviews-content { + display: none; +} + +.document-reviews.htmx-request .document-reviews-loading { + display: block; +} + +.review-star { + position: relative; + display: inline-block; +} + +.review-star-fill { + position: absolute; + top: 0; + left: 0; + overflow: hidden; + white-space: nowrap; +} + #searchbox-container .input-group .form-control:focus { box-shadow: none !important; } diff --git a/internal/webserver/embedded/images/goodreads.svg b/internal/webserver/embedded/images/goodreads.svg new file mode 100644 index 00000000..e42f54b6 --- /dev/null +++ b/internal/webserver/embedded/images/goodreads.svg @@ -0,0 +1 @@ + diff --git a/internal/webserver/embedded/js/datetime.js b/internal/webserver/embedded/js/datetime.js index d7a62386..476d0f0a 100644 --- a/internal/webserver/embedded/js/datetime.js +++ b/internal/webserver/embedded/js/datetime.js @@ -9,7 +9,7 @@ import(`./luxon.min.js${versionParam}`).then(({ DateTime }) => { const datetimeFormatter = () => { const datetime = document.querySelectorAll('time.locale'); datetime.forEach(function(element) { - let dt = DateTime.fromISO(element.textContent); + let dt = DateTime.fromISO(element.getAttribute('datetime') || element.textContent); if (dt.isValid) { if (element.classList.contains('relative')) { element.textContent = dt.toRelative({ locale: document.documentElement.lang }); @@ -25,7 +25,8 @@ import(`./luxon.min.js${versionParam}`).then(({ DateTime }) => { }); } - document.addEventListener('DOMContentLoaded', datetimeFormatter()); + datetimeFormatter(); + document.body.addEventListener('htmx:afterSwap', datetimeFormatter); const observer = new MutationObserver(datetimeFormatter); diff --git a/internal/webserver/embedded/js/document-reviews.js b/internal/webserver/embedded/js/document-reviews.js new file mode 100644 index 00000000..c7474fcb --- /dev/null +++ b/internal/webserver/embedded/js/document-reviews.js @@ -0,0 +1,31 @@ +"use strict"; + +const reviewsObserver = new ResizeObserver(entries => { + for (const { target: content } of entries) { + const button = content.nextElementSibling; + const expanded = button.getAttribute('aria-expanded') === 'true'; + button.hidden = !expanded && content.scrollHeight <= content.clientHeight; + } +}); + +function observeReviews() { + document.querySelectorAll('.review-content').forEach(content => { + reviewsObserver.observe(content); + }); +} + +observeReviews(); +document.body.addEventListener('htmx:afterSwap', observeReviews); + +document.body.addEventListener('click', event => { + const button = event.target.closest('.review-toggle'); + if (!button) { + return; + } + + const content = document.getElementById(button.getAttribute('aria-controls')); + const expanded = button.getAttribute('aria-expanded') !== 'true'; + content.classList.toggle('clamp', !expanded); + button.setAttribute('aria-expanded', String(expanded)); + button.textContent = expanded ? button.dataset.showLess : button.dataset.showMore; +}); diff --git a/internal/webserver/embedded/js/feedback.js b/internal/webserver/embedded/js/feedback.js index 466dbcd1..1ac58ff9 100644 --- a/internal/webserver/embedded/js/feedback.js +++ b/internal/webserver/embedded/js/feedback.js @@ -115,6 +115,28 @@ htmx.on('htmx:afterRequest', (evt) => { } }); +function showBackgroundProgressError(evt) { + const progress = evt.detail?.elt + if (!(progress instanceof HTMLElement) || progress.id !== 'background-progress') { + return + } + + const message = progress.dataset.errorMessage + if (!message) { + return + } + + const alert = document.createElement('div') + alert.className = 'alert alert-warning' + alert.setAttribute('role', 'alert') + alert.textContent = message + progress.replaceChildren(alert) +} + +htmx.on('htmx:sendError', showBackgroundProgressError) +htmx.on('htmx:responseError', showBackgroundProgressError) +htmx.on('htmx:timeout', showBackgroundProgressError) + const setUpWarningsListener = () => { document.querySelectorAll('[data-warning-once]').forEach((el) => { el.addEventListener('click', () => { diff --git a/internal/webserver/embedded/js/home-search.js b/internal/webserver/embedded/js/home-search.js index 0ff89fb5..0530e7ae 100644 --- a/internal/webserver/embedded/js/home-search.js +++ b/internal/webserver/embedded/js/home-search.js @@ -51,7 +51,7 @@ function collectPanelParams(panel, composeDateControls, type) { if (!el.name || el.disabled) return if ((el.type === "checkbox" || el.type === "radio") && !el.checked) return const value = String(el.value ?? "").trim() - if (value === "" || value === "0") return + if (value === "" || (value === "0" && el.tagName !== "SELECT")) return params.append(el.name, value) }) return params diff --git a/internal/webserver/embedded/js/search-filter-utils.js b/internal/webserver/embedded/js/search-filter-utils.js index 6e4e77c9..cf94a027 100644 --- a/internal/webserver/embedded/js/search-filter-utils.js +++ b/internal/webserver/embedded/js/search-filter-utils.js @@ -99,7 +99,7 @@ export function applyHiddenDatesToVisible(container) { } function hasValue(input) { - return input.value.trim() !== '' && input.value !== '0' + return input.value.trim() !== '' && (input.tagName === 'SELECT' || input.value !== '0') } function updateDateClearButtonState(dateControl) { @@ -112,7 +112,7 @@ function updateDateClearButtonState(dateControl) { function updateRangeClearButtonState(group) { const clearButton = group.querySelector('.clear-range-control') if (!clearButton) return - clearButton.disabled = !Array.from(group.querySelectorAll('input')).some(hasValue) + clearButton.disabled = !Array.from(group.querySelectorAll('input, select')).some(hasValue) } function resetDateControl(dateControl) { @@ -128,7 +128,7 @@ function resetDateControl(dateControl) { } function resetRangeGroup(group) { - group.querySelectorAll('input').forEach(input => { input.value = '' }) + group.querySelectorAll('input, select').forEach(input => { input.value = '' }) updateRangeClearButtonState(group) } @@ -201,14 +201,15 @@ export function initClearRangeControls(searchFilters) { const group = clearLink.closest('.input-group') if (!group) return - group.querySelectorAll('input').forEach(input => { + group.querySelectorAll('input, select').forEach(input => { input.addEventListener('input', () => updateRangeClearButtonState(group)) + input.addEventListener('change', () => updateRangeClearButtonState(group)) }) clearLink.addEventListener('click', (e) => { e.preventDefault() resetRangeGroup(group) - const firstInput = group.querySelector('input') + const firstInput = group.querySelector('input, select') if (firstInput) firstInput.dispatchEvent(new Event('input', { bubbles: true })) }) diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index de06e22e..5e32fd4b 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -17,6 +17,14 @@ "Illustrated": "Illustriert" "Illustrations": "Illustrationen" "Only illustrated": "Nur illustrierte" +"%s score": "%s-Bewertung" +"Review score": "Rezensionsbewertung" +"Clear %s score": "%s-Bewertung löschen" +"Clear review score": "Rezensionsbewertung löschen" +"%s score (ascending)": "%s-Bewertung (aufsteigend)" +"%s score (descending)": "%s-Bewertung (absteigend)" +"Review score (ascending)": "Rezensionsbewertung (aufsteigend)" +"Review score (descending)": "Rezensionsbewertung (absteigend)" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Lesezeit berechnet mit einer Geschwindigkeit von %s Wörtern pro Minute. Sie können die Lesegeschwindigkeit in Ihrem Profil ändern." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Lesezeit berechnet mit einer Geschwindigkeit von %s Wörtern pro Minute. Melden Sie sich an, um diesen Wert anzupassen." "This filter only applies to EPUB documents": "Dieser Filter gilt nur für EPUB-Dokumente" @@ -163,7 +171,7 @@ "Click to upload a custom image": "Klicken Sie, um ein benutzerdefiniertes Bild hochzuladen" "Indexing in progress search results may not be accurate.": "Indexierung läuft die Suchergebnisse sind möglicherweise nicht genau." "Indexing in progress, search results may not be accurate.": "Indexierung läuft, Suchergebnisse sind möglicherweise nicht genau." -"Retrieving author information, author pages may not be complete yet.": "Autoreninformationen werden abgerufen, Autorenseiten sind möglicherweise noch unvollständig." +"Retrieving author information. Author pages may be incomplete, and author filters may not work yet.": "Autoreninformationen werden abgerufen. Autorenseiten sind möglicherweise unvollständig, und Autorenfilter funktionieren möglicherweise noch nicht." "Analyzing documents, search results and recommendations may not be accurate yet.": "Dokumente werden analysiert, Suchergebnisse und Empfehlungen sind möglicherweise noch nicht genau." "Author information progress": "Fortschritt der Autoreninformationen" "Related content analysis progress": "Fortschritt der Analyse verwandter Inhalte" @@ -171,7 +179,7 @@ "Keyword clean-up progress": "Fortschritt der Schlüsselwortbereinigung" "Indexing progress": "Indexierungsfortschritt" "Remaining time": "Verbleibende Zeit" -"Remaining time: %s": "Verbleibende Zeit: %s" +"%s left": "noch %s" "There was an error deleting the user please try again later": "Beim Löschen des Benutzers ist ein Fehler aufgetreten bitte versuchen Sie es später erneut." "A user with this username already exists": "Ein Benutzer mit diesem Benutzernamen existiert bereits." "A user with this email address already exists": "Ein Benutzer mit dieser E-Mail-Adresse existiert bereits." @@ -404,3 +412,20 @@ "A newer version of this document is available.": "Eine neuere Version dieses Dokuments ist verfügbar." "This document is no longer available. Its offline copy has been removed.": "Dieses Dokument ist nicht mehr verfügbar. Seine Offline-Kopie wurde entfernt." "Reload document": "Dokument neu laden" +"Edit %s link": "%s-Link bearbeiten" +"%s book URL": "%s-Buch-URL" +"Enter a valid %s book URL.": "Geben Sie eine gültige %s-Buch-URL ein." +"Unable to load %s score": "%s-Bewertung konnte nicht geladen werden" +"%s rating": "%s-Bewertung" +"%s user reviews": "%s-Nutzerrezensionen" +"Go to %s author page": "Zur Autorenseite auf %s" +"Show spoiler": "Spoiler anzeigen" +"Show more": "Mehr anzeigen" +"Show less": "Weniger anzeigen" +"No user reviews available in the current language": "Keine Nutzerrezensionen in der aktuellen Sprache verfügbar" +"No score": "Keine Bewertung" +"Not scored": "Nicht bewertet" +"%s ratings": "%s Bewertungen" +"Review information progress": "Fortschritt der Bewertungsinformationen" +"Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved.": "Bewertungsinformationen sind unvollständig. Bewertungsfilter können Dokumente auslassen, deren Bewertungen noch nicht abgerufen wurden." +"Unable to retrieve background progress.": "Der Fortschritt der Hintergrundaufgaben konnte nicht abgerufen werden." diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index a14d6f25..bcfea15c 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -17,6 +17,14 @@ "Illustrated": "Ilustrado" "Illustrations": "Ilustraciones" "Only illustrated": "Solo ilustrados" +"%s score": "Puntuación de %s" +"Review score": "Puntuación de reseñas" +"Clear %s score": "Borrar puntuación de %s" +"Clear review score": "Borrar puntuación de reseñas" +"%s score (ascending)": "Puntuación de %s (ascendente)" +"%s score (descending)": "Puntuación de %s (descendente)" +"Review score (ascending)": "Puntuación de reseñas (ascendente)" +"Review score (descending)": "Puntuación de reseñas (descendente)" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Tiempo de lectura calculado en base a una velocidad de %s palabras por minuto. Puedes cambiarla en tu perfil." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Tiempo de lectura calculado en base a una velocidad de %s palabras por minuto. Inicia sesión para personalizar este valor." "This filter only applies to EPUB documents": "Este filtro solo se aplica a documentos EPUB" @@ -160,14 +168,14 @@ "An error occurred while uploading the image": "Ocurrió un error al subir la imagen" "Click to upload a custom image": "Haz clic para subir una imagen personalizada" "Indexing in progress, search results may not be accurate.": "Indexando documentos, los resultados de búsqueda pueden no ser precisos." -"Retrieving author information, author pages may not be complete yet.": "Recuperando información de autores, las páginas de autor pueden estar incompletas." +"Retrieving author information. Author pages may be incomplete, and author filters may not work yet.": "Recuperando información de autores. Las páginas de autor pueden estar incompletas y los filtros de autor quizá no funcionen todavía." "Analyzing documents, search results and recommendations may not be accurate yet.": "Analizando documentos, los resultados de búsqueda y las recomendaciones pueden no ser precisos todavía." "Author information progress": "Progreso de información de autores" "Related content analysis progress": "Progreso del análisis de contenido relacionado" "Cleaning up overly common keywords, search results and recommendations may not be accurate yet.": "Limpiando palabras clave demasiado comunes, los resultados de búsqueda y las recomendaciones pueden no ser precisos todavía." "Keyword clean-up progress": "Progreso de la limpieza de palabras clave" "Indexing progress": "Progreso de indexación" -"Remaining time: %s": "Tiempo restante: %s" +"%s left": "quedan %s" "There was an error deleting the user, please try again later": "Hubo un error al borrar el usuario, por favor, vuelva a intentarlo más tarde" "A user with this username already exists": "Ya existe un usuario con ese nombre de usuario" "A user with this email address already exists": "Ya existe un usuario con esa dirección de correo electrónico" @@ -400,3 +408,20 @@ "A newer version of this document is available.": "Hay una versión más reciente de este documento disponible." "This document is no longer available. Its offline copy has been removed.": "Este documento ya no está disponible. Se ha eliminado su copia sin conexión." "Reload document": "Recargar documento" +"Edit %s link": "Editar enlace de %s" +"%s book URL": "URL del libro de %s" +"Enter a valid %s book URL.": "Introduce una URL válida de un libro de %s." +"Unable to load %s score": "No se pudo cargar la puntuación de %s" +"%s rating": "Puntuación en %s" +"%s user reviews": "Reseñas de usuarios de %s" +"Go to %s author page": "Ir a la página del autor en %s" +"Show spoiler": "Mostrar spoiler" +"Show more": "Mostrar más" +"Show less": "Mostrar menos" +"No user reviews available in the current language": "No hay reseñas de usuarios disponibles en el idioma actual" +"No score": "Sin puntuación" +"Not scored": "Sin puntuar" +"%s ratings": "%s valoraciones" +"Review information progress": "Progreso de la información de valoraciones" +"Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved.": "La información de valoraciones está incompleta. Los filtros de puntuación pueden omitir documentos cuyas valoraciones aún no se han obtenido." +"Unable to retrieve background progress.": "No se pudo obtener el progreso de las tareas en segundo plano." diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index 71859694..1346ad3f 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -17,6 +17,14 @@ "Illustrated": "Illustré" "Illustrations": "Illustrations" "Only illustrated": "Seulement illustrés" +"%s score": "Note %s" +"Review score": "Note des avis" +"Clear %s score": "Effacer la note %s" +"Clear review score": "Effacer la note des avis" +"%s score (ascending)": "Note %s (croissante)" +"%s score (descending)": "Note %s (décroissante)" +"Review score (ascending)": "Note des avis (croissante)" +"Review score (descending)": "Note des avis (décroissante)" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Temps de lecture calculé avec une vitesse de %s mots par minute. Vous pouvez modifier votre vitesse de lecture dans votre profil." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Temps de lecture calculé avec une vitesse de %s mots par minute. Connectez-vous pour personnaliser cette valeur." "This filter only applies to EPUB documents": "Ce filtre s'applique uniquement aux documents EPUB" @@ -161,14 +169,14 @@ "An error occurred while uploading the image": "Une erreur s'est produite lors du téléchargement de l'image" "Click to upload a custom image": "Cliquez pour télécharger une image personnalisée" "Indexing in progress, search results may not be accurate.": "Indexation en cours, les résultats de la recherche peuvent ne pas être précis." -"Retrieving author information, author pages may not be complete yet.": "Récupération des informations sur les auteurs, les pages auteur peuvent être incomplètes." +"Retrieving author information. Author pages may be incomplete, and author filters may not work yet.": "Récupération des informations sur les auteurs. Les pages auteur peuvent être incomplètes et les filtres par auteur peuvent ne pas encore fonctionner." "Analyzing documents, search results and recommendations may not be accurate yet.": "Analyse des documents, les résultats de recherche et les recommandations peuvent ne pas être précis pour le moment." "Author information progress": "Progression des informations sur les auteurs" "Related content analysis progress": "Progression de l'analyse du contenu associé" "Cleaning up overly common keywords, search results and recommendations may not be accurate yet.": "Nettoyage des mots-clés trop courants, les résultats de recherche et les recommandations peuvent ne pas être précis pour le moment." "Keyword clean-up progress": "Progression du nettoyage des mots-clés" "Indexing progress": "Progression de l'indexation" -"Remaining time: %s": "Temps restant: %s" +"%s left": "%s restant" "There was an error deleting the user, please try again later": "Une erreur s'est produite lors de supprimer l'utilisateur, veuillez réessayer ultérieurement" "A user with this username already exists": "Un utilisateur avec ce nom d'utilisateur existe déjà" "A user with this email address already exists": "Un utilisateur avec cette adresse e-mail existe déjà" @@ -402,3 +410,20 @@ "A newer version of this document is available.": "Une version plus récente de ce document est disponible." "This document is no longer available. Its offline copy has been removed.": "Ce document n'est plus disponible. Sa copie hors connexion a été supprimée." "Reload document": "Recharger le document" +"Edit %s link": "Modifier le lien %s" +"%s book URL": "URL du livre sur %s" +"Enter a valid %s book URL.": "Saisissez une URL de livre %s valide." +"Unable to load %s score": "Impossible de charger la note %s" +"%s rating": "Note %s" +"%s user reviews": "Avis des utilisateurs de %s" +"Go to %s author page": "Voir la page de l'auteur sur %s" +"Show spoiler": "Afficher le spoiler" +"Show more": "Afficher plus" +"Show less": "Afficher moins" +"No user reviews available in the current language": "Aucun avis utilisateur disponible dans la langue actuelle" +"No score": "Aucune note" +"Not scored": "Non noté" +"%s ratings": "%s notes" +"Review information progress": "Progression des informations d'évaluation" +"Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved.": "Les informations d'évaluation sont incomplètes. Les filtres de note peuvent omettre les documents dont les évaluations n'ont pas encore été récupérées." +"Unable to retrieve background progress.": "Impossible de récupérer la progression des tâches en arrière-plan." diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index ae8fbbd6..7fd89c59 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -17,6 +17,14 @@ "Illustrated": "Иллюстрированное" "Illustrations": "Иллюстрации" "Only illustrated": "Только с иллюстрациями" +"%s score": "Оценка %s" +"Review score": "Оценка отзывов" +"Clear %s score": "Очистить оценку %s" +"Clear review score": "Очистить оценку отзывов" +"%s score (ascending)": "Оценка %s (по возрастанию)" +"%s score (descending)": "Оценка %s (по убыванию)" +"Review score (ascending)": "Оценка отзывов (по возрастанию)" +"Review score (descending)": "Оценка отзывов (по убыванию)" "Reading time calculated using a speed of %s words per minute. You can change your reading speed in your profile.": "Время чтения рассчитано при скорости %s слов в минуту. Изменить скорость чтения можно в профиле." "Reading time calculated using a speed of %s words per minute. Log in to customize this value": "Время чтения рассчитано при скорости %s слов в минуту. Войдите, чтобы изменить это значение." "This filter only applies to EPUB documents": "Этот фильтр применяется только к документам EPUB" @@ -162,14 +170,14 @@ "An error occurred while uploading the image": "Произошла ошибка при загрузке изображения" "Click to upload a custom image": "Нажмите, чтобы загрузить пользовательское изображение" "Indexing in progress, search results may not be accurate.": "Идёт индексация; результаты поиска могут быть неточными." -"Retrieving author information, author pages may not be complete yet.": "Получение информации об авторах; страницы авторов могут быть неполными." +"Retrieving author information. Author pages may be incomplete, and author filters may not work yet.": "Получение информации об авторах. Страницы авторов могут быть неполными, а фильтры по авторам могут пока не работать." "Analyzing documents, search results and recommendations may not be accurate yet.": "Анализ документов; результаты поиска и рекомендации могут быть пока неточными." "Author information progress": "Прогресс получения информации об авторах" "Related content analysis progress": "Прогресс анализа похожего контента" "Cleaning up overly common keywords, search results and recommendations may not be accurate yet.": "Очистка слишком часто встречающихся ключевых слов; результаты поиска и рекомендации могут быть пока неточными." "Keyword clean-up progress": "Прогресс очистки ключевых слов" "Indexing progress": "Прогресс индексации" -"Remaining time: %s": "Оставшееся время: %s" +"%s left": "осталось %s" "There was an error deleting the user, please try again later": "Ошибка при удалении пользователя. Повторите попытку позже." "A user with this username already exists": "Пользователь с таким именем уже существует" "A user with this email address already exists": "Пользователь с таким e‑mail уже существует" @@ -396,3 +404,20 @@ "A newer version of this document is available.": "Доступна новая версия этого документа." "This document is no longer available. Its offline copy has been removed.": "Этот документ больше недоступен. Его копия для чтения без подключения к сети удалена." "Reload document": "Перезагрузить документ" +"Edit %s link": "Изменить ссылку %s" +"%s book URL": "URL книги на %s" +"Enter a valid %s book URL.": "Введите корректный URL книги на %s." +"Unable to load %s score": "Не удалось загрузить рейтинг %s" +"%s rating": "Рейтинг %s" +"%s user reviews": "Отзывы пользователей %s" +"Go to %s author page": "Перейти на страницу автора в %s" +"Show spoiler": "Показать спойлер" +"Show more": "Показать больше" +"Show less": "Показать меньше" +"No user reviews available in the current language": "Нет отзывов пользователей на текущем языке" +"No score": "Нет оценки" +"Not scored": "Не оценено" +"%s ratings": "Оценок: %s" +"Review information progress": "Ход получения информации об оценках" +"Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved.": "Информация об оценках неполная. Фильтры по оценкам могут пропускать документы, оценки которых ещё не получены." +"Unable to retrieve background progress.": "Не удалось получить состояние фоновых задач." diff --git a/internal/webserver/embedded/views/author/list.html b/internal/webserver/embedded/views/author/list.html index 043f3113..d715aea1 100644 --- a/internal/webserver/embedded/views/author/list.html +++ b/internal/webserver/embedded/views/author/list.html @@ -1,6 +1,6 @@ {{if ne .Author.Name ""}} {{if not .Author.RetrievedOn.IsZero}} - {{template "partials/author-summary" dict "Lang" .Lang "Author" .Author "Session" .Session "ImageVersion" .ImageVersion}} + {{template "partials/author-summary" dict "Lang" .Lang "Author" .Author "Session" .Session "ImageVersion" .ImageVersion "DataSourceLinks" .DataSourceLinks}} {{else}}
@@ -22,7 +22,7 @@

{{.Author.Name}}

{{end}}
- {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}}
diff --git a/internal/webserver/embedded/views/document/detail.html b/internal/webserver/embedded/views/document/detail.html index e1c32a99..d8eac799 100644 --- a/internal/webserver/embedded/views/document/detail.html +++ b/internal/webserver/embedded/views/document/detail.html @@ -31,7 +31,7 @@

{{.Document.Title}}

{{template "partials/actions" dict "Lang" .Lang "Document" .Document "Session" .Session "FQDN" .fqdn "EmailSendingConfigured" .EmailSendingConfigured "DefaultAction" .DefaultAction "CanShare" .CanShare "PreferredEpub" .PreferredEpub "EmailFrom" .EmailFrom "ShareMaxRecipients" .ShareMaxRecipients "ShareCommentMaxSize" .ShareCommentMaxSize "ButtonSize" "btn-lg" "ButtonStyle" "btn-primary"}}
- {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount}} + {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount "ReviewsEnabled" .ReviewsEnabled}}
@@ -60,7 +60,7 @@

{{.Document.Title}}


{{range .AuthorSummaries}} {{if not .Author.RetrievedOn.IsZero}} - {{template "partials/author-summary-doc-detail" dict "Lang" $.Lang "Author" .Author "Session" $.Session "ImageVersion" .ImageVersion}} + {{template "partials/author-summary-doc-detail" dict "Lang" $.Lang "Author" .Author "Session" $.Session "ImageVersion" .ImageVersion "DataSourceLinks" .DataSourceLinks}} {{else}} {{template "partials/author-summary-doc-detail-placeholder" dict "AuthorSlug" .Author.Slug}} {{end}} @@ -71,7 +71,7 @@

{{.Document.Title}}

{{range $i, $entry := .IllustratorSummaries}} {{if eq $i 0}}
{{end}} {{if not $entry.Author.RetrievedOn.IsZero}} - {{template "partials/author-summary-doc-detail" dict "Lang" $.Lang "Author" $entry.Author "Session" $.Session "ImageVersion" $entry.ImageVersion}} + {{template "partials/author-summary-doc-detail" dict "Lang" $.Lang "Author" $entry.Author "Session" $.Session "ImageVersion" $entry.ImageVersion "DataSourceLinks" $entry.DataSourceLinks}} {{else}} {{template "partials/author-summary-doc-detail-placeholder" dict "AuthorSlug" $entry.Author.Slug}} {{end}} @@ -119,6 +119,13 @@

Words

{{end}} + {{if .ReviewsEnabled}} +
+ {{if .DocumentLink.URL}} + {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .DocumentLink.Source "BookURL" .DocumentLink.URL "Slug" .Document.Slug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .Lang}} + {{end}} +
+ {{end}}
@@ -126,3 +133,6 @@

Words

+{{if .ReviewsEnabled}} + +{{end}} diff --git a/internal/webserver/embedded/views/highlight/list.html b/internal/webserver/embedded/views/highlight/list.html index 97165746..af37ef6f 100644 --- a/internal/webserver/embedded/views/highlight/list.html +++ b/internal/webserver/embedded/views/highlight/list.html @@ -1,7 +1,7 @@

{{t .Lang "Highlights"}}

- {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "highlight"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "highlight"}}
@@ -28,7 +28,7 @@
{{t .L
- {{template "partials/document-metadata-filters" dict "Lang" .Lang "FilterIdPrefix" "highlight" "SearchFields" .SearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} + {{template "partials/document-metadata-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "ReviewSourceConfigured" .ReviewSourceConfigured "FilterIdPrefix" "highlight" "SearchFields" .SearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} {{template "partials/search-filters-actions" dict "Lang" .Lang "ShowSearchButton" false}} diff --git a/internal/webserver/embedded/views/layout.html b/internal/webserver/embedded/views/layout.html index 027cd25d..8d2c1d5e 100644 --- a/internal/webserver/embedded/views/layout.html +++ b/internal/webserver/embedded/views/layout.html @@ -34,7 +34,7 @@ {{template "partials/navbar" .}}
- {{template "partials/main" dict "Lang" .Lang "IndexingInProgress" .IndexingInProgress "IndexingProgressKind" .IndexingProgressKind "RemainingIndexingTime" .RemainingIndexingTime "IndexingProgressPercentage" .IndexingProgressPercentage "Error" .Error "Warning" .Warning "Success" .Success "Embed" .Embed}} + {{template "partials/main" dict "RequireAuth" .RequireAuth "Session" .Session "Lang" .Lang "IndexingInProgress" .IndexingInProgress "IndexingProgressKind" .IndexingProgressKind "RemainingIndexingTime" .RemainingIndexingTime "IndexingProgressPercentage" .IndexingProgressPercentage "Error" .Error "Warning" .Warning "Success" .Success "Embed" .Embed}}
{{template "partials/share-modal" .}} {{template "partials/keyboard-shortcuts-modal" .}} diff --git a/internal/webserver/embedded/views/partials/author-data-source-link.html b/internal/webserver/embedded/views/partials/author-data-source-link.html new file mode 100644 index 00000000..64026771 --- /dev/null +++ b/internal/webserver/embedded/views/partials/author-data-source-link.html @@ -0,0 +1,7 @@ +{{$lang := .Lang}} +{{range .DataSourceLinks}} +
  • + {{if .IconClass}}{{else if .Icon}}{{end}} + {{if .Label}}{{t $lang .Label}}{{else}}{{t $lang "Go to %s author page" .Source}}{{end}} +
  • +{{end}} diff --git a/internal/webserver/embedded/views/partials/author-summary-doc-detail.html b/internal/webserver/embedded/views/partials/author-summary-doc-detail.html index a0e2c970..cf8d944a 100644 --- a/internal/webserver/embedded/views/partials/author-summary-doc-detail.html +++ b/internal/webserver/embedded/views/partials/author-summary-doc-detail.html @@ -70,20 +70,17 @@

    {{.Author.Name}}

    {{end}} + {{end}} + {{if or (index .Author.DataSourceLinks "website") .DataSourceLinks}} {{end}} diff --git a/internal/webserver/embedded/views/partials/author-summary.html b/internal/webserver/embedded/views/partials/author-summary.html index 83dd8f6d..42ca8889 100644 --- a/internal/webserver/embedded/views/partials/author-summary.html +++ b/internal/webserver/embedded/views/partials/author-summary.html @@ -70,20 +70,17 @@

    {{.Author.Name}}

    {{end}} + {{end}} + {{if or (index .Author.DataSourceLinks "website") .DataSourceLinks}} {{end}} diff --git a/internal/webserver/embedded/views/partials/background-progress.html b/internal/webserver/embedded/views/partials/background-progress.html new file mode 100644 index 00000000..45ee5747 --- /dev/null +++ b/internal/webserver/embedded/views/partials/background-progress.html @@ -0,0 +1,38 @@ +{{if or (not .RequireAuth) (and .Session .Session.Uuid)}} +{{$progress := backgroundProgress}} +
    + {{if $progress.Error}} + + {{end}} + {{if $progress.Tasks}} +
    + {{range $progress.Tasks}} + {{$label := "Indexing progress"}} + {{if eq .Kind "authors"}}{{$label = "Author information progress"}} + {{else if eq .Kind "textrank"}}{{$label = "Related content analysis progress"}} + {{else if eq .Kind "pruning"}}{{$label = "Keyword clean-up progress"}} + {{else if eq .Kind "data-sources"}}{{$label = "Review information progress"}} + {{end}} +
    +

    {{t $.Lang $label}}{{if .Name}} ({{.Name}}){{end}}

    + {{$remaining := ""}}{{if gt .RemainingTime 0}}{{$remaining = t $.Lang "%s left" (remainingTime .RemainingTime)}}{{end}} +
    +
    {{.Percentage}}%{{if $remaining}} ({{$remaining}}){{end}}
    +
    + {{if eq .Kind "data-sources"}} +

    {{t $.Lang "Review information is incomplete. Score filters may omit documents whose ratings have not been retrieved."}}

    + {{else if eq .Kind "authors"}} +

    {{t $.Lang "Retrieving author information. Author pages may be incomplete, and author filters may not work yet."}}

    + {{else if eq .Kind "textrank"}} +

    {{t $.Lang "Analyzing documents, search results and recommendations may not be accurate yet."}}

    + {{else if eq .Kind "pruning"}} +

    {{t $.Lang "Cleaning up overly common keywords, search results and recommendations may not be accurate yet."}}

    + {{else}} +

    {{t $.Lang "Indexing in progress, search results may not be accurate."}}

    + {{end}} +
    + {{end}} +
    + {{end}} +
    +{{end}} diff --git a/internal/webserver/embedded/views/partials/completed-list.html b/internal/webserver/embedded/views/partials/completed-list.html index bf1304cd..b63e483c 100644 --- a/internal/webserver/embedded/views/partials/completed-list.html +++ b/internal/webserver/embedded/views/partials/completed-list.html @@ -12,7 +12,7 @@
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}} diff --git a/internal/webserver/embedded/views/partials/count-and-sort.html b/internal/webserver/embedded/views/partials/count-and-sort.html index 1bfd8ca7..544e211f 100644 --- a/internal/webserver/embedded/views/partials/count-and-sort.html +++ b/internal/webserver/embedded/views/partials/count-and-sort.html @@ -8,6 +8,6 @@ {{if gt .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}} diff --git a/internal/webserver/embedded/views/partials/docs-list-fragments.html b/internal/webserver/embedded/views/partials/docs-list-fragments.html index 1230fc1a..a396824a 100644 --- a/internal/webserver/embedded/views/partials/docs-list-fragments.html +++ b/internal/webserver/embedded/views/partials/docs-list-fragments.html @@ -1,10 +1,10 @@ {{if and .Author (ne .Author.Name "")}}
    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "author"}}
    {{else if .SeriesTitle}}
    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}}
    {{else}}
    {{template "partials/search-type-tabs" .}}
    diff --git a/internal/webserver/embedded/views/partials/docs-list-item.html b/internal/webserver/embedded/views/partials/docs-list-item.html index 94beb0ed..70dd7a84 100644 --- a/internal/webserver/embedded/views/partials/docs-list-item.html +++ b/internal/webserver/embedded/views/partials/docs-list-item.html @@ -71,7 +71,7 @@

    {{end}} - {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount}} + {{template "partials/document-metadata" dict "Lang" .Lang "Document" .Document "Session" .Session "WordsPerMinute" .WordsPerMinute "IllustratedMinAmount" .IllustratedMinAmount "ShowScore" true}} {{ if .Document.Subjects }}
      diff --git a/internal/webserver/embedded/views/partials/document-metadata-filters.html b/internal/webserver/embedded/views/partials/document-metadata-filters.html index 6ea67bad..8af85003 100644 --- a/internal/webserver/embedded/views/partials/document-metadata-filters.html +++ b/internal/webserver/embedded/views/partials/document-metadata-filters.html @@ -19,6 +19,28 @@ {{end}} +{{if .ReviewSourceConfigured}} +
      + {{if .DataSourceName}}{{t .Lang "%s score" .DataSourceName}}{{else}}{{t .Lang "Review score"}}{{end}} +
      +
      +
      + {{$reviewScoreFrom := 0.0}} + {{if .SearchFields}} + {{$reviewScoreFrom = .SearchFields.DataSourceScoreFrom}} + {{end}} + {{$reviewScoreTo := 5.0}} + {{if .SearchFields}} + {{$reviewScoreTo = .SearchFields.DataSourceScoreUpperBound}} + {{end}} + {{template "partials/document-score-filter-control" dict "Lang" .Lang "IdPrefix" $idPrefix "Name" "review-score-from" "Label" "From" "Score" $reviewScoreFrom "Default" 0.0}} + {{template "partials/document-score-filter-control" dict "Lang" .Lang "IdPrefix" $idPrefix "Name" "review-score-to" "Label" "To" "Score" $reviewScoreTo "Default" 5.0 "RoundedEnd" true}} + +
      +
      +
      +
      +{{end}}
      {{t .Lang "Subjects"}}
      diff --git a/internal/webserver/embedded/views/partials/document-metadata.html b/internal/webserver/embedded/views/partials/document-metadata.html index 8d32a311..42d31b6b 100644 --- a/internal/webserver/embedded/views/partials/document-metadata.html +++ b/internal/webserver/embedded/views/partials/document-metadata.html @@ -12,6 +12,15 @@ {{ if .Document.Pages }}
    • {{.Document.Pages}}
    • {{ end }} + {{ if .ReviewsEnabled }} +
    • + {{ end }} + {{if .ShowScore}} + {{$score := storedDocumentScore .Document.Document}} + {{if $score.URL}} + {{template "partials/document-score" dict "Lang" .Lang "Source" $score.Source "Rating" $score.Value "RatingCount" $score.Count "Scored" (gt .Document.DataSourceScoreValue 0.0) "URL" $score.URL}} + {{end}} + {{end}} {{ if and .IllustratedMinAmount (ge .Document.Illustrations .IllustratedMinAmount) }}
    • {{t .Lang "Illustrated"}}
    • {{ end }} diff --git a/internal/webserver/embedded/views/partials/document-review-link-form.html b/internal/webserver/embedded/views/partials/document-review-link-form.html new file mode 100644 index 00000000..c70a4fd1 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-review-link-form.html @@ -0,0 +1,10 @@ + diff --git a/internal/webserver/embedded/views/partials/document-reviews-header.html b/internal/webserver/embedded/views/partials/document-reviews-header.html new file mode 100644 index 00000000..30aa41f9 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-reviews-header.html @@ -0,0 +1,14 @@ +
      +

      {{t .Lang "%s user reviews" .Source}}

      + {{if gt (len .SupportedLanguages) 1}} + + + {{end}} +
      diff --git a/internal/webserver/embedded/views/partials/document-reviews-skeleton.html b/internal/webserver/embedded/views/partials/document-reviews-skeleton.html new file mode 100644 index 00000000..b1c14a15 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-reviews-skeleton.html @@ -0,0 +1,16 @@ +{{range 2}} +
      +
      +
      +
      + {{range 5}}{{end}} + +
      +
      +
      +
      +
      +
      +
      +
      +{{end}} diff --git a/internal/webserver/embedded/views/partials/document-reviews.html b/internal/webserver/embedded/views/partials/document-reviews.html new file mode 100644 index 00000000..fc8827a8 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-reviews.html @@ -0,0 +1,30 @@ +
      + {{template "partials/document-reviews-header" .}} +
      + {{if not .Unavailable}} + {{range $i, $review := .Reviews}} +
      +
      + {{.Reviewer}} +
      + {{if gt .Rating 0.0}}{{range .StarFills}}{{if eq . 100}}{{else if eq . 0}}{{else}}{{end}}{{end}}{{else}}{{t $.Lang "Not scored"}}{{end}} + {{if .URL}}{{if .CreatedOn}}{{else}}{{t $.Lang "Read"}}{{end}}{{end}} +
      +
      + {{if .Spoiler}} +
      + {{t $.Lang "Show spoiler"}} + {{end}} +
      {{.HTML}}
      + + {{if .Spoiler}}
      {{end}} +
      + {{else}} +

      {{t .Lang "No user reviews available in the current language"}}

      + {{end}} + {{end}} +
      + +
      diff --git a/internal/webserver/embedded/views/partials/document-score-filter-control.html b/internal/webserver/embedded/views/partials/document-score-filter-control.html new file mode 100644 index 00000000..b9251974 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-score-filter-control.html @@ -0,0 +1,16 @@ +{{$id := .Name}} +{{if .IdPrefix}} + {{$id = printf "%s-%s" .IdPrefix .Name}} +{{end}} +
      + + +
      diff --git a/internal/webserver/embedded/views/partials/document-score.html b/internal/webserver/embedded/views/partials/document-score.html new file mode 100644 index 00000000..24267558 --- /dev/null +++ b/internal/webserver/embedded/views/partials/document-score.html @@ -0,0 +1,19 @@ +
    • + {{if .ScoreUnavailable}}{{t .Lang "Unable to load %s score" .Source}}{{else if .Scored}}{{.Rating}}{{if .RatingCount}} ({{t .Lang "%s ratings" (html .RatingCount)}}){{end}}{{else}}{{t .Lang "Not scored"}}{{end}} + {{if .CanEditDocumentLink}} + + + {{end}} +
    • +{{if and .URL .DocumentSlug}} +
      + {{template "partials/document-reviews-header" dict "Lang" .Lang "Source" .Source "BookURL" .URL "Slug" .DocumentSlug "SupportedLanguages" .SupportedLanguages "ReviewLanguage" .Lang}} + {{if .Rating}} + + {{end}} +
      +{{end}} diff --git a/internal/webserver/embedded/views/partials/highlights-list-fragments.html b/internal/webserver/embedded/views/partials/highlights-list-fragments.html index 4df094f7..c87b6a25 100644 --- a/internal/webserver/embedded/views/partials/highlights-list-fragments.html +++ b/internal/webserver/embedded/views/partials/highlights-list-fragments.html @@ -1,4 +1,4 @@
      - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "highlight"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "highlight"}}
      {{template "partials/highlights-list" .}}
      diff --git a/internal/webserver/embedded/views/partials/main.html b/internal/webserver/embedded/views/partials/main.html index 364d559d..95eefb9a 100644 --- a/internal/webserver/embedded/views/partials/main.html +++ b/internal/webserver/embedded/views/partials/main.html @@ -1,28 +1,5 @@
      - {{if .IndexingInProgress}} -
      -
    - {{template "partials/document-search-filters" dict "Lang" .Lang "FilterIdPrefix" "sidebar" "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" true "HideSearch" true}} + {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "ReviewSourceConfigured" .ReviewSourceConfigured "FilterIdPrefix" "sidebar" "SearchFields" .DocumentSearchFields "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs "DocumentsSearchPage" true "HideSearch" true}}
    {{if not .DocumentsOnly}}
    diff --git a/internal/webserver/embedded/views/partials/search-type-tabs.html b/internal/webserver/embedded/views/partials/search-type-tabs.html index 1caa82da..cf833dad 100644 --- a/internal/webserver/embedded/views/partials/search-type-tabs.html +++ b/internal/webserver/embedded/views/partials/search-type-tabs.html @@ -16,7 +16,7 @@
    {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}}
    diff --git a/internal/webserver/embedded/views/partials/searchbox.html b/internal/webserver/embedded/views/partials/searchbox.html index 61becca4..d45dbcbd 100644 --- a/internal/webserver/embedded/views/partials/searchbox.html +++ b/internal/webserver/embedded/views/partials/searchbox.html @@ -32,7 +32,7 @@
    - {{template "partials/document-search-filters" dict "Lang" .Lang "HomeSearch" true "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}} + {{template "partials/document-search-filters" dict "Lang" .Lang "DataSourceName" .DataSourceName "ReviewSourceConfigured" .ReviewSourceConfigured "HomeSearch" true "AvailableLanguages" .AvailableLanguages "HasReflowableDocs" .HasReflowableDocs "HasFixedDocs" .HasFixedDocs}}
    {{template "partials/author-search-filters" dict "Lang" .Lang "HomeSearch" true}} diff --git a/internal/webserver/embedded/views/partials/similar-to-banner.html b/internal/webserver/embedded/views/partials/similar-to-banner.html index 3622aa9f..cc5c2205 100644 --- a/internal/webserver/embedded/views/partials/similar-to-banner.html +++ b/internal/webserver/embedded/views/partials/similar-to-banner.html @@ -9,7 +9,7 @@
    {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}}
    @@ -19,7 +19,7 @@ {{if ne .Results.TotalHits 0}}
    - {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "AdditionalSortOptions" .AdditionalSortOptions}} + {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" .Lang "DataSourceName" .DataSourceName "AdditionalSortOptions" .AdditionalSortOptions}}
    {{end}} diff --git a/internal/webserver/embedded/views/partials/sort.html b/internal/webserver/embedded/views/partials/sort.html index 4bdc09d2..0bd2862e 100644 --- a/internal/webserver/embedded/views/partials/sort.html +++ b/internal/webserver/embedded/views/partials/sort.html @@ -4,7 +4,7 @@ diff --git a/internal/webserver/embedded/views/series/list.html b/internal/webserver/embedded/views/series/list.html index 7409ef2b..cdea5070 100644 --- a/internal/webserver/embedded/views/series/list.html +++ b/internal/webserver/embedded/views/series/list.html @@ -1,6 +1,6 @@

    {{t .Lang "Collection \"%s\"" .Title}}

    - {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}} + {{template "partials/count-and-sort" dict "Results" .Results "Lang" .Lang "DataSourceName" .DataSourceName "SortURL" .SortURL "SortBy" .SortBy "AdditionalSortOptions" .AdditionalSortOptions "FilterIdPrefix" "series"}}
    diff --git a/internal/webserver/highlights_test.go b/internal/webserver/highlights_test.go index aa7bbf52..f43d0390 100644 --- a/internal/webserver/highlights_test.go +++ b/internal/webserver/highlights_test.go @@ -8,6 +8,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v3" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/infrastructure" "github.com/svera/coreander/v5/internal/webserver/model" @@ -162,6 +163,87 @@ func TestHighlights(t *testing.T) { }) } +func TestHighlightsReviewScoreFilter(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + app, idx := bootstrapAppWithIndexer(db, &infrastructure.NoEmail{}, loadFilesInMemoryFs([]string{"testdata/library/metadata.epub"}), webserver.Config{}) + t.Cleanup(func() { _ = idx.Close() }) + document, err := idx.Document("john-doe-test-epub") + if err != nil { + t.Fatal(err) + } + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + response, err := highlight(adminCookie, app, "john-doe-test-epub", fiber.MethodPost, t) + if err != nil { + t.Fatalf("Unexpected error highlighting document: %v", err) + } + response.Body.Close() + if response.StatusCode != fiber.StatusOK { + t.Fatalf("Expected status %d, received %d", fiber.StatusOK, response.StatusCode) + } + + response, err = getRequest(adminCookie, app, "/highlights", t) + if err != nil { + t.Fatal(err) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + response.Body.Close() + if err != nil { + t.Fatal(err) + } + if doc.Find(`select[name="review-score-from"], select[name="review-score-to"]`).Length() != 0 { + t.Fatal("review score filter must be hidden on highlights when no review source is configured") + } + + for _, test := range []struct { + name string + query string + score float64 + expectedCount int + }{ + {name: "default range", expectedCount: 1}, + {name: "full range", query: "review-score-from=0&review-score-to=5", expectedCount: 1}, + {name: "zero lower bound", query: "review-score-from=0", expectedCount: 1}, + {name: "no score", query: "review-score-from=0&review-score-to=0", expectedCount: 1}, + {name: "legacy no score", query: "review-score-from=none", expectedCount: 1}, + {name: "numeric score range", query: "review-score-from=1", expectedCount: 0}, + {name: "fractional score included", query: "review-score-to=1", score: 0.5, expectedCount: 1}, + {name: "lower boundary included", query: "review-score-from=1", score: 1, expectedCount: 1}, + {name: "upper boundary included", query: "review-score-from=1&review-score-to=5", score: 5, expectedCount: 1}, + {name: "rated excluded from unrated", query: "review-score-to=0", score: 1, expectedCount: 0}, + {name: "rated below range", query: "review-score-from=1", score: 0.5, expectedCount: 0}, + {name: "rated above range", query: "review-score-to=1", score: 5, expectedCount: 0}, + } { + t.Run(test.name, func(t *testing.T) { + if err := idx.UpdateDocument(document.ID, func(update *index.DocumentUpdate) (bool, error) { + update.DataSourceScoreValue = test.score + return true, nil + }); err != nil { + t.Fatal(err) + } + response, err := getRequest(adminCookie, app, "/highlights?"+test.query, t) + if err != nil { + t.Fatalf("Unexpected error fetching highlights: %v", err) + } + defer response.Body.Close() + if response.StatusCode != fiber.StatusOK { + t.Fatalf("Expected status %d, received %d", fiber.StatusOK, response.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if got := doc.Find("#list .list-group-item").Length(); got != test.expectedCount { + t.Errorf("Expected %d highlighted documents, got %d", test.expectedCount, got) + } + }) + } +} + func highlight(cookie *http.Cookie, app *fiber.App, slug string, method string, t *testing.T) (*http.Response, error) { t.Helper() req, err := http.NewRequest(method, fmt.Sprintf("/highlights/%s", slug), nil) diff --git a/internal/webserver/js-tests/home-search.test.mjs b/internal/webserver/js-tests/home-search.test.mjs new file mode 100644 index 00000000..beafe054 --- /dev/null +++ b/internal/webserver/js-tests/home-search.test.mjs @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' +import vm from 'node:vm' + +const source = await readFile(new URL('../embedded/js/home-search.js', import.meta.url), 'utf8') +const context = vm.createContext({ + URLSearchParams, + document: { getElementById: () => null }, +}) +vm.runInContext(source.replace(/^import .* from .*$/m, ''), context) + +test('home advanced search preserves the unrated-only upper bound', () => { + let composed = false + const panel = { + querySelectorAll: () => [ + { name: 'review-score-from', value: '', tagName: 'SELECT' }, + { name: 'review-score-to', value: '0', tagName: 'SELECT' }, + { name: 'pages-from', value: '0', tagName: 'INPUT' }, + { name: 'review-score-to', value: '5', tagName: 'SELECT', disabled: true }, + { name: 'illustrated', value: '1', type: 'checkbox', checked: false }, + ], + } + const params = context.collectPanelParams(panel, () => { composed = true }, 'documents') + assert.equal(composed, true) + assert.equal(params.toString(), 'type=documents&review-score-to=0') + assert.equal(context.hasFilterParams(params), true) +}) + +test('empty home advanced search does not submit', () => { + const panel = { + querySelectorAll: () => [ + { name: 'review-score-from', value: '', tagName: 'SELECT' }, + { name: 'review-score-to', value: '', tagName: 'SELECT' }, + { name: 'pages-from', value: '0', tagName: 'INPUT' }, + ], + } + const params = context.collectPanelParams(panel, () => {}, 'documents') + assert.equal(params.toString(), 'type=documents') + assert.equal(context.hasFilterParams(params), false) +}) diff --git a/internal/webserver/js-tests/search-filter-utils.test.mjs b/internal/webserver/js-tests/search-filter-utils.test.mjs new file mode 100644 index 00000000..85063ed5 --- /dev/null +++ b/internal/webserver/js-tests/search-filter-utils.test.mjs @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' + +const source = await readFile(new URL('../embedded/js/search-filter-utils.js', import.meta.url), 'utf8') +const { initClearRangeControls } = await import(`data:text/javascript;base64,${Buffer.from(source).toString('base64')}`) + +for (const tag of ['input', 'select']) { + test(`range clear controls track and reset ${tag} values`, () => { + const controls = [new EventTarget(), new EventTarget()] + controls.forEach(control => { + control.value = '' + control.tagName = tag.toUpperCase() + }) + const button = new EventTarget() + const group = { + querySelectorAll: selector => selector.split(', ').includes(tag) ? controls : [], + querySelector: selector => selector === '.clear-range-control' + ? button + : selector.split(', ').includes(tag) ? controls[0] : null, + } + button.closest = () => group + const filters = { + querySelectorAll: selector => selector === '.clear-range-control' ? [button] : [], + } + const reset = initClearRangeControls(filters) + assert.equal(button.disabled, true) + + controls[1].value = '0' + controls[1].dispatchEvent(new Event('change')) + assert.equal(button.disabled, tag !== 'select') + if (tag === 'select') { + button.dispatchEvent(new Event('click', { cancelable: true })) + assert.deepEqual(controls.map(control => control.value), ['', '']) + assert.equal(button.disabled, true) + } + + controls[1].value = '5' + controls[1].dispatchEvent(new Event('change')) + assert.equal(button.disabled, false) + + let inputEvents = 0 + controls[0].addEventListener('input', () => { inputEvents++ }) + button.dispatchEvent(new Event('click', { cancelable: true })) + assert.deepEqual(controls.map(control => control.value), ['', '']) + assert.equal(button.disabled, true) + assert.equal(inputEvents, 1) + + controls[0].value = '1' + controls[0].dispatchEvent(new Event('input')) + assert.equal(button.disabled, false) + reset() + assert.deepEqual(controls.map(control => control.value), ['', '']) + assert.equal(button.disabled, true) + }) +} diff --git a/internal/webserver/middleware.go b/internal/webserver/middleware.go index da187350..a8e06a12 100644 --- a/internal/webserver/middleware.go +++ b/internal/webserver/middleware.go @@ -37,10 +37,12 @@ func RequireAdmin(c fiber.Ctx) error { // SetConfigLocals sets config values in c.Locals() for template access func SetConfigLocals(cfg Config) func(fiber.Ctx) error { return func(c fiber.Ctx) error { + c.Locals("RequireAuth", cfg.RequireAuth) c.Locals("CoverMaxWidth", cfg.CoverMaxWidth) c.Locals("ShareCommentMaxSize", cfg.ShareCommentMaxSize) c.Locals("ShareMaxRecipients", cfg.ShareMaxRecipients) c.Locals("IllustratedMinAmount", cfg.IllustratedMinAmount) + c.Locals("ReviewSourceConfigured", cfg.ReviewSourceConfigured) return c.Next() } } @@ -81,9 +83,9 @@ func fmtRemainingTime(d time.Duration) string { // SetProgress retrieves indexing progress information from the index and sets it // as a local variable of the request -func SetProgress(progress IndexInfo) func(fiber.Ctx) error { +func SetProgress(idx IndexInfo) func(fiber.Ctx) error { return func(c fiber.Ctx) error { - progress, err := progress.IndexingProgress() + progress, err := idx.IndexingProgress() if err != nil { fmt.Println(err) } diff --git a/internal/webserver/model/highlight_repository.go b/internal/webserver/model/highlight_repository.go index d7339a15..fb60655f 100644 --- a/internal/webserver/model/highlight_repository.go +++ b/internal/webserver/model/highlight_repository.go @@ -141,6 +141,8 @@ func hasSearchFields(sf index.SearchFields) bool { sf.EstReadTimeTo != 0 || sf.PagesFrom != 0 || sf.PagesTo != 0 || + sf.DataSourceScoreFrom != 0 || + sf.DataSourceScoreUpperBound() != 5 || sf.IllustratedOnly } @@ -191,6 +193,12 @@ func matchesSearchFields(doc AugmentedDocument, sf index.SearchFields, illustrat return false } + if max := sf.DataSourceScoreUpperBound(); sf.DataSourceScoreFrom != 0 || max != 5 { + if score := doc.DataSourceScore(); score < sf.DataSourceScoreFrom || score > max { + return false + } + } + if sf.IllustratedOnly && illustratedMinAmount > 0 && doc.Illustrations < illustratedMinAmount { return false } diff --git a/internal/webserver/routes.go b/internal/webserver/routes.go index 36904f2e..631e0904 100644 --- a/internal/webserver/routes.go +++ b/internal/webserver/routes.go @@ -52,6 +52,7 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se c.Locals("Version", c.App().Config().AppName) c.Locals("SupportedLanguages", supportedLanguages) c.Locals("Lang", chooseBestLanguage(c)) + c.Locals("DataSourceName", controllers.Documents.DataSourceName()) q := c.Queries() delete(q, "l") c.Locals("URLPath", c.Path()) @@ -59,7 +60,7 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se return c.Next() }) - app.Use(SetConfigLocals(cfg)) + app.Use(SetConfigLocals(cfg), SetProgress(idx)) // Set available languages for the language filter app.Use(SetAvailableLanguages(idx)) @@ -70,6 +71,11 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se // Set email sending configuration (must be early so it's available in all routes) app.Use(SetEmailSendingConfigured(sender)) + app.Get("/background-progress", configurableAuthentication, func(c fiber.Ctx) error { + c.Set("Cache-Control", "no-store") + return c.Render("partials/background-progress", fiber.Map{}) + }) + app.Get("/sessions/new", allowIfNotLoggedIn, controllers.Auth.Login) app.Post("/sessions", allowIfNotLoggedIn, controllers.Auth.SignIn) app.Get("/recover", allowIfNotLoggedIn, controllers.Auth.Recover) @@ -98,6 +104,7 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se app.Get("/upload", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UploadForm) docsGroup.Post("/", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Upload) docsGroup.Delete("/:slug", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Delete) + docsGroup.Put("/:slug/review-link", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UpdateReviewLink) // Authentication requirement is configurable for all routes below this middleware app.Use(configurableAuthentication) @@ -125,6 +132,8 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se c.Locals("SimilarToSlug", c.Params("slug")) return controllers.Search.SearchDocuments(c) }) + docsGroup.Get("/:slug/score", controllers.Documents.Score) + docsGroup.Get("/:slug/reviews", controllers.Documents.Reviews) docsGroup.Get("/:slug/read", controllers.Documents.Reader) docsGroup.Get("/:slug/annotations", alwaysRequireAuthentication, controllers.Annotations.List) docsGroup.Post("/:slug/annotations", alwaysRequireAuthentication, controllers.Annotations.Save) diff --git a/internal/webserver/search_test.go b/internal/webserver/search_test.go index 15ee46fc..86dc4d36 100644 --- a/internal/webserver/search_test.go +++ b/internal/webserver/search_test.go @@ -2,12 +2,16 @@ package webserver_test import ( "net/http" + "slices" "strings" "testing" "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v3" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/webserver" "github.com/svera/coreander/v5/internal/webserver/infrastructure" ) @@ -145,6 +149,340 @@ func TestUnifiedSearch(t *testing.T) { } } +func TestDataSourceScoreSortOptions(t *testing.T) { + paths := []struct { + name string + url string + }{ + {name: "document search", url: "/search?type=documents&"}, + {name: "author documents", url: "/authors/john-doe?"}, + {name: "series documents", url: "/series/the-lord-of-the-rings?"}, + } + modes := []struct { + name string + htmx bool + }{ + {name: "full page"}, + {name: "fragment", htmx: true}, + } + + for _, configured := range []bool{false, true} { + name := "without review sources" + directions := []string{""} + if configured { + name = "with review sources" + directions = []string{"asc", "desc"} + } + t.Run(name, func(t *testing.T) { + cfg := defaultTestConfig() + cfg.ReviewSourceConfigured = configured + app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, + loadDirInMemoryFs("testdata/library"), cfg) + + for _, path := range paths { + for _, mode := range modes { + for _, direction := range directions { + name := path.name + "/" + mode.name + selected := "" + if direction != "" { + selected = "review-score-" + direction + name += "/" + direction + } + t.Run(name, func(t *testing.T) { + url := path.url + if selected != "" { + url += "sort-by=" + selected + } + req := mustGetRequest(t, url) + if mode.htmx { + req.Header.Set("HX-Request", "true") + } + response, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("unexpected status %d", response.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + wantCount := 0 + if configured { + wantCount = 1 + } + for _, value := range []string{"review-score-asc", "review-score-desc"} { + if got := doc.Find(`#sort-by option[value="` + value + `"]`).Length(); got != wantCount { + t.Errorf("score sort option %q count = %d, want %d", value, got, wantCount) + } + } + if selected != "" && doc.Find(`#sort-by option[value="`+selected+`"][selected]`).Length() != 1 { + t.Errorf("expected selected option %q", selected) + } + }) + } + } + } + }) + } +} + +func TestReviewScoreFilter(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + cfg := defaultTestConfig() + cfg.ReviewSourceConfigured = true + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), cfg) + + homeResponse, err := app.Test(mustGetRequest(t, "/")) + if err != nil { + t.Fatal(err) + } + defer homeResponse.Body.Close() + homeDoc, err := goquery.NewDocumentFromReader(homeResponse.Body) + if err != nil { + t.Fatal(err) + } + if homeDoc.Find(`select[name="review-score-from"]`).Length() == 0 { + t.Fatal("review score filter must be visible on home search when a review source is configured") + } + + response, err := app.Test(mustGetRequest(t, "/search?type=documents&review-score-from=4&review-score-to=5")) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("Expected status %d, received %d", http.StatusOK, response.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + for _, field := range []struct { + name string + value string + }{ + {name: "review-score-from", value: "4"}, + {name: "review-score-to", value: ""}, + } { + selects := doc.Find(`select[name="` + field.name + `"]`) + if selects.Length() == 0 { + t.Errorf("Expected data-source score filter %q", field.name) + continue + } + selects.Each(func(_ int, selectBox *goquery.Selection) { + if got, _ := selectBox.Find("option[selected]").Attr("value"); got != field.value { + t.Errorf("Expected %s value %q, got %q", field.name, field.value, got) + } + options := selectBox.Find("option").Map(func(i int, option *goquery.Selection) string { + value, _ := option.Attr("value") + wantLabel := "" + switch value { + case "0": + if strings.TrimSpace(option.Text()) == "" { + t.Errorf("Expected a label for %s no-score option", field.name) + } + wantLabel = option.Text() + case "1", "2", "3", "4", "5": + wantLabel = value + " " + strings.Repeat("\u2605", i-1) + } + if option.Text() != wantLabel { + t.Errorf("Expected %s option %q label %q, got %q", field.name, value, wantLabel, option.Text()) + } + return value + }) + if strings.Join(options, ",") != ",0,1,2,3,4,5" { + t.Errorf("Expected blank, no-score, and scores 1 to 5 for %s, got %v", field.name, options) + } + }) + if doc.Find(`input[name="`+field.name+`"]`).Length() != 0 { + t.Errorf("Expected no numeric input for %s", field.name) + } + + } + + response, err = app.Test(mustGetRequest(t, "/search?type=documents&review-score-from=0&review-score-to=0")) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + doc, err = goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"review-score-from", "review-score-to"} { + selected, _ := doc.Find(`select[name="` + name + `"] option[selected]`).Attr("value") + want := "0" + if name == "review-score-from" { + want = "" + } + if selected != want { + t.Errorf("Expected %q selected for %s, got %q", want, name, selected) + } + } +} + +func TestReviewScoreFilterHiddenWithoutConfiguredInstances(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs("testdata/library"), webserver.Config{}) + + for _, path := range []string{"/", "/search?type=documents"} { + response, err := app.Test(mustGetRequest(t, path)) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("Expected status %d for %s, received %d", http.StatusOK, path, response.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(response.Body) + if err != nil { + t.Fatal(err) + } + if doc.Find(`select[name="review-score-from"], select[name="review-score-to"]`).Length() != 0 { + t.Fatalf("review score filter must be hidden for %s when no review source is configured", path) + } + } +} + +func TestSearchResultsShowStoredDataSourceScore(t *testing.T) { + for _, tt := range []struct { + score float64 + want string + }{ + {score: 3.82, want: "3.82"}, + {score: 4.1, want: "4.10"}, + {score: 5, want: "5.00"}, + } { + t.Run(tt.want, func(t *testing.T) { + app, _ := newSearchResultsTestApp(t, []index.Document{withBiblioReadsData(t, index.Document{ + ID: "linked.epub", Slug: "linked", + Metadata: metadata.Metadata{Title: "La Reina Dragon", Format: "EPUB"}, + + DataSourceScoreValue: tt.score, + }, biblioreads.DocumentData{URL: "/book/show/linked", RatingCount: "2,285"}), + }) + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents&search=La"))) + if err != nil { + t.Fatal(err) + } + score := page.Find("#list .list-group-item li").FilterFunction(func(_ int, item *goquery.Selection) bool { + text := strings.TrimSpace(item.Text()) + return strings.Contains(text, tt.want) && strings.Contains(text, "2,285") + }) + if score.Length() != 1 || score.Find("i.bi-star-fill").Length() != 1 { + t.Fatalf("expected generic data-source score and count in search result, got %q", score.Text()) + } + }) + } +} + +func TestSearchResultsSortByDataSourceScore(t *testing.T) { + app, _ := newSearchResultsTestApp(t, []index.Document{withBiblioReadsData(t, index.Document{ + ID: "linked.epub", Slug: "linked", + Metadata: metadata.Metadata{Title: "Linked", Format: "EPUB"}, + DataSourceScoreValue: 3.2, + }, biblioreads.DocumentData{URL: "/book/show/linked"}), withBiblioReadsData(t, index.Document{ + ID: "unlinked.epub", Slug: "unlinked", + Metadata: metadata.Metadata{Title: "Unlinked", Format: "EPUB"}, + DataSourceScoreValue: 4.1, + }, biblioreads.DocumentData{URL: "/book/show/unlinked"}), withBiblioReadsData(t, index.Document{ + ID: "unmatched.epub", Slug: "unmatched", + Metadata: metadata.Metadata{Title: "Unmatched", Format: "EPUB"}, + DataSourceScoreValue: 4.8, + }, biblioreads.DocumentData{URL: "/book/show/unmatched"}), + }) + for _, tt := range []struct { + direction string + want []string + }{ + {direction: "asc", want: []string{"linked", "unlinked", "unmatched"}}, + {direction: "desc", want: []string{"unmatched", "unlinked", "linked"}}, + } { + t.Run(tt.direction, func(t *testing.T) { + body := getBody(t, app, "/search?type=documents&sort-by=review-score-"+tt.direction) + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + var got []string + page.Find("#list .list-group-item h2 a").Each(func(_ int, link *goquery.Selection) { + href, _ := link.Attr("href") + got = append(got, strings.TrimPrefix(href, "/documents/")) + }) + if !slices.Equal(got, tt.want) { + t.Fatalf("score sort %s: got %v, want %v", tt.direction, got, tt.want) + } + }) + } +} + +func TestSearchResultsFilterByInclusiveScoreRange(t *testing.T) { + documents := []index.Document{ + {ID: "unrated.epub", Slug: "unrated", DataSourceScoreValue: 0}, + {ID: "fractional.epub", Slug: "fractional", DataSourceScoreValue: 0.5}, + {ID: "one.epub", Slug: "one", DataSourceScoreValue: 1}, + {ID: "five.epub", Slug: "five", DataSourceScoreValue: 5}, + } + for i := range documents { + documents[i].Metadata = metadata.Metadata{Title: documents[i].Slug, Format: "EPUB"} + } + app, _ := newSearchResultsTestApp(t, documents) + for _, tt := range []struct { + name string + query string + want []string + }{ + {name: "default", want: []string{"five", "fractional", "one", "unrated"}}, + {name: "full range", query: "&review-score-from=0&review-score-to=5", want: []string{"five", "fractional", "one", "unrated"}}, + {name: "zero lower bound", query: "&review-score-from=0", want: []string{"five", "fractional", "one", "unrated"}}, + {name: "unrated", query: "&review-score-from=0&review-score-to=0", want: []string{"unrated"}}, + {name: "zero upper bound", query: "&review-score-to=0", want: []string{"unrated"}}, + {name: "positive range", query: "&review-score-from=1&review-score-to=5", want: []string{"five", "one"}}, + {name: "upper bound includes unrated", query: "&review-score-to=1", want: []string{"fractional", "one", "unrated"}}, + {name: "exact score", query: "&review-score-from=1&review-score-to=1", want: []string{"one"}}, + {name: "reversed bounds", query: "&review-score-from=5&review-score-to=1", want: []string{"five", "one"}}, + {name: "legacy unrated", query: "&review-score-from=none", want: []string{"unrated"}}, + } { + t.Run(tt.name, func(t *testing.T) { + page, err := goquery.NewDocumentFromReader(strings.NewReader(getBody(t, app, "/search?type=documents"+tt.query))) + if err != nil { + t.Fatal(err) + } + var got []string + page.Find("#list .list-group-item h2 a").Each(func(_ int, link *goquery.Selection) { + href, _ := link.Attr("href") + got = append(got, strings.TrimPrefix(href, "/documents/")) + }) + slices.Sort(got) + if !slices.Equal(got, tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestSearchResultsShowStoredZeroRatingAsNotScored(t *testing.T) { + app, _ := newSearchResultsTestApp(t, []index.Document{withBiblioReadsData(t, index.Document{ + ID: "unscored.epub", Slug: "unscored", + Metadata: metadata.Metadata{Title: "Unscored", Format: "EPUB"}, + }, biblioreads.DocumentData{URL: "/book/show/unscored"}), + }) + body := getBody(t, app, "/search?type=documents&search=Unscored") + page, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + row := page.Find("#list .list-group-item li").FilterFunction(func(_ int, item *goquery.Selection) bool { + return strings.TrimSpace(item.Text()) == "Not scored" && item.Find("i.bi-star-fill").Length() == 1 + }) + if row.Length() == 0 || row.Find("a").Length() != 0 { + t.Fatalf("expected generic zero score to render as Not scored, got %q", row.Text()) + } +} + // TestSimilarDocumentsFullPage checks that navigating to a document's // /similar route (the "Similar documents" > "See all" link) renders the same // full search UI as a regular search, scoped to that document, with any diff --git a/internal/webserver/series_test.go b/internal/webserver/series_test.go index cea8191a..0167e201 100644 --- a/internal/webserver/series_test.go +++ b/internal/webserver/series_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/PuerkitoBio/goquery" + "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/precisiondate" "github.com/svera/coreander/v5/internal/webserver" @@ -70,9 +71,21 @@ func TestSeriesFilters(t *testing.T) { third.Words = 0 catalog[testLibraryDir+"/metadata.pdf"] = third reader := catalogReader{byPath: catalog} - app := bootstrapApp(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, + app, idx := bootstrapAppWithIndexer(infrastructure.Connect(":memory:", 250), &infrastructure.NoEmail{}, loadDirInMemoryFs(testLibraryDir), webserver.Config{}, map[string]metadata.Reader{".epub": reader, ".pdf": reader}) + seriesDocuments, err := idx.SearchBySeries(index.SearchFields{Keywords: "the-lord-of-the-rings"}, 1, 10) + if err != nil { + t.Fatal(err) + } + scores := map[string]string{ + "First volume": "3.20", + "Second volume": "4.50", + "Third volume": "4.50", + } + for _, doc := range seriesDocuments.Hits() { + setDataSourceScore(t, idx, doc, scores[doc.Title], "") + } cases := []struct { name string @@ -81,6 +94,8 @@ func TestSeriesFilters(t *testing.T) { }{ {"default series order", "", []string{"First volume", "Second volume", "Third volume"}}, {"descending series order", "?sort-by=number-desc", []string{"Third volume", "Second volume", "First volume"}}, + {"review score ascending", "?sort-by=review-score-asc", []string{"First volume", "Second volume", "Third volume"}}, + {"review score descending with series-order tie-break", "?sort-by=review-score-desc", []string{"Second volume", "Third volume", "First volume"}}, {"language", "?language=es", []string{"Second volume"}}, {"subjects", "?subjects=fiction", []string{"First volume", "Third volume"}}, {"publishing date", "?pub-date-from=2021-01-01", []string{"Second volume"}}, diff --git a/internal/webserver/author_update_auth_test.go b/internal/webserver/update_auth_test.go similarity index 54% rename from internal/webserver/author_update_auth_test.go rename to internal/webserver/update_auth_test.go index def10854..dcda24e1 100644 --- a/internal/webserver/author_update_auth_test.go +++ b/internal/webserver/update_auth_test.go @@ -59,3 +59,50 @@ func TestAuthorUpdateRequiresAdmin(t *testing.T) { }) } } + +func TestDocumentUpdateDataSourceLinkRequiresAdmin(t *testing.T) { + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + app := bootstrapApp(db, &infrastructure.NoEmail{}, loadDirInMemoryFs(testLibraryDir), webserver.Config{}) + t.Cleanup(func() { _ = app.Shutdown() }) + users := &model.UserRepository{DB: db} + if err := users.Create(&model.User{ + Uuid: "regular-review-link-user", Name: "Regular user", Username: "regular-review-link", + Email: "regular@example.com", Password: model.Hash("regular-password"), + Role: model.RoleRegular, WordsPerMinute: 250, + }); err != nil { + t.Fatal(err) + } + adminCookie, err := login(app, "admin@example.com", "admin", t) + if err != nil { + t.Fatal(err) + } + regularCookie, err := login(app, "regular@example.com", "regular-password", t) + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + cookie *http.Cookie + status int + }{ + {name: "anonymous", cookie: &http.Cookie{}, status: http.StatusForbidden}, + {name: "regular user", cookie: regularCookie, status: http.StatusForbidden}, + {name: "administrator without an editable review provider", cookie: adminCookie, status: http.StatusNotFound}, + } { + t.Run(tt.name, func(t *testing.T) { + response, err := putRequest(url.Values{ + "bookURL": {"https://www.goodreads.com/book/show/1368314.El_Cercano_Oriente"}, + }, tt.cookie, app, "/documents/john-doe-test-epub/review-link", t) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + mustReturnStatus(response, tt.status, t) + }) + } +} diff --git a/internal/webserver/upload_test.go b/internal/webserver/upload_test.go index 863eda13..07009355 100644 --- a/internal/webserver/upload_test.go +++ b/internal/webserver/upload_test.go @@ -46,7 +46,7 @@ func (uploadErrorView) Render(out io.Writer, _ string, binding any, _ ...string) } func TestUploadRejectedDuringBulkIndexing(t *testing.T) { - controller := document.NewController(nil, nil, nil, nil, nil, busyUploadIndex{}, nil, document.Config{}, nil) + controller := document.NewController(nil, nil, nil, nil, nil, busyUploadIndex{}, nil, document.Config{}, nil, nil, nil) app := fiber.New(fiber.Config{Views: uploadErrorView{}}) app.Post("/documents", controller.Upload) app.Delete("/documents/:slug", controller.Delete) diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go index da3ce5ae..6939d14b 100644 --- a/internal/webserver/webserver.go +++ b/internal/webserver/webserver.go @@ -57,6 +57,8 @@ type Config struct { ClientDynamicImageCacheTTL int ServerStaticCacheTTL int ServerDynamicImageCacheTTL int + ReviewSourceConfigured bool + ReviewSourceCacheTTL int ShareCommentMaxSize int ShareMaxRecipients int IllustratedMinAmount int @@ -74,6 +76,7 @@ type Sender interface { type IndexInfo interface { IndexingProgress() (index.Progress, error) + BackgroundProgress() ([]index.Progress, error) Languages() ([]string, error) Formats() ([]string, error) } @@ -134,6 +137,15 @@ func New(cfg Config, controllers Controllers, sender Sender, idx IndexInfo, user if err != nil { log.Fatal(err) } + engine.AddFunc("storedDocumentScore", controllers.Documents.StoredScore) + engine.AddFunc("remainingTime", fmtRemainingTime) + engine.AddFunc("backgroundProgress", func() fiber.Map { + tasks, err := idx.BackgroundProgress() + if err != nil { + log.Printf("Error reading background progress: %s", err) + } + return fiber.Map{"Tasks": tasks, "Error": err != nil} + }) // UploadDocumentMaxSize <= 0 means "unlimited", but fiber has no such // sentinel for BodyLimit - it coerces any value <= 0 to its own 4MB @@ -154,9 +166,11 @@ func New(cfg Config, controllers Controllers, sender Sender, idx IndexInfo, user app.Use( SetFQDN(cfg), - SetProgress(idx), favicon.New(), cache.New(cache.Config{ + Next: func(c fiber.Ctx) bool { + return c.Path() == "/background-progress" + }, ExpirationGenerator: func(c fiber.Ctx, cfg *cache.Config) time.Duration { newCacheTime, _ := strconv.Atoi(c.GetRespHeader("Cache-Time", "0")) return time.Second * time.Duration(newCacheTime) diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index 6ca66b33..00cb3c8d 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -16,6 +16,7 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/gofiber/fiber/v3" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" @@ -56,6 +57,11 @@ func TestGET(t *testing.T) { } func bootstrapApp(db *gorm.DB, sender webserver.Sender, appFs afero.Fs, webserverConfig webserver.Config, metadataReaders ...map[string]metadata.Reader) *fiber.App { + app, _ := bootstrapAppWithIndexer(db, sender, appFs, webserverConfig, metadataReaders...) + return app +} + +func bootstrapAppWithIndexer(db *gorm.DB, sender webserver.Sender, appFs afero.Fs, webserverConfig webserver.Config, metadataReaders ...map[string]metadata.Reader) (*fiber.App, *index.BleveIndexer) { readers := testMetadataReaders() if len(metadataReaders) > 0 { readers = metadataReaders[0] @@ -89,9 +95,69 @@ func bootstrapApp(db *gorm.DB, sender webserver.Sender, appFs afero.Fs, webserve if err != nil { log.Fatal(err) } - controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource) + adapter := biblioreads.NewEnricher(nil) + adapter.SetIndex(idx) + controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, nil, dataSource, biblioreads.NewReviewProvider(nil, adapter)) usersRepository := &model.UserRepository{DB: db} - return webserver.New(webserverConfig, controllers, sender, idx, usersRepository) + return webserver.New(webserverConfig, controllers, sender, idx, usersRepository), idx +} + +func newSearchResultsTestApp(t *testing.T, documents []index.Document) (*fiber.App, *index.BleveIndexer) { + t.Helper() + db := infrastructure.Connect(":memory:", 250) + sqlDB, err := db.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sqlDB.Close() }) + + documentIndex, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authorsIndex, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + appFS := afero.NewMemMapFs() + cfg := defaultTestConfig() + idx := index.NewBleve(documentIndex, authorsIndex, appFS, testLibraryDir, nil, index.Config{}) + t.Cleanup(func() { _ = idx.Close() }) + for _, doc := range documents { + if err := index.IndexDocument(documentIndex, doc.ID, doc); err != nil { + t.Fatal(err) + } + } + + sender := &infrastructure.NoEmail{} + dataSource := wikidata.NewWikidataSource(wikidata.Gowikidata{}) + controllers := webserver.SetupControllers(cfg, db, idx, sender, appFS, dataSource, nil, dataSource, biblioreads.NewReviewProvider(nil, nil)) + app := webserver.New(cfg, controllers, sender, idx, &model.UserRepository{DB: db}) + t.Cleanup(func() { _ = app.Shutdown() }) + return app, idx +} + +// setReviewScore seeds a document's review score through the current provider-backed index storage. +func setDataSourceScore(t *testing.T, idx *index.BleveIndexer, doc index.Document, score, count string) { + t.Helper() + adapter := biblioreads.NewEnricher(nil) + adapter.SetIndex(idx) + bookURL := mustBiblioReadsData(t, doc).URL + if bookURL == "" { + bookURL = "https://www.goodreads.com/book/show/1-" + doc.Slug + if err := adapter.UpdateDataSourceLink(doc.ID, bookURL); err != nil { + t.Fatalf("set review link for %s: %v", doc.Slug, err) + } + } + updated, err := idx.Document(doc.Slug) + if err != nil { + t.Fatalf("reload %s: %v", doc.Slug, err) + } + if err := adapter.StoreDetails(updated, mustBiblioReadsData(t, updated).URL, biblioreads.BookResponse{ + Rating: score, RatingCount: count, + }); err != nil { + t.Fatalf("set review score for %s: %v", doc.Slug, err) + } } func loadFilesInMemoryFs(files []string) afero.Fs { diff --git a/main.go b/main.go index 51077b19..ffaffb42 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "log" "os" "runtime" + "slices" "strings" "time" @@ -15,11 +16,13 @@ import ( "github.com/alecthomas/kong" "github.com/spf13/afero" + "github.com/svera/coreander/v5/internal/datasource/biblioreads" "github.com/svera/coreander/v5/internal/datasource/wikidata" "github.com/svera/coreander/v5/internal/index" "github.com/svera/coreander/v5/internal/metadata" "github.com/svera/coreander/v5/internal/versioncheck" "github.com/svera/coreander/v5/internal/webserver" + "github.com/svera/coreander/v5/internal/webserver/controller/document" "github.com/svera/coreander/v5/internal/webserver/infrastructure" "github.com/svera/coreander/v5/internal/webserver/model" ) @@ -34,6 +37,7 @@ var ( input CLIInput appFs afero.Fs idx *index.BleveIndexer + biblioreadsClient *biblioreads.Client db *gorm.DB homeDir string err error @@ -42,7 +46,7 @@ var ( resolvedIndexWorkers int ) -func init() { +func main() { ctx := kong.Parse(&input, kong.Description(` Coreander is a document management system which indexes metadata from documents in a library and allows users to search and read them through a web interface. `), @@ -98,6 +102,18 @@ func init() { var documentsIndex, authorsIndex bleve.Index var needsReindex bool documentsIndex, authorsIndex, needsReindex = getIndexes(appFs, input.IllustratedMinSize, input.MinOccurrenceRatio, maxTextRankWords) + var documentEnrichers []index.DocumentEnricher + biblioReadsAdapter := biblioreads.NewEnricher(nil) + if len(input.ReviewSourceInstances) > 0 { + biblioreadsClient = biblioreads.NewClient( + biblioreads.WithInstances(input.ReviewSourceInstances), + biblioreads.WithCacheTTL(time.Duration(input.ReviewSourceCacheTTL)*time.Second), + ) + biblioReadsAdapter = biblioreads.NewEnricher(biblioreadsClient) + documentEnrichers = append(documentEnrichers, biblioReadsAdapter) + } else { + log.Println("Review lookups disabled: configure review-source-instances with self-hosted endpoints or endpoints whose operators permit automation.") + } idx = index.NewBleve(documentsIndex, authorsIndex, appFs, input.LibPath, metadataReaders, index.Config{ IllustratedMinAmount: input.IllustratedMinAmount, IllustratedMinSize: input.IllustratedMinSize, @@ -110,17 +126,15 @@ func init() { MinCommonTextRankAbsoluteCount: input.MinCommonTextRankAbsoluteCount, PruneChangeTriggerRatio: input.PruneChangeTriggerRatio, TextRankEnrichWorkers: resolvedIndexWorkers, + DocumentEnrichers: documentEnrichers, }) - + biblioReadsAdapter.SetIndex(idx) // If index was newly created or recreated, force reindexing if needsReindex { input.ForceIndexing = true } db = infrastructure.Connect(homeDir+databasePath, input.WordsPerMinute) -} - -func main() { defer idx.Close() idx.BeginIndexing() @@ -156,6 +170,8 @@ func main() { ClientDynamicImageCacheTTL: input.ClientDynamicImageCacheTTL, ServerStaticCacheTTL: input.ServerStaticCacheTTL, ServerDynamicImageCacheTTL: input.ServerDynamicImageCacheTTL, + ReviewSourceConfigured: len(input.ReviewSourceInstances) > 0, + ReviewSourceCacheTTL: input.ReviewSourceCacheTTL, ShareCommentMaxSize: input.ShareCommentMaxSize, ShareMaxRecipients: input.ShareMaxRecipients, IllustratedMinAmount: input.IllustratedMinAmount, @@ -194,7 +210,12 @@ func main() { dataSource := wikidata.NewWikidataSource(wikidata.Gowikidata{}) - controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource) + var reviewProvider document.ReviewProvider + biblioReads := biblioreads.NewReviewProvider(biblioreadsClient, biblioReadsAdapter) + if biblioreadsClient != nil { + reviewProvider = biblioReads + } + controllers := webserver.SetupControllers(webserverConfig, db, idx, sender, appFs, dataSource, reviewProvider, dataSource, biblioReads) usersRepository := &model.UserRepository{DB: db} app := webserver.New(webserverConfig, controllers, sender, idx, usersRepository) if strings.ToLower(input.FQDN) == "localhost" { @@ -219,6 +240,13 @@ func startIndex(idx *index.BleveIndexer, batchSize int, libPath string, indexWor dur, _ := time.ParseDuration(fmt.Sprintf("%ds", end-start)) log.Printf("Indexing finished, took %d seconds", int(dur.Seconds())) + // Network-bound, so it runs alongside the CPU-bound TextRank enrichment. + go func() { + if err := idx.EnrichDocuments(batchSize); err != nil { + log.Printf("Error enriching documents with provider data: %s", err) + } + }() + if err := idx.EnrichTextRankKeywords(batchSize, indexWorkers); err != nil { log.Printf("Error enriching documents with TextRank keywords: %s", err) } @@ -233,21 +261,30 @@ func startIndex(idx *index.BleveIndexer, batchSize int, libPath string, indexWor func getIndexes(fs afero.Fs, illustratedMinSize, minOccurrenceRatio float64, maxTextRankWords int) (bleve.Index, bleve.Index, bool) { needsReindex := false + authorsIndexCreated := false // Open or create documents index - documentsIndex, err := bleve.Open(homeDir + documentsIndexPath) + documentsPath := homeDir + documentsIndexPath + if err := index.RecoverDocumentsIndex(documentsPath); err != nil { + log.Fatal(err) + } + documentsIndex, err := bleve.Open(documentsPath) if err == bleve.ErrorIndexPathDoesNotExist { log.Println("No documents index found, creating a new one.") documentsIndex = index.CreateDocumentsIndex(homeDir + documentsIndexPath) needsReindex = true + } else if err != nil { + log.Fatal(err) } // Open or create authors index - authorsIndex, err := bleve.Open(homeDir + authorsIndexPath) + authorsIndex, err := index.OpenAuthorsIndex(homeDir+authorsIndexPath, input.BatchSize) if err == bleve.ErrorIndexPathDoesNotExist { log.Println("No authors index found, creating a new one.") authorsIndex = index.CreateAuthorsIndex(homeDir + authorsIndexPath) - needsReindex = true + authorsIndexCreated = true + } else if err != nil { + log.Fatal(err) } // Check documents index version @@ -255,36 +292,46 @@ func getIndexes(fs afero.Fs, illustratedMinSize, minOccurrenceRatio float64, max if err != nil { log.Fatal(err) } - if string(version) == "" || string(version) != index.DocumentVersion { - log.Println("Old version documents index found, recreating with new mapping.") - if err = documentsIndex.Close(); err != nil { - log.Fatal(err) - } - if err = fs.RemoveAll(homeDir + documentsIndexPath); err != nil { + if string(version) == "v13" && !input.ForceIndexing { + log.Printf("Migrating documents index from v13 to %s.", index.DocumentVersion) + documentsIndex, err = index.MigrateDocumentsIndexV13(documentsIndex, documentsPath, input.BatchSize) + if err != nil { log.Fatal(err) } - documentsIndex = index.CreateDocumentsIndex(homeDir + documentsIndexPath) - needsReindex = true + version = []byte(index.DocumentVersion) + } + reindexDocuments := slices.Contains([]string{ + "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", + }, string(version)) + if !reindexDocuments && string(version) != index.DocumentVersion { + log.Fatalf("Unsupported documents index version %q; preserving the index instead of recreating it.", version) + } + if reindexDocuments { + log.Printf("Documents index version %s found, recreating with %s mapping and reindexing the library.", version, index.DocumentVersion) } // Rebuild index if TextRank or illustration configuration changed (stored in index metadata). - if !needsReindex { + if !needsReindex && !reindexDocuments { reindexForConfig, err := index.NeedsReindex(documentsIndex, illustratedMinSize, minOccurrenceRatio, maxTextRankWords) if err != nil { log.Fatal(err) } if reindexForConfig { log.Println("Illustrated min size, min occurrence ratio, or max TextRank words config changed, recreating documents index.") - if err = documentsIndex.Close(); err != nil { - log.Fatal(err) - } - if err = fs.RemoveAll(homeDir + documentsIndexPath); err != nil { - log.Fatal(err) - } - documentsIndex = index.CreateDocumentsIndex(homeDir + documentsIndexPath) - needsReindex = true + reindexDocuments = true } } + if reindexDocuments { + if err = documentsIndex.Close(); err != nil { + log.Fatal(err) + } + if err = fs.RemoveAll(homeDir + documentsIndexPath); err != nil { + log.Fatal(err) + } + documentsIndex = index.CreateDocumentsIndex(homeDir + documentsIndexPath) + needsReindex = true + } // Check authors index version version, err = authorsIndex.GetInternal([]byte("version")) @@ -307,6 +354,13 @@ func getIndexes(fs afero.Fs, illustratedMinSize, minOccurrenceRatio float64, max log.Fatal(err) } } + } else if authorsIndexCreated && !needsReindex { + // A missing authors index can be rebuilt from the existing documents index, + // without forcing a scan of the library files. + tmpIdx := index.NewBleve(documentsIndex, authorsIndex, nil, "", nil, index.Config{}) + if err = tmpIdx.RebuildAuthorsFromDocuments(input.BatchSize); err != nil { + log.Fatal(err) + } } return documentsIndex, authorsIndex, needsReindex