From 8d254d06be2f4b2f03c6c44511c0f558c0d5f092 Mon Sep 17 00:00:00 2001 From: liseli Date: Thu, 9 Jul 2026 15:05:28 -0500 Subject: [PATCH 1/9] Adding solr cloud service to babel --- common/docker/solr.yml | 49 ++++++++++++++++++++++++++++++++---------- docker-compose.yml | 10 +++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/common/docker/solr.yml b/common/docker/solr.yml index 9ed3af339..8d6348bcb 100644 --- a/common/docker/solr.yml +++ b/common/docker/solr.yml @@ -9,21 +9,45 @@ x-healthcheck-defaults: &healthcheck-defaults services: solr-lss-dev: - # This is another core service - image: solr:6.6.6-alpine + image: ghcr.io/hathitrust/full-text-search-cloud:shards-docker + container_name: solr-lss-dev ports: - - "8983:8983" - user: ${CURRENT_USER} + - "8981:8983" + environment: + - ZK_HOST=zoo1:2181 + - SOLR_OPTS=-XX:-UseLargePages + depends_on: + zoo1: + condition: service_healthy volumes: - - ${BABEL_HOME}/lss_solr_configs/solr6_standalone/lss-dev/core-x:/opt/solr/server/solr/core-x - - ${BABEL_HOME}/lss_solr_configs/solr6_standalone/lss-dev/core-y:/opt/solr/server/solr/core-y - - ${BABEL_HOME}/lss_solr_configs/solr6_standalone:/opt/lss_solr_configs - - ${BABEL_HOME}/lss_solr_configs/solr6_standalone/lib:/opt/solr/server/solr/lib - - ${BABEL_HOME}/logs/solr:/opt/solr/server/logs + - solr1_data:/var/solr/data + command: [ "sh", "-c", 'solr-foreground -c & sleep 150 && export SOLR_AUTHENTICATION_OPTS=-Dbasicauth=admin:solrRocks && solr create_collection -d /opt/solr/core-x -c core-x -shards 1 -replicationFactor 1 -p 8983 && wait' ] healthcheck: <<: *healthcheck-defaults - test: "ping -c 1 solr-lss-dev:8983" - + test: [ "CMD-SHELL", "curl -s -u admin:solrRocks 'http://solr-lss-dev:8983/solr/admin/collections?action=CLUSTERSTATUS&collection=core-x' | grep -q '\"state\":\"active\"'" ] + interval: 30s + start_period: 30s + timeout: 30s + retries: 10 + zoo1: # Zookeeper run in standalone mode for development and testing purposes. In production, we would use a Zookeeper ensemble for high availability. + image: zookeeper:3.8.0 + container_name: zoo1 + restart: "no" + hostname: zoo1 + ports: + - 2181:2181 + environment: + ZOO_MY_ID: 1 + JVMFLAGS: "-XX:-UseContainerSupport" + volumes: + - zoo1_data:/data + healthcheck: + <<: *healthcheck-defaults + test: [ "CMD", "nc", "-z", "-w", "2", "localhost", "2181" ] # Check if ZooKeeper is listening on port 2181 + interval: 30s + start_period: 30s + timeout: 30s + retries: 10 solr-ptsearch: # This is a core service build: ../../ptsearch-solr #TODO: Build from image in ghcr.io @@ -35,3 +59,6 @@ services: healthcheck: <<: *healthcheck-defaults test: "curl -f --retry-connrefused http://solr-ptsearch:8983/solr/ptsearch/admin/ping" +volumes: + solr1_data: null + zoo1_data: null \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 67a477254..315289f05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,6 +99,14 @@ services: working_dir: /app command: composer install + zoo1: + extends: + file: common/docker/solr.yml + service: zoo1 + profiles: + - backend + - stage_item + solr-lss-dev: extends: file: common/docker/solr.yml @@ -155,3 +163,5 @@ services: volumes: gem_cache: + solr1_data: + zoo1_data: From 3e4fdbda2ce20de79534ecbe5599671f51ee9953 Mon Sep 17 00:00:00 2001 From: liseli Date: Fri, 10 Jul 2026 10:29:27 -0500 Subject: [PATCH 2/9] Declare Zookeeper dependency when Solr is extended --- common/docker/solr.yml | 3 --- docker-compose.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/docker/solr.yml b/common/docker/solr.yml index 8d6348bcb..bc6aa500d 100644 --- a/common/docker/solr.yml +++ b/common/docker/solr.yml @@ -16,9 +16,6 @@ services: environment: - ZK_HOST=zoo1:2181 - SOLR_OPTS=-XX:-UseLargePages - depends_on: - zoo1: - condition: service_healthy volumes: - solr1_data:/var/solr/data command: [ "sh", "-c", 'solr-foreground -c & sleep 150 && export SOLR_AUTHENTICATION_OPTS=-Dbasicauth=admin:solrRocks && solr create_collection -d /opt/solr/core-x -c core-x -shards 1 -replicationFactor 1 -p 8983 && wait' ] diff --git a/docker-compose.yml b/docker-compose.yml index 315289f05..826383d82 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,6 +114,9 @@ services: profiles: - backend - stage_item + depends_on: + zoo1: + condition: service_healthy solr-ptsearch: extends: From 868ba2752c11d9cffda7469d98f15380e972c421 Mon Sep 17 00:00:00 2001 From: liseli Date: Fri, 17 Jul 2026 13:49:10 -0500 Subject: [PATCH 3/9] Update solr.yml to start the Solr server using the image with Solr 9 --- common/docker/solr.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/common/docker/solr.yml b/common/docker/solr.yml index bc6aa500d..8596e3ccb 100644 --- a/common/docker/solr.yml +++ b/common/docker/solr.yml @@ -9,16 +9,17 @@ x-healthcheck-defaults: &healthcheck-defaults services: solr-lss-dev: - image: ghcr.io/hathitrust/full-text-search-cloud:shards-docker + image: ghcr.io/hathitrust/full-text-search-cloud:solr9-shards-docker container_name: solr-lss-dev ports: - "8981:8983" environment: - ZK_HOST=zoo1:2181 - SOLR_OPTS=-XX:-UseLargePages + - SOLR_MODULES=analysis-extras volumes: - - solr1_data:/var/solr/data - command: [ "sh", "-c", 'solr-foreground -c & sleep 150 && export SOLR_AUTHENTICATION_OPTS=-Dbasicauth=admin:solrRocks && solr create_collection -d /opt/solr/core-x -c core-x -shards 1 -replicationFactor 1 -p 8983 && wait' ] + - solr1_data:/var/solr + command: [ "sh", "-c", "solr-foreground & sleep 150 && export SOLR_AUTHENTICATION_OPTS=-Dbasicauth=admin:solrRocks && solr create_collection -d /opt/solr/core-x -c core-x -shards 1 -replicationFactor 1 -p 8983 && wait" ] healthcheck: <<: *healthcheck-defaults test: [ "CMD-SHELL", "curl -s -u admin:solrRocks 'http://solr-lss-dev:8983/solr/admin/collections?action=CLUSTERSTATUS&collection=core-x' | grep -q '\"state\":\"active\"'" ] @@ -27,15 +28,17 @@ services: timeout: 30s retries: 10 zoo1: # Zookeeper run in standalone mode for development and testing purposes. In production, we would use a Zookeeper ensemble for high availability. - image: zookeeper:3.8.0 + image: zookeeper:3.9.0 container_name: zoo1 - restart: "no" + restart: always hostname: zoo1 - ports: - - 2181:2181 + #ports: + # - 2181:2181 environment: ZOO_MY_ID: 1 - JVMFLAGS: "-XX:-UseContainerSupport" + # JVMFLAGS: "-XX:-UseContainerSupport" + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 + ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok volumes: - zoo1_data:/data healthcheck: From 2820bcf9baebcee0870c77ab8cfcbf3b39589f46 Mon Sep 17 00:00:00 2001 From: liseli Date: Tue, 21 Jul 2026 13:16:16 -0500 Subject: [PATCH 4/9] Removed uncommented lines on docker-compose --- common/docker/solr.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/common/docker/solr.yml b/common/docker/solr.yml index 8596e3ccb..3ba354dff 100644 --- a/common/docker/solr.yml +++ b/common/docker/solr.yml @@ -32,13 +32,12 @@ services: container_name: zoo1 restart: always hostname: zoo1 - #ports: - # - 2181:2181 + ports: + - 2181:2181 environment: ZOO_MY_ID: 1 - # JVMFLAGS: "-XX:-UseContainerSupport" ZOO_SERVERS: server.1=zoo1:2888:3888;2181 - ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok + ZOO_4LW_COMMANDS_WHITELIST: mntr,conf,ruok volumes: - zoo1_data:/data healthcheck: From d293c9c32bb926dd0ccd2a9c57d3420e4dd70260 Mon Sep 17 00:00:00 2001 From: liseli Date: Tue, 11 Aug 2026 18:21:16 -0500 Subject: [PATCH 5/9] Set up Solr 6 and Solr 9 in docker environment --- README.md | 12 ++++++++++++ apache/000-default.conf | 6 ++++++ common/docker/apache.yml | 2 ++ common/docker/perl.yml | 2 ++ common/docker/solr.yml | 19 +++++++++++++++++-- docker-compose.yml | 18 ++++++++++++------ ls/lib/Config/docker-local.conf | 7 ++++--- mdp-lib/Institutions.pm | 2 +- 8 files changed, 56 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index eea74ae7e..fbbb2c50c 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,12 @@ In your workdir: docker compose --profile frontend build docker compose --profile backend build ``` +If you want to build the application with Solr 9, you can use the `solr9` and `backend` profiles: + +``` +LSS_SOLR_HOST=solr-lss-dev-solr9 docker compose --profile backend --profile solr9 build +``` + ## Step 4: Build assets Build the CSS and JavaScript for `firebird-common` and `pt`: @@ -57,6 +63,12 @@ docker compose --profile frontend up docker compose --profile backend up ``` +If you want to build the application with Solr 9, you can use the `solr9` and `backend` profiles: + +``` +LSS_SOLR_HOST=solr-lss-dev-solr9 docker compose --profile backend --profile solr9 up +``` + In your browser: * catalog: `http://localhost:8080/Search/Home` diff --git a/apache/000-default.conf b/apache/000-default.conf index 66fef5891..47232561e 100644 --- a/apache/000-default.conf +++ b/apache/000-default.conf @@ -26,6 +26,12 @@ Listen 8080 SetEnv PTSEARCH_SOLR http://solr-ptsearch:8983/solr/ptsearch SetEnv PTSEARCH_SOLR_BASIC_AUTH "" + +# docker-compose.yml/apache.yml set the variables LSS_SOLR_ENGINE_URL_A/LSS_SOLR_ENGINE_URL_B with +# solr URL. We should add these variables here so the CGI process can access them. +# Use PassEnv to pass the environment variables to the CGI process so we can manage both Solr 6 and Solr 9 URLs + PassEnv LSS_SOLR_ENGINE_URL_A + PassEnv LSS_SOLR_ENGINE_URL_B Options Indexes FollowSymLinks diff --git a/common/docker/apache.yml b/common/docker/apache.yml index 2b6be167b..07dc66929 100644 --- a/common/docker/apache.yml +++ b/common/docker/apache.yml @@ -19,6 +19,8 @@ services: environment: - SDRROOT=/htapps/babel - SDRDATAROOT=/sdr1 + - LSS_SOLR_ENGINE_URL_A=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-x + - LSS_SOLR_ENGINE_URL_B=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-y healthcheck: interval: 5s timeout: 10s diff --git a/common/docker/perl.yml b/common/docker/perl.yml index 4fb2c2a32..505e90c1d 100644 --- a/common/docker/perl.yml +++ b/common/docker/perl.yml @@ -16,6 +16,8 @@ services: - MARIADB_USER=ht_web - HTDEV=test - PERL5LIB=/htapps/babel/mdp-lib + - LSS_SOLR_ENGINE_URL_A=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-x + - LSS_SOLR_ENGINE_URL_B=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-y depends_on: mysql-sdr: condition: service_healthy diff --git a/common/docker/solr.yml b/common/docker/solr.yml index 3ba354dff..22c9a79fd 100644 --- a/common/docker/solr.yml +++ b/common/docker/solr.yml @@ -9,8 +9,23 @@ x-healthcheck-defaults: &healthcheck-defaults services: solr-lss-dev: + # This is another core service + image: solr:6.6.6-alpine + ports: + - "8983:8983" + user: ${CURRENT_USER} + volumes: + - ${BABEL_HOME}/lss_solr_configs/solr6_standalone/lss-dev/core-x:/opt/solr/server/solr/core-x + - ${BABEL_HOME}/lss_solr_configs/solr6_standalone/lss-dev/core-y:/opt/solr/server/solr/core-y + - ${BABEL_HOME}/lss_solr_configs/solr6_standalone:/opt/lss_solr_configs + - ${BABEL_HOME}/lss_solr_configs/solr6_standalone/lib:/opt/solr/server/solr/lib + - ${BABEL_HOME}/logs/solr:/opt/solr/server/logs + healthcheck: + <<: *healthcheck-defaults + test: "ping -c 1 solr-lss-dev:8983" + solr-lss-dev-solr-solr9: image: ghcr.io/hathitrust/full-text-search-cloud:solr9-shards-docker - container_name: solr-lss-dev + container_name: solr-lss-dev-solr9 ports: - "8981:8983" environment: @@ -22,7 +37,7 @@ services: command: [ "sh", "-c", "solr-foreground & sleep 150 && export SOLR_AUTHENTICATION_OPTS=-Dbasicauth=admin:solrRocks && solr create_collection -d /opt/solr/core-x -c core-x -shards 1 -replicationFactor 1 -p 8983 && wait" ] healthcheck: <<: *healthcheck-defaults - test: [ "CMD-SHELL", "curl -s -u admin:solrRocks 'http://solr-lss-dev:8983/solr/admin/collections?action=CLUSTERSTATUS&collection=core-x' | grep -q '\"state\":\"active\"'" ] + test: [ "CMD-SHELL", "curl -s -u admin:solrRocks 'http://solr-lss-dev-solr9:8983/solr/admin/collections?action=CLUSTERSTATUS&collection=core-x' | grep -q '\"state\":\"active\"'" ] interval: 30s start_period: 30s timeout: 30s diff --git a/docker-compose.yml b/docker-compose.yml index 826383d82..d47ece9d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,22 +98,28 @@ services: - ${BABEL_HOME}/catalog:/app working_dir: /app command: composer install + # This is Solr 6 + solr-lss-dev: + extends: + file: common/docker/solr.yml + service: solr-lss-dev + profiles: + - backend + - stage_item zoo1: extends: file: common/docker/solr.yml service: zoo1 profiles: - - backend - - stage_item + - solr9 - solr-lss-dev: + solr-lss-dev-solr9: extends: file: common/docker/solr.yml - service: solr-lss-dev + service: solr-lss-dev-solr-solr9 profiles: - - backend - - stage_item + - solr9 depends_on: zoo1: condition: service_healthy diff --git a/ls/lib/Config/docker-local.conf b/ls/lib/Config/docker-local.conf index cde40d830..f5e399b53 100644 --- a/ls/lib/Config/docker-local.conf +++ b/ls/lib/Config/docker-local.conf @@ -1,5 +1,6 @@ num_shards_list = 1 -mbooks_solr_engines = http://solr-lss-dev:8983/solr/core-x -A_mbooks_solr_engines = http://solr-lss-dev:8983/solr/core-x -B_mbooks_solr_engines = http://solr-lss-dev:8983/solr/core-y +mbooks_solr_engines = ENV[LSS_SOLR_ENGINE_URL_A] +A_mbooks_solr_engines = ENV[LSS_SOLR_ENGINE_URL_A] +B_mbooks_solr_engines = ENV[LSS_SOLR_ENGINE_URL_B] + # debug_enabled = 1 diff --git a/mdp-lib/Institutions.pm b/mdp-lib/Institutions.pm index 189da5300..662f5179b 100644 --- a/mdp-lib/Institutions.pm +++ b/mdp-lib/Institutions.pm @@ -329,7 +329,7 @@ sub get_idp_list { $seen{$$hash_ref{inst_id}} = 1; my $host = $ENV{'HTTP_HOST'} || 'localhost'; - my $idp_url = $hash_ref->{template}; + my $idp_url = $hash_ref->{template} // ''; $idp_url =~ s,___HOST___,$host,; $idp_url =~ s,&,&,; From 6d31165a29225ae52180c1cf32ed160c674cfaeb Mon Sep 17 00:00:00 2001 From: liseli Date: Wed, 12 Aug 2026 15:40:37 -0500 Subject: [PATCH 6/9] add add_basic_auth to send Authorization header to ls has access to Solr9 server --- common/docker/apache.yml | 1 + common/docker/perl.yml | 1 + ls/lib/Config/docker-local.conf | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/common/docker/apache.yml b/common/docker/apache.yml index 07dc66929..27d4f6b29 100644 --- a/common/docker/apache.yml +++ b/common/docker/apache.yml @@ -21,6 +21,7 @@ services: - SDRDATAROOT=/sdr1 - LSS_SOLR_ENGINE_URL_A=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-x - LSS_SOLR_ENGINE_URL_B=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-y + - LSS_SOLR_BASIC_AUTH=YWRtaW46c29sclJvY2tz healthcheck: interval: 5s timeout: 10s diff --git a/common/docker/perl.yml b/common/docker/perl.yml index 505e90c1d..eb8f476f5 100644 --- a/common/docker/perl.yml +++ b/common/docker/perl.yml @@ -18,6 +18,7 @@ services: - PERL5LIB=/htapps/babel/mdp-lib - LSS_SOLR_ENGINE_URL_A=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-x - LSS_SOLR_ENGINE_URL_B=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-y + - LSS_SOLR_BASIC_AUTH=YWRtaW46c29sclJvY2tz depends_on: mysql-sdr: condition: service_healthy diff --git a/ls/lib/Config/docker-local.conf b/ls/lib/Config/docker-local.conf index f5e399b53..71433ef7a 100644 --- a/ls/lib/Config/docker-local.conf +++ b/ls/lib/Config/docker-local.conf @@ -2,5 +2,5 @@ num_shards_list = 1 mbooks_solr_engines = ENV[LSS_SOLR_ENGINE_URL_A] A_mbooks_solr_engines = ENV[LSS_SOLR_ENGINE_URL_A] B_mbooks_solr_engines = ENV[LSS_SOLR_ENGINE_URL_B] - +solr_basic_auth_token = ENV[LSS_SOLR_BASIC_AUTH] # debug_enabled = 1 From 77eba6775f324a89409ac849e144e184ef9906ca Mon Sep 17 00:00:00 2001 From: "Brian \"Moses\" Hall" Date: Thu, 13 Aug 2026 10:33:26 -0400 Subject: [PATCH 7/9] Pass `Context` object `$C` to `__get_request_object` so that mdp-config can be queried for Solr auth token --- ls/lib/LS/Searcher/Facets.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ls/lib/LS/Searcher/Facets.pm b/ls/lib/LS/Searcher/Facets.pm index 8866007f0..528841d13 100644 --- a/ls/lib/LS/Searcher/Facets.pm +++ b/ls/lib/LS/Searcher/Facets.pm @@ -57,7 +57,7 @@ sub __Solr_result { my ($C, $query_string, $rs, $AB) = @_; my $url = $self->__get_Solr_select_url($C, $query_string, $AB); - my $req = $self->__get_request_object($url); + my $req = $self->__get_request_object($url, $C); my $ua = $self->__create_user_agent(); if (DEBUG('query')) { From 269997319e10d8fa5589ea8e20426c791a98dbec Mon Sep 17 00:00:00 2001 From: liseli Date: Thu, 13 Aug 2026 10:13:20 -0500 Subject: [PATCH 8/9] adding the context to __get_request_object and LSS_SOLR_BASIC_AUTH to 00-default.conf --- apache/000-default.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/apache/000-default.conf b/apache/000-default.conf index 47232561e..ac7de405d 100644 --- a/apache/000-default.conf +++ b/apache/000-default.conf @@ -32,6 +32,7 @@ Listen 8080 # Use PassEnv to pass the environment variables to the CGI process so we can manage both Solr 6 and Solr 9 URLs PassEnv LSS_SOLR_ENGINE_URL_A PassEnv LSS_SOLR_ENGINE_URL_B + PassEnv LSS_SOLR_BASIC_AUTH Options Indexes FollowSymLinks From 014ebfc1146d3ef3e6d54afc01c1afa30a880347 Mon Sep 17 00:00:00 2001 From: liseli Date: Thu, 13 Aug 2026 14:13:41 -0500 Subject: [PATCH 9/9] Document Solr credentials in the config files --- common/docker/apache.yml | 1 + common/docker/perl.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/common/docker/apache.yml b/common/docker/apache.yml index 27d4f6b29..f71a6a365 100644 --- a/common/docker/apache.yml +++ b/common/docker/apache.yml @@ -21,6 +21,7 @@ services: - SDRDATAROOT=/sdr1 - LSS_SOLR_ENGINE_URL_A=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-x - LSS_SOLR_ENGINE_URL_B=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-y + # This is the basic auth for Solr. The default 'admin:solrRocks' credential - LSS_SOLR_BASIC_AUTH=YWRtaW46c29sclJvY2tz healthcheck: interval: 5s diff --git a/common/docker/perl.yml b/common/docker/perl.yml index eb8f476f5..ab95ba006 100644 --- a/common/docker/perl.yml +++ b/common/docker/perl.yml @@ -18,6 +18,7 @@ services: - PERL5LIB=/htapps/babel/mdp-lib - LSS_SOLR_ENGINE_URL_A=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-x - LSS_SOLR_ENGINE_URL_B=http://${LSS_SOLR_HOST:-solr-lss-dev}:8983/solr/core-y + # This is the basic auth for Solr. The default 'admin:solrRocks' credential - LSS_SOLR_BASIC_AUTH=YWRtaW46c29sclJvY2tz depends_on: mysql-sdr: