From 69d903cd71e006371ea9163061c81bf743f008ef Mon Sep 17 00:00:00 2001 From: Peter Hoffmann <954078+p-hoffmann@users.noreply.github.com> Date: Mon, 9 Feb 2026 00:21:29 +0800 Subject: [PATCH] Extract trexsql into external plugin --- .gitignore | 1 + Dockerfile | 25 ++-- docker/auth-test/run-tests.sh | 2 +- docker/integration-test/run-tests.sh | 2 +- pom.xml | 37 +----- .../plugins/PluginsConfigurationInfo.java | 15 ++- .../ohdsi/webapi/plugins/WebApiPlugin.java | 8 ++ .../org/ohdsi/webapi/service/UserService.java | 8 +- .../ohdsi/webapi/trexsql/TrexSQLConfig.java | 38 ------ .../trexsql/TrexSQLInstanceManager.java | 117 ------------------ .../webapi/trexsql/TrexSQLSearchProvider.java | 116 ----------------- .../ohdsi/webapi/trexsql/TrexSQLService.java | 61 --------- .../webapi/trexsql/TrexSQLServletConfig.java | 45 ------- .../VocabularySearchProviderType.java | 3 +- 14 files changed, 48 insertions(+), 430 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/plugins/WebApiPlugin.java delete mode 100644 src/main/java/org/ohdsi/webapi/trexsql/TrexSQLConfig.java delete mode 100644 src/main/java/org/ohdsi/webapi/trexsql/TrexSQLInstanceManager.java delete mode 100644 src/main/java/org/ohdsi/webapi/trexsql/TrexSQLSearchProvider.java delete mode 100644 src/main/java/org/ohdsi/webapi/trexsql/TrexSQLService.java delete mode 100644 src/main/java/org/ohdsi/webapi/trexsql/TrexSQLServletConfig.java diff --git a/.gitignore b/.gitignore index 65d97f4691..0ecf984c8a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ specs/ docs .claude *.bak +**/test-results/ ### Developer's personal properties ### **/resources/config/application*-dev-*.properties diff --git a/Dockerfile b/Dockerfile index 528dfadec0..e1e2eecdf1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,6 @@ FROM maven:3.9-eclipse-temurin-21 AS builder WORKDIR /code -ARG MAVEN_PROFILE=trexsql ARG MAVEN_PARAMS="" # can use maven options, e.g. -DskipTests=true -DskipUnitTests=true ARG OPENTELEMETRY_JAVA_AGENT_VERSION=1.17.0 @@ -17,8 +16,7 @@ COPY src /code/src RUN mvn package ${MAVEN_PARAMS} \ -Dpackaging.type=jar \ -Dgit.branch=${GIT_BRANCH} \ - -Dgit.commit.id.abbrev=${GIT_COMMIT_ID_ABBREV} \ - -P${MAVEN_PROFILE} + -Dgit.commit.id.abbrev=${GIT_COMMIT_ID_ABBREV} # OHDSI WebAPI running as a Spring Boot executable JAR with Java 21 FROM index.docker.io/library/eclipse-temurin:21-jre @@ -40,15 +38,24 @@ RUN apt-get update && apt-get install -y unzip && rm -rf /var/lib/apt/lists/* COPY --from=builder /code/opentelemetry-javaagent.jar . COPY --from=builder /code/target/WebAPI.jar . +# Plugin setup: download trexsql-ext plugin JAR +ARG TREXSQL_VERSION=v0.1.23 +RUN mkdir -p /opt/webapi/plugins && \ + if curl -fL -o /opt/webapi/plugins/trexsql-ext.jar \ + "https://github.com/p-hoffmann/trexsql-ext/releases/download/${TREXSQL_VERSION}/trexsql-ext.jar"; then \ + echo "Downloaded trexsql-ext plugin ${TREXSQL_VERSION}"; \ + else \ + echo "WARNING: Failed to download trexsql-ext plugin ${TREXSQL_VERSION}, trexsql will be unavailable"; \ + fi + +# Extract native lib from plugin JAR RUN mkdir -p /tmp/trexsql && \ - unzip -j WebAPI.jar 'BOOT-INF/lib/trexsql-ext-*.jar' -d /tmp && \ - unzip -j /tmp/trexsql-ext-*.jar 'libtrexsql_java.so_linux_amd64' -d /tmp/trexsql 2>/dev/null || true && \ - mv /tmp/trexsql/libtrexsql_java.so_linux_amd64 /tmp/trexsql/libtrexsql_java.so 2>/dev/null || true && \ - rm -f /tmp/trexsql-ext-*.jar + unzip -j /opt/webapi/plugins/trexsql-ext.jar 'libtrexsql_java.so_linux_amd64' -d /tmp/trexsql 2>/dev/null || true && \ + mv /tmp/trexsql/libtrexsql_java.so_linux_amd64 /tmp/trexsql/libtrexsql_java.so 2>/dev/null || true EXPOSE 8080 USER 101 -# Run the executable JAR with TrexSQL native library path -CMD ["sh", "-c", "exec java ${DEFAULT_JAVA_OPTS} ${JAVA_OPTS} -Dorg.duckdb.lib_path=/tmp/trexsql/libtrexsql_java.so --add-opens java.naming/com.sun.jndi.ldap=ALL-UNNAMED -jar WebAPI.jar"] +# Run the executable JAR with plugin directory and TrexSQL native library path +CMD ["sh", "-c", "exec java ${DEFAULT_JAVA_OPTS} ${JAVA_OPTS} -Dloader.path=/opt/webapi/plugins -Dorg.duckdb.lib_path=/tmp/trexsql/libtrexsql_java.so --add-opens java.naming/com.sun.jndi.ldap=ALL-UNNAMED -jar WebAPI.jar"] diff --git a/docker/auth-test/run-tests.sh b/docker/auth-test/run-tests.sh index 0feebf9e8e..ce2009cc09 100755 --- a/docker/auth-test/run-tests.sh +++ b/docker/auth-test/run-tests.sh @@ -60,7 +60,7 @@ fi if [ "$BUILD_WEBAPI" = true ]; then log_info "Building WebAPI..." cd ../.. - mvn clean package -DskipTests -Dpackaging.type=jar -P webapi-postgresql,trexsql -B + mvn clean package -DskipTests -Dpackaging.type=jar -P webapi-postgresql -B cd "$SCRIPT_DIR" fi diff --git a/docker/integration-test/run-tests.sh b/docker/integration-test/run-tests.sh index bf25616409..766f42cf98 100755 --- a/docker/integration-test/run-tests.sh +++ b/docker/integration-test/run-tests.sh @@ -45,7 +45,7 @@ fi if [ "$BUILD_WEBAPI" = true ]; then log_info "Building WebAPI..." cd ../.. - mvn clean package -DskipTests -Dpackaging.type=jar -P webapi-postgresql,trexsql -B + mvn clean package -DskipTests -Dpackaging.type=jar -P webapi-postgresql -B cd "$SCRIPT_DIR" fi diff --git a/pom.xml b/pom.xml index 7653c5c55c..44cbf91fb6 100644 --- a/pom.xml +++ b/pom.xml @@ -128,6 +128,7 @@ spring-boot-maven-plugin ${spring.boot.version} + ZIP false false org.ohdsi.webapi.WebApi @@ -182,10 +183,6 @@ -parameters - - - **/trexsql/** - @@ -900,38 +897,6 @@ - - trexsql - - true - - - - com.github.p-hoffmann - trexsql-ext - v0.1.23 - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - true - - - - - webapi-oracle diff --git a/src/main/java/org/ohdsi/webapi/plugins/PluginsConfigurationInfo.java b/src/main/java/org/ohdsi/webapi/plugins/PluginsConfigurationInfo.java index e666410504..e25009bc42 100644 --- a/src/main/java/org/ohdsi/webapi/plugins/PluginsConfigurationInfo.java +++ b/src/main/java/org/ohdsi/webapi/plugins/PluginsConfigurationInfo.java @@ -4,11 +4,24 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Collections; +import java.util.List; + @Component public class PluginsConfigurationInfo extends ConfigurationInfo { private static final String KEY = "plugins"; - public PluginsConfigurationInfo(@Value("${atlasgis.enabled}") Boolean atlasgisEnabled) { + public PluginsConfigurationInfo( + @Autowired(required = false) List plugins, + @Value("${atlasgis.enabled}") Boolean atlasgisEnabled) { + if (plugins == null) { + plugins = Collections.emptyList(); + } + for (WebApiPlugin plugin : plugins) { + properties.put(plugin.getId() + "Enabled", plugin.isActive()); + } properties.put("atlasgisEnabled", atlasgisEnabled); } diff --git a/src/main/java/org/ohdsi/webapi/plugins/WebApiPlugin.java b/src/main/java/org/ohdsi/webapi/plugins/WebApiPlugin.java new file mode 100644 index 0000000000..8975eb3d3e --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/plugins/WebApiPlugin.java @@ -0,0 +1,8 @@ +package org.ohdsi.webapi.plugins; + +public interface WebApiPlugin { + String getId(); + String getName(); + String getVersion(); + boolean isActive(); +} diff --git a/src/main/java/org/ohdsi/webapi/service/UserService.java b/src/main/java/org/ohdsi/webapi/service/UserService.java index 41ab8c8ba0..b62da26fb4 100644 --- a/src/main/java/org/ohdsi/webapi/service/UserService.java +++ b/src/main/java/org/ohdsi/webapi/service/UserService.java @@ -1,6 +1,7 @@ package org.ohdsi.webapi.service; import org.ohdsi.webapi.arachne.logging.event.*; +import org.ohdsi.webapi.plugins.WebApiPlugin; import org.ohdsi.webapi.shiro.Entities.PermissionEntity; import org.ohdsi.webapi.shiro.Entities.RoleEntity; import org.ohdsi.webapi.shiro.Entities.UserEntity; @@ -31,8 +32,8 @@ public class UserService { @Autowired private ApplicationEventPublisher eventPublisher; - @Value("${trexsql.enabled:false}") - private boolean trexsqlCacheEnabled; + @Autowired(required = false) + private List plugins = Collections.emptyList(); @Value("${security.auth.ad.default.import.group}#{T(java.util.Collections).emptyList()}") private List defaultRoles; @@ -114,7 +115,8 @@ public User getCurrentUser() throws Exception { user.name = currentUser.getName(); user.permissions = convertPermissions(permissions); user.permissionIdx = authorizer.queryUserPermissions(currentUser.getLogin()).permissions; - user.trexsqlCacheEnabled = trexsqlCacheEnabled; + user.trexsqlCacheEnabled = plugins.stream() + .anyMatch(p -> "trexsql".equals(p.getId()) && p.isActive()); return user; } diff --git a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLConfig.java b/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLConfig.java deleted file mode 100644 index 2829256af5..0000000000 --- a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.ohdsi.webapi.trexsql; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -/** - * Global trexsql configuration. Per-source config is in the source table (is_cache_enabled). - */ -@ConfigurationProperties(prefix = "trexsql") -public class TrexSQLConfig { - - private boolean enabled = false; - private String cachePath = "./data/cache"; - private String extensionsPath; - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getCachePath() { - return cachePath; - } - - public void setCachePath(String cachePath) { - this.cachePath = cachePath; - } - - public String getExtensionsPath() { - return extensionsPath; - } - - public void setExtensionsPath(String extensionsPath) { - this.extensionsPath = extensionsPath; - } -} diff --git a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLInstanceManager.java b/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLInstanceManager.java deleted file mode 100644 index e6112a8414..0000000000 --- a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLInstanceManager.java +++ /dev/null @@ -1,117 +0,0 @@ -package org.ohdsi.webapi.trexsql; - -import org.trex.Trexsql; -import jakarta.annotation.PreDestroy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.locks.ReentrantLock; - -@Component -@ConditionalOnProperty(name = "trexsql.enabled", havingValue = "true", matchIfMissing = false) -public class TrexSQLInstanceManager { - - private static final Logger log = LoggerFactory.getLogger(TrexSQLInstanceManager.class); - - private final TrexSQLConfig config; - private volatile boolean initialized = false; - private volatile boolean initFailed = false; - private final ReentrantLock initLock = new ReentrantLock(); - - public TrexSQLInstanceManager(TrexSQLConfig config) { - this.config = config; - } - - public void ensureInitialized() { - if (!config.isEnabled()) { - throw new IllegalStateException("TrexSQL is not enabled"); - } - - if (initFailed) { - return; - } - - if (!initialized) { - initLock.lock(); - try { - if (!initialized && !initFailed) { - log.info("Initializing TrexSQL instance"); - try { - Trexsql.init(buildConfig()); - initialized = true; - log.info("TrexSQL instance initialized successfully"); - } catch (Exception | Error e) { - log.error("Failed to initialize TrexSQL: {}. TrexSQL features will be unavailable.", e.getMessage()); - initFailed = true; - } - } - } finally { - initLock.unlock(); - } - } - } - - public boolean isAvailable() { - if (!config.isEnabled() || !initialized) { - return false; - } - try { - return Trexsql.isRunning(); - } catch (Exception e) { - log.warn("Error checking TrexSQL status: {}", e.getMessage()); - return false; - } - } - - public boolean isAttached(String databaseCode) { - if (!initialized) { - return false; - } - try { - return Trexsql.isAttached(databaseCode); - } catch (Exception e) { - log.warn("Error checking if database {} is attached: {}", databaseCode, e.getMessage()); - return false; - } - } - - private Map buildConfig() { - Map initConfig = new HashMap<>(); - - if (config.getExtensionsPath() != null && !config.getExtensionsPath().isEmpty()) { - initConfig.put("extensions-path", config.getExtensionsPath()); - } - - if (config.getCachePath() != null && !config.getCachePath().isEmpty()) { - initConfig.put("cache-path", config.getCachePath()); - } - - initConfig.put("allow-unsigned-extensions", true); - - return initConfig; - } - - @PreDestroy - public void shutdown() { - initLock.lock(); - try { - if (initialized) { - log.info("Shutting down TrexSQL instance"); - try { - Trexsql.shutdown(); - log.info("TrexSQL instance shut down successfully"); - } catch (Exception e) { - log.error("Error shutting down TrexSQL instance: {}", e.getMessage(), e); - } finally { - initialized = false; - } - } - } finally { - initLock.unlock(); - } - } -} diff --git a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLSearchProvider.java b/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLSearchProvider.java deleted file mode 100644 index 3dd6e3eed8..0000000000 --- a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLSearchProvider.java +++ /dev/null @@ -1,116 +0,0 @@ -package org.ohdsi.webapi.trexsql; - -import org.ohdsi.vocabulary.Concept; -import org.ohdsi.vocabulary.SearchProvider; -import org.ohdsi.vocabulary.SearchProviderConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -/** - * SearchProvider implementation using TrexSQL. - */ -@Component -@ConditionalOnProperty(name = "trexsql.enabled", havingValue = "true", matchIfMissing = false) -public class TrexSQLSearchProvider implements SearchProvider { - - private static final Logger log = LoggerFactory.getLogger(TrexSQLSearchProvider.class); - - private static final int TREXSQL_PRIORITY = 1; - - private final TrexSQLService trexsqlService; - private final TrexSQLConfig config; - - public TrexSQLSearchProvider(TrexSQLService trexsqlService, TrexSQLConfig config) { - this.trexsqlService = trexsqlService; - this.config = config; - } - - @Override - public boolean supports(String vocabularyVersionKey) { - return config.isEnabled(); - } - - @Override - public int getPriority() { - return TREXSQL_PRIORITY; - } - - @Override - public Collection executeSearch(SearchProviderConfig searchConfig, String query, String rows) throws Exception { - String sourceKey = searchConfig.getSourceKey(); - - if (!trexsqlService.isCacheAvailable(sourceKey)) { - log.debug("Cache not available for source {}", sourceKey); - throw new IllegalStateException("TrexSQL cache not available for source: " + sourceKey); - } - - int maxRows = parseRows(rows); - log.debug("TrexSQL search for source {} with query: {}", sourceKey, query); - - try { - List> results = trexsqlService.searchVocab(sourceKey, query, maxRows); - return mapToConcepts(results); - } catch (Exception e) { - log.error("TrexSQL search failed for source {}: {}", sourceKey, e.getMessage(), e); - throw new RuntimeException("TrexSQL search failed: " + e.getMessage(), e); - } - } - - private int parseRows(String rows) { - if (rows == null || rows.isEmpty()) { - return 1000; - } - try { - int parsed = Integer.parseInt(rows); - return Math.min(parsed, 10000); - } catch (NumberFormatException e) { - return 1000; - } - } - - private Collection mapToConcepts(List> results) { - List concepts = new ArrayList<>(); - - for (Map row : results) { - Concept concept = new Concept(); - - Object conceptId = row.get("concept_id"); - if (conceptId != null) { - concept.conceptId = ((Number) conceptId).longValue(); - } - - concept.conceptName = (String) row.get("concept_name"); - concept.domainId = (String) row.get("domain_id"); - concept.vocabularyId = (String) row.get("vocabulary_id"); - concept.conceptClassId = (String) row.get("concept_class_id"); - concept.standardConcept = (String) row.get("standard_concept"); - concept.conceptCode = (String) row.get("concept_code"); - concept.invalidReason = (String) row.get("invalid_reason"); - - Object validStartDate = row.get("valid_start_date"); - if (validStartDate instanceof java.sql.Date) { - concept.validStartDate = new java.util.Date(((java.sql.Date) validStartDate).getTime()); - } else if (validStartDate instanceof java.util.Date) { - concept.validStartDate = (java.util.Date) validStartDate; - } - - Object validEndDate = row.get("valid_end_date"); - if (validEndDate instanceof java.sql.Date) { - concept.validEndDate = new java.util.Date(((java.sql.Date) validEndDate).getTime()); - } else if (validEndDate instanceof java.util.Date) { - concept.validEndDate = (java.util.Date) validEndDate; - } - - concepts.add(concept); - } - - return concepts; - } -} diff --git a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLService.java b/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLService.java deleted file mode 100644 index 395bfc7017..0000000000 --- a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLService.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.ohdsi.webapi.trexsql; - -import org.trex.Trexsql; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; - -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Service for TrexSQL operations. Cache is available if file exists. - */ -@Service -@ConditionalOnProperty(name = "trexsql.enabled", havingValue = "true", matchIfMissing = false) -public class TrexSQLService { - - private static final Logger log = LoggerFactory.getLogger(TrexSQLService.class); - - private final TrexSQLConfig config; - private final TrexSQLInstanceManager instanceManager; - - public TrexSQLService(TrexSQLConfig config, TrexSQLInstanceManager instanceManager) { - this.config = config; - this.instanceManager = instanceManager; - } - - /** - * Check if cache file exists for source. - */ - public boolean isCacheAvailable(String sourceKey) { - return Paths.get(config.getCachePath(), sourceKey + ".db").toFile().exists(); - } - - @SuppressWarnings("unchecked") - public List> searchVocab(String sourceKey, String searchTerm, int maxRows) { - log.debug("Searching vocabulary for source {} with term: {}", sourceKey, searchTerm); - - if (!isCacheAvailable(sourceKey)) { - throw new IllegalStateException("TrexSQL cache not available for source: " + sourceKey); - } - - Map options = new HashMap<>(); - options.put("database-code", sourceKey); - options.put("max-rows", maxRows); - options.put("cache-path", config.getCachePath()); - - try { - instanceManager.ensureInitialized(); - List> results = Trexsql.searchVocab(searchTerm, options); - log.debug("Vocabulary search returned {} results", results.size()); - return results; - } catch (Exception e) { - log.error("Error searching vocabulary for source {}: {}", sourceKey, e.getMessage(), e); - throw new RuntimeException("Vocabulary search failed: " + e.getMessage(), e); - } - } -} diff --git a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLServletConfig.java b/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLServletConfig.java deleted file mode 100644 index 106f7bf32c..0000000000 --- a/src/main/java/org/ohdsi/webapi/trexsql/TrexSQLServletConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.ohdsi.webapi.trexsql; - -import org.trex.TrexServlet; -import jakarta.servlet.http.HttpServlet; -import org.ohdsi.webapi.source.SourceRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.web.servlet.ServletRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.HashMap; -import java.util.Map; - -@Configuration -@ConditionalOnProperty(name = "trexsql.enabled", havingValue = "true") -@EnableConfigurationProperties(TrexSQLConfig.class) -public class TrexSQLServletConfig { - - private static final Logger log = LoggerFactory.getLogger(TrexSQLServletConfig.class); - - @Bean - public ServletRegistrationBean trexServlet( - TrexSQLInstanceManager instanceManager, - TrexSQLConfig trexConfig, - SourceRepository sourceRepository) { - - instanceManager.ensureInitialized(); - - TrexServlet servlet = new TrexServlet(); - Map config = new HashMap<>(); - String cachePath = trexConfig.getCachePath(); - log.info("TrexSQL cache path configured as: {}", cachePath); - config.put("cache-path", cachePath); - - servlet.initTrex(sourceRepository, config); - - ServletRegistrationBean registration = - new ServletRegistrationBean<>(servlet, "/trexsql/*"); - registration.setLoadOnStartup(1); - return registration; - } -} diff --git a/src/main/java/org/ohdsi/webapi/vocabulary/VocabularySearchProviderType.java b/src/main/java/org/ohdsi/webapi/vocabulary/VocabularySearchProviderType.java index 09607a3fdf..fe1205bf68 100644 --- a/src/main/java/org/ohdsi/webapi/vocabulary/VocabularySearchProviderType.java +++ b/src/main/java/org/ohdsi/webapi/vocabulary/VocabularySearchProviderType.java @@ -2,6 +2,5 @@ public enum VocabularySearchProviderType { DATABASE, - SOLR, - TREXSQL + SOLR }