diff --git a/bot/pom.xml b/bot/pom.xml index 0064b4b..f7ae0b6 100644 --- a/bot/pom.xml +++ b/bot/pom.xml @@ -73,6 +73,12 @@ + + org.junit.vintage + junit-vintage-engine + 5.10.1 + test + org.springframework.boot spring-boot-starter-test @@ -130,10 +136,31 @@ test - org.junit.vintage - junit-vintage-engine - 5.10.1 - test + com.giffing.bucket4j.spring.boot.starter + bucket4j-spring-boot-starter + 0.12.5 + + + org.springframework.boot + spring-boot-starter-cache + + + javax.cache + cache-api + + + com.github.ben-manes.caffeine + caffeine + + + com.github.ben-manes.caffeine + jcache + + + edu.java + retry + 0.1 + compile diff --git a/bot/src/main/java/edu/java/bot/BotApplication.java b/bot/src/main/java/edu/java/bot/BotApplication.java index e8fd914..510b644 100644 --- a/bot/src/main/java/edu/java/bot/BotApplication.java +++ b/bot/src/main/java/edu/java/bot/BotApplication.java @@ -1,12 +1,15 @@ package edu.java.bot; +import edu.java.RetryQueryConfiguration; import edu.java.bot.configuration.ApplicationConfig; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.annotation.EnableCaching; @SpringBootApplication -@EnableConfigurationProperties(ApplicationConfig.class) +@EnableCaching +@EnableConfigurationProperties({ApplicationConfig.class, RetryQueryConfiguration.class}) public class BotApplication { public static void main(String[] args) { SpringApplication.run(BotApplication.class, args); diff --git a/bot/src/main/java/edu/java/bot/configuration/ScrapperClientConfiguration.java b/bot/src/main/java/edu/java/bot/configuration/ScrapperClientConfiguration.java index 48677ba..992b2d5 100644 --- a/bot/src/main/java/edu/java/bot/configuration/ScrapperClientConfiguration.java +++ b/bot/src/main/java/edu/java/bot/configuration/ScrapperClientConfiguration.java @@ -1,5 +1,7 @@ package edu.java.bot.configuration; +import edu.java.RetryFactory; +import edu.java.RetryQueryConfiguration; import edu.java.bot.client.scrapper.ScrapperClient; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Value; @@ -9,6 +11,7 @@ import org.springframework.web.reactive.function.client.support.WebClientAdapter; import org.springframework.web.service.invoker.HttpServiceProxyFactory; import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; @Configuration @Log4j2 @@ -18,10 +21,12 @@ public class ScrapperClientConfiguration { private String scrapperUrl; @Bean - public ScrapperClient scrapperClient() { + public ScrapperClient scrapperClient(RetryQueryConfiguration retryQueryConfiguration) { + Retry retry = RetryFactory.createRetry(retryQueryConfiguration, "scrapper"); WebClient webClient = WebClient.builder() .defaultStatusHandler(httpStatusCode -> true, clientResponse -> Mono.empty()) .defaultHeader("Content-Type", "application/json") + .filter(RetryFactory.createFilter(retry)) .baseUrl(scrapperUrl).build(); HttpServiceProxyFactory httpServiceProxyFactory = HttpServiceProxyFactory diff --git a/bot/src/main/resources/application.yml b/bot/src/main/resources/application.yml index ddc18c1..175d44a 100644 --- a/bot/src/main/resources/application.yml +++ b/bot/src/main/resources/application.yml @@ -6,6 +6,13 @@ spring: name: bot jackson: time-zone: UTC + cache: + jcache: + provider: com.github.benmanes.caffeine.jcache.spi.CaffeineCachingProvider + caffeine: + spec: maximumSize=100000,expireAfterAccess=3600s + cache-names: + - rate-limit-bucket server: port: 8090 @@ -19,3 +26,26 @@ springdoc: scrapper: url: http://localhost:8080 + +retry-query: + retries: + - target: scrapper + type: exponential + max-attempts: 3 + min-delay: 1s + max-delay: 10s + codes: 429 + +bucket4j: + enabled: true + filters: + - cache-name: rate-limit-bucket + url: .* + http-status-code: too_many_requests + rate-limits: + - bandwidths: + - capacity: 1000 + time: 1 + unit: hours + refill-speed: interval + cache-key: getRemoteAddr() diff --git a/pom.xml b/pom.xml index 7c7d0b3..531ee0d 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ bot scrapper scrapper-jooq + retry diff --git a/retry/pom.xml b/retry/pom.xml new file mode 100644 index 0000000..9f21383 --- /dev/null +++ b/retry/pom.xml @@ -0,0 +1,47 @@ + + + 4.0.0 + + edu.java + root + ${revision} + + + retry + ${revision} + + + 21 + 21 + UTF-8 + + + + io.projectreactor + reactor-core + + + org.springframework + spring-webflux + + + org.springframework + spring-context + + + org.springframework.boot + spring-boot + + + org.springframework.retry + spring-retry + + + org.projectlombok + lombok + + + + diff --git a/retry/src/main/java/edu/java/ErrorFilterPredicate.java b/retry/src/main/java/edu/java/ErrorFilterPredicate.java new file mode 100644 index 0000000..7ee702c --- /dev/null +++ b/retry/src/main/java/edu/java/ErrorFilterPredicate.java @@ -0,0 +1,21 @@ +package edu.java; + +import java.util.List; +import java.util.function.Predicate; +import org.springframework.web.reactive.function.client.WebClientResponseException; + +public class ErrorFilterPredicate implements Predicate { + private final List retryCodes; + + public ErrorFilterPredicate(List retryCodes) { + this.retryCodes = retryCodes; + } + + @Override + public boolean test(Throwable throwable) { + if (throwable instanceof WebClientResponseException e) { + return retryCodes.contains(e.getStatusCode().value()); + } + return true; + } +} diff --git a/retry/src/main/java/edu/java/LinearRetryBackoffSpec.java b/retry/src/main/java/edu/java/LinearRetryBackoffSpec.java new file mode 100644 index 0000000..cf41479 --- /dev/null +++ b/retry/src/main/java/edu/java/LinearRetryBackoffSpec.java @@ -0,0 +1,93 @@ +package edu.java; + +import java.time.Duration; +import java.util.function.Predicate; +import java.util.function.Supplier; +import lombok.RequiredArgsConstructor; +import org.reactivestreams.Publisher; +import org.springframework.retry.ExhaustedRetryException; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; +import reactor.core.scheduler.Schedulers; +import reactor.util.retry.Retry; + + +@RequiredArgsConstructor +public class LinearRetryBackoffSpec extends Retry { + private static final Duration MAX_BACKOFF = Duration.ofMillis(Long.MAX_VALUE); + private final Duration minBackoff; + private final Duration maxBackoff; + private final double factor; + private final int maxAttempts; + private final Predicate errorFilter; + private final Supplier schedulerSupplier; + + public LinearRetryBackoffSpec factor(double factor) { + return new LinearRetryBackoffSpec( + this.minBackoff, + this.maxBackoff, + factor, + this.maxAttempts, + this.errorFilter, + this.schedulerSupplier + ); + } + + public LinearRetryBackoffSpec filter(Predicate errorFilter) { + return new LinearRetryBackoffSpec( + this.minBackoff, + this.maxBackoff, + this.factor, + this.maxAttempts, + errorFilter, + this.schedulerSupplier + ); + } + + public static LinearRetryBackoffSpec linear(int maxAttempts, Duration minDelay) { + return new LinearRetryBackoffSpec( + minDelay, + MAX_BACKOFF, + 1.0, + maxAttempts, + e -> true, + Schedulers::parallel + ); + } + + @Override + public Publisher generateCompanion(Flux retrySignals) { + return Flux.deferContextual(cv -> + retrySignals.contextWrite(cv) + .concatMap(retryWhenState -> { + RetrySignal copy = retryWhenState.copy(); + Throwable currentFailure = copy.failure(); + long iteration = copy.totalRetries(); + if (currentFailure == null) { + return Mono.error(new IllegalStateException( + "Retry.RetrySignal#failure() not expected to be null")); + } + if (!errorFilter.test(currentFailure)) { + return Mono.error(currentFailure); + } + if (iteration >= maxAttempts) { + return Mono.error(new ExhaustedRetryException("Retry exhausted: " + this)); + } + + Duration nextBackoff; + try { + nextBackoff = minBackoff.multipliedBy((long) (iteration * factor)); + if (nextBackoff.compareTo(maxBackoff) > 0) { + nextBackoff = maxBackoff; + } + } catch (ArithmeticException overflow) { + nextBackoff = maxBackoff; + } + + return Mono.delay(nextBackoff, schedulerSupplier.get()).contextWrite(cv); + }) + .onErrorStop() + ); + } +} diff --git a/retry/src/main/java/edu/java/RetryElement.java b/retry/src/main/java/edu/java/RetryElement.java new file mode 100644 index 0000000..2a46f6b --- /dev/null +++ b/retry/src/main/java/edu/java/RetryElement.java @@ -0,0 +1,16 @@ +package edu.java; + +import java.time.Duration; +import java.util.List; +import org.jetbrains.annotations.NotNull; + +public record RetryElement( + @NotNull String target, + @NotNull String type, + int maxAttempts, + double factor, + Duration minDelay, + Duration maxDelay, + List codes +) { +} diff --git a/retry/src/main/java/edu/java/RetryFactory.java b/retry/src/main/java/edu/java/RetryFactory.java new file mode 100644 index 0000000..a7df46c --- /dev/null +++ b/retry/src/main/java/edu/java/RetryFactory.java @@ -0,0 +1,41 @@ +package edu.java; + +import edu.java.builders.ExponentialRetryBuilder; +import edu.java.builders.FixedRetryBuilder; +import edu.java.builders.LinearRetryBuilder; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import lombok.experimental.UtilityClass; +import org.springframework.web.reactive.function.client.ExchangeFilterFunction; +import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; + +@UtilityClass +public class RetryFactory { + + private static final Map> RETRY_BUILDERS = new HashMap<>(); + + static { + RETRY_BUILDERS.put("fixed", new FixedRetryBuilder()); + RETRY_BUILDERS.put("linear", new LinearRetryBuilder()); + RETRY_BUILDERS.put("exponential", new ExponentialRetryBuilder()); + } + + public static ExchangeFilterFunction createFilter(Retry retry) { + return (response, next) -> next.exchange(response) + .flatMap(clientResponse -> { + if (clientResponse.statusCode().isError()) { + return clientResponse.createError(); + } else { + return Mono.just(clientResponse); + } + }).retryWhen(retry); + } + + public static Retry createRetry(RetryQueryConfiguration config, String target) { + return config.retries().stream().filter(element -> element.target().equals(target)).findFirst() + .map(element -> RETRY_BUILDERS.get(element.type()).apply(element)) + .orElseThrow(() -> new RuntimeException("Unknown target " + target)); + } +} diff --git a/retry/src/main/java/edu/java/RetryQueryConfiguration.java b/retry/src/main/java/edu/java/RetryQueryConfiguration.java new file mode 100644 index 0000000..b65789c --- /dev/null +++ b/retry/src/main/java/edu/java/RetryQueryConfiguration.java @@ -0,0 +1,10 @@ +package edu.java; + +import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Validated +@ConfigurationProperties(prefix = "retry-query", ignoreUnknownFields = false) +public record RetryQueryConfiguration(List retries) { +} diff --git a/retry/src/main/java/edu/java/builders/ExponentialRetryBuilder.java b/retry/src/main/java/edu/java/builders/ExponentialRetryBuilder.java new file mode 100644 index 0000000..16de20d --- /dev/null +++ b/retry/src/main/java/edu/java/builders/ExponentialRetryBuilder.java @@ -0,0 +1,16 @@ +package edu.java.builders; + +import edu.java.ErrorFilterPredicate; +import edu.java.RetryElement; +import java.util.function.Function; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +public class ExponentialRetryBuilder implements Function { + @Override + public Retry apply(RetryElement retryElement) { + return RetryBackoffSpec.backoff(retryElement.maxAttempts(), retryElement.minDelay()) + .maxBackoff(retryElement.maxDelay()) + .filter(new ErrorFilterPredicate(retryElement.codes())); + } +} diff --git a/retry/src/main/java/edu/java/builders/FixedRetryBuilder.java b/retry/src/main/java/edu/java/builders/FixedRetryBuilder.java new file mode 100644 index 0000000..676816b --- /dev/null +++ b/retry/src/main/java/edu/java/builders/FixedRetryBuilder.java @@ -0,0 +1,15 @@ +package edu.java.builders; + +import edu.java.ErrorFilterPredicate; +import edu.java.RetryElement; +import java.util.function.Function; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +public class FixedRetryBuilder implements Function { + @Override + public Retry apply(RetryElement retryElement) { + return RetryBackoffSpec.fixedDelay(retryElement.maxAttempts(), retryElement.minDelay()) + .filter(new ErrorFilterPredicate(retryElement.codes())); + } +} diff --git a/retry/src/main/java/edu/java/builders/LinearRetryBuilder.java b/retry/src/main/java/edu/java/builders/LinearRetryBuilder.java new file mode 100644 index 0000000..88206de --- /dev/null +++ b/retry/src/main/java/edu/java/builders/LinearRetryBuilder.java @@ -0,0 +1,15 @@ +package edu.java.builders; + +import edu.java.ErrorFilterPredicate; +import edu.java.LinearRetryBackoffSpec; +import edu.java.RetryElement; +import java.util.function.Function; +import reactor.util.retry.Retry; + +public class LinearRetryBuilder implements Function { + @Override + public Retry apply(RetryElement retryElement) { + return LinearRetryBackoffSpec.linear(retryElement.maxAttempts(), retryElement.minDelay()) + .factor(retryElement.factor()).filter(new ErrorFilterPredicate(retryElement.codes())); + } +} diff --git a/scrapper/pom.xml b/scrapper/pom.xml index 6e0c622..53fac52 100644 --- a/scrapper/pom.xml +++ b/scrapper/pom.xml @@ -42,10 +42,21 @@ 2.3.0 + org.springframework.kafka spring-kafka + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + @@ -67,6 +78,12 @@ + + org.junit.vintage + junit-vintage-engine + 5.10.1 + test + org.springframework.boot spring-boot-starter-test @@ -82,6 +99,10 @@ spring-kafka-test test + + ch.qos.logback + logback-core + ch.qos.logback logback-classic @@ -122,6 +143,16 @@ org.testcontainers kafka test + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + @@ -129,22 +160,46 @@ org.springframework.boot spring-boot-starter-jdbc - org.postgresql postgresql runtime - org.jooq jooq - org.springframework.boot spring-boot-starter-data-jpa + + com.giffing.bucket4j.spring.boot.starter + bucket4j-spring-boot-starter + 0.12.5 + + + org.springframework.boot + spring-boot-starter-cache + + + javax.cache + cache-api + + + com.github.ben-manes.caffeine + caffeine + + + com.github.ben-manes.caffeine + jcache + + + edu.java + retry + 0.1 + compile + diff --git a/scrapper/src/main/java/edu/java/ScrapperApplication.java b/scrapper/src/main/java/edu/java/ScrapperApplication.java index 4d12505..b77624a 100644 --- a/scrapper/src/main/java/edu/java/ScrapperApplication.java +++ b/scrapper/src/main/java/edu/java/ScrapperApplication.java @@ -6,9 +6,12 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.annotation.EnableCaching; @SpringBootApplication -@EnableConfigurationProperties({ApplicationConfig.class, GithubConfig.class, StackOverflowConfig.class }) +@EnableCaching +@EnableConfigurationProperties({ApplicationConfig.class, GithubConfig.class, StackOverflowConfig.class, + RetryQueryConfiguration.class}) public class ScrapperApplication { public static void main(String[] args) { SpringApplication.run(ScrapperApplication.class, args); diff --git a/scrapper/src/main/java/edu/java/configuration/BotClientConfiguration.java b/scrapper/src/main/java/edu/java/configuration/BotClientConfiguration.java index 906ade3..83dbc42 100644 --- a/scrapper/src/main/java/edu/java/configuration/BotClientConfiguration.java +++ b/scrapper/src/main/java/edu/java/configuration/BotClientConfiguration.java @@ -1,5 +1,7 @@ package edu.java.configuration; +import edu.java.RetryFactory; +import edu.java.RetryQueryConfiguration; import edu.java.client.bot.BotClient; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -8,6 +10,7 @@ import org.springframework.web.reactive.function.client.support.WebClientAdapter; import org.springframework.web.service.invoker.HttpServiceProxyFactory; import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; @Configuration public class BotClientConfiguration { @@ -16,10 +19,12 @@ public class BotClientConfiguration { private String botUrl; @Bean - public BotClient botClient() { + public BotClient botClient(RetryQueryConfiguration retryQueryConfiguration) { + Retry retry = RetryFactory.createRetry(retryQueryConfiguration, "bot"); WebClient webClient = WebClient.builder() .defaultStatusHandler(httpStatusCode -> true, clientResponse -> Mono.empty()) .defaultHeader("Content-Type", "application/json") + .filter(RetryFactory.createFilter(retry)) .baseUrl(botUrl).build(); HttpServiceProxyFactory httpServiceProxyFactory = HttpServiceProxyFactory diff --git a/scrapper/src/main/java/edu/java/service/jpa/JpaLinkService.java b/scrapper/src/main/java/edu/java/service/jpa/JpaLinkService.java index 14bf3bc..6cc40d2 100644 --- a/scrapper/src/main/java/edu/java/service/jpa/JpaLinkService.java +++ b/scrapper/src/main/java/edu/java/service/jpa/JpaLinkService.java @@ -76,6 +76,7 @@ public LinkResponse addTrackingLink(URL link, Long chatId) { linkEntity.setLastUpdate(lastUpdate); linkEntity.setLastCheck(OffsetDateTime.now()); linkEntity.setMetaInfo(linkInfo.metaInfo()); + linkRepository.save(linkEntity); chat.addLink(linkEntity); return new LinkResponse(linkEntity.getId(), link); } @@ -85,6 +86,7 @@ public LinkResponse addTrackingLink(URL link, Long chatId) { OffsetDateTime.now(), linkInfo.metaInfo() ); + linkRepository.save(linkEntity); chat.addLink(linkEntity); return new LinkResponse(linkEntity.getId(), link); } diff --git a/scrapper/src/main/java/edu/java/supplier/api/WebClientInfoSupplier.java b/scrapper/src/main/java/edu/java/supplier/api/WebClientInfoSupplier.java index c74922f..064a2b2 100644 --- a/scrapper/src/main/java/edu/java/supplier/api/WebClientInfoSupplier.java +++ b/scrapper/src/main/java/edu/java/supplier/api/WebClientInfoSupplier.java @@ -1,8 +1,10 @@ package edu.java.supplier.api; +import edu.java.RetryFactory; import java.time.OffsetDateTime; import org.springframework.http.MediaType; import org.springframework.web.reactive.function.client.WebClient; +import reactor.util.retry.Retry; public abstract class WebClientInfoSupplier implements InfoSupplier { private final WebClient webClient; @@ -15,9 +17,26 @@ public WebClientInfoSupplier(WebClient webClient) { this.webClient = webClient; } + public WebClientInfoSupplier(String baseUrl, Retry retry) { + this(WebClient.builder() + .baseUrl(baseUrl) + .filter(RetryFactory.createFilter(retry)) + .build() + ); + } + public T executeRequestGet(String uri, Class type, T defaultValue) { - return webClient.get().uri(uri).accept(MediaType.APPLICATION_JSON).retrieve().bodyToMono(type) - .onErrorReturn(defaultValue).block(); + try { + return webClient + .get() + .uri(uri) + .accept(MediaType.APPLICATION_JSON) + .retrieve() + .bodyToMono(type) + .block(); + } catch (Exception e) { + return defaultValue; + } } public abstract LinkInfo filterByDateTime(LinkInfo linkInfo, OffsetDateTime afterDateTime, String context); diff --git a/scrapper/src/main/java/edu/java/supplier/github/GithubInfoSupplier.java b/scrapper/src/main/java/edu/java/supplier/github/GithubInfoSupplier.java index b2860fe..6ac1a62 100644 --- a/scrapper/src/main/java/edu/java/supplier/github/GithubInfoSupplier.java +++ b/scrapper/src/main/java/edu/java/supplier/github/GithubInfoSupplier.java @@ -1,5 +1,7 @@ package edu.java.supplier.github; +import edu.java.RetryFactory; +import edu.java.RetryQueryConfiguration; import edu.java.configuration.ApplicationConfig; import edu.java.configuration.supplier.GithubConfig; import edu.java.supplier.api.LinkInfo; @@ -29,7 +31,11 @@ public class GithubInfoSupplier extends WebClientInfoSupplier { private final Pattern repositoryPattern; @Autowired - public GithubInfoSupplier(GithubConfig githubConfig, ApplicationConfig applicationConfig) { + public GithubInfoSupplier( + GithubConfig githubConfig, + ApplicationConfig applicationConfig, + RetryQueryConfiguration retryQueryConfiguration + ) { super(WebClient.builder() .baseUrl(githubConfig.url()) .defaultHeaders(headers -> { @@ -37,20 +43,13 @@ public GithubInfoSupplier(GithubConfig githubConfig, ApplicationConfig applicati headers.set("Authorization", "Bearer " + applicationConfig.githubToken()); } }) + .filter(RetryFactory.createFilter(RetryFactory.createRetry(retryQueryConfiguration, TYPE_SUPPLIER))) .build() ); repositoryPattern = Pattern.compile(githubConfig.patterns().repository()); eventResolver = new GithubEventResolver(); } - public GithubInfoSupplier( - GithubConfig githubConfig - ) { - super(githubConfig.url()); - this.eventResolver = new GithubEventResolver(); - repositoryPattern = Pattern.compile(githubConfig.patterns().repository()); - } - public String getTypeSupplier() { return TYPE_SUPPLIER; } diff --git a/scrapper/src/main/java/edu/java/supplier/stackoverflow/StackOverflowInfoSupplier.java b/scrapper/src/main/java/edu/java/supplier/stackoverflow/StackOverflowInfoSupplier.java index 1869f5d..cd1ac8b 100644 --- a/scrapper/src/main/java/edu/java/supplier/stackoverflow/StackOverflowInfoSupplier.java +++ b/scrapper/src/main/java/edu/java/supplier/stackoverflow/StackOverflowInfoSupplier.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import edu.java.RetryFactory; +import edu.java.RetryQueryConfiguration; import edu.java.configuration.supplier.StackOverflowConfig; import edu.java.supplier.api.EventResolver; import edu.java.supplier.api.LinkInfo; @@ -36,8 +38,12 @@ public class StackOverflowInfoSupplier extends WebClientInfoSupplier { private final ObjectMapper mapper; - public StackOverflowInfoSupplier(StackOverflowConfig config, ObjectMapper mapper) { - super(config.url()); + public StackOverflowInfoSupplier( + StackOverflowConfig config, + ObjectMapper mapper, + RetryQueryConfiguration retryQueryConfiguration + ) { + super(config.url(), RetryFactory.createRetry(retryQueryConfiguration, TYPE_SUPPLIER)); questionsPattern = Pattern.compile(config.patterns().questions()); this.mapper = mapper; eventResolver = new StackOverflowEventResolver(); diff --git a/scrapper/src/main/resources/application.yml b/scrapper/src/main/resources/application.yml index 4274471..cf9d148 100644 --- a/scrapper/src/main/resources/application.yml +++ b/scrapper/src/main/resources/application.yml @@ -5,7 +5,7 @@ app: force-check-delay: 20s max-links-per-check: 100 github-token: ${GITHUB_TOKEN} - database-access-type: jdbc + database-access-type: jpa supplier: github: @@ -30,6 +30,13 @@ spring: hibernate: ddl-auto: validate open-in-view: false + cache: + jcache: + provider: com.github.benmanes.caffeine.jcache.spi.CaffeineCachingProvider + caffeine: + spec: maximumSize=100000,expireAfterAccess=3600s + cache-names: + - rate-limit-bucket server: port: 8080 @@ -43,3 +50,38 @@ springdoc: bot: url: http://localhost:8090 + +retry-query: + retries: + - target: github + type: exponential + max-attempts: 3 + min-delay: 1s + max-delay: 10s + codes: 429 + - target: stackoverflow + type: exponential + max-attempts: 3 + min-delay: 1s + max-delay: 10s + codes: 429 + - target: bot + type: exponential + max-attempts: 3 + min-delay: 1s + max-delay: 10s + codes: 429 + +bucket4j: + enabled: true + filters: + - cache-name: rate-limit-bucket + url: .* + http-status-code: too_many_requests + rate-limits: + - bandwidths: + - capacity: 1000 + time: 1 + unit: hours + refill-speed: interval + cache-key: getRemoteAddr() diff --git a/scrapper/src/test/java/edu/java/scrapper/IntegrationEnvironment.java b/scrapper/src/test/java/edu/java/scrapper/IntegrationEnvironment.java index 336989d..a54fc45 100644 --- a/scrapper/src/test/java/edu/java/scrapper/IntegrationEnvironment.java +++ b/scrapper/src/test/java/edu/java/scrapper/IntegrationEnvironment.java @@ -10,6 +10,7 @@ import liquibase.database.jvm.JdbcConnection; import liquibase.resource.DirectoryResourceAccessor; import lombok.SneakyThrows; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.JdbcDatabaseContainer; @@ -17,6 +18,7 @@ import org.testcontainers.junit.jupiter.Testcontainers; @Testcontainers +@DirtiesContext public abstract class IntegrationEnvironment { public static PostgreSQLContainer POSTGRES; diff --git a/scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcChatLinkTest.java b/scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcChatLinkTest.java similarity index 98% rename from scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcChatLinkTest.java rename to scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcChatLinkTest.java index 19c6053..5449311 100644 --- a/scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcChatLinkTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcChatLinkTest.java @@ -1,4 +1,4 @@ -package edu.java.scrapper.jdbc; +package edu.java.scrapper.repository.jdbc; import edu.java.dto.Chat; import edu.java.dto.Link; diff --git a/scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcChatTest.java b/scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcChatTest.java similarity index 96% rename from scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcChatTest.java rename to scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcChatTest.java index 40710d4..b325e23 100644 --- a/scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcChatTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcChatTest.java @@ -1,4 +1,4 @@ -package edu.java.scrapper.jdbc; +package edu.java.scrapper.repository.jdbc; import edu.java.dto.Chat; import edu.java.repository.jdbc.JdbcChatRepository; diff --git a/scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcLinkTest.java b/scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcLinkTest.java similarity index 98% rename from scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcLinkTest.java rename to scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcLinkTest.java index 7516a75..8cb5260 100644 --- a/scrapper/src/test/java/edu/java/scrapper/jdbc/JdbcLinkTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/repository/jdbc/JdbcLinkTest.java @@ -1,4 +1,4 @@ -package edu.java.scrapper.jdbc; +package edu.java.scrapper.repository.jdbc; import edu.java.dto.Link; import edu.java.repository.jdbc.JdbcLinkRepository; @@ -18,6 +18,7 @@ @SpringBootTest public class JdbcLinkTest extends IntegrationEnvironment { + @Autowired private JdbcLinkRepository jdbcLinkRepository; diff --git a/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqChatLinkTest.java b/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqChatLinkTest.java new file mode 100644 index 0000000..d9a43d7 --- /dev/null +++ b/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqChatLinkTest.java @@ -0,0 +1,82 @@ +package edu.java.scrapper.repository.jooq; + +import edu.java.dto.Chat; +import edu.java.dto.Link; +import edu.java.repository.jooq.JooqChatLinkRepository; +import edu.java.repository.jooq.JooqChatRepository; +import edu.java.repository.jooq.JooqLinkRepository; +import edu.java.scrapper.IntegrationEnvironment; +import edu.java.util.URLCreator; +import java.time.OffsetDateTime; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.transaction.annotation.Transactional; + +@SpringBootTest +public class JooqChatLinkTest extends IntegrationEnvironment { + + private static final OffsetDateTime MIN = + OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, OffsetDateTime.now().getOffset()); + + private static final OffsetDateTime MAX = + OffsetDateTime.of(2025, 1, 1, 0, 0, 0, 0, OffsetDateTime.now().getOffset()); + + + @Autowired + private JooqChatRepository jooqChatRepository; + + @Autowired + private JooqLinkRepository jooqLinkRepository; + + @Autowired + private JooqChatLinkRepository jooqChatLinkRepository; + + private Link link; + + @BeforeEach + void setUp() { + link = new Link(0L, URLCreator.createURL("https://google.com"), MIN, MAX, ""); + } + + @Test + @Transactional + @Rollback + void findAllLinkByChatIdTest() { + jooqChatRepository.add(41L); + Long linkId = jooqLinkRepository.add(link); + jooqChatLinkRepository.add(41L, linkId); + Assertions.assertThat(jooqChatLinkRepository.findAllLinkByChatId(41L)).contains( + link = new Link( + linkId, URLCreator.createURL("https://google.com"), MIN, MAX, "" + ) + ); + } + + @Test + @Transactional + @Rollback + void findAllChatByLinkIdTest() { + jooqChatRepository.add(41L); + Long linkId = jooqLinkRepository.add(link); + jooqChatLinkRepository.add(41L, linkId); + Assertions.assertThat(jooqChatLinkRepository.findAllChatByLinkId(linkId)).contains(new Chat(41L)); + } + + @Test + @Transactional + @Rollback + void removeLinkFromDatabaseTest() { + jooqChatRepository.add(41L); + Long linkId = jooqLinkRepository.add(link); + jooqChatLinkRepository.add(41L, linkId); + jooqChatLinkRepository.remove(41L, linkId); + Assertions.assertThat(jooqChatLinkRepository.findAllLinkByChatId(41L)).doesNotContain( + link = + new Link(linkId, URLCreator.createURL("https://google.com"), MIN, MAX, "") + ); + } +} diff --git a/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqChatTest.java b/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqChatTest.java new file mode 100644 index 0000000..1a4c3fd --- /dev/null +++ b/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqChatTest.java @@ -0,0 +1,36 @@ +package edu.java.scrapper.repository.jooq; + +import edu.java.dto.Chat; +import edu.java.repository.jooq.JooqChatRepository; +import edu.java.scrapper.IntegrationEnvironment; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.transaction.annotation.Transactional; + +@SpringBootTest +public class JooqChatTest extends IntegrationEnvironment { + + @Autowired + private JooqChatRepository jooqChatRepository; + + @Test + @Transactional + @Rollback + void addChatTest() { + jooqChatRepository.add(41L); + Assertions.assertThat(jooqChatRepository.findAll()).contains(new Chat(41L)); + } + + @Test + @Transactional + @Rollback + void addAndDeleteChatTest() { + jooqChatRepository.add(41L); + jooqChatRepository.remove(41L); + Assertions.assertThat(jooqChatRepository.findAll()).doesNotContain(new Chat(41L)); + } + +} diff --git a/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqLinkTest.java b/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqLinkTest.java new file mode 100644 index 0000000..73a0ac0 --- /dev/null +++ b/scrapper/src/test/java/edu/java/scrapper/repository/jooq/JooqLinkTest.java @@ -0,0 +1,87 @@ +package edu.java.scrapper.repository.jooq; + +import edu.java.dto.Link; +import edu.java.repository.jooq.JooqLinkRepository; +import edu.java.scrapper.IntegrationEnvironment; +import edu.java.util.URLCreator; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.transaction.annotation.Transactional; + +@SpringBootTest +public class JooqLinkTest extends IntegrationEnvironment { + + private static final OffsetDateTime MIN = + OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, OffsetDateTime.now().getOffset()); + + private static final OffsetDateTime MAX = + OffsetDateTime.of(2025, 1, 1, 0, 0, 0, 0, OffsetDateTime.now().getOffset()); + + @Autowired + private JooqLinkRepository jooqLinkRepository; + + private Link link; + private Link link2; + + @BeforeEach + void setUp() { + link = new Link(0L, URLCreator.createURL("https://google.com"), MIN, MAX, ""); + link2 = new Link(0L, URLCreator.createURL("https://tinkoff.ru"), MIN, MAX, ""); + } + + @Test + @Transactional + @Rollback + void addLinkTest() { + Long id = jooqLinkRepository.add(link); + Optional linkFromDatabase = jooqLinkRepository.findById(id); + Assertions.assertThat(linkFromDatabase.get().url()).isEqualTo(link.url()); + } + + @Test + @Transactional + @Rollback + void removeLinkTest() { + jooqLinkRepository.add(link); + Optional linkFromDatabase = jooqLinkRepository.findByUrl(link.url()); + jooqLinkRepository.remove(linkFromDatabase.get().linkId()); + Optional actualResult = jooqLinkRepository.findByUrl(link.url()); + Assertions.assertThat(actualResult).isEmpty(); + } + + @Test + @Transactional + @Rollback + void getAllLinks() { + jooqLinkRepository.add(link); + jooqLinkRepository.add(link2); + List linksFromDatabase = jooqLinkRepository.findAll(); + Assertions.assertThat(linksFromDatabase).map(Link::url).contains(link.url(), link2.url()); + } + + @Test + @Transactional + @Rollback + void oldLinksTest() { + link = new Link(0L, URLCreator.createURL("https://google.com"), MIN, OffsetDateTime.now(), ""); + link2 = new Link( + 0L, + URLCreator.createURL("https://tinkoff.ru"), + MIN, + OffsetDateTime.now().minus(Duration.ofMinutes(60)), + "" + ); + jooqLinkRepository.add(link); + jooqLinkRepository.add(link2); + List linksFromDatabase = jooqLinkRepository.findLinksCheckedAfter(Duration.ofMinutes(10), 2); + Assertions.assertThat(linksFromDatabase).map(Link::url).contains(link2.url()); + } +} diff --git a/scrapper/src/test/java/edu/java/scrapper/service/JdbcLinkServiceTest.java b/scrapper/src/test/java/edu/java/scrapper/service/jdbc/JdbcLinkServiceTest.java similarity index 55% rename from scrapper/src/test/java/edu/java/scrapper/service/JdbcLinkServiceTest.java rename to scrapper/src/test/java/edu/java/scrapper/service/jdbc/JdbcLinkServiceTest.java index 9f0c37e..ff47005 100644 --- a/scrapper/src/test/java/edu/java/scrapper/service/JdbcLinkServiceTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/service/jdbc/JdbcLinkServiceTest.java @@ -1,23 +1,31 @@ -package edu.java.scrapper.service; +package edu.java.scrapper.service.jdbc; import edu.java.dto.Chat; import edu.java.dto.Link; import edu.java.dto.response.ListChatsResponse; import edu.java.exception.LinkNotFoundException; import edu.java.repository.jdbc.JdbcChatLinkRepository; +import edu.java.repository.jdbc.JdbcChatRepository; import edu.java.repository.jdbc.JdbcLinkRepository; import edu.java.scrapper.IntegrationEnvironment; import edu.java.service.LinkService; import edu.java.service.TelegramChatService; +import edu.java.supplier.InfoSuppliers; +import edu.java.supplier.api.LinkInfo; +import edu.java.supplier.github.GithubInfoSupplier; import edu.java.util.URLCreator; import java.net.URL; import java.time.OffsetDateTime; import java.util.List; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; import org.springframework.transaction.annotation.Transactional; @SpringBootTest @@ -29,17 +37,44 @@ public class JdbcLinkServiceTest extends IntegrationEnvironment { @Autowired private TelegramChatService chatService; + @Autowired + private JdbcChatRepository chatRepository; + @Autowired private JdbcLinkRepository linkRepository; @Autowired private JdbcChatLinkRepository chatLinkRepository; + @MockBean + private GithubInfoSupplier supplier; + + @MockBean + private InfoSuppliers suppliers; + @Test @Transactional @Rollback - void getSubscribersTest() { + void addLinkTest() { + String url = "https://github.com/yuuusha/java-course-2023-backend"; + Mockito.when(supplier.fetchInfo(Mockito.any())) + .thenReturn(new LinkInfo(URLCreator.createURL(url), "github", List.of(), "")); + Mockito.when(supplier.isSupported(Mockito.any())).thenReturn(true); + Mockito.when(suppliers.getSupplierByTypeHost(Mockito.any())).thenReturn(supplier); chatService.registerChat(41L); + + var response = linkService.addTrackingLink(URLCreator.createURL(url), 41L); + Assertions.assertThat(linkRepository.findById(response.id()).get().url().toString()) + .isEqualTo(url); + Assertions.assertThat(chatLinkRepository.findAllLinkByChatId(41L)).map(Link::url) + .contains(URLCreator.createURL(url)); + } + + @Test + @Transactional + @Rollback + void getSubscribersTest() { + chatRepository.add(41L); URL url = URLCreator.createURL("https://github.com"); Long linkId = linkRepository.add(new Link(0L, url, OffsetDateTime.now(), OffsetDateTime.now(), "")); chatLinkRepository.add(41L, linkId); @@ -58,4 +93,8 @@ void removeNotExistedLinkTests() { .isInstanceOf(LinkNotFoundException.class); } + @DynamicPropertySource + static void jdbcProperties(DynamicPropertyRegistry registry) { + registry.add("app.database-access-type", () -> "jdbc"); + } } diff --git a/scrapper/src/test/java/edu/java/scrapper/service/JdbcTelegramChatServiceTest.java b/scrapper/src/test/java/edu/java/scrapper/service/jdbc/JdbcTelegramChatServiceTest.java similarity index 83% rename from scrapper/src/test/java/edu/java/scrapper/service/JdbcTelegramChatServiceTest.java rename to scrapper/src/test/java/edu/java/scrapper/service/jdbc/JdbcTelegramChatServiceTest.java index d4ee76b..84497ef 100644 --- a/scrapper/src/test/java/edu/java/scrapper/service/JdbcTelegramChatServiceTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/service/jdbc/JdbcTelegramChatServiceTest.java @@ -1,4 +1,4 @@ -package edu.java.scrapper.service; +package edu.java.scrapper.service.jdbc; import edu.java.exception.ChatAlreadyRegisteredException; import edu.java.exception.ChatNotFoundException; @@ -10,6 +10,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; import org.springframework.transaction.annotation.Transactional; @SpringBootTest @@ -17,6 +19,7 @@ public class JdbcTelegramChatServiceTest extends IntegrationEnvironment { @Autowired private TelegramChatService chatService; + @Autowired private JdbcChatRepository chatRepository; @@ -53,4 +56,9 @@ void deleteChatNotFoundTest() { Assertions.assertThatThrownBy(() -> chatService.deleteChat(41L)) .isInstanceOf(ChatNotFoundException.class); } + + @DynamicPropertySource + static void jdbcProperties(DynamicPropertyRegistry registry) { + registry.add("app.database-access-type", () -> "jdbc"); + } } diff --git a/scrapper/src/test/java/edu/java/scrapper/service/jpa/JpaChatServiceTest.java b/scrapper/src/test/java/edu/java/scrapper/service/jpa/JpaChatServiceTest.java new file mode 100644 index 0000000..f86bf6b --- /dev/null +++ b/scrapper/src/test/java/edu/java/scrapper/service/jpa/JpaChatServiceTest.java @@ -0,0 +1,65 @@ +package edu.java.scrapper.service.jpa; + +import edu.java.exception.ChatAlreadyRegisteredException; +import edu.java.exception.ChatNotFoundException; +import edu.java.repository.jpa.JpaChatRepository; +import edu.java.scrapper.IntegrationEnvironment; +import edu.java.service.jpa.JpaChatService; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.transaction.annotation.Transactional; + +@SpringBootTest +public class JpaChatServiceTest extends IntegrationEnvironment { + + @Autowired + private JpaChatService chatService; + + @Autowired + private JpaChatRepository chatRepository; + + + @Test + @Transactional + @Rollback + void registerChatTest() { + chatService.registerChat(41L); + Assertions.assertThat(chatRepository.findById(41L).isPresent()).isTrue(); + } + + @Test + @Transactional + @Rollback + void registerChatAlreadyExistTest() { + chatService.registerChat(41L); + Assertions.assertThatThrownBy(() -> chatService.registerChat(41L)) + .isInstanceOf(ChatAlreadyRegisteredException.class); + } + + @Test + @Transactional + @Rollback + void deleteChatTest() { + chatService.registerChat(41L); + chatService.deleteChat(41L); + Assertions.assertThat(chatRepository.findById(41L).isPresent()).isFalse(); + } + + @Test + @Transactional + @Rollback + void deleteChatNotFoundTest() { + Assertions.assertThatThrownBy(() -> chatService.deleteChat(41L)) + .isInstanceOf(ChatNotFoundException.class); + } + + @DynamicPropertySource + static void jpaProperties(DynamicPropertyRegistry registry) { + registry.add("app.database-access-type", () -> "jpa"); + } +} diff --git a/scrapper/src/test/java/edu/java/scrapper/service/jpa/JpaLinkServiceTest.java b/scrapper/src/test/java/edu/java/scrapper/service/jpa/JpaLinkServiceTest.java new file mode 100644 index 0000000..ba2f71e --- /dev/null +++ b/scrapper/src/test/java/edu/java/scrapper/service/jpa/JpaLinkServiceTest.java @@ -0,0 +1,98 @@ +package edu.java.scrapper.service.jpa; + +import edu.java.dto.Chat; +import edu.java.dto.response.ListChatsResponse; +import edu.java.exception.LinkNotFoundException; +import edu.java.repository.jpa.JpaChatRepository; +import edu.java.repository.jpa.JpaLinkRepository; +import edu.java.scrapper.IntegrationEnvironment; +import edu.java.service.TelegramChatService; +import edu.java.service.jpa.JpaLinkService; +import edu.java.supplier.InfoSuppliers; +import edu.java.supplier.api.LinkInfo; +import edu.java.supplier.github.GithubInfoSupplier; +import edu.java.util.URLCreator; +import java.net.URL; +import java.util.List; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.transaction.annotation.Transactional; + +@SpringBootTest +public class JpaLinkServiceTest extends IntegrationEnvironment { + + @Autowired + private JpaLinkService linkService; + + @Autowired + private TelegramChatService chatService; + + @Autowired + private JpaLinkRepository linkRepository; + + @Autowired + private JpaChatRepository chatRepository; + + @MockBean + private GithubInfoSupplier supplier; + + @MockBean + private InfoSuppliers suppliers; + + @Test + @Transactional + @Rollback + void addLinkTest() { + URL url = URLCreator.createURL("https://github.com/yuuusha/java-course-2023-backend"); + Mockito.when(supplier.fetchInfo(Mockito.any())) + .thenReturn(new LinkInfo(url, "github", List.of(), "")); + Mockito.when(supplier.isSupported(Mockito.any())).thenReturn(true); + Mockito.when(suppliers.getSupplierByTypeHost(Mockito.any())).thenReturn(supplier); + chatService.registerChat(41L); + + linkService.addTrackingLink(url, 41L); + Assertions.assertThat(linkRepository.findByUrl(String.valueOf(url)).get().toDto().url()) + .isEqualTo(url); + } + + @Test + @Transactional + @Rollback + void getSubscribersTest() { + chatService.registerChat(41L); + URL url = URLCreator.createURL("https://github.com/yuuusha/java-course-2023-backend"); + Mockito.when(supplier.fetchInfo(Mockito.any())) + .thenReturn(new LinkInfo(url, "github", List.of(), "")); + Mockito.when(supplier.isSupported(Mockito.any())).thenReturn(true); + Mockito.when(suppliers.getSupplierByTypeHost(Mockito.any())).thenReturn(supplier); + linkService.addTrackingLink(url, 41L); + + ListChatsResponse linkSubscribers = linkService.getLinkSubscribers(url); + Assertions.assertThat(linkSubscribers).extracting(ListChatsResponse::chats).isEqualTo(List.of(new Chat(41L))); + } + + @Test + @Transactional + @Rollback + void removeNotExistedLinkTests() { + chatService.registerChat(41L); + + Assertions.assertThatThrownBy(() -> linkService.deleteTrackingLink( + URLCreator.createURL("https://github.com"), + 41L + )) + .isInstanceOf(LinkNotFoundException.class); + } + + @DynamicPropertySource + static void jpaProperties(DynamicPropertyRegistry registry) { + registry.add("app.database-access-type", () -> "jpa"); + } +} diff --git a/scrapper/src/test/java/edu/java/scrapper/supplier/github/GithubInfoSupplierTest.java b/scrapper/src/test/java/edu/java/scrapper/supplier/github/GithubInfoSupplierTest.java index d514e26..8ec94cd 100644 --- a/scrapper/src/test/java/edu/java/scrapper/supplier/github/GithubInfoSupplierTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/supplier/github/GithubInfoSupplierTest.java @@ -1,11 +1,16 @@ package edu.java.scrapper.supplier.github; import com.github.tomakehurst.wiremock.WireMockServer; +import edu.java.RetryElement; +import edu.java.RetryQueryConfiguration; +import edu.java.configuration.ApplicationConfig; import edu.java.configuration.supplier.GithubConfig; import edu.java.configuration.supplier.GithubPatternConfig; import edu.java.supplier.api.LinkInfo; import edu.java.supplier.github.GithubInfoSupplier; import java.net.URI; +import java.time.Duration; +import java.util.List; import lombok.SneakyThrows; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -20,6 +25,25 @@ public class GithubInfoSupplierTest { private static WireMockServer server; + private static final ApplicationConfig NULL_APPLICATION_CONFIG = new ApplicationConfig( + null, + null, + null + ); + + private static final RetryQueryConfiguration RETRY_QUERY_CONFIGURATION = new RetryQueryConfiguration( + List.of(new RetryElement( + "github", + "fixed", + 1, + 1, + Duration.ofSeconds(1), + null, + List.of(429) + ) + ) + ); + @BeforeAll public static void setUp() { server = new WireMockServer(wireMockConfig().dynamicPort()); @@ -42,7 +66,7 @@ public void returnRightInformationTest() { GithubConfig config = new GithubConfig(server.baseUrl(), githubPatternConfig); - GithubInfoSupplier supplier = new GithubInfoSupplier(config); + GithubInfoSupplier supplier = new GithubInfoSupplier(config, NULL_APPLICATION_CONFIG, RETRY_QUERY_CONFIGURATION); LinkInfo info = supplier.fetchInfo( new URI("https://github.com/yuuusha/java-course-2023-backend").toURL() ); @@ -62,7 +86,7 @@ public void returnNullInformationWhenRepositoryWrongTest() { GithubConfig config = new GithubConfig(server.baseUrl(), githubPatternConfig); - GithubInfoSupplier supplier = new GithubInfoSupplier(config); + GithubInfoSupplier supplier = new GithubInfoSupplier(config, NULL_APPLICATION_CONFIG, RETRY_QUERY_CONFIGURATION); LinkInfo info = supplier.fetchInfo( new URI("https://github.com/yuuusha/java-course-2023-backend").toURL() ); @@ -77,7 +101,7 @@ public void returnNullInformationWithWrongUrlTest() { GithubConfig config = new GithubConfig(server.baseUrl(), githubPatternConfig); - GithubInfoSupplier supplier = new GithubInfoSupplier(config); + GithubInfoSupplier supplier = new GithubInfoSupplier(config, NULL_APPLICATION_CONFIG, RETRY_QUERY_CONFIGURATION); LinkInfo info = supplier.fetchInfo( new URI("https://github.com/yuuusha/test/test").toURL() ); diff --git a/scrapper/src/test/java/edu/java/scrapper/supplier/stackoverflow/StackOverflowInfoSupplierTest.java b/scrapper/src/test/java/edu/java/scrapper/supplier/stackoverflow/StackOverflowInfoSupplierTest.java index b96d4dc..fdbcb02 100644 --- a/scrapper/src/test/java/edu/java/scrapper/supplier/stackoverflow/StackOverflowInfoSupplierTest.java +++ b/scrapper/src/test/java/edu/java/scrapper/supplier/stackoverflow/StackOverflowInfoSupplierTest.java @@ -2,11 +2,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.WireMockServer; +import edu.java.RetryElement; +import edu.java.RetryQueryConfiguration; import edu.java.configuration.supplier.StackOverflowConfig; import edu.java.configuration.supplier.StackOverflowPatternConfig; import edu.java.supplier.api.LinkInfo; import edu.java.supplier.stackoverflow.StackOverflowInfoSupplier; import java.net.URI; +import java.time.Duration; +import java.util.List; import lombok.SneakyThrows; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -20,6 +24,19 @@ public class StackOverflowInfoSupplierTest { private static WireMockServer server; + private static final RetryQueryConfiguration RETRY_QUERY_CONFIGURATION = new RetryQueryConfiguration( + List.of(new RetryElement( + "stackoverflow", + "fixed", + 1, + 1, + Duration.ofSeconds(1), + null, + List.of(429) + ) + ) + ); + @BeforeAll public static void setUp() { server = new WireMockServer(wireMockConfig().dynamicPort()); @@ -50,7 +67,7 @@ public void returnRightInformationTest() { Mockito.when(stackOverflowPatternConfig.questions()).thenReturn("https://stackoverflow.com/questions/(\\d+).*"); StackOverflowConfig config = new StackOverflowConfig(server.baseUrl(), stackOverflowPatternConfig); - StackOverflowInfoSupplier supplier = new StackOverflowInfoSupplier(config, new ObjectMapper()); + StackOverflowInfoSupplier supplier = new StackOverflowInfoSupplier(config, new ObjectMapper(), RETRY_QUERY_CONFIGURATION); LinkInfo info = supplier.fetchInfo( new URI( "https://stackoverflow.com/questions/69228850/spring-boot-with-postgres-hikaripool-1-exception-during-pool-initializatio").toURL() @@ -71,7 +88,7 @@ public void returnNullInformationWhenRepositoryWrongTest() { Mockito.when(stackOverflowPatternConfig.questions()).thenReturn("https://stackoverflow.com/wrongUrl/(\\d+).*"); StackOverflowConfig config = new StackOverflowConfig(server.baseUrl(), stackOverflowPatternConfig); - StackOverflowInfoSupplier supplier = new StackOverflowInfoSupplier(config, new ObjectMapper()); + StackOverflowInfoSupplier supplier = new StackOverflowInfoSupplier(config, new ObjectMapper(), RETRY_QUERY_CONFIGURATION); LinkInfo info = supplier.fetchInfo( new URI("https://stackoverflow.com/questions/69228850/spring-boot-with-postgres-hikaripool-1-exception-during-pool-initializatio").toURL() );