Merge pull request '[Fix] Movie missing translations' (#2) from fix/movie-missing-translations into main

Reviewed-on: #2

.env.example

@@ -0,0 +1,3 @@
+KODIK_TOKEN=YOUR_KODIK_TOKEN
+EUREKA_SCHEMA=http
+EUREKA_HOST=anyame-vue-bff:8080

Dockerfile.prod

@@ -1,26 +1,40 @@
+# Build
+FROM maven:3.9.6-eclipse-temurin-21 AS builder
+WORKDIR /workspace
+COPY pom.xml .
+RUN --mount=type=cache,target=/root/.m2 mvn dependency:go-offline -B
+COPY src ./src
+RUN --mount=type=cache,target=/root/.m2 mvn clean package -DskipTests
+
+# Create optimized runtime
 FROM eclipse-temurin:21 AS app-build
 ENV RELEASE=21
 
 WORKDIR /opt/build
-COPY ./target/*.jar ./application.jar
+COPY --from=builder /workspace/target/*.jar ./application.jar
 
 RUN java -Djarmode=layertools -jar application.jar extract
 RUN $JAVA_HOME/bin/jlink \
-         --add-modules `jdeps --ignore-missing-deps -q -recursive --multi-release ${RELEASE} --print-module-deps -cp 'dependencies/BOOT-INF/lib/*' application.jar` \
-         --strip-debug \
-         --no-man-pages \
-         --no-header-files \
-         --compress=2 \
-         --output jdk
+    --add-modules $(jdeps --ignore-missing-deps -q -recursive --multi-release ${RELEASE} --print-module-deps -cp 'dependencies/BOOT-INF/lib/*' application.jar),jdk.crypto.ec,jdk.security.auth,jdk.crypto.cryptoki \
+    --strip-debug \
+    --no-man-pages \
+    --no-header-files \
+    --compress=2 \
+    --output jdk
 
-FROM debian:buster-slim
+# Run
+FROM debian:bookworm-slim
 
 ARG BUILD_PATH=/opt/build
 ENV JAVA_HOME=/opt/jdk
 ENV PATH="${JAVA_HOME}/bin:${PATH}"
 
 RUN groupadd --gid 1000 spring-app \
-  && useradd --uid 1000 --gid spring-app --shell /bin/bash --create-home spring-app
+    && useradd --uid 1000 --gid spring-app --shell /bin/bash --create-home spring-app
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
 
 USER spring-app:spring-app
 WORKDIR /opt/workspace

compose.yml

@@ -0,0 +1,15 @@
+services:
+  kodik-extractor:
+    image: anyame-kodik-extractor:latest
+    ports:
+      - 8081:8080
+    env_file: .env
+    networks:
+      - anyame-shared
+      - elk-network
+
+networks:
+  anyame-shared:
+    external: true
+  elk-network:
+    external: true

pom.xml

@@ -6,13 +6,13 @@
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
-        <version>3.5.0</version>
+        <version>3.5.5</version>
         <relativePath /> <!-- lookup parent from repository -->
     </parent>
     <groupId>com.backend.extractor.kodik.service</groupId>
-    <artifactId>anyame-backend</artifactId>
+    <artifactId>anyame-kodik-extractor-backend</artifactId>
     <version>0.0.1-SNAPSHOT</version>
-    <name>anyame-backend</name>
+    <name>anyame-kodik-extractor-backend</name>
     <description>Kodik video URL extractor for anyame</description>
     <url />
     <licenses>
@@ -34,6 +34,8 @@
         <jsoup.version>1.20.1</jsoup.version>
         <spring-dotenv.version>4.0.0</spring-dotenv.version>
         <springdoc-openapi-starter.version>2.8.9</springdoc-openapi-starter.version>
+        <spring-eureka-client.version>4.3.0</spring-eureka-client.version>
+        <logstash-logback-encoder.version>8.1</logstash-logback-encoder.version>
     </properties>
     <dependencies>
         <dependency>
@@ -66,6 +68,16 @@
             <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
             <version>${springdoc-openapi-starter.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
+            <version>${spring-eureka-client.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>net.logstash.logback</groupId>
+            <artifactId>logstash-logback-encoder</artifactId>
+            <version>${logstash-logback-encoder.version}</version>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>

src/main/java/com/backend/extractor/kodik/service/anyame_backend/component/KodikAPITokenProvider.java

@@ -5,7 +5,7 @@ import org.springframework.stereotype.Component;
 
 @Component
 public class KodikAPITokenProvider {
-    @Value("${KODIK_TOKEN}")
+    @Value("${kodik.token}")
     private String kodikToken;
 
     public String getKodikToken() {

src/main/java/com/backend/extractor/kodik/service/anyame_backend/config/CacheConfig.java

@@ -0,0 +1,21 @@
+package com.backend.extractor.kodik.service.anyame_backend.config;
+
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class CacheConfig {
+
+    @Bean
+    public CacheManager cacheManager() {
+        return new ConcurrentMapCacheManager(
+                "kodikIDPlayerLink",
+                "shikimoriIDPlayerLink",
+                "kinopoiskIDPlayerLink",
+                "imdbIDPlayerLink",
+                "kodikPage");
+    }
+
+}

src/main/java/com/backend/extractor/kodik/service/anyame_backend/controller/ExtractController.java

@@ -2,6 +2,8 @@ package com.backend.extractor.kodik.service.anyame_backend.controller;
 
 import java.io.IOException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -23,7 +25,7 @@ import com.backend.extractor.kodik.service.anyame_backend.service.KodikURLDecode
 @RestController
 @RequestMapping("/extract")
 public class ExtractController {
-
+    Logger logger = LoggerFactory.getLogger(ExtractController.class);
     private final KodikExtractService extractService;
     private final KodikURLDecoderService decoderService;
 
@@ -41,14 +43,18 @@ public class ExtractController {
             links.decodeLinks(decoderService);
             return links;
         } catch (KodikExtractionException e) {
-            throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "cannot retrieve metadat", e);
+            logger.error("cannot retrieve metadata", e);
+            throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "cannot retrieve metadata", e);
         } catch (IOException e) {
             throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "i/o error");
         } catch (KodikPlayerNotFoundException e) {
+            logger.error("cannoot find player", e);
             throw new ResponseStatusException(HttpStatus.NOT_FOUND, "player not found");
         } catch (KodikAPINotFoundException e) {
+            logger.error("api endpoint not found", e);
             throw new ResponseStatusException(HttpStatus.NOT_FOUND, "api endpoint not found");
         } catch (KodikDecryptionException e) {
+            logger.error("cannot decrypt link", e);
             throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "cannot decode links");
         }
     }

src/main/java/com/backend/extractor/kodik/service/anyame_backend/controller/MetadataController.java

@@ -2,6 +2,8 @@ package com.backend.extractor.kodik.service.anyame_backend.controller;
 
 import java.io.IOException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -9,63 +11,70 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.server.ResponseStatusException;
 
-import com.backend.extractor.kodik.service.anyame_backend.api.KodikPlayerAPI;
-import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikPlayerResponse;
-import com.backend.extractor.kodik.service.anyame_backend.component.KodikAPITokenProvider;
 import com.backend.extractor.kodik.service.anyame_backend.exception.KodikExtractionException;
 import com.backend.extractor.kodik.service.anyame_backend.service.KodikExtractService;
+import com.backend.extractor.kodik.service.anyame_backend.service.KodikPlayerLinkResolveService;
 import com.backend.extractor.kodik.service.anyame_backend.service.model.KodikMetadata;
 
-import retrofit2.Response;
-
 @RestController
 @RequestMapping("/metadata")
 public class MetadataController {
-    private final KodikPlayerAPI kodikPlayerAPI;
-    private final KodikAPITokenProvider tokenProvider;
+    Logger logger = LoggerFactory.getLogger(MetadataController.class);
     private final KodikExtractService extractService;
+    private final KodikPlayerLinkResolveService linkResolveService;
 
-    public MetadataController(
-            KodikPlayerAPI kodikPlayerAPI,
-            KodikAPITokenProvider tokenProvider, KodikExtractService extractService) {
-        this.kodikPlayerAPI = kodikPlayerAPI;
-        this.tokenProvider = tokenProvider;
+    public MetadataController(KodikExtractService extractService, KodikPlayerLinkResolveService linkResolveService) {
         this.extractService = extractService;
+        this.linkResolveService = linkResolveService;
     }
 
     @GetMapping("/shikimori")
     public KodikMetadata shikimori(@RequestParam("id") String shikimoriId) {
-        try {
-            Response<KodikPlayerResponse> response = kodikPlayerAPI
-                    .getShikimoriPlayer(shikimoriId, tokenProvider.getKodikToken()).execute();
-            if (!response.isSuccessful()) {
-                throw new ResponseStatusException(HttpStatus.NOT_FOUND, "cannot find player, check id validity");
-            }
-            KodikPlayerResponse responseResult = response.body();
-            String url = responseResult.getLink();
+        return findMetadata(() -> {
+            String url = linkResolveService.resolveShikimoriLink(shikimoriId);
             return extractService.getMetadata(url);
-        } catch (KodikExtractionException e) {
-            throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "cannot retrieve metadat", e);
-        } catch (IOException e) {
-            throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "i/o error");
-        }
+        });
     }
 
     @GetMapping("/kodik")
     public KodikMetadata kodik(@RequestParam("id") String kodikId) {
-        try {
-            Response<KodikPlayerResponse> response = kodikPlayerAPI
-                    .getKodikIDPlayer(kodikId, tokenProvider.getKodikToken()).execute();
-            if (!response.isSuccessful()) {
-                throw new ResponseStatusException(HttpStatus.NOT_FOUND, "cannot find player, check id validity");
-            }
-            KodikPlayerResponse responseResult = response.body();
-            String url = responseResult.getLink();
+        return findMetadata(() -> {
+            String url = linkResolveService.resolveKodikLink(kodikId);
             return extractService.getMetadata(url);
+        });
+    }
+
+    @GetMapping("/kinopoisk")
+    public KodikMetadata kinopoisk(@RequestParam("id") String kinopoiskId) {
+        return findMetadata(() -> {
+            String url = linkResolveService.resolveKinopoiskLink(kinopoiskId);
+            return extractService.getMetadata(url);
+        });
+    }
+
+    @GetMapping("/imdb")
+    public KodikMetadata imdb(@RequestParam("id") String imdbId) {
+        return findMetadata(() -> {
+            String url = linkResolveService.resolveIMDBLink(imdbId);
+            return extractService.getMetadata(url);
+        });
+    }
+
+    private KodikMetadata findMetadata(SupplierWithException<KodikMetadata> supplier) {
+        try {
+            return supplier.get();
         } catch (KodikExtractionException e) {
+            logger.error("cannot retrieve metadata", e);
             throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "cannot retrieve metadata", e);
         } catch (IOException e) {
             throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "i/o error");
+        } catch (Exception e) {
+            logger.error("Unknown error", e);
+            throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "unknown error");
         }
     }
+
+    private static interface SupplierWithException<T> {
+        T get() throws Exception;
+    }
 }

src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikExtractService.java

@@ -4,7 +4,6 @@ import java.io.IOException;
 
 import org.springframework.stereotype.Service;
 
-import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikTranslation;
 import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikVideoLinks;
 import com.backend.extractor.kodik.service.anyame_backend.exception.KodikAPINotFoundException;
 import com.backend.extractor.kodik.service.anyame_backend.exception.KodikExtractionException;

src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikHtmlParserService.java

@@ -10,6 +10,8 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
 import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikEpisode;
@@ -22,6 +24,7 @@ import com.backend.extractor.kodik.service.anyame_backend.service.model.KodikMet
  */
 @Service
 public class KodikHtmlParserService {
+    private static final Logger log = LoggerFactory.getLogger(KodikHtmlParserService.class);
     private static final Pattern VIDEO_INFO_TYPE = Pattern.compile("https://kodik\\.info/([^\\s/]+)");
 
     public KodikMetadata parseMetadata(String html) {
@@ -37,7 +40,10 @@ public class KodikHtmlParserService {
 
     private List<KodikTranslation> parseTranslations(Document doc) {
         List<KodikTranslation> translations = new ArrayList<>();
-        Elements translationOptions = doc.select(".serial-translations-box select option");
+        String possibleMediaType = findVideoType(doc.html()).orElse("serial");
+        boolean isMovie = possibleMediaType.equals("video");
+        String videoType = isMovie ? "movie" : possibleMediaType;
+        Elements translationOptions = doc.select(String.format(".%s-translations-box select option", videoType));
 
         for (Element option : translationOptions) {
             String id = option.attr("value");
@@ -46,24 +52,28 @@ public class KodikHtmlParserService {
             String mediaHash = option.attr("data-media-hash");
             String mediaType = option.attr("data-media-type");
             String translationType = option.attr("data-translation-type");
-            int episodeCount = parseIntSafely(option.attr("data-episode-count"));
+            int episodeCount = isMovie ? 1 : parseIntSafely(option.attr("data-episode-count"));
 
             translations.add(new KodikTranslation(id, title, mediaId, mediaHash,
                     mediaType, translationType, episodeCount));
         }
 
-        Elements seasonOptions = doc.select(".serial-seasons-box select option");
-        Elements seriesOptions = doc.select(".serial-series-box select option");
-        for (Element option : seasonOptions) {
-            if (!option.hasAttr("selected")) {
-                continue;
+        Elements seasonOptions = doc.select(String.format(".%s-seasons-box select option", videoType));
+        Elements seriesOptions = doc.select(String.format(".%s-series-box select option", videoType));
+        if (translations.isEmpty()) {
+            for (Element option : seasonOptions) {
+                if (!option.hasAttr("selected")) {
+                    continue;
+                }
+                String id = option.attr("value");
+                String title = option.attr("data-translation-title");
+                String mediaId = option.attr("data-serial-id");
+                String mediaHash = option.attr("data-serial-hash");
+
+                translations
+                        .add(new KodikTranslation(id, title, mediaId, mediaHash, possibleMediaType, "",
+                                seriesOptions.size()));
             }
-            String id = option.attr("value");
-            String title = option.attr("data-translation-title");
-            String mediaId = option.attr("data-serial-id");
-            String mediaHash = option.attr("data-serial-hash");
-            String mediaType = findVideoType(doc.html()).orElse("serial");
-            translations.add(new KodikTranslation(id, title, mediaId, mediaHash, mediaType, "", seriesOptions.size()));
         }
 
         return translations;
@@ -93,8 +103,12 @@ public class KodikHtmlParserService {
 
     private static int parseIntSafely(String value) {
         try {
+            if (value.contains("~")) {
+                return Integer.parseInt(value.split("~")[1]);
+            }
             return Integer.parseInt(value);
         } catch (NumberFormatException e) {
+            log.error(String.format("invalid number format %s", value), e);
             return 0;
         }
     }

src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikNetworkService.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
 
+import org.springframework.cache.annotation.Cacheable;
 import org.springframework.stereotype.Service;
 
 import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikVideoLinks;
@@ -28,6 +29,7 @@ public class KodikNetworkService {
         this.objectMapper = objectMapper;
     }
 
+    @Cacheable("kodikPage")
     public String fetchPage(String url) throws IOException {
         Request request = new Request.Builder()
                 .url(normalizeUrl(url))

src/main/java/com/backend/extractor/kodik/service/anyame_backend/service/KodikPlayerLinkResolveService.java

@@ -0,0 +1,61 @@
+package com.backend.extractor.kodik.service.anyame_backend.service;
+
+import java.io.IOException;
+
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Service;
+import org.springframework.web.server.ResponseStatusException;
+
+import com.backend.extractor.kodik.service.anyame_backend.api.KodikPlayerAPI;
+import com.backend.extractor.kodik.service.anyame_backend.api.model.KodikPlayerResponse;
+import com.backend.extractor.kodik.service.anyame_backend.component.KodikAPITokenProvider;
+
+import retrofit2.Response;
+
+@Service
+public class KodikPlayerLinkResolveService {
+    private final KodikPlayerAPI kodikPlayerAPI;
+    private final KodikAPITokenProvider tokenProvider;
+
+    public KodikPlayerLinkResolveService(KodikPlayerAPI kodikPlayerAPI, KodikAPITokenProvider tokenProvider) {
+        this.kodikPlayerAPI = kodikPlayerAPI;
+        this.tokenProvider = tokenProvider;
+    }
+
+    @Cacheable("kodikIDPlayerLink")
+    public String resolveKodikLink(String kodikID) throws IOException {
+        Response<KodikPlayerResponse> response = kodikPlayerAPI.getKodikIDPlayer(kodikID, tokenProvider.getKodikToken())
+                .execute();
+        return retrieveLink(response);
+    }
+
+    @Cacheable("shikimoriIDPlayerLink")
+    public String resolveShikimoriLink(String shikimoriID) throws IOException {
+        Response<KodikPlayerResponse> response = kodikPlayerAPI
+                .getShikimoriPlayer(shikimoriID, tokenProvider.getKodikToken()).execute();
+        return retrieveLink(response);
+    }
+
+    @Cacheable("kinopoiskIDPlayerLink")
+    public String resolveKinopoiskLink(String kinopoiskID) throws IOException {
+        Response<KodikPlayerResponse> response = kodikPlayerAPI
+                .getKinopoiskPlayer(kinopoiskID, tokenProvider.getKodikToken()).execute();
+        return retrieveLink(response);
+    }
+
+    @Cacheable("imdbIDPlayerLink")
+    public String resolveIMDBLink(String imdbID) throws IOException {
+        Response<KodikPlayerResponse> response = kodikPlayerAPI.getIMDBPlayer(imdbID, tokenProvider.getKodikToken())
+                .execute();
+        return retrieveLink(response);
+    }
+
+    private String retrieveLink(Response<KodikPlayerResponse> response) {
+        if (response.isSuccessful()) {
+            return response.body().getLink();
+        } else {
+            throw new ResponseStatusException(HttpStatus.NOT_FOUND, "cannot find player, check id validity");
+        }
+    }
+}

src/main/resources/application.properties

@@ -1,2 +1,4 @@
-spring.application.name=anyame-backend
-server.error.include-message=always
+spring.application.name=anyame-kodik-extractor-backend
+kodik.token=${KODIK_TOKEN}
+eureka.client.serviceUrl.defaultZone: ${EUREKA_SCHEMA}://${EUREKA_HOST}/eureka/
+

src/main/resources/logback.xml

@@ -0,0 +1,11 @@
+<configuration>
+    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+        <destination>logstash:5044</destination>
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="LOGSTASH" />
+    </root>
+</configuration>
+